From ee6a2783466828184932c6f1dbfecb070f72f770 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:14:00 -0400 Subject: [PATCH 01/14] fix(migrations): remove unnecessary migration scripts --- ...90000-correct-canonical-x-mitre-domains.js | 34 ----- ...0-repair-release-track-bundle-integrity.js | 136 ------------------ 2 files changed, 170 deletions(-) delete mode 100644 migrations/20260803190000-correct-canonical-x-mitre-domains.js delete mode 100644 migrations/20260805150000-repair-release-track-bundle-integrity.js diff --git a/migrations/20260803190000-correct-canonical-x-mitre-domains.js b/migrations/20260803190000-correct-canonical-x-mitre-domains.js deleted file mode 100644 index d94822d7..00000000 --- a/migrations/20260803190000-correct-canonical-x-mitre-domains.js +++ /dev/null @@ -1,34 +0,0 @@ -'use strict'; - -/** - * Correct canonical-domain successor revisions created from legacy collection - * appearance metadata. - * - * The original backfill now uses exact canonical collection TOC membership. - * Deployments that already ran its earlier form may contain domain-only - * successor revisions with domains inherited from secondary bundle - * appearances. This forward migration recognizes only semantic domain-only - * successors whose historical predecessor has an exact canonical TOC pin and - * creates another immutable revision with that authoritative domain union. - */ - -const logger = require('../app/lib/logger'); -const canonicalDomainMigration = require('./20260730230000-backfill-canonical-x-mitre-domains'); - -const MIGRATION_NAME = '20260803190000-correct-canonical-x-mitre-domains'; - -module.exports = { - async up(db, client) { - const report = await canonicalDomainMigration._private.run(db, client, { - migrationName: MIGRATION_NAME, - correctIncorrect: true, - }); - logger.info(`[${MIGRATION_NAME}] ${JSON.stringify(report)}`); - }, - - async down() { - logger.info( - `[${MIGRATION_NAME}] down migration is a no-op: immutable correction revisions are retained`, - ); - }, -}; diff --git a/migrations/20260805150000-repair-release-track-bundle-integrity.js b/migrations/20260805150000-repair-release-track-bundle-integrity.js deleted file mode 100644 index fee6e61a..00000000 --- a/migrations/20260805150000-repair-release-track-bundle-integrity.js +++ /dev/null @@ -1,136 +0,0 @@ -'use strict'; - -/** - * Repair frozen x-mitre-collection entries for persisted release-track graph - * manifests and recompute the exact STIX 2.0/2.1 download hashes for tagged - * snapshots. Draft snapshots may contain historical baseline manifests, but - * their exports remain live and therefore do not receive deterministic hashes. - */ - -const { isDeepStrictEqual } = require('node:util'); -const mongoose = require('mongoose'); -const logger = require('../app/lib/logger'); - -const MIGRATION_NAME = '20260805150000-repair-release-track-bundle-integrity'; - -function ensureMongooseUsesClient(client) { - if (client && mongoose.connection.readyState === 0) { - mongoose.connection.setClient(client); - } -} - -async function organizationIdentityRef(db) { - const systemConfig = await db - .collection('systemconfigurations') - .findOne({}, { sort: { created_at: -1 }, projection: { organization_identity_ref: 1 } }); - if (!systemConfig?.organization_identity_ref) { - throw new Error( - 'System configuration is missing organization_identity_ref; cannot repair graph bundles.', - ); - } - return systemConfig.organization_identity_ref; -} - -function expectedCollectionId(trackId) { - return `x-mitre-collection--${trackId.split('--')[1]}`; -} - -async function linkedGraphSnapshots(db) { - const manifests = await db - .collection('releaseTrackGraphManifests') - .find({ state: { $in: ['pending', 'active'] } }) - .sort({ track_id: 1, created_at: 1, _id: 1 }) - .toArray(); - const collectionNames = new Set( - (await db.listCollections({}, { nameOnly: true }).toArray()).map((entry) => entry.name), - ); - const linked = []; - - for (const manifest of manifests) { - if (!collectionNames.has(manifest.track_id)) continue; - const snapshot = await db.collection(manifest.track_id).findOne({ - graph_manifest_id: manifest.manifest_id, - modified: manifest.snapshot_modified, - }); - if (snapshot) linked.push({ manifest, snapshot }); - } - return { manifests, linked }; -} - -async function run(db, client, options = {}) { - ensureMongooseUsesClient(client); - const graphManifestService = require('../app/services/release-tracks/graph-manifest-service'); - const bundleHashService = require('../app/services/release-tracks/bundle-hash-service'); - const createdByRef = await organizationIdentityRef(db); - const { manifests, linked } = await linkedGraphSnapshots(db); - const report = { - manifests_scanned: manifests.length, - linked_snapshots: linked.length, - collection_entries_repaired: 0, - bundle_hashes_recomputed: 0, - draft_hashes_cleared: 0, - orphaned_manifests_skipped: manifests.length - linked.length, - dry_run: options.dryRun === true, - }; - - for (const { manifest, snapshot } of linked) { - const collectionEntry = await db.collection('releaseTrackGraphManifestEntries').findOne({ - manifest_id: manifest.manifest_id, - kind: 'collection', - }); - const collectionNeedsRepair = - !collectionEntry || - collectionEntry.object_ref !== expectedCollectionId(manifest.track_id) || - collectionEntry.revision_key !== `${expectedCollectionId(manifest.track_id)}::collection` || - collectionEntry.frozen_stix?.id !== expectedCollectionId(manifest.track_id) || - collectionEntry.frozen_stix?.created_by_ref !== createdByRef; - if (collectionNeedsRepair) report.collection_entries_repaired++; - - if (options.dryRun) { - if (typeof snapshot.version === 'string') report.bundle_hashes_recomputed++; - else if (snapshot.bundle_hashes) report.draft_hashes_cleared++; - continue; - } - - await graphManifestService.refreshCollectionEntry(snapshot, manifest); - - if (typeof snapshot.version !== 'string') { - if (snapshot.bundle_hashes) { - await db - .collection(manifest.track_id) - .updateOne({ _id: snapshot._id }, { $unset: { bundle_hashes: '' } }); - report.draft_hashes_cleared++; - } - continue; - } - - const bundleHashes = await bundleHashService.generateBundleHashes(snapshot); - if (!isDeepStrictEqual(snapshot.bundle_hashes, bundleHashes)) { - report.bundle_hashes_recomputed++; - await db - .collection(manifest.track_id) - .updateOne({ _id: snapshot._id }, { $set: { bundle_hashes: bundleHashes } }); - } - } - - return report; -} - -module.exports = { - async up(db, client) { - const report = await run(db, client); - logger.info(`[${MIGRATION_NAME}] ${JSON.stringify(report)}`); - }, - - async down() { - logger.info( - `[${MIGRATION_NAME}] down migration is a no-op: corrected collection identities and hashes are retained`, - ); - }, - - _private: { - run, - linkedGraphSnapshots, - expectedCollectionId, - }, -}; From c57cee508f85689731198af471a6388fa2b40a7e Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:54:10 -0400 Subject: [PATCH 02/14] chore(migrations): remove nightly compatibility remnants Remove tests, correction logic, and documentation tied to retired alpha/beta migrations. Document that prerelease databases must be recreated and reserve migrations for stable release upgrade paths. --- AGENTS.md | 3 + CONTRIBUTING.md | 8 + .../canonical-domain-migration.spec.js | 30 ---- .../deterministic-graph-migration.spec.js | 109 ------------- docs/README.md | 1 - docs/admin/canonical-domain-migration.md | 17 -- ...elease-track-bundle-integrity-migration.md | 27 ---- docs/developer/TODO.md | 24 +++ docs/developer/data-model.md | 4 +- .../developer/release-tracks/bundle-export.md | 7 - .../release-tracks/implementation-notes.md | 10 +- ...0000-backfill-canonical-x-mitre-domains.js | 146 +++--------------- 12 files changed, 60 insertions(+), 326 deletions(-) delete mode 100644 docs/admin/release-track-bundle-integrity-migration.md diff --git a/AGENTS.md b/AGENTS.md index 7c8fa090..49aa14f3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -148,6 +148,9 @@ parameter semantics in the `docs { }` block. ## Gotchas +- Database migrations support stable-release upgrade paths. Alpha and beta + databases are ephemeral and should be reset or recreated rather than carried + forward by permanent nightly-only migration scripts. - STIX version rules: the bundle envelope carries `spec_version` only in STIX 2.0 (2.1 removed it; each 2.1 *object* declares its own `spec_version`). Marking definitions have no `stix.modified`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45400760..0ef8503c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -136,6 +136,14 @@ When commits are pushed to a release branch (main, next, etc.), semantic-release Pre-release branches (alpha, beta) will generate pre-release versions with appropriate suffixes. +### Pre-release database compatibility + +Alpha and beta builds do not provide a persistent database upgrade contract. +Their schemas and stored data may change rapidly, so developers should reset or +recreate pre-release databases instead of adding long-lived migrations solely +to carry nightly data forward. Database migrations are reserved for supported +upgrade paths between stable releases. + ## Docker Image Publishing The project publishes Docker images to the GitHub Container Registry (ghcr.io) with these tags: diff --git a/app/tests/api/release-tracks/canonical-domain-migration.spec.js b/app/tests/api/release-tracks/canonical-domain-migration.spec.js index c5ed262c..9089e78f 100644 --- a/app/tests/api/release-tracks/canonical-domain-migration.spec.js +++ b/app/tests/api/release-tracks/canonical-domain-migration.spec.js @@ -457,7 +457,6 @@ describe('Canonical ATT&CK domain migration', function () { }); expect(report.verification).toEqual({ remaining_latest_domainless_target_objects: 0, - remaining_latest_incorrect_domain_objects: 0, remaining_domain_validation_bypasses: 0, }); @@ -526,35 +525,6 @@ describe('Canonical ATT&CK domain migration', function () { ); }); - it('corrects a previously generated domain-only successor from its exact TOC predecessor', async function () { - const latest = await mongoose.connection.db - .collection('attackObjects') - .findOne({ 'stix.id': campaignFixture.id }, { sort: { 'stix.modified': -1 } }); - const incorrect = structuredClone(latest); - delete incorrect._id; - incorrect.stix.modified = new Date(new Date(latest.stix.modified).getTime() + 1); - incorrect.stix.x_mitre_domains = ['enterprise-attack', 'ics-attack']; - incorrect.stix.x_mitre_modified_by_ref = 'identity--ffffffff-ffff-4fff-8fff-ffffffffffff'; - await mongoose.connection.db.collection('attackObjects').insertOne(incorrect); - - const report = await migration._private.run(migrationDb, migrationClient, { - migrationName: 'test-correct-canonical-x-mitre-domains', - correctIncorrect: true, - }); - expect(report.counts).toMatchObject({ - scanned_candidates: 1, - active_reposts: 1, - updated: 1, - failed: 0, - }); - expect(report.verification.remaining_latest_incorrect_domain_objects).toBe(0); - - const corrected = await mongoose.connection.db - .collection('attackObjects') - .findOne({ 'stix.id': campaignFixture.id }, { sort: { 'stix.modified': -1 } }); - expect(corrected.stix.x_mitre_domains).toEqual(['enterprise-attack']); - }); - it('is idempotent after canonical revisions and bypass removal are complete', async function () { const report = await migration._private.run(migrationDb, migrationClient); expect(report.counts.scanned_candidates).toBe(0); diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js index 0df0a791..2bd2ff5b 100644 --- a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -1,6 +1,5 @@ 'use strict'; -const crypto = require('node:crypto'); const mongoose = require('mongoose'); const request = require('supertest'); const { expect } = require('expect'); @@ -10,7 +9,6 @@ const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const migration = require('../../../../migrations/20260730180000-backfill-deterministic-snapshot-graphs'); -const bundleIntegrityMigration = require('../../../../migrations/20260805150000-repair-release-track-bundle-integrity'); const Relationship = require('../../../models/relationship-model'); const { ReleaseTrackGraphManifest, @@ -244,113 +242,6 @@ describe('Deterministic snapshot graph migration', function () { expect(objectIds).toContain(relationship.stix.id); }); - it('repairs graph collection identities and recomputes tagged bundle hashes', async function () { - const organizationIdentity = ( - await request(app) - .get('/api/config/organization-identity') - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) - ).body; - const manifests = await ReleaseTrackGraphManifest.find({ track_id: trackId }) - .sort({ created_at: 1 }) - .lean() - .exec(); - const collectionEntries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: { $in: manifests.map((manifest) => manifest.manifest_id) }, - kind: 'collection', - }) - .sort({ snapshot_modified: 1 }) - .lean() - .exec(); - - for (const [index, entry] of collectionEntries.entries()) { - const badId = `x-mitre-collection--00000000-0000-4000-8000-${String(index).padStart( - 12, - '0', - )}`; - await ReleaseTrackGraphManifestEntry.updateOne( - { _id: entry._id }, - { - $set: { - object_ref: badId, - revision_key: `${badId}::collection`, - 'frozen_stix.id': badId, - 'frozen_stix.created_by_ref': 'identity--00000000-0000-4000-8000-000000000000', - }, - }, - ).exec(); - } - await mongoose.connection.db.collection(trackId).updateMany( - { graph_manifest_id: { $in: manifests.map((manifest) => manifest.manifest_id) } }, - { - $set: { - bundle_hashes: { - manifest_id: manifests[0].manifest_id, - stix_2_0: '0'.repeat(64), - stix_2_1: '0'.repeat(64), - }, - }, - }, - ); - - const preview = await bundleIntegrityMigration._private.run(mongoose.connection.db, null, { - dryRun: true, - }); - expect(preview.collection_entries_repaired).toBe(collectionEntries.length); - expect(preview.bundle_hashes_recomputed).toBeGreaterThan(0); - - const report = await bundleIntegrityMigration._private.run(mongoose.connection.db); - expect(report.collection_entries_repaired).toBe(collectionEntries.length); - expect(report.bundle_hashes_recomputed).toBeGreaterThan(0); - - const repairedEntries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: { $in: manifests.map((manifest) => manifest.manifest_id) }, - kind: 'collection', - }) - .lean() - .exec(); - const expectedCollectionId = `x-mitre-collection--${trackId.split('--')[1]}`; - expect(new Set(repairedEntries.map((entry) => entry.frozen_stix.id))).toEqual( - new Set([expectedCollectionId]), - ); - expect( - repairedEntries.every( - (entry) => entry.frozen_stix.created_by_ref === organizationIdentity.stix.id, - ), - ).toBe(true); - - const taggedSnapshots = await mongoose.connection.db - .collection(trackId) - .find({ graph_manifest_id: { $exists: true }, version: { $type: 'string' } }) - .toArray(); - for (const snapshot of taggedSnapshots) { - for (const stixVersion of ['2.0', '2.1']) { - const bundle = ( - await request(app) - .get( - `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent( - snapshot.modified.toISOString(), - )}?format=bundle&stixVersion=${stixVersion}`, - ) - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200) - ).body; - if (stixVersion === '2.0') { - expect(bundle.objects.some((object) => object.type === 'x-mitre-collection')).toBe(false); - } - const hash = crypto - .createHash('sha256') - .update(JSON.stringify(bundle, null, 4), 'utf8') - .digest('hex'); - expect(hash).toBe(snapshot.bundle_hashes?.[`stix_2_${stixVersion.split('.')[1]}`]); - } - } - - const rerun = await bundleIntegrityMigration._private.run(mongoose.connection.db); - expect(rerun.collection_entries_repaired).toBe(0); - expect(rerun.bundle_hashes_recomputed).toBe(0); - }); - it('replays and activates a complete linked pending manifest after interruption', async function () { const snapshot = await mongoose.connection.db .collection(trackId) diff --git a/docs/README.md b/docs/README.md index 505fdb69..9666db1a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -64,7 +64,6 @@ Configuration, deployment, and identity provider setup. - [Release-Track Membership Reconciliation](admin/release-track-reconciliation.md): Inspect and repair durable object-backref protection failures - [Release-Track Destructive Audit Events](admin/release-track-audit.md): Inspect administrator track-deletion attempts - [Release-Track Deterministic Graph Migration](admin/release-track-graph-migration.md): Preview and operate the relationship-pin and snapshot-manifest backfill -- [Release-Track Bundle Integrity Migration](admin/release-track-bundle-integrity-migration.md): Repair frozen collection identities and deterministic bundle hashes - [ATT&CK Canonical-Domain Migration](admin/canonical-domain-migration.md): Understand the release-agnostic startup repair, inactive-revision handling, strict validation, and verification procedure ### Authentication diff --git a/docs/admin/canonical-domain-migration.md b/docs/admin/canonical-domain-migration.md index 9c7c1989..22728242 100644 --- a/docs/admin/canonical-domain-migration.md +++ b/docs/admin/canonical-domain-migration.md @@ -78,22 +78,6 @@ automation audit records are inserted together with stable sequence numbers. The old revision is never updated or deleted in either path. -### Forward correction for earlier deployments - -Migration `20260803190000-correct-canonical-x-mitre-domains.js` repairs -deployments that already ran the earlier collection-appearance inference. It -only selects a latest revision when: - -- an exact historical predecessor is present in a canonical collection TOC; -- the latest revision is substantively identical to that predecessor after - ignoring the fields controlled by a domain repair; and -- the latest domain array differs from the predecessor's exact TOC union. - -This recognizes migration/bootstrap-generated domain-only successors without -overwriting a later operator-authored revision that changed substantive STIX -content. The correction creates another immutable revision through the same -active/inactive paths described above. - ## Unmapped-object handling Before creating any object revision, the migration resolves the complete @@ -142,7 +126,6 @@ above: ```javascript { remaining_latest_domainless_target_objects: 0, - remaining_latest_incorrect_domain_objects: 0, remaining_domain_validation_bypasses: 0 } ``` diff --git a/docs/admin/release-track-bundle-integrity-migration.md b/docs/admin/release-track-bundle-integrity-migration.md deleted file mode 100644 index dadf0fca..00000000 --- a/docs/admin/release-track-bundle-integrity-migration.md +++ /dev/null @@ -1,27 +0,0 @@ -# Release-Track Bundle Integrity Migration - -The `20260805150000-repair-release-track-bundle-integrity` migration repairs -bundle metadata persisted by earlier deterministic release-track graph -implementations. - -For every active or pending graph manifest that is still linked to a snapshot, -the migration creates or refreshes its frozen `x-mitre-collection` entry. The -entry uses one ID derived from the release-track UUID across the track's full -history, and its `created_by_ref` is the STIX ID returned by the configured -organization-identity service. - -For tagged snapshots, the migration then recomputes `bundle_hashes.stix_2_0` -and `bundle_hashes.stix_2_1` from the exact four-space-indented download bytes. -STIX 2.0 serialization never includes the `x-mitre-collection` object; STIX -2.1 includes the repaired frozen object. Historical draft graphs are live -exports rather than deterministic caches, so any stale hashes on them are -removed. - -The migration runs during normal startup when -`WB_REST_DATABASE_MIGRATION_ENABLE=true`. It is rerunnable: already-correct -collection entries and hashes are retained. Orphaned manifests that are no -longer linked from their recorded snapshot are reported and skipped. - -The down migration is intentionally a no-op because restoring inconsistent -identifiers, creator references, or hashes would reintroduce invalid integrity -metadata. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 4eb48f11..4c0fbb8e 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,29 @@ # Release Track TODOs +## Remove nightly-only migration compatibility + +- [x] Remove regression code that imports the retired beta bundle-integrity + migration. +- [x] Remove canonical-domain correction logic used only to carry flawed beta + migration output forward. +- [x] Remove current documentation for the retired nightly migrations and + document the alpha/beta database reset policy. +- [x] Run the focused migration and current graph-invariant regression specs. +- [x] Run the complete `npm test` suite and propose a conventional commit + message without committing. + +Verification (2026-08-18): + +- Focused canonical-domain, deterministic-graph, opt-in-graph, and bundle + regressions pass (39); ESLint and whitespace checks also pass. +- Four complete `npm test` attempts reached 1001-1009 passing API tests but + each encountered the documented roaming shared-server failure (HTTP parse + error, transient 404, `ECONNRESET`, or socket hang-up). Every affected spec + passes independently, including techniques conversion (24), software + pagination (13), the grouped release-track cases (15), and change capture + (10). OpenAPI (2) and configuration (22) passed on every complete attempt. +- Proposed commit: `chore(migrations): remove nightly compatibility remnants`. + ## Frontend and REST API build information - [x] Source REST API build metadata from the Docker/runtime build variables, diff --git a/docs/developer/data-model.md b/docs/developer/data-model.md index 23b6b060..d674a9d7 100644 --- a/docs/developer/data-model.md +++ b/docs/developer/data-model.md @@ -49,9 +49,7 @@ collection `x_mitre_contents` TOCs. Broad `workspace.collections` appearance backrefs are not authoritative because legacy imports also attached them to secondary graph objects. Unmappable content is left unchanged and reported; the migration retains legacy validation bypasses rather than fabricate -Enterprise membership. Forward -migration `20260803190000-correct-canonical-x-mitre-domains.js` corrects -domain-only successors created by the older inference. See the +Enterprise membership. See the [operator guide](../admin/canonical-domain-migration.md). ## Database Structure diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index f0ad1482..ca547126 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -148,13 +148,6 @@ STIX version serialization. The pipeline: until the graph is deleted; callers then edit the notes and regenerate the graph and hashes. -The `20260805150000-repair-release-track-bundle-integrity` forward migration -applies these invariants to existing graph manifests. It creates or rewrites -each frozen collection entry with the track-derived ID and current configured -organization identity, then recomputes both hashes for every linked tagged -snapshot. Historical draft graphs remain live exports and therefore do not -retain deterministic hashes. - ### Canonical domains and the legacy graph renderer Domain membership is object data, not an export projection. A cross-domain diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index ca89c0f1..7e6ab153 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -19,12 +19,10 @@ whose `version` is a string. Drafts therefore remain unlimited at `version: null`, while the database—not an application-level preflight—decides which concurrent release may claim a version. -Release tracks are still pre-release, and no shared deployment retains track -data written under the former non-unique index. Existing personal development -tracks are therefore reset or recreated instead of establishing a permanent -upgrade contract for beta data. Once release tracks are formally released, -future index or persistence changes must include an appropriate migration for -supported deployments. +Alpha and beta builds are ephemeral and do not establish a database upgrade +contract. Development databases created by those builds are reset or recreated +rather than carried forward by permanent migration scripts. Migrations are +reserved for upgrade paths between stable releases. ## Validation Rules diff --git a/migrations/20260730230000-backfill-canonical-x-mitre-domains.js b/migrations/20260730230000-backfill-canonical-x-mitre-domains.js index e2831c35..82d3dbef 100644 --- a/migrations/20260730230000-backfill-canonical-x-mitre-domains.js +++ b/migrations/20260730230000-backfill-canonical-x-mitre-domains.js @@ -27,7 +27,6 @@ */ const mongoose = require('mongoose'); -const _ = require('lodash'); const config = require('../app/config/config'); const { createAutomationRunRecorder, @@ -200,70 +199,6 @@ function resolveCandidates(documents, domainsByRevision = new Map()) { return candidates; } -function normalizedRepairStix(stix) { - const normalized = JSON.parse(JSON.stringify(stix)); - delete normalized.modified; - delete normalized.x_mitre_domains; - delete normalized.x_mitre_attack_spec_version; - delete normalized.x_mitre_modified_by_ref; - if (normalized.revoked === false) delete normalized.revoked; - return normalized; -} - -function isDomainOnlySuccessor(document, predecessor) { - return _.isEqual(normalizedRepairStix(document.stix), normalizedRepairStix(predecessor.stix)); -} - -function normalizedDomains(value) { - return Array.isArray(value) ? [...new Set(value)].sort() : []; -} - -async function latestIncorrectTargetDocuments(db, domainsByRevision) { - const latestDocuments = await db - .collection('attackObjects') - .aggregate(latestTargetDocumentsPipeline()) - .toArray(); - const revisions = await db - .collection('attackObjects') - .find({ 'stix.id': { $in: latestDocuments.map((document) => document.stix.id) } }) - .sort({ 'stix.id': 1, 'stix.modified': -1 }) - .toArray(); - const revisionsById = new Map(); - for (const revision of revisions) { - const lineage = revisionsById.get(revision.stix.id) || []; - lineage.push(revision); - revisionsById.set(revision.stix.id, lineage); - } - - const candidates = []; - for (const document of latestDocuments) { - let domains = domainsFromCanonicalToc(document, domainsByRevision); - let domainSource = 'canonical-collection-toc'; - - if (domains.length === 0) { - const predecessor = (revisionsById.get(document.stix.id) || []) - .slice(1) - .find( - (revision) => - domainsFromCanonicalToc(revision, domainsByRevision).length > 0 && - isDomainOnlySuccessor(document, revision), - ); - if (!predecessor) continue; - domains = domainsFromCanonicalToc(predecessor, domainsByRevision); - domainSource = 'canonical-collection-toc-predecessor'; - } - - if (_.isEqual(normalizedDomains(document.stix.x_mitre_domains), domains)) continue; - candidates.push({ - document, - domains, - domainSource, - lifecycle: isInactive(document) ? 'inactive' : 'active', - }); - } - return candidates; -} - function ensureMongooseUsesClient(client) { if (client && mongoose.connection.readyState === 0) { mongoose.connection.setClient(client); @@ -342,7 +277,7 @@ function removeResolvedDomainValidation(workspace) { return replacement; } -async function repostActive(candidate, recorder, migrationName = MIGRATION_NAME) { +async function repostActive(candidate, recorder) { const { document, domains } = candidate; const service = serviceFor(document.stix.type); const modified = nextModifiedTimestamp(document.stix.modified); @@ -350,7 +285,7 @@ async function repostActive(candidate, recorder, migrationName = MIGRATION_NAME) const created = await service.create(repost, { import: false, automationContext: { - automationName: migrationName, + automationName: MIGRATION_NAME, runId: recorder.runId, }, }); @@ -384,7 +319,7 @@ function prepareInactiveClone(candidate) { }; } -async function syncInactiveClone(candidate, result, recorder, migrationName = MIGRATION_NAME) { +async function syncInactiveClone(candidate, result, recorder) { const { document } = candidate; // The direct clone is intentionally not presented as a generic create. It // still advances any standard track that references this object, matching @@ -395,23 +330,18 @@ async function syncInactiveClone(candidate, result, recorder, migrationName = MI modifiedBy: 'system', trigger: document.stix.revoked === true ? 'revocation' : 'new-revision', automationContext: { - automationName: migrationName, + automationName: MIGRATION_NAME, runId: recorder.runId, }, }); } -async function processActiveBatch( - candidates, - recorder, - concurrency, - migrationName = MIGRATION_NAME, -) { +async function processActiveBatch(candidates, recorder, concurrency) { return mapWithConcurrency(candidates, concurrency, async (candidate) => { try { return { candidate, - result: await repostActive(candidate, recorder, migrationName), + result: await repostActive(candidate, recorder), }; } catch (error) { return { candidate, error }; @@ -419,7 +349,7 @@ async function processActiveBatch( }); } -async function processInactiveBatch(db, candidates, recorder, migrationName = MIGRATION_NAME) { +async function processInactiveBatch(db, candidates, recorder) { return mapWithConcurrency(candidates, ACTIVE_CONCURRENCY, async (candidate) => { try { const result = prepareInactiveClone(candidate); @@ -428,7 +358,7 @@ async function processInactiveBatch(db, candidates, recorder, migrationName = MI // the native driver performing the insert create its own ObjectId. const insertResult = await db.collection('attackObjects').insertOne(result.document); result.document._id = insertResult.insertedId; - await syncInactiveClone(candidate, result, recorder, migrationName); + await syncInactiveClone(candidate, result, recorder); return { candidate, result }; } catch (error) { return { candidate, error }; @@ -604,11 +534,6 @@ async function countRemainingDomainlessTargets(db) { return (await latestDomainlessTargetDocuments(db)).length; } -async function countRemainingIncorrectTargets(db) { - const domainsByRevision = await buildCanonicalTocDomainIndex(db); - return (await latestIncorrectTargetDocuments(db, domainsByRevision)).length; -} - async function countStaleDomainBypasses(db) { return db.collection('validationbypassrules').countDocuments({ fieldPath: ['x_mitre_domains'], @@ -625,33 +550,17 @@ async function removeStaleDomainBypasses(db) { }); } -async function run(db, client, options = {}) { - const migrationName = options.migrationName || MIGRATION_NAME; - const correctIncorrect = options.correctIncorrect === true; +async function run(db, client) { const domainsByRevision = await buildCanonicalTocDomainIndex(db); - const domainlessDocuments = correctIncorrect ? [] : await latestDomainlessTargetDocuments(db); - const incorrectCandidates = await latestIncorrectTargetDocuments(db, domainsByRevision); - const incorrectIds = new Set(incorrectCandidates.map((candidate) => candidate.document.stix.id)); - const unresolvedDomainless = correctIncorrect - ? [] - : domainlessDocuments.filter( - (document) => - !incorrectIds.has(document.stix.id) && - domainsFromCanonicalToc(document, domainsByRevision).length === 0, - ); - const candidates = [ - ...incorrectCandidates, - ...(correctIncorrect - ? [] - : resolveCandidates( - domainlessDocuments.filter((document) => !incorrectIds.has(document.stix.id)), - domainsByRevision, - )), - ]; + const domainlessDocuments = await latestDomainlessTargetDocuments(db); + const unresolvedDomainless = domainlessDocuments.filter( + (document) => domainsFromCanonicalToc(document, domainsByRevision).length === 0, + ); + const candidates = resolveCandidates(domainlessDocuments, domainsByRevision); const recorder = await createAutomationRunRecorder(db, { automationType: 'migration', - name: migrationName, + name: MIGRATION_NAME, trigger: { source: 'startup', runner: 'migrate-mongo' }, scope: { collections: ['attackObjects', 'validationbypassrules'], @@ -662,7 +571,6 @@ async function run(db, client, options = {}) { domain_source: 'exact-canonical-collection-toc-membership', canonical_collection_domains: Object.fromEntries(CANONICAL_COLLECTION_DOMAINS), unmapped_policy: 'leave-unchanged-and-retain-validation-bypasses', - correct_incorrect_successors: correctIncorrect, active_method: 'service-create', inactive_method: 'immutable-direct-clone', batch_size: BATCH_SIZE, @@ -714,12 +622,7 @@ async function run(db, client, options = {}) { size: batch.length, concurrency: ACTIVE_CONCURRENCY, }); - const processed = await processActiveBatch( - batch, - recorder, - ACTIVE_CONCURRENCY, - migrationName, - ); + const processed = await processActiveBatch(batch, recorder, ACTIVE_CONCURRENCY); await finalizeBatch(db, processed, recorder, counts, failures); } @@ -734,7 +637,7 @@ async function run(db, client, options = {}) { concurrency: 1, stix_types: [...new Set(batch.map((candidate) => candidate.document.stix.type))], }); - const processed = await processActiveBatch(batch, recorder, 1, migrationName); + const processed = await processActiveBatch(batch, recorder, 1); await finalizeBatch(db, processed, recorder, counts, failures); } @@ -745,22 +648,19 @@ async function run(db, client, options = {}) { size: batch.length, concurrency: ACTIVE_CONCURRENCY, }); - const processed = await processInactiveBatch(db, batch, recorder, migrationName); + const processed = await processInactiveBatch(db, batch, recorder); await finalizeBatch(db, processed, recorder, counts, failures); } const remainingDomainless = await countRemainingDomainlessTargets(db); - const remainingIncorrect = await countRemainingIncorrectTargets(db); - const remainingCandidates = remainingIncorrect; - if (failures.length > 0 || remainingCandidates > 0) { + if (failures.length > 0) { const failureSample = failures .slice(0, 5) .map((failure) => `${failure.stix_id}: ${failure.error}`) .join('; '); throw new Error( `Canonical-domain object repair is incomplete: ${failures.length} failed item(s), ` + - `${remainingCandidates} remaining target object(s). Validation bypasses ` + - `were retained.${failureSample ? ` Failures: ${failureSample}` : ''}`, + `validation bypasses were retained.${failureSample ? ` Failures: ${failureSample}` : ''}`, ); } @@ -774,7 +674,6 @@ async function run(db, client, options = {}) { verification = { remaining_latest_domainless_target_objects: remainingDomainless, - remaining_latest_incorrect_domain_objects: remainingIncorrect, remaining_domain_validation_bypasses: await countStaleDomainBypasses(db), }; @@ -817,9 +716,6 @@ async function run(db, client, options = {}) { remaining_latest_domainless_target_objects: verification.remaining_latest_domainless_target_objects ?? (await countRemainingDomainlessTargets(db).catch(() => null)), - remaining_latest_incorrect_domain_objects: - verification.remaining_latest_incorrect_domain_objects ?? - (await countRemainingIncorrectTargets(db).catch(() => null)), remaining_domain_validation_bypasses: verification.remaining_domain_validation_bypasses ?? (await countStaleDomainBypasses(db).catch(() => null)), @@ -857,14 +753,12 @@ module.exports = { TARGET_TYPES, chunkItems, countRemainingDomainlessTargets, - countRemainingIncorrectTargets, countStaleDomainBypasses, buildCanonicalTocDomainIndex, domainsFromCanonicalToc, hasCanonicalDomains, isInactive, latestDomainlessTargetDocuments, - latestIncorrectTargetDocuments, mapWithConcurrency, nextModifiedTimestamp, prepareInactiveClone, From 2fffeb76099a109d9f060878f6ff77ca4ee9586f Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:13:26 -0400 Subject: [PATCH 03/14] feat(release-tracks): seal snapshot content manifests Make every snapshot deterministic from birth. A content manifest is sealed whenever a snapshot's members are written (track creation, release commit, virtual materialization, bundle import, quarantine promotion, track clone) and inherited by reference by every other clone; it is discarded only when no snapshot references it. One closed-member algorithm selects relationships whose source and target IDs are both members and pins them to the member revisions; supporting identities and marking definitions and non-emitted LinkById targets are recorded as dependencies. No secondary SDO is ever discovered through a relationship, and bundle export has exactly one path: replay the manifest. The opt-in graph cache and its create/delete endpoints are removed; the admin-only source-attested reconstruction remains and requires replace_manifest_id. Stop cloning relationships when an endpoint revision advances. Exact pairing for a release lives in the sealed manifest; the create-time endpoint pins stay as authoring context and the release preview reports relationships added, removed, and authored against other revisions. Project the x-mitre-collection object from the snapshot and a new config.publication rule instead of storing it: identity and markings inherit the organization settings unless overridden, collection id and created default to track-derived values and lock after the first release, modified is the snapshot timestamp, and drafts omit x_mitre_version. Release commit freezes the resolved values, assigns a stable bundle id, and stores SHA-256 hashes of both serializations; include is a draft-only preview and includeToc is gone. Notes are immutable once released. Name storage for what it holds: releaseTrackContentManifests and releaseTrackContentManifestEntries with release-track-content-manifest ids, a required seal_reason instead of resolver_version and baseline_reconstruction, outstanding-work-only releaseTrackReconciliations, and no config.include_secondary_objects. The 20260902120000 migration upgrades existing databases in place (dry run: npm run preview:content-manifests), skips unregistered release-track collections, and names the failing snapshot and missing references on any integrity error; the 20260730180000 migration no longer creates manifests. Co-Authored-By: Claude Fable 5.1 --- AGENTS.md | 6 + .../definitions/components/release-tracks.yml | 180 ++- app/api/definitions/openapi.yml | 3 - .../paths/release-tracks-paths.yml | 133 +- app/controllers/release-tracks-controller.js | 54 +- app/lib/release-tracks/export-schemas.js | 128 +- .../release-tracks/release-track-schemas.js | 38 +- .../release-track-validators.js | 7 + .../release-track-content-manifest-model.js | 125 ++ .../release-track-graph-manifest-model.js | 92 -- .../release-track-reconciliation-model.js | 5 +- .../release-track-snapshot-schema.js | 83 +- app/repository/relationships-repository.js | 70 +- .../release-track-dynamic.repository.js | 47 +- ...release-track-reconciliation.repository.js | 32 +- app/routes/release-tracks-routes.js | 15 +- app/services/meta-classes/base.service.js | 8 +- .../release-tracks/bundle-hash-service.js | 2 +- .../content-manifest-service.js | 965 +++++++++++++ app/services/release-tracks/export-service.js | 134 +- .../release-tracks/graph-manifest-service.js | 1262 ----------------- .../release-tracks/member-sync-service.js | 5 +- .../release-tracks/publication-service.js | 180 +++ .../release-tracks/release-tracks-service.js | 22 +- .../release-tracks/snapshot-service.js | 282 ++-- .../release-tracks/versioning-service.js | 110 +- app/services/stix/relationships-service.js | 96 +- .../api/attack-objects/attack-objects.spec.js | 6 +- .../relationship-endpoint-pins.spec.js | 20 +- .../release-tracks/content-manifests.spec.js | 599 ++++++++ .../deterministic-graph-migration.spec.js | 399 ++++-- .../api/release-tracks/opt-in-graphs.spec.js | 699 --------- .../release-tracks/publication-config.spec.js | 255 ++++ .../reconciliation-durability.spec.js | 6 +- .../release-tracks-bundle.spec.js | 64 +- .../release-tracks-release.spec.js | 14 +- .../api/release-tracks/release-tracks.spec.js | 17 +- .../snapshot-descriptions.spec.js | 90 +- .../release-tracks/snapshot-history.spec.js | 35 +- .../virtual-graph-integrity.spec.js | 56 +- docs/README.md | 1 + docs/developer/TODO.md | 318 ++++- .../release-tracks/backref-reconciliation.md | 11 +- .../developer/release-tracks/bundle-export.md | 393 +++-- docs/developer/release-tracks/entities.md | 127 +- .../release-tracks/implementation-notes.md | 39 +- .../release-tracks/member-sync-strategies.md | 2 +- .../sealed-content-manifests.md | 126 ++ docs/user/release-tracks/api-reference.md | 198 +-- docs/user/release-tracks/output-formats.md | 75 +- docs/user/release-tracks/versioning.md | 20 +- docs/user/release-tracks/virtual-tracks.md | 31 +- ...-backfill-deterministic-snapshot-graphs.js | 92 +- ...00-seal-release-track-content-manifests.js | 542 +++++++ package.json | 1 + scripts/previewContentManifestMigration.js | 22 + 56 files changed, 4873 insertions(+), 3469 deletions(-) create mode 100644 app/models/release-tracks/release-track-content-manifest-model.js delete mode 100644 app/models/release-tracks/release-track-graph-manifest-model.js create mode 100644 app/services/release-tracks/content-manifest-service.js delete mode 100644 app/services/release-tracks/graph-manifest-service.js create mode 100644 app/services/release-tracks/publication-service.js create mode 100644 app/tests/api/release-tracks/content-manifests.spec.js delete mode 100644 app/tests/api/release-tracks/opt-in-graphs.spec.js create mode 100644 app/tests/api/release-tracks/publication-config.spec.js create mode 100644 docs/developer/release-tracks/sealed-content-manifests.md create mode 100644 migrations/20260902120000-seal-release-track-content-manifests.js create mode 100644 scripts/previewContentManifestMigration.js diff --git a/AGENTS.md b/AGENTS.md index 49aa14f3..ba9f8cdc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,6 +159,12 @@ parameter semantics in the `docs { }` block. - Legacy endpoints under deprecation (e.g. `GET /api/stix-bundles`) are replaced by release-tracks equivalents — check `docs/developer/release-tracks/bundle-export.md` before extending them. +- Release-track bundle export has one content path: replay the snapshot's + sealed content manifest (`docs/developer/release-tracks/sealed-content-manifests.md`). + Never add live relationship discovery, secondary-SDO expansion, or a + deletable "graph cache" to release-track exports; drafts inherit their + predecessor's manifest and only member-changing writes seal a new one. The + `x-mitre-collection` object is a projection, not a stored object. - Historic full-suite flake (fixed 2026-07-10): per-spec-file mongod restarts hit "Port already in use", failing a random file's `before` hook (visible as `loginAnonymous` 404s). `database-in-memory.js` now reuses one diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index e412dccb..ecde61f5 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -23,24 +23,31 @@ components: nullable: true description: 'Semantic version (e.g., "1.0", "2.1") if tagged, null for draft snapshots' example: '1.0' - graph_manifest_id: + content_manifest_id: type: string readOnly: true description: | - Server-controlled identifier for an opt-in deterministic member - graph on a tagged snapshot. Absent on drafts and graphless tagged - snapshots. Clients should treat this value as opaque. + Server-controlled identifier of the sealed content manifest that + describes this snapshot's exact member graph. Present on every + snapshot; clients should treat this value as opaque. + publication: + $ref: '#/components/schemas/frozen-publication' + bundle_id: + type: string + readOnly: true + description: | + Stable STIX bundle envelope identifier assigned when the snapshot + was released. Drafts derive a deterministic identifier at export. bundle_hashes: $ref: '#/components/schemas/bundle-hashes' snapshot_description: type: string maxLength: 4000 description: | - User-authored, snapshot-local notes. Editors may change this - workspace annotation without changing the snapshot identity or - release tag while no pinned member graph exists. Cached snapshots - reject note edits until their graph is deleted and regenerated; - graphless exports fall back to the track description when absent. + User-authored, snapshot-local notes emitted as the collection + object's description. Editable on drafts; immutable once the + snapshot is released. Exports fall back to the track description + when absent. name: type: string pattern: '^[a-zA-Z0-9 &]+$' @@ -57,11 +64,6 @@ components: type: string description: 'STIX ID of the user who created the track' example: 'identity--12345678-1234-1234-1234-123456789012' - object_marking_refs: - type: array - items: - type: string - description: 'STIX marking definition references' members: type: array description: 'Released objects (promoted from staged during tagging)' @@ -116,10 +118,10 @@ components: items: $ref: '#/components/schemas/version-history-entry' - graph-statistics: + content-statistics: type: object readOnly: true - description: 'Counts of exact revision pointers by role in a materialized snapshot graph' + description: 'Counts of exact revision pointers by role in a sealed content manifest' required: - primary_count - secondary_count @@ -135,15 +137,15 @@ components: secondary_count: type: integer minimum: 0 - description: 'Source-attested historical non-member objects; zero for ordinary closed-member graphs' + description: 'Legacy source-attested historical non-member objects; zero for sealed closed-member manifests' relationship_count: type: integer minimum: 0 - description: 'Relationships connecting objects in the resolved graph' + description: 'Relationships whose source and target are both members' supporting_count: type: integer minimum: 0 - description: 'Supporting identities and marking definitions required by cached objects' + description: 'Supporting identities and marking definitions required by emitted objects' link_target_count: type: integer minimum: 0 @@ -158,8 +160,8 @@ components: readOnly: true description: | SHA-256 digests of the exact UTF-8, four-space-indented JSON files - downloaded for a deterministic snapshot. The manifest ID binds the - digests to the cached graph that produced them. + downloaded for a released snapshot. The manifest ID binds the digests + to the sealed content manifest that produced them. required: - manifest_id - stix_2_0 @@ -200,26 +202,26 @@ components: type: string nullable: true description: 'Tagged version, or null for an untagged draft' - graph_manifest_id: + content_manifest_id: type: string readOnly: true - description: | - Opaque identifier for the tagged snapshot's deterministic member - graph. Omitted when the snapshot has not been materialized. + description: "Opaque identifier of the snapshot's sealed content manifest" + bundle_id: + type: string + readOnly: true + description: 'Stable bundle envelope identifier of a released snapshot' bundle_hashes: $ref: '#/components/schemas/bundle-hashes' - graph_statistics: - $ref: '#/components/schemas/graph-statistics' - description: | - High-level statistics for the materialized graph. Omitted when the - snapshot does not reference a graph manifest. + content_statistics: + $ref: '#/components/schemas/content-statistics' + description: 'Counts of sealed manifest entries by role' snapshot_description: type: string maxLength: 4000 description: | User-authored notes attached only to this snapshot and mapped to - x-mitre-collection.description during bundle export. Notes cannot - be changed while the snapshot has a deterministic graph cache. + x-mitre-collection.description during bundle export. Editable on + drafts; immutable once released. name: type: string description: @@ -375,6 +377,10 @@ components: type: object description: 'Release track configuration' properties: + publication: + $ref: '#/components/schemas/publication-config' + publication_resolved: + $ref: '#/components/schemas/publication-resolved' auto_promote: type: boolean description: 'Whether to automatically promote candidates that meet the threshold' @@ -855,6 +861,116 @@ components: minItems: 1 items: $ref: '#/components/schemas/source-graph-entry' + replace_manifest_id: + type: string + description: | + The content manifest currently attached to the snapshot that this + reconstruction replaces. Required unless that manifest already + carries the same source attestation. + + inherited-identity: + type: object + additionalProperties: false + required: + - inherit + description: | + Publication identity rule. inherit=true uses the organization identity + from system configuration; inherit=false uses value. + properties: + inherit: + type: boolean + value: + type: string + description: 'STIX identity ID used when inherit is false' + + inherited-marking-refs: + type: object + additionalProperties: false + required: + - inherit + description: | + Publication marking rule. inherit=true uses the default marking + definitions from system configuration; inherit=false uses value. + properties: + inherit: + type: boolean + value: + type: array + items: + type: string + description: 'Marking definition IDs used when inherit is false' + + publication-config: + type: object + additionalProperties: false + description: | + Track-scoped publication metadata for the emitted x-mitre-collection + object. Collection identity and creation time default to values + derived from the track and become immutable once the track has a + tagged release. Identity and markings inherit from the global scope + unless overridden. + properties: + collection_id: + type: string + nullable: true + description: 'Explicit collection object ID; null or absent derives it from the track UUID' + created: + type: string + format: date-time + nullable: true + description: 'Explicit collection created timestamp; null or absent uses the track creation time' + created_by_ref: + $ref: '#/components/schemas/inherited-identity' + object_marking_refs: + $ref: '#/components/schemas/inherited-marking-refs' + + publication-resolved: + type: object + readOnly: true + description: | + The publication values currently in effect for the latest snapshot and + the scope each one was resolved from (track, global, or derived). + properties: + collection_id: + type: string + created: + type: string + format: date-time + created_by_ref: + type: string + object_marking_refs: + type: array + items: + type: string + attack_spec_version: + type: string + sources: + type: object + additionalProperties: + type: string + enum: + - track + - global + - derived + + frozen-publication: + type: object + readOnly: true + description: 'Publication values frozen onto a released snapshot at commit time' + properties: + collection_id: + type: string + created: + type: string + format: date-time + created_by_ref: + type: string + object_marking_refs: + type: array + items: + type: string + attack_spec_version: + type: string source-graph-entry: type: object diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index 736d5c85..4d0ecd1f 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -412,9 +412,6 @@ paths: /api/release-tracks/{id}/snapshots/{modified}/description: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1description' - /api/release-tracks/{id}/snapshots/{modified}/graph: - $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1graph' - /api/release-tracks/{id}/snapshots/{modified}/graph/reconstruct: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1graph~1reconstruct' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 649338c9..f63c96fb 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -335,7 +335,8 @@ paths: summary: 'Update metadata on the latest snapshot' operationId: 'release-tracks-update-meta-latest' description: | - Update name, description, or object_marking_refs on the latest snapshot. + Update name or description on the latest snapshot. Publication + markings are configured through the track configuration. Creates a new snapshot clone with updated metadata. Request body validated via Zod in controller. tags: @@ -498,11 +499,6 @@ paths: schema: type: string enum: ['2.0', '2.1'] - - name: includeToc - in: query - description: 'Include the x-mitre-collection TOC in STIX 2.1 bundle previews; STIX 2.0 always omits it' - schema: - type: boolean responses: '200': description: 'Release preview generated' @@ -972,10 +968,10 @@ paths: ordered by modified timestamp from newest to oldest. Standard snapshot summaries contain members, staged, and candidates counts. Virtual snapshot summaries contain members and quarantine counts, plus - scheduled_materialization when present. Tagged summaries also expose - graph_manifest_id when their deterministic member graph has been - materialized, together with graph_statistics counts for primary, - secondary, relationship, supporting, and LinkById entries. + scheduled_materialization when present. Every summary exposes its + content_manifest_id together with content_statistics counts for + primary, relationship, supporting, and LinkById entries. Tagged + summaries also expose bundle_id and bundle_hashes. tags: - 'Release Tracks' parameters: @@ -1066,7 +1062,9 @@ paths: description: | Format-sensitive tier selector. For workbench responses, selects members, staged, candidates, quarantine, or all. For bundle - responses, selects staged and/or candidates in addition to members. + responses, selects staged and/or candidates in addition to members; + this is a draft-only preview option that resolves the included + tiers live, and tagged snapshots reject it with 400. allowReserved: true schema: oneOf: @@ -1106,12 +1104,6 @@ paths: - '2.0' - '2.1' default: '2.1' - - name: includeToc - in: query - description: 'Include the x-mitre-collection TOC in STIX 2.1 bundle responses; STIX 2.0 bundles never include it' - schema: - type: boolean - default: true responses: '200': description: 'Latest snapshot retrieved successfully' @@ -1159,7 +1151,8 @@ paths: are returned — members | staged | candidates | quarantine | all (default: all). For format=bundle: a list of additional tiers (staged and/or candidates, comma-separated or repeated) to include alongside members. If omitted, only - members are included in the bundle. + members are included in the bundle. Including tiers is a draft-only preview + option; tagged snapshots reject it with 400. allowReserved: true schema: oneOf: @@ -1204,15 +1197,6 @@ paths: - '2.0' - '2.1' default: '2.1' - - name: includeToc - in: query - description: | - Whether to include a table-of-contents object (of type `x-mitre-collection`) - derived from the release-track metadata (bundle format only). - This applies only to STIX 2.1; STIX 2.0 bundles never include it. - schema: - type: boolean - default: true responses: '200': description: 'Snapshot retrieved successfully' @@ -1291,12 +1275,12 @@ paths: summary: 'Set or clear a snapshot description' operationId: 'release-tracks-snapshot-description-update' description: | - Replace the user-authored notes on one draft or tagged snapshot. - Whitespace is trimmed; an empty string clears the notes. This mutable - workspace annotation does not change the snapshot modified timestamp, - semantic version, tier contents, or release-track metadata. A snapshot - with a graph manifest is immutable and returns 409 until its bundle - cache is deleted. + Replace the user-authored notes on one draft snapshot. Whitespace is + trimmed; an empty string clears the notes. This annotation does not + change the snapshot modified timestamp, tier contents, or + release-track metadata. The notes become the emitted collection + object's description, so a tagged snapshot is immutable and returns + 409; set release notes through the release request instead. tags: - 'Release Tracks' parameters: @@ -1338,72 +1322,9 @@ paths: '409': description: 'Delete the snapshot bundle cache before editing its notes' - /api/release-tracks/{id}/snapshots/{modified}/graph: - post: - summary: 'Make a tagged snapshot member graph deterministic' - operationId: 'release-tracks-snapshot-graph-create' - description: | - Resolve the tagged snapshot's members into a deterministic graph and - persist exact-revision pointers for its primary objects, - relationships, and secondary objects. The referenced revisions are - write-protected until the graph is deleted. Draft snapshots cannot - have persisted graphs. Repeating this operation for a snapshot that - already has a graph is idempotent and returns the existing snapshot. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: modified - in: path - required: true - schema: - type: string - responses: - '200': - description: 'The tagged snapshot already had a deterministic graph' - '201': - description: 'Deterministic member graph created successfully' - '404': - description: 'Snapshot not found' - '409': - description: 'The snapshot is untagged, changed concurrently, or references missing revisions' - - delete: - summary: 'Remove a tagged snapshot deterministic graph' - operationId: 'release-tracks-snapshot-graph-delete' - description: | - Remove the opt-in deterministic member graph and release its - exact-revision deletion protections. Subsequent exports resolve the live - graph. This operation is idempotent when no graph exists. Draft - snapshots cannot have persisted graphs. - tags: - - 'Release Tracks' - parameters: - - name: id - in: path - required: true - schema: - type: string - - name: modified - in: path - required: true - schema: - type: string - responses: - '204': - description: 'Deterministic member graph absent after the request' - '404': - description: 'Snapshot not found' - '409': - description: 'The snapshot is untagged or its graph changed concurrently' - /api/release-tracks/{id}/snapshots/{modified}/graph/reconstruct: post: - summary: 'Reconstruct a historical deterministic graph from source-bundle pointers' + summary: 'Reconstruct a historical content manifest from source-bundle pointers' operationId: 'release-tracks-snapshot-graph-reconstruct' description: | Administrative recovery operation for a tagged historical snapshot. @@ -1413,8 +1334,11 @@ paths: relationship endpoint pins match the stored relationship, and all referenced endpoint and supporting objects are present. Versioned objects remain pointer-only; only unversioned marking definitions may - carry a frozen payload. Repeating the same attestation is idempotent; - an ordinary graph or a different attestation is rejected. + carry a frozen payload. Repeating the same attestation is idempotent. + Every tagged snapshot already has a sealed content manifest, so the + request must name that manifest in `replace_manifest_id` to replace + it; a different current manifest is rejected. Replacement recomputes + the snapshot's bundle hashes. tags: - 'Release Tracks' parameters: @@ -1436,15 +1360,15 @@ paths: $ref: '../components/release-tracks.yml#/components/schemas/source-graph-reconstruction' responses: '200': - description: 'The tagged snapshot already had a deterministic graph' + description: 'The tagged snapshot already references a manifest with this attestation' '201': - description: 'Source-attested deterministic graph created successfully' + description: 'Source-attested content manifest replaced the previous manifest' '400': description: 'Malformed reconstruction plan' '404': description: 'Snapshot or exact object revision not found' '409': - description: 'The snapshot is untagged or the source plan violates graph integrity' + description: 'The snapshot is untagged, replace_manifest_id does not match, or the source plan violates graph integrity' /api/release-tracks/{id}/snapshots/{modified}/release: post: @@ -1565,11 +1489,6 @@ paths: schema: type: string enum: ['2.0', '2.1'] - - name: includeToc - in: query - description: 'Include the x-mitre-collection TOC in STIX 2.1 bundle previews; STIX 2.0 always omits it' - schema: - type: boolean responses: '200': description: 'Release preview generated' diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 3f501e60..3988e994 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -116,8 +116,9 @@ function rejectFilesystemStoreFormat(format, methodName) { * - format=bundle: list of additional tiers ('staged' and/or 'candidates') * to hydrate into the bundle alongside members. Omitted → members only. * - * The `state`, `stixVersion`, and `includeToc` parameters only apply to - * format=bundle. + * The `state` and `stixVersion` parameters only apply to format=bundle. + * `include` for bundles is a draft-only preview option; the service rejects it + * for tagged snapshots. */ function parseSnapshotQueryParams(query) { const format = parseOptionalQueryStrict(query.format, formatQuerySchema, 'workbench', 'format'); @@ -147,12 +148,6 @@ function parseSnapshotQueryParams(query) { '2.1', 'stixVersion', ), - includeToc: parseOptionalQueryStrict( - query.includeToc, - booleanQuerySchema, - true, - 'includeToc', - ), }; } @@ -198,12 +193,6 @@ function parseReleasePreviewQueryParams(query) { '2.1', 'stixVersion', ), - includeToc: parseOptionalQueryStrict( - query.includeToc, - booleanQuerySchema, - true, - 'includeToc', - ), }; } if (format === 'workbench') { @@ -644,54 +633,27 @@ exports.cloneByModified = async function cloneByModified(req, res, next) { } }; -/** POST /api/release-tracks/:id/snapshots/:modified/graph */ -exports.createSnapshotGraph = async function createSnapshotGraph(req, res, next) { - try { - const result = await releaseTracksService.createSnapshotGraph( - req.params.id, - req.params.modified, - ); - logger.debug(`Success: Created graph for snapshot ${req.params.modified}`); - return res.status(result.created ? 201 : 200).send(result.snapshot); - } catch (err) { - logger.error('Failed to create snapshot graph: ' + err); - return next(err); - } -}; - /** POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct */ -exports.reconstructSnapshotGraph = async function reconstructSnapshotGraph(req, res, next) { +exports.reconstructSnapshotManifest = async function reconstructSnapshotManifest(req, res, next) { try { const bodyResult = reconstructSnapshotGraphBodySchema.safeParse(req.body); if (!bodyResult.success) { return next( new BadRequestError({ - message: 'Invalid source graph reconstruction request', + message: 'Invalid source manifest reconstruction request', details: bodyResult.error.errors, }), ); } - const result = await releaseTracksService.reconstructSnapshotGraph( + const result = await releaseTracksService.reconstructSnapshotManifest( req.params.id, req.params.modified, bodyResult.data, ); - logger.debug(`Success: Reconstructed graph for snapshot ${req.params.modified}`); + logger.debug(`Success: Reconstructed content manifest for snapshot ${req.params.modified}`); return res.status(result.created ? 201 : 200).send(result.snapshot); } catch (err) { - logger.error('Failed to reconstruct snapshot graph: ' + err); - return next(err); - } -}; - -/** DELETE /api/release-tracks/:id/snapshots/:modified/graph */ -exports.deleteSnapshotGraph = async function deleteSnapshotGraph(req, res, next) { - try { - await releaseTracksService.deleteSnapshotGraph(req.params.id, req.params.modified); - logger.debug(`Success: Deleted graph for snapshot ${req.params.modified}`); - return res.status(204).end(); - } catch (err) { - logger.error('Failed to delete snapshot graph: ' + err); + logger.error('Failed to reconstruct snapshot content manifest: ' + err); return next(err); } }; diff --git a/app/lib/release-tracks/export-schemas.js b/app/lib/release-tracks/export-schemas.js index 2b87f89a..819a30d8 100644 --- a/app/lib/release-tracks/export-schemas.js +++ b/app/lib/release-tracks/export-schemas.js @@ -38,7 +38,6 @@ const snapshotSchema = z.looseObject({ snapshot_description: z.string().optional(), created: z.date().or(z.string()).optional(), created_by_ref: z.string().optional(), - object_marking_refs: z.array(z.string()).optional(), modified: z.date().or(z.string()), members: z.array(tierEntrySchema).default([]), staged: z.array(tierEntrySchema).optional(), @@ -50,16 +49,20 @@ const hydratedObjectSchema = z.looseObject({ workspace: z.looseObject({}).optional(), }); +const publicationSchema = z.looseObject({ + collection_id: z.string(), + created: z.date().or(z.string()), + created_by_ref: z.string(), + object_marking_refs: z.array(z.string()), + attack_spec_version: z.string(), +}); + const exportOptionsSchema = z .looseObject({ include: z.array(z.enum(['staged', 'candidates'])).optional(), state: z.array(z.enum(['work-in-progress', 'awaiting-review'])).optional(), stixVersion: z.enum(['2.0', '2.1']).default('2.1'), - includeToc: z.boolean().default(true), - attackSpecVersion: z.string().optional(), - collectionObject: z.looseObject({}).optional(), - collectionId: z.string().optional(), - createdByRef: z.string().optional(), + publication: publicationSchema.optional(), bundleId: z.string().optional(), }) .optional() @@ -94,54 +97,66 @@ function buildTierLookup(snapshot) { } // ----------------------------------------------------------------------------- -// Helper: Build the x-mitre-collection table-of-contents (TOC) object +// Helper: Build the x-mitre-collection object // -// The x-mitre-collection object is effectively a table of contents for the -// bundle. For release-track exports it is derived from the track/snapshot -// metadata rather than from user-supplied query parameters: -// - id: stable per track (reuses the track UUID) -// - x_mitre_version: the snapshot's tagged version, or '0.1' for drafts -// - modified: the snapshot's modified timestamp -// - x_mitre_contents: every bundle object except marking definitions, -// which are recorded in object_marking_refs instead +// The collection object is the bundle's bill of materials. It is a projection +// of the snapshot and its publication metadata, never a stored object: +// - id, created, created_by_ref, object_marking_refs, and +// x_mitre_attack_spec_version come from the resolved (draft) or frozen +// (tagged) publication values +// - modified is the snapshot's modified timestamp +// - x_mitre_version is the tagged version; drafts omit the key because a +// draft has no publication version and a placeholder would collide with a +// legitimate first release +// - x_mitre_contents lists every bundle object except marking definitions +// - object_marking_refs falls back to the marking definitions referenced by +// the bundle's contents when neither the track nor the global scope +// configures any, so the object never ships without markings // ----------------------------------------------------------------------------- -function buildTocObject(snapshot, bundleObjects, options) { - const trackUuid = snapshot.id.split('--')[1]; - - const tocObject = { +function buildCollectionObject(snapshot, bundleObjects, options) { + const publication = options.publication || {}; + const created = publication.created || snapshot.created || snapshot.modified; + const configuredMarkingRefs = publication.object_marking_refs || []; + const contentMarkingRefs = [ + ...new Set( + bundleObjects + .filter((bundleObject) => bundleObject.type === 'marking-definition') + .map((bundleObject) => bundleObject.id), + ), + ].sort(); + + const collectionObject = { type: 'x-mitre-collection', - id: options.collectionId || `x-mitre-collection--${trackUuid}`, - x_mitre_attack_spec_version: options.attackSpecVersion, + id: publication.collection_id || `x-mitre-collection--${snapshot.id.split('--')[1]}`, + x_mitre_attack_spec_version: publication.attack_spec_version, name: snapshot.name, - x_mitre_version: snapshot.version || '0.1', + ...(snapshot.version ? { x_mitre_version: snapshot.version } : {}), description: snapshot.snapshot_description ?? snapshot.description, - created_by_ref: options.createdByRef || snapshot.created_by_ref || '', - created: options.created || snapshot.created || snapshot.modified, - modified: options.modified || snapshot.modified, + created_by_ref: publication.created_by_ref || '', + created: new Date(created).toISOString(), + modified: new Date(snapshot.modified).toISOString(), x_mitre_contents: [], - object_marking_refs: [], + object_marking_refs: + configuredMarkingRefs.length > 0 ? [...configuredMarkingRefs] : contentMarkingRefs, }; for (const bundleObject of bundleObjects) { - if (bundleObject.type === 'marking-definition') { - tocObject.object_marking_refs.push(bundleObject.id); - } else { - tocObject.x_mitre_contents.push({ - object_ref: bundleObject.id, - object_modified: bundleObject.modified, - }); - } + if (bundleObject.type === 'marking-definition') continue; + collectionObject.x_mitre_contents.push({ + object_ref: bundleObject.id, + object_modified: bundleObject.modified, + }); } if (options.stixVersion === '2.1') { - tocObject.spec_version = '2.1'; + collectionObject.spec_version = '2.1'; } // Sort x_mitre_contents by id for deterministic output - tocObject.x_mitre_contents.sort((x, y) => x.object_ref.localeCompare(y.object_ref)); + collectionObject.x_mitre_contents.sort((x, y) => x.object_ref.localeCompare(y.object_ref)); - return tocObject; + return collectionObject; } // ----------------------------------------------------------------------------- @@ -155,24 +170,17 @@ function buildTocObject(snapshot, bundleObjects, options) { // the requested STIX version. The bundle envelope carries spec_version // only for STIX 2.0 — the STIX 2.1 specification removed spec_version // from the bundle object (objects declare their own spec_version). -// - includeToc (default true): prepend an x-mitre-collection object derived -// from the snapshot metadata for STIX 2.1; STIX 2.0 always omits it -// - attackSpecVersion: x_mitre_attack_spec_version for the TOC object +// - publication: resolved or frozen collection metadata. STIX 2.1 bundles +// always begin with the x-mitre-collection object; STIX 2.0 bundles never +// contain this ATT&CK extension object. +// - bundleId: stable envelope identifier // // Notes are Workbench-native objects, not STIX objects, so they are never // included in emitted bundles. // ----------------------------------------------------------------------------- const bundleTransformSchema = exportInputSchema.transform((input) => { - const { - stixVersion, - includeToc, - attackSpecVersion, - collectionObject, - collectionId, - createdByRef, - bundleId, - } = input.options; + const { stixVersion, publication, bundleId } = input.options; const objects = input.hydratedObjects .map((doc) => doc.stix) @@ -182,19 +190,13 @@ const bundleTransformSchema = exportInputSchema.transform((input) => { conformToStixVersion(stixObject, stixVersion); } - // x-mitre-collection is a STIX 2.1 ATT&CK extension object. It must never be - // emitted in a STIX 2.0 bundle, even when includeToc retains its default. - if (includeToc && stixVersion === '2.1') { - const tocObject = collectionObject - ? structuredClone(collectionObject) - : buildTocObject(input.snapshot, objects, { - stixVersion, - attackSpecVersion, - collectionId, - createdByRef, - }); - conformToStixVersion(tocObject, stixVersion); - objects.unshift(tocObject); + if (stixVersion === '2.1') { + const collectionObject = buildCollectionObject(input.snapshot, objects, { + stixVersion, + publication, + }); + conformToStixVersion(collectionObject, stixVersion); + objects.unshift(collectionObject); } return { @@ -290,5 +292,5 @@ module.exports = { // Helpers (exported for testing) buildTierLookup, - buildTocObject, + buildCollectionObject, }; diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 12d95569..a44fa698 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -261,11 +261,43 @@ const promotionConflictsSchema = z.object({ staged_to_members: conflictPolicySchema.optional(), }); +// Publication metadata inheritance. Each attribute either inherits the global +// system-configuration value or carries an explicit track-scoped override. +const inheritedIdentitySchema = z.discriminatedUnion('inherit', [ + z.object({ inherit: z.literal(true) }).strict(), + z + .object({ + inherit: z.literal(false), + value: createStixIdValidator('identity'), + }) + .strict(), +]); + +const inheritedMarkingRefsSchema = z.discriminatedUnion('inherit', [ + z.object({ inherit: z.literal(true) }).strict(), + z + .object({ + inherit: z.literal(false), + value: z.array(createStixIdValidator('marking-definition')), + }) + .strict(), +]); + +const publicationConfigSchema = z + .object({ + collection_id: createStixIdValidator('x-mitre-collection').nullable().optional(), + created: z.iso.datetime().nullable().optional(), + created_by_ref: inheritedIdentitySchema.optional(), + object_marking_refs: inheritedMarkingRefsSchema.optional(), + }) + .strict(); + const updateConfigBodySchema = z.object({ candidacy_threshold: candidacyThresholdSchema.optional(), auto_promote: z.boolean().optional(), promotion_conflicts: promotionConflictsSchema.optional(), member_sync: memberSyncConfigSchema.optional(), + publication: publicationConfigSchema.optional(), }); // ============================================================================= @@ -396,7 +428,6 @@ const createTrackBodySchema = z description: z.string().optional(), snapshot_description: snapshotDescriptionSchema.optional(), type: trackTypeQuerySchema.default('standard'), - object_marking_refs: z.array(stixIdentifierSchema).optional(), composition: compositionSchema.optional(), snapshot_schedule: snapshotScheduleSchema.optional(), scheduled_materialization: scheduledMaterializationSchema.optional(), @@ -431,7 +462,6 @@ const createFromBundleBodySchema = z.object({ const updateMetadataBodySchema = z.object({ name: trackNameSchema.optional(), description: z.string().optional(), - object_marking_refs: z.array(stixIdentifierSchema).optional(), }); /** PUT /release-tracks/:id/snapshots/:modified/description */ @@ -584,6 +614,9 @@ const reconstructSnapshotGraphBodySchema = z }) .strict(), entries: z.array(sourceGraphEntrySchema).min(1), + // The content manifest the caller expects to replace. Required when the + // snapshot's current manifest was not produced from the same attestation. + replace_manifest_id: z.string().optional(), }) .strict(); @@ -641,6 +674,7 @@ module.exports = { updateMetadataBodySchema, updateSnapshotDescriptionBodySchema, releaseBodySchema, + publicationConfigSchema, cloneBodySchema, addCandidatesBodySchema, reviewCandidatesBodySchema, diff --git a/app/lib/release-tracks/release-track-validators.js b/app/lib/release-tracks/release-track-validators.js index 24e771fa..164c3492 100644 --- a/app/lib/release-tracks/release-track-validators.js +++ b/app/lib/release-tracks/release-track-validators.js @@ -49,6 +49,12 @@ const validateIdentityRef = { `"${props.value}" is not a valid identity reference (expected "identity--")`, }; +const validateCollectionId = { + validator: (v) => createStixIdValidator('x-mitre-collection').safeParse(v).success, + message: (props) => + `"${props.value}" is not a valid collection identifier (expected "x-mitre-collection--")`, +}; + const validateMarkingDefRefs = { validator: (v) => v.every((ref) => createStixIdValidator('marking-definition').safeParse(ref).success), @@ -102,6 +108,7 @@ module.exports = { validateStixId, validateIdentityRef, validateMarkingDefRefs, + validateCollectionId, validateVersion, validateCron, validateSnapshotSchedule, diff --git a/app/models/release-tracks/release-track-content-manifest-model.js b/app/models/release-tracks/release-track-content-manifest-model.js new file mode 100644 index 00000000..71523d9b --- /dev/null +++ b/app/models/release-tracks/release-track-content-manifest-model.js @@ -0,0 +1,125 @@ +'use strict'; + +const mongoose = require('mongoose'); + +const exactRevisionSchema = new mongoose.Schema( + { + object_ref: { type: String, required: true }, + object_modified: { type: Date, required: true }, + }, + { _id: false }, +); + +// A content manifest is the sealed bill of materials for one exact member +// set. Snapshots reference it by `manifest_id`; several snapshots may share +// one manifest. See docs/developer/release-tracks/sealed-content-manifests.md. +const manifestSchema = new mongoose.Schema( + { + manifest_id: { type: String, required: true, unique: true }, + track_id: { type: String, required: true }, + // The snapshot whose write sealed this manifest (later clones may inherit it). + snapshot_modified: { type: Date, required: true }, + // `pending` while entries are being written and verified; `active` once a + // snapshot references it. Both states protect referenced revisions from + // deletion, so an interrupted seal never leaves an unprotected manifest. + state: { + type: String, + enum: ['pending', 'active'], + required: true, + default: 'pending', + }, + // 1: legacy manifests from the July 2026 backfill whose relationship + // entries carry frozen STIX payloads and may include relationship- + // discovered `secondary` objects (replayed read-only). + // 2: pointer-only manifests closed over exact members. + schema_version: { type: Number, required: true, default: 2 }, + // Which write sealed the manifest. + seal_reason: { + type: String, + required: true, + enum: [ + 'track_creation', + 'members_written', + 'release', + 'materialization', + 'track_clone', + 'source_reconstruction', + 'migration', + 'legacy_graph', + ], + }, + // Present only for source_reconstruction: the externally verified bundle + // an administrator attested the pointers were derived from. + source_attestation: { type: mongoose.Schema.Types.Mixed }, + created_at: { type: Date, required: true, default: Date.now }, + }, + { collection: 'releaseTrackContentManifests' }, +); + +manifestSchema.index( + { track_id: 1, snapshot_modified: 1, state: 1 }, + { name: 'manifest_by_snapshot' }, +); + +// One entry per exact object revision a manifest emits or depends on. +// root a member revision (tier is always `members`) +// relationship an SRO whose source and target are both members; `source` +// and `target` record the member revisions it ships with +// supporting an identity or marking definition referenced by emitted +// objects or by the collection object; unversioned marking +// definitions are frozen by value in `frozen_stix` +// link_target a non-emitted object hydrated only to render LinkById tags +// secondary legacy (schema 1) relationship-discovered object +const entrySchema = new mongoose.Schema( + { + manifest_id: { type: String, required: true }, + track_id: { type: String, required: true }, + snapshot_modified: { type: Date, required: true }, + revision_key: { type: String, required: true }, + kind: { + type: String, + enum: ['root', 'relationship', 'secondary', 'supporting', 'link_target'], + required: true, + }, + tier: { + type: String, + enum: ['members', 'staged', 'candidates', 'quarantine'], + }, + object_ref: { type: String, required: true }, + object_modified: { type: Date }, + source: { type: exactRevisionSchema }, + target: { type: exactRevisionSchema }, + // Source-attested serialization hints: false-valued defaults the attested + // publication omitted. + omitted_optional_defaults: { + type: [String], + enum: ['revoked', 'x_mitre_remote_support'], + default: undefined, + }, + // Legacy (schema 1): which member revision discovered a secondary object. + discovered_from: { type: [exactRevisionSchema], default: undefined }, + frozen_stix: { type: mongoose.Schema.Types.Mixed }, + }, + { collection: 'releaseTrackContentManifestEntries' }, +); + +entrySchema.index( + { manifest_id: 1, revision_key: 1, kind: 1, tier: 1 }, + { unique: true, name: 'unique_manifest_entry' }, +); +entrySchema.index( + { object_ref: 1, object_modified: 1, manifest_id: 1 }, + { name: 'manifest_revision_protection' }, +); +entrySchema.index({ manifest_id: 1, kind: 1, tier: 1 }); + +const ReleaseTrackContentManifest = mongoose.model('ReleaseTrackContentManifest', manifestSchema); +const ReleaseTrackContentManifestEntry = mongoose.model( + 'ReleaseTrackContentManifestEntry', + entrySchema, +); + +module.exports = { + ReleaseTrackContentManifest, + ReleaseTrackContentManifestEntry, +}; diff --git a/app/models/release-tracks/release-track-graph-manifest-model.js b/app/models/release-tracks/release-track-graph-manifest-model.js deleted file mode 100644 index 1356ee58..00000000 --- a/app/models/release-tracks/release-track-graph-manifest-model.js +++ /dev/null @@ -1,92 +0,0 @@ -'use strict'; - -const mongoose = require('mongoose'); - -const exactRevisionSchema = new mongoose.Schema( - { - object_ref: { type: String, required: true }, - object_modified: { type: Date, required: true }, - }, - { _id: false }, -); - -const manifestSchema = new mongoose.Schema( - { - manifest_id: { type: String, required: true, unique: true }, - track_id: { type: String, required: true }, - snapshot_modified: { type: Date, required: true }, - state: { - type: String, - enum: ['pending', 'active'], - required: true, - default: 'pending', - }, - schema_version: { type: Number, required: true, default: 1 }, - resolver_version: { type: String, required: true }, - baseline_reconstruction: { type: Boolean, required: true, default: false }, - source_attestation: { type: mongoose.Schema.Types.Mixed }, - created_at: { type: Date, required: true, default: Date.now }, - }, - { collection: 'releaseTrackGraphManifests' }, -); - -manifestSchema.index( - { track_id: 1, snapshot_modified: 1, state: 1 }, - { name: 'manifest_by_snapshot' }, -); - -const entrySchema = new mongoose.Schema( - { - manifest_id: { type: String, required: true }, - track_id: { type: String, required: true }, - snapshot_modified: { type: Date, required: true }, - revision_key: { type: String, required: true }, - kind: { - type: String, - enum: ['root', 'relationship', 'secondary', 'supporting', 'link_target', 'collection'], - required: true, - }, - tier: { - type: String, - enum: ['members', 'staged', 'candidates', 'quarantine'], - }, - object_status: { type: String }, - object_ref: { type: String, required: true }, - object_modified: { type: Date }, - source: { type: exactRevisionSchema }, - target: { type: exactRevisionSchema }, - omitted_optional_defaults: { - type: [String], - enum: ['revoked', 'x_mitre_remote_support'], - default: undefined, - }, - discovered_from: { type: [exactRevisionSchema], default: undefined }, - // Schema-v2 relationships are exact-revision pointers. Marking - // definitions are not STIX-versioned, so their complete payload is frozen - // for the same replay guarantee. Schema-v1 relationships retain frozen - // payloads for backwards-compatible replay. - frozen_stix: { type: mongoose.Schema.Types.Mixed }, - }, - { collection: 'releaseTrackGraphManifestEntries' }, -); - -entrySchema.index( - { manifest_id: 1, revision_key: 1, kind: 1, tier: 1 }, - { unique: true, name: 'unique_manifest_entry' }, -); -entrySchema.index( - { object_ref: 1, object_modified: 1, manifest_id: 1 }, - { name: 'manifest_revision_protection' }, -); -entrySchema.index({ manifest_id: 1, kind: 1, tier: 1 }); - -const ReleaseTrackGraphManifest = mongoose.model('ReleaseTrackGraphManifest', manifestSchema); -const ReleaseTrackGraphManifestEntry = mongoose.model( - 'ReleaseTrackGraphManifestEntry', - entrySchema, -); - -module.exports = { - ReleaseTrackGraphManifest, - ReleaseTrackGraphManifestEntry, -}; diff --git a/app/models/release-tracks/release-track-reconciliation-model.js b/app/models/release-tracks/release-track-reconciliation-model.js index 397847fe..3a9b07b5 100644 --- a/app/models/release-tracks/release-track-reconciliation-model.js +++ b/app/models/release-tracks/release-track-reconciliation-model.js @@ -28,10 +28,13 @@ const releaseTrackReconciliationSchema = new mongoose.Schema( required: true, enum: ['contents_changed', 'repair', 'full_scan'], }, + // Only outstanding work is stored: a record is created before the + // backref listeners run and deleted when they succeed, so every document + // in this collection is a reconciliation that still needs repair. status: { type: String, required: true, - enum: ['pending', 'completed', 'failed'], + enum: ['pending', 'failed'], default: 'pending', }, attempts: { diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 1f94acd2..5f3acebd 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -8,6 +8,7 @@ const { validateStixId, validateIdentityRef, validateMarkingDefRefs, + validateCollectionId, validateVersion, validateObjectTypesFilter, } = require('../../lib/release-tracks/release-track-validators'); @@ -239,18 +240,6 @@ const promotionConflictsDefinition = { }; const promotionConflictsSchema = new mongoose.Schema(promotionConflictsDefinition, { _id: false }); -const includeSecondaryObjectsDefinition = { - enabled: { type: Boolean, default: true }, - status_threshold: { - type: String, - enum: ['work-in-progress', 'awaiting-review', 'reviewed'], - default: 'reviewed', - }, -}; -const includeSecondaryObjectsSchema = new mongoose.Schema(includeSecondaryObjectsDefinition, { - _id: false, -}); - // --- Member sync sub-schemas --- const memberSyncSupplantDefinition = { @@ -280,6 +269,56 @@ const memberSyncDefinition = { }; const memberSyncSchema = new mongoose.Schema(memberSyncDefinition, { _id: false }); +// --- Publication sub-schemas --- +// +// Publication metadata follows an inheritance rule: each attribute either +// inherits the global (system configuration) value or carries an explicit +// track-scoped override. Collection identity and creation time default to +// track-derived values and become immutable once the track has a release. + +const inheritedIdentityDefinition = { + inherit: { type: Boolean, required: true, default: true }, + value: { + type: String, + validate: validateIdentityRef, + }, +}; +const inheritedIdentitySchema = new mongoose.Schema(inheritedIdentityDefinition, { _id: false }); + +const inheritedMarkingRefsDefinition = { + inherit: { type: Boolean, required: true, default: true }, + value: { + type: [String], + default: undefined, + validate: validateMarkingDefRefs, + }, +}; +const inheritedMarkingRefsSchema = new mongoose.Schema(inheritedMarkingRefsDefinition, { + _id: false, +}); + +const publicationConfigDefinition = { + collection_id: { + type: String, + validate: validateCollectionId, + }, + created: { type: Date }, + created_by_ref: { type: inheritedIdentitySchema, default: () => ({ inherit: true }) }, + object_marking_refs: { type: inheritedMarkingRefsSchema, default: () => ({ inherit: true }) }, +}; +const publicationConfigSchema = new mongoose.Schema(publicationConfigDefinition, { _id: false }); + +// Values frozen onto a tagged snapshot at release commit. They are the exact +// inputs used to render the x-mitre-collection object for that release. +const frozenPublicationDefinition = { + collection_id: { type: String, required: true, validate: validateCollectionId }, + created: { type: Date, required: true }, + created_by_ref: { type: String, required: true, validate: validateIdentityRef }, + object_marking_refs: { type: [String], required: true, validate: validateMarkingDefRefs }, + attack_spec_version: { type: String, required: true }, +}; +const frozenPublicationSchema = new mongoose.Schema(frozenPublicationDefinition, { _id: false }); + const configDefinition = { candidacy_threshold: { type: String, @@ -287,7 +326,6 @@ const configDefinition = { default: 'reviewed', }, auto_promote: { type: Boolean, default: true }, - include_secondary_objects: { type: includeSecondaryObjectsSchema, default: undefined }, promotion_conflicts: { type: promotionConflictsSchema, default: () => ({}), @@ -296,6 +334,10 @@ const configDefinition = { type: memberSyncSchema, default: () => ({}), }, + publication: { + type: publicationConfigSchema, + default: () => ({}), + }, }; const configSchema = new mongoose.Schema(configDefinition, { _id: false }); @@ -373,7 +415,13 @@ const releaseTrackSnapshotDefinition = { default: null, validate: validateVersion, }, - graph_manifest_id: { type: String }, + // Every snapshot references the sealed content manifest that describes its + // exact member graph. Member-changing writes seal a new manifest; other + // clones inherit their predecessor's manifest by reference. + content_manifest_id: { type: String, required: true }, + // Release-only fields frozen at commit. + publication: { type: frozenPublicationSchema }, + bundle_id: { type: String }, bundle_hashes: { type: bundleHashesSchema }, snapshot_description: { type: String, @@ -392,11 +440,6 @@ const releaseTrackSnapshotDefinition = { type: String, validate: validateIdentityRef, }, - object_marking_refs: { - type: [String], - default: undefined, - validate: validateMarkingDefRefs, - }, // --- Standard track tiers --- members: { type: [memberEntrySchema], default: [] }, @@ -483,5 +526,7 @@ module.exports = { compositionResolutionSchema, scheduledMaterializationSchema, configSchema, + publicationConfigSchema, + frozenPublicationSchema, versionHistoryEntrySchema, }; diff --git a/app/repository/relationships-repository.js b/app/repository/relationships-repository.js index c601d736..0bec975c 100644 --- a/app/repository/relationships-repository.js +++ b/app/repository/relationships-repository.js @@ -156,42 +156,50 @@ class RelationshipsRepository extends BaseRepository { } /** - * Retrieve every relationship revision whose stored source or target pin - * exactly matches one of the supplied object revisions. + * Retrieve the newest revision of every relationship lineage whose source + * and target IDs are both in the supplied object set, regardless of + * lifecycle state. Sealing chooses each lineage's globally latest revision + * before applying active/deprecated filters so an older active revision is + * never resurrected by a newer inactive one. * - * The caller deliberately receives inactive and superseded relationship - * revisions. Deterministic graph capture must choose the newest revision - * for an exact endpoint pair before applying active/deprecated filters, or - * an older active revision could be resurrected. + * @param {Array} objectRefs - Member STIX IDs + * @param {Object} [options] + * @param {number} [options.batchSize] + * @returns {Promise>} Lean relationship documents */ - async retrieveRevisionsTouchingExactEndpoints(endpointRevisions, options = {}) { - if (!Array.isArray(endpointRevisions) || endpointRevisions.length === 0) return []; + async retrieveLatestBetween(objectRefs, options = {}) { + if (!Array.isArray(objectRefs) || objectRefs.length === 0) return []; - const batchSize = options.batchSize || 250; - const revisionsByKey = new Map(); + const batchSize = options.batchSize || 2000; try { - for (let offset = 0; offset < endpointRevisions.length; offset += batchSize) { - const batch = endpointRevisions.slice(offset, offset + batchSize); - const exactEndpointQueries = batch.flatMap((entry) => { - const objectModified = new Date(entry.object_modified); - return [ - { - 'workspace.relationship_endpoints.source.object_ref': entry.object_ref, - 'workspace.relationship_endpoints.source.object_modified': objectModified, - }, - { - 'workspace.relationship_endpoints.target.object_ref': entry.object_ref, - 'workspace.relationship_endpoints.target.object_modified': objectModified, - }, - ]; - }); - const relationships = await this.model.find({ $or: exactEndpointQueries }).lean().exec(); - for (const relationship of relationships) { - const key = `${relationship.stix.id}::${new Date(relationship.stix.modified).getTime()}`; - revisionsByKey.set(key, relationship); - } + const lineageIds = new Set(); + for (let offset = 0; offset < objectRefs.length; offset += batchSize) { + const batch = objectRefs.slice(offset, offset + batchSize); + const ids = await this.model + .distinct('stix.id', { + 'stix.source_ref': { $in: batch }, + 'stix.target_ref': { $in: objectRefs }, + }) + .exec(); + for (const id of ids) lineageIds.add(id); + } + if (lineageIds.size === 0) return []; + + const results = []; + const allIds = [...lineageIds]; + for (let offset = 0; offset < allIds.length; offset += batchSize) { + const batch = allIds.slice(offset, offset + batchSize); + const latest = await this.model + .aggregate([ + { $match: { 'stix.id': { $in: batch } } }, + { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, + { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, + { $replaceRoot: { newRoot: '$document' } }, + ]) + .exec(); + results.push(...latest); } - return [...revisionsByKey.values()]; + return results; } catch (err) { throw new DatabaseError(err); } diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 4b4ad57a..600c7d4a 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -288,7 +288,8 @@ class ReleaseTrackDynamicRepository { type: 1, modified: 1, version: 1, - graph_manifest_id: 1, + content_manifest_id: 1, + bundle_id: 1, bundle_hashes: 1, snapshot_description: 1, name: 1, @@ -397,7 +398,12 @@ class ReleaseTrackDynamicRepository { } } - async attachGraphManifest(trackId, modified, manifestId) { + /** + * Replace a tagged snapshot's content manifest. Used by administrative + * source-attested reconstruction, which must name the manifest it expects + * to replace so a concurrent change is detected. + */ + async replaceContentManifest(trackId, modified, expectedManifestId, manifestId) { try { const Model = this._getModel(trackId); return await Model.findOneAndUpdate( @@ -405,9 +411,9 @@ class ReleaseTrackDynamicRepository { id: trackId, modified, version: { $type: 'string' }, - graph_manifest_id: { $exists: false }, + content_manifest_id: expectedManifestId, }, - { $set: { graph_manifest_id: manifestId } }, + { $set: { content_manifest_id: manifestId }, $unset: { bundle_hashes: '' } }, { new: true, runValidators: true, lean: true }, ).exec(); } catch (err) { @@ -415,7 +421,7 @@ class ReleaseTrackDynamicRepository { } } - async detachGraphManifest(trackId, modified, manifestId) { + async attachBundleHashes(trackId, modified, manifestId, bundleHashes) { try { const Model = this._getModel(trackId); return await Model.findOneAndUpdate( @@ -423,9 +429,9 @@ class ReleaseTrackDynamicRepository { id: trackId, modified, version: { $type: 'string' }, - graph_manifest_id: manifestId, + content_manifest_id: manifestId, }, - { $unset: { graph_manifest_id: '', bundle_hashes: '' } }, + { $set: { bundle_hashes: bundleHashes } }, { new: true, runValidators: true, lean: true }, ).exec(); } catch (err) { @@ -433,19 +439,19 @@ class ReleaseTrackDynamicRepository { } } - async attachBundleHashes(trackId, modified, manifestId, bundleHashes) { + /** + * Return the subset of manifest IDs still referenced by any snapshot in the + * track. Manifests are shared by reference between a sealing snapshot and + * the clones that inherit it. + */ + async findReferencedManifestIds(trackId, manifestIds) { + if (!Array.isArray(manifestIds) || manifestIds.length === 0) return []; try { const Model = this._getModel(trackId); - return await Model.findOneAndUpdate( - { - id: trackId, - modified, - version: { $type: 'string' }, - graph_manifest_id: manifestId, - }, - { $set: { bundle_hashes: bundleHashes } }, - { new: true, runValidators: true, lean: true }, - ).exec(); + return await Model.distinct('content_manifest_id', { + id: trackId, + content_manifest_id: { $in: manifestIds }, + }).exec(); } catch (err) { throw new DatabaseError(err); } @@ -455,7 +461,10 @@ class ReleaseTrackDynamicRepository { try { const Model = this._getModel(trackId); const query = { id: trackId, version: null, modified: { $lt: modified } }; - const snapshots = await Model.find(query).select('modified graph_manifest_id').lean().exec(); + const snapshots = await Model.find(query) + .select('modified content_manifest_id') + .lean() + .exec(); if (snapshots.length > 0) { await Model.deleteMany({ _id: { $in: snapshots.map((snapshot) => snapshot._id) } }).exec(); } diff --git a/app/repository/release-tracks/release-track-reconciliation.repository.js b/app/repository/release-tracks/release-track-reconciliation.repository.js index 154dec54..52f817c8 100644 --- a/app/repository/release-tracks/release-track-reconciliation.repository.js +++ b/app/repository/release-tracks/release-track-reconciliation.repository.js @@ -44,22 +44,28 @@ class ReleaseTrackReconciliationRepository { } } + /** + * A completed reconciliation needs no record: the collection holds only + * outstanding work (pending or failed attempts) so it stays small and its + * contents always mean "repair me". The completed summary is returned to + * the caller without being persisted. + */ async complete(reconciliationId, snapshotModified) { const now = new Date(); try { - return await ReleaseTrackReconciliation.findOneAndUpdate( - { reconciliation_id: reconciliationId }, - { - $set: { - status: 'completed', - reconciled_snapshot_modified: snapshotModified || null, - updated_at: now, - completed_at: now, - last_error: null, - }, - }, - { new: true, lean: true }, - ).exec(); + const record = await ReleaseTrackReconciliation.findOneAndDelete({ + reconciliation_id: reconciliationId, + }) + .lean() + .exec(); + return { + ...(record || { reconciliation_id: reconciliationId }), + status: 'completed', + reconciled_snapshot_modified: snapshotModified || null, + updated_at: now, + completed_at: now, + last_error: null, + }; } catch (error) { throw new DatabaseError(error); } diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 16dc7ca9..c44ff750 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -278,20 +278,7 @@ router .post( authn.authenticate, authz.requireRole(authz.admin), - releaseTracksController.reconstructSnapshotGraph, - ); - -router - .route('/release-tracks/:id/snapshots/:modified/graph') - .post( - authn.authenticate, - authz.requireRole(authz.editorOrHigher), - releaseTracksController.createSnapshotGraph, - ) - .delete( - authn.authenticate, - authz.requireRole(authz.editorOrHigher), - releaseTracksController.deleteSnapshotGraph, + releaseTracksController.reconstructSnapshotManifest, ); router diff --git a/app/services/meta-classes/base.service.js b/app/services/meta-classes/base.service.js index 225ac0ce..8ea36891 100644 --- a/app/services/meta-classes/base.service.js +++ b/app/services/meta-classes/base.service.js @@ -780,8 +780,8 @@ class BaseService extends ServiceWithHooks { * exempt from this guard. */ static async assertNotGraphPinned(document, operation) { - const graphManifestService = require('../release-tracks/graph-manifest-service'); - const pins = await graphManifestService.findPinsForRevision( + const contentManifestService = require('../release-tracks/content-manifest-service'); + const pins = await contentManifestService.findPinsForRevision( document.stix.id, document.stix.modified, ); @@ -798,8 +798,8 @@ class BaseService extends ServiceWithHooks { } static async assertNoGraphPinnedVersions(stixId, operation) { - const graphManifestService = require('../release-tracks/graph-manifest-service'); - const pins = await graphManifestService.findPinsForObject(stixId); + const contentManifestService = require('../release-tracks/content-manifest-service'); + const pins = await contentManifestService.findPinsForObject(stixId); if (pins.length === 0) return; throw new SnapshotGraphPinnedRevisionError({ diff --git a/app/services/release-tracks/bundle-hash-service.js b/app/services/release-tracks/bundle-hash-service.js index aad2e0e7..b37b4765 100644 --- a/app/services/release-tracks/bundle-hash-service.js +++ b/app/services/release-tracks/bundle-hash-service.js @@ -16,7 +16,7 @@ async function generateBundleHashes(snapshot) { exportService.exportSnapshot(snapshot, 'bundle', { stixVersion: '2.1' }), ]); return { - manifest_id: snapshot.graph_manifest_id, + manifest_id: snapshot.content_manifest_id, stix_2_0: hashDownloadPayload(stix20Bundle), stix_2_1: hashDownloadPayload(stix21Bundle), }; diff --git a/app/services/release-tracks/content-manifest-service.js b/app/services/release-tracks/content-manifest-service.js new file mode 100644 index 00000000..c1d3fe81 --- /dev/null +++ b/app/services/release-tracks/content-manifest-service.js @@ -0,0 +1,965 @@ +'use strict'; + +// ============================================================================= +// Content Manifest Service +// +// A content manifest is the sealed bill of materials for one exact member set. +// It records, as exact-revision pointers, every object a member-only bundle +// export emits: member roots, relationships closed over those members, +// supporting identities and marking definitions, and non-emitted LinkById +// render targets. Every snapshot references a manifest from birth; a new one +// is sealed whenever a snapshot's members are written and inherited otherwise. +// +// One graph algorithm (resolveClosedGraph) serves sealing, release previews, +// and draft exports that add live workflow tiers. It never discovers +// secondary SDOs through relationships: an SRO is selected only when both of +// its endpoint IDs are members, and the member revisions become its pins. +// +// ============================================================================= + +const { isDeepStrictEqual } = require('node:util'); +const { v4: uuidv4 } = require('uuid'); +const linkById = require('../../lib/linkById'); +const logger = require('../../lib/logger'); +const bundleRelationships = require('../../lib/stix-bundle-relationships'); +const attackObjectsRepository = require('../../repository/attack-objects-repository'); +const relationshipsRepository = require('../../repository/relationships-repository'); +const dynamicRepo = require('../../repository/release-tracks/release-track-dynamic.repository'); +const { + ReleaseTrackContentManifest, + ReleaseTrackContentManifestEntry, +} = require('../../models/release-tracks/release-track-content-manifest-model'); +const { ReleaseContentIntegrityError } = require('../../exceptions'); +const primaryRevisionService = require('./primary-revision-service'); +const publicationService = require('./publication-service'); + +const MANIFEST_SCHEMA_VERSION = 2; +const MANIFEST_ID_PREFIX = 'release-track-content-manifest--'; +const STATISTIC_FIELDS_BY_KIND = { + root: 'primary_count', + secondary: 'secondary_count', + relationship: 'relationship_count', + supporting: 'supporting_count', + link_target: 'link_target_count', +}; +const MUTATION_PROTECTED_ENTRY_FILTER = { + $or: [ + { kind: { $ne: 'root' } }, + { kind: 'root', tier: { $in: ['members', 'quarantine'] } }, + { kind: 'root', 'discovered_from.0': { $exists: true } }, + ], +}; + +function revisionKey(objectRef, objectModified) { + return `${objectRef}::${new Date(objectModified).getTime()}`; +} + +function manifestUuid(manifestId) { + return manifestId?.split('--')[1]; +} + +function exactMemberMap(entries) { + const membersByObjectRef = new Map(); + for (const entry of entries) { + const existing = membersByObjectRef.get(entry.object_ref); + if ( + existing && + revisionKey(existing.object_ref, existing.object_modified) !== + revisionKey(entry.object_ref, entry.object_modified) + ) { + throw new ReleaseContentIntegrityError( + [ + { + object_ref: entry.object_ref, + object_modified: new Date(entry.object_modified).toISOString(), + dependency: 'unique_member_revision', + }, + ], + { details: 'A sealed snapshot cannot select two revisions of one STIX object.' }, + ); + } + membersByObjectRef.set(entry.object_ref, entry); + } + return membersByObjectRef; +} + +function memberPin(member) { + return { object_ref: member.object_ref, object_modified: new Date(member.object_modified) }; +} + +function authoredPin(relationship, side) { + const endpoint = relationship.workspace?.relationship_endpoints?.[side]; + if (!endpoint?.object_ref || !endpoint.object_modified) return null; + return { object_ref: endpoint.object_ref, object_modified: new Date(endpoint.object_modified) }; +} + +// ============================================================================= +// Closed-member graph resolution (the one algorithm) +// ============================================================================= + +/** + * Resolve the exact graph emitted for a member set. + * + * @param {Array<{object_ref: string, object_modified: Date|string}>} memberEntries + * @param {Object} [options] + * @param {Array} [options.extraSupportingRefs] - Identity and marking + * definition IDs the collection object itself references, so the bundle + * stays self-contained + * @returns {Promise<{ + * roots: { entries: Array, documents: Array }, + * relationships: Array<{ relationship: Object, source: Object, target: Object, + * stale_endpoints: Array }>, + * supportingDocuments: Array, + * linkTargetDocuments: Array, + * }>} + */ +async function resolveClosedGraph(memberEntries, options = {}) { + const roots = await primaryRevisionService.assertStoredEntries(memberEntries || []); + const membersByObjectRef = exactMemberMap(roots.entries); + + const candidates = await relationshipsRepository.retrieveLatestBetween([ + ...membersByObjectRef.keys(), + ]); + const relationships = []; + for (const relationship of candidates) { + const sourceMember = membersByObjectRef.get(relationship.stix.source_ref); + const targetMember = membersByObjectRef.get(relationship.stix.target_ref); + if (!sourceMember || !targetMember) continue; + if ( + !bundleRelationships.relationshipIsActive(relationship) || + bundleRelationships.isDeprecatedPattern(relationship.stix) + ) { + continue; + } + const source = memberPin(sourceMember); + const target = memberPin(targetMember); + const staleEndpoints = []; + for (const [side, pin] of [ + ['source', source], + ['target', target], + ]) { + const authored = authoredPin(relationship, side); + if ( + authored && + authored.object_ref === pin.object_ref && + authored.object_modified.getTime() !== pin.object_modified.getTime() + ) { + staleEndpoints.push(side); + } + } + relationships.push({ relationship, source, target, stale_endpoints: staleEndpoints }); + } + relationships.sort((left, right) => + left.relationship.stix.id.localeCompare(right.relationship.stix.id), + ); + + const emitted = [...roots.documents, ...relationships.map((candidate) => candidate.relationship)]; + const supportingDocuments = await loadSupportingDocuments( + emitted, + membersByObjectRef, + options.extraSupportingRefs || [], + ); + const linkTargetDocuments = await loadLinkTargets(emitted, roots.documents); + + return { roots, relationships, supportingDocuments, linkTargetDocuments }; +} + +/** + * Identity and marking definitions referenced by a snapshot's collection + * object, which must ship alongside the content they describe. + */ +async function publicationSupportingRefs(snapshot) { + const publication = await publicationService.publicationForExport(snapshot); + return [publication.created_by_ref, ...(publication.object_marking_refs || [])].filter(Boolean); +} + +async function loadSupportingDocuments(documents, selectedByObjectRef, extraRefs = []) { + const supportingRefs = new Set(extraRefs); + for (const document of documents) { + if (document.stix.created_by_ref) supportingRefs.add(document.stix.created_by_ref); + for (const markingRef of document.stix.object_marking_refs || []) { + supportingRefs.add(markingRef); + } + } + + const supporting = []; + for (const objectRef of supportingRefs) { + if (selectedByObjectRef.has(objectRef)) continue; + const document = await attackObjectsRepository.retrieveLatestByStixIdLean(objectRef); + if (document) { + supporting.push(document); + } else { + logger.warn(`ContentManifestService: Referenced supporting object not found: ${objectRef}`); + } + } + return supporting; +} + +async function loadLinkTargets(documents, rootDocuments) { + const selectedByAttackId = new Map(); + for (const document of rootDocuments) { + const attackId = linkById.getAttackId(document.stix); + if (attackId) selectedByAttackId.set(attackId, document); + } + const linkTargets = new Map(); + for (const document of documents) { + for (const attackId of linkById.extractLinkByIds(document.stix)) { + if (selectedByAttackId.has(attackId) || linkTargets.has(attackId)) continue; + const target = await linkById.getAttackObjectFromDatabase(attackId); + if (target) linkTargets.set(attackId, target); + } + } + return [...linkTargets.values()]; +} + +function entriesFromGraph(graph) { + const entries = graph.roots.documents.map((document) => ({ + revision_key: revisionKey(document.stix.id, document.stix.modified), + kind: 'root', + tier: 'members', + object_ref: document.stix.id, + object_modified: document.stix.modified, + })); + for (const candidate of graph.relationships) { + entries.push({ + revision_key: revisionKey( + candidate.relationship.stix.id, + candidate.relationship.stix.modified, + ), + kind: 'relationship', + object_ref: candidate.relationship.stix.id, + object_modified: candidate.relationship.stix.modified, + source: candidate.source, + target: candidate.target, + }); + } + for (const document of graph.supportingDocuments) { + const isVersioned = Boolean(document.stix.modified); + entries.push({ + revision_key: isVersioned + ? revisionKey(document.stix.id, document.stix.modified) + : `${document.stix.id}::unversioned`, + kind: 'supporting', + object_ref: document.stix.id, + object_modified: document.stix.modified, + frozen_stix: isVersioned ? undefined : document.stix, + }); + } + for (const document of graph.linkTargetDocuments) { + entries.push({ + revision_key: revisionKey(document.stix.id, document.stix.modified), + kind: 'link_target', + object_ref: document.stix.id, + object_modified: document.stix.modified, + }); + } + return entries; +} + +/** + * Bundle-shaped view of a resolved graph, matching replay output. + */ +function graphFromResolution(graph) { + return { + documents: [ + ...graph.roots.documents, + ...graph.relationships.map((candidate) => candidate.relationship), + ...graph.supportingDocuments, + ], + linkTargetDocuments: graph.linkTargetDocuments, + sourceOmittedDefaults: new Map(), + manifest: null, + }; +} + +// ============================================================================= +// Sealing +// ============================================================================= + +async function persistManifest(snapshot, manifest, entries) { + const common = { + manifest_id: manifest.manifest_id, + track_id: snapshot.id, + snapshot_modified: snapshot.modified, + }; + await ReleaseTrackContentManifest.create({ ...common, ...manifest }); + try { + if (entries.length > 0) { + await ReleaseTrackContentManifestEntry.insertMany( + entries.map((entry) => ({ ...common, ...entry })), + ); + } + // The pending manifest now protects every pointer from deletion. Verify + // once inside that window so a revision deleted during resolution cannot + // leave an attachable dangling manifest. + await primaryRevisionService.assertStoredEntries( + entries + .filter((entry) => entry.object_modified && !entry.frozen_stix) + .map((entry) => ({ object_ref: entry.object_ref, object_modified: entry.object_modified })), + ); + } catch (err) { + await discard(manifest.manifest_id); + throw err; + } + return manifest.manifest_id; +} + +/** + * Seal a content manifest for a snapshot's member set. + * + * @param {Object} snapshot - Snapshot document (may be unsaved) + * @param {Object} options + * @param {string} options.reason - seal_reason enum value + * @param {Array} [options.members] - Member entries when they differ + * from snapshot.members (release planning) + * @returns {Promise} The pending manifest ID + */ +async function seal(snapshot, options = {}) { + const graph = await resolveClosedGraph(options.members ?? snapshot.members ?? [], { + extraSupportingRefs: await publicationSupportingRefs(snapshot), + }); + const entries = entriesFromGraph(graph); + return persistManifest( + snapshot, + { + manifest_id: `${MANIFEST_ID_PREFIX}${uuidv4()}`, + state: 'pending', + schema_version: MANIFEST_SCHEMA_VERSION, + seal_reason: options.reason, + created_at: new Date(), + }, + entries, + ); +} + +async function activate(manifestId) { + await ReleaseTrackContentManifest.updateOne( + { manifest_id: manifestId, state: 'pending' }, + { $set: { state: 'active' } }, + ).exec(); +} + +async function discard(manifestId) { + if (!manifestId) return; + await Promise.all([ + ReleaseTrackContentManifestEntry.deleteMany({ manifest_id: manifestId }).exec(), + ReleaseTrackContentManifest.deleteOne({ manifest_id: manifestId }).exec(), + ]); +} + +/** + * Discard manifests that no snapshot in the track references any more. + * Manifests are shared by reference between a sealing snapshot and the + * clones that inherit it, so callers must never discard by snapshot alone. + */ +async function discardUnreferenced(trackId, manifestIds) { + const candidates = [...new Set((manifestIds || []).filter(Boolean))]; + if (candidates.length === 0) return []; + const referenced = new Set(await dynamicRepo.findReferencedManifestIds(trackId, candidates)); + const unreferenced = candidates.filter((manifestId) => !referenced.has(manifestId)); + await Promise.all(unreferenced.map((manifestId) => discard(manifestId))); + return unreferenced; +} + +async function discardTrack(trackId) { + const manifests = await ReleaseTrackContentManifest.find({ track_id: trackId }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec(); + const manifestIds = manifests.map((manifest) => manifest.manifest_id); + + await Promise.all([ + manifestIds.length > 0 + ? ReleaseTrackContentManifestEntry.deleteMany({ manifest_id: { $in: manifestIds } }).exec() + : Promise.resolve(), + ReleaseTrackContentManifest.deleteMany({ track_id: trackId }).exec(), + ]); +} + +/** + * Remove manifests owned by a track that no surviving snapshot references. + * Used by deletion recovery paths. + */ +async function discardOrphans(trackId) { + const manifests = await ReleaseTrackContentManifest.find({ track_id: trackId }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec(); + return discardUnreferenced( + trackId, + manifests.map((manifest) => manifest.manifest_id), + ); +} + +// ============================================================================= +// Source-attested reconstruction (administrative) +// ============================================================================= + +function sourcePlanIntegrityError(details, references = []) { + return new ReleaseContentIntegrityError(references, { details }); +} + +async function buildSourceManifestEntries(snapshot, plan) { + const seenObjectRefs = new Set(); + const planned = []; + + for (const input of plan.entries) { + if (seenObjectRefs.has(input.object_ref)) { + throw sourcePlanIntegrityError( + `Source bundle contains more than one revision for '${input.object_ref}'.`, + [{ object_ref: input.object_ref, dependency: 'unique_source_revision' }], + ); + } + seenObjectRefs.add(input.object_ref); + + const isVersioned = input.object_modified != null; + if (isVersioned && input.frozen_stix) { + throw sourcePlanIntegrityError( + 'Versioned source-bundle entries must be exact database pointers, not frozen payloads.', + [{ object_ref: input.object_ref, dependency: 'pointer_only_manifest' }], + ); + } + if (!isVersioned) { + if ( + input.kind !== 'supporting' || + input.frozen_stix?.type !== 'marking-definition' || + input.frozen_stix?.id !== input.object_ref || + input.frozen_stix?.modified != null + ) { + throw sourcePlanIntegrityError( + 'Only unversioned marking definitions may be frozen in a schema-v2 manifest.', + [{ object_ref: input.object_ref, dependency: 'unversioned_supporting_object' }], + ); + } + } + if (input.kind === 'relationship') { + if (!input.source || !input.target || !isVersioned) { + throw sourcePlanIntegrityError( + 'Relationship entries require an exact relationship pointer and exact endpoint pins.', + [{ object_ref: input.object_ref, dependency: 'relationship_endpoints' }], + ); + } + } else if (input.source || input.target) { + throw sourcePlanIntegrityError( + 'Only relationship entries may declare source and target endpoint pins.', + [{ object_ref: input.object_ref, dependency: 'relationship_endpoints' }], + ); + } + + planned.push({ + ...input, + object_modified: isVersioned ? new Date(input.object_modified) : undefined, + source: input.source + ? { ...input.source, object_modified: new Date(input.source.object_modified) } + : undefined, + target: input.target + ? { ...input.target, object_modified: new Date(input.target.object_modified) } + : undefined, + revision_key: isVersioned + ? revisionKey(input.object_ref, input.object_modified) + : `${input.object_ref}::unversioned`, + }); + } + + const expectedRoots = new Map( + (snapshot.members || []).map((entry) => [ + revisionKey(entry.object_ref, entry.object_modified), + entry, + ]), + ); + const suppliedRoots = planned.filter((entry) => entry.kind === 'root'); + const suppliedRootKeys = new Set(suppliedRoots.map((entry) => entry.revision_key)); + if ( + suppliedRoots.length !== expectedRoots.size || + [...expectedRoots.keys()].some((key) => !suppliedRootKeys.has(key)) + ) { + throw sourcePlanIntegrityError( + 'Source bundle root pointers must exactly equal the tagged snapshot members.', + [{ track_id: snapshot.id, dependency: 'snapshot_members' }], + ); + } + + const versioned = planned.filter((entry) => entry.object_modified); + const hydrated = await primaryRevisionService.assertStoredEntries(versioned); + const documentsByRevision = new Map( + hydrated.documents.map((document) => [ + revisionKey(document.stix.id, document.stix.modified), + document, + ]), + ); + const selectableKeys = new Set( + planned + .filter((entry) => ['root', 'secondary'].includes(entry.kind)) + .map((entry) => entry.revision_key), + ); + + for (const entry of planned) { + if (!entry.object_modified) continue; + const document = documentsByRevision.get(entry.revision_key); + if (entry.kind === 'relationship') { + if (document.stix.type !== 'relationship') { + throw sourcePlanIntegrityError(`'${entry.object_ref}' is not a relationship revision.`, [ + { object_ref: entry.object_ref, dependency: 'relationship_type' }, + ]); + } + for (const side of ['source', 'target']) { + const endpoint = entry[side]; + if (document.stix[`${side}_ref`] !== endpoint.object_ref) { + throw sourcePlanIntegrityError( + `Relationship '${entry.object_ref}' has a mismatched ${side} pointer.`, + [{ object_ref: entry.object_ref, dependency: `${side}_ref` }], + ); + } + if (!selectableKeys.has(revisionKey(endpoint.object_ref, endpoint.object_modified))) { + throw sourcePlanIntegrityError( + `Relationship '${entry.object_ref}' references an endpoint revision absent from the source graph.`, + [{ ...endpoint, dependency: `${side}_revision` }], + ); + } + } + } else if (document.stix.type === 'relationship') { + throw sourcePlanIntegrityError( + `Relationship revision '${entry.object_ref}' must use kind 'relationship'.`, + [{ object_ref: entry.object_ref, dependency: 'entry_kind' }], + ); + } + } + + const includedObjectRefs = new Set(planned.map((entry) => entry.object_ref)); + for (const document of hydrated.documents) { + const supportingRefs = [ + document.stix.created_by_ref, + ...(document.stix.object_marking_refs || []), + ].filter(Boolean); + const missingRef = supportingRefs.find((objectRef) => !includedObjectRefs.has(objectRef)); + if (missingRef) { + throw sourcePlanIntegrityError(`Source graph omits supporting object '${missingRef}'.`, [ + { object_ref: missingRef, dependency: 'supporting_object' }, + ]); + } + } + + return planned.map((entry) => { + if (entry.kind !== 'root') return entry; + return { ...entry, tier: 'members' }; + }); +} + +async function prepareSourceReconstruction(snapshot, plan) { + const entries = await buildSourceManifestEntries(snapshot, plan); + return persistManifest( + snapshot, + { + manifest_id: `${MANIFEST_ID_PREFIX}${uuidv4()}`, + state: 'pending', + schema_version: MANIFEST_SCHEMA_VERSION, + source_attestation: plan.source_attestation, + seal_reason: 'source_reconstruction', + created_at: new Date(), + }, + entries, + ); +} + +async function findManifest(manifestId) { + return ReleaseTrackContentManifest.findOne({ + manifest_id: manifestId, + state: { $in: ['pending', 'active'] }, + }) + .lean() + .exec(); +} + +async function isSameSourceReconstruction(manifestId, sourceAttestation) { + const manifest = await findManifest(manifestId); + return Boolean( + manifest && + manifest.seal_reason === 'source_reconstruction' && + isDeepStrictEqual(manifest.source_attestation, sourceAttestation), + ); +} + +// ============================================================================= +// Replay +// ============================================================================= + +function legacySelectedRevisionKeys(entries) { + const selected = new Set( + entries + .filter((entry) => entry.kind === 'root' && entry.tier === 'members') + .map((entry) => entry.revision_key), + ); + // Schema-v1 manifests recorded relationship-discovered secondaries with + // the revision that discovered them. Replay only follows frozen edges. + let added; + do { + added = false; + for (const entry of entries) { + if (!['root', 'secondary'].includes(entry.kind) || selected.has(entry.revision_key)) { + continue; + } + if ( + (entry.discovered_from || []).some((source) => + selected.has(revisionKey(source.object_ref, source.object_modified)), + ) + ) { + selected.add(entry.revision_key); + added = true; + } + } + } while (added); + return selected; +} + +async function replayEntries(entries, manifest) { + const pointerOnly = manifest.schema_version >= MANIFEST_SCHEMA_VERSION; + const versionedEntries = entries.filter( + (entry) => + entry.object_modified && + (entry.kind !== 'relationship' || (pointerOnly && !entry.frozen_stix)), + ); + const hydrated = await primaryRevisionService.assertStoredEntries( + versionedEntries.map((entry) => ({ + object_ref: entry.object_ref, + object_modified: entry.object_modified, + })), + ); + const documentsByRevision = new Map( + hydrated.documents.map((document) => [ + revisionKey(document.stix.id, document.stix.modified), + document, + ]), + ); + for (const entry of entries) { + if (entry.frozen_stix) { + documentsByRevision.set(entry.revision_key, { stix: entry.frozen_stix }); + } + } + + const selectedRevisionKeys = pointerOnly + ? new Set( + entries + .filter((entry) => ['root', 'secondary'].includes(entry.kind)) + .map((entry) => entry.revision_key), + ) + : legacySelectedRevisionKeys(entries); + + const selectedRelationships = entries.filter( + (entry) => + entry.kind === 'relationship' && + selectedRevisionKeys.has( + revisionKey(entry.source.object_ref, entry.source.object_modified), + ) && + selectedRevisionKeys.has(revisionKey(entry.target.object_ref, entry.target.object_modified)), + ); + for (const entry of selectedRelationships) { + selectedRevisionKeys.add(entry.revision_key); + } + + const selectedDocuments = [...selectedRevisionKeys] + .map((key) => documentsByRevision.get(key)) + .filter(Boolean); + const supportingRefs = new Set(); + for (const document of selectedDocuments) { + if (document.stix.created_by_ref) supportingRefs.add(document.stix.created_by_ref); + for (const objectRef of document.stix.object_marking_refs || []) { + supportingRefs.add(objectRef); + } + } + + const supportingDocuments = entries + .filter((entry) => entry.kind === 'supporting' && supportingRefs.has(entry.object_ref)) + .map((entry) => + entry.object_modified + ? documentsByRevision.get(entry.revision_key) + : { stix: entry.frozen_stix }, + ) + .filter(Boolean); + const linkTargetDocuments = entries + .filter((entry) => entry.kind === 'link_target') + .map((entry) => documentsByRevision.get(entry.revision_key)) + .filter(Boolean); + const sourceOmittedDefaults = new Map( + entries + .filter((entry) => entry.omitted_optional_defaults?.length) + .map((entry) => [entry.object_ref, entry.omitted_optional_defaults]), + ); + + const emittedByRevision = new Map(); + for (const document of [...selectedDocuments, ...supportingDocuments]) { + const key = document.stix.modified + ? revisionKey(document.stix.id, document.stix.modified) + : `${document.stix.id}::unversioned`; + emittedByRevision.set(key, document); + } + + return { + documents: [...emittedByRevision.values()], + linkTargetDocuments, + sourceOmittedDefaults, + manifest, + }; +} + +async function loadEntries(manifestId) { + return ReleaseTrackContentManifestEntry.find({ manifest_id: manifestId }) + .sort({ _id: 1 }) + .lean() + .exec(); +} + +/** + * Replay a snapshot's sealed manifest. + */ +async function replay(snapshot) { + if (!snapshot.content_manifest_id) { + throw new ReleaseContentIntegrityError( + [ + { + track_id: snapshot.id, + snapshot_modified: new Date(snapshot.modified).toISOString(), + dependency: 'content_manifest', + }, + ], + { details: 'Snapshot does not reference a sealed content manifest.' }, + ); + } + + const manifest = await ReleaseTrackContentManifest.findOne({ + manifest_id: snapshot.content_manifest_id, + track_id: snapshot.id, + state: { $in: ['pending', 'active'] }, + }) + .lean() + .exec(); + if (!manifest) { + throw new ReleaseContentIntegrityError( + [{ manifest_id: snapshot.content_manifest_id, dependency: 'content_manifest' }], + { details: 'Snapshot content manifest is missing.' }, + ); + } + + // A snapshot link is the durable commit record. If the process stopped + // after linking a complete pending manifest but before activation, replay + // remains deterministic and repairs the visibility marker opportunistically. + if (manifest.state === 'pending') { + await activate(manifest.manifest_id); + manifest.state = 'active'; + } + + return replayEntries(await loadEntries(manifest.manifest_id), manifest); +} + +/** + * Resolve the graph live for a member set plus optional extra tier entries. + * Used by release previews (unsaved planned snapshots) and by draft exports + * that add workflow tiers. Not deterministic by design. + */ +async function resolveLive(snapshot, extraEntries = []) { + const graph = await resolveClosedGraph([...(snapshot.members || []), ...extraEntries], { + extraSupportingRefs: await publicationSupportingRefs(snapshot), + }); + return graphFromResolution(graph); +} + +/** + * Append the identity and marking definitions the collection object + * references when a replayed manifest predates the current publication rule + * (for example a virtual materialization sealed before a configuration + * change). Sealed releases normally already contain them. + */ +async function ensurePublicationSupport(documents, publication) { + const present = new Set(documents.map((document) => document.stix.id)); + const appended = []; + for (const objectRef of [ + publication.created_by_ref, + ...(publication.object_marking_refs || []), + ].filter(Boolean)) { + if (present.has(objectRef)) continue; + const document = await attackObjectsRepository.retrieveLatestByStixIdLean(objectRef); + if (document) { + appended.push(document); + present.add(objectRef); + } else { + logger.warn(`ContentManifestService: Publication supporting object not found: ${objectRef}`); + } + } + return appended; +} + +// ============================================================================= +// Preview: relationship changes between a manifest and a fresh resolution +// ============================================================================= + +function relationshipSummary(relationship, source, target, extra = {}) { + return { + object_ref: relationship.stix.id, + object_modified: new Date(relationship.stix.modified).toISOString(), + relationship_type: relationship.stix.relationship_type, + source_ref: source.object_ref, + target_ref: target.object_ref, + ...extra, + }; +} + +/** + * Compare the relationships a fresh seal would select against the manifest a + * snapshot currently references. + * + * @param {Object} snapshot - Snapshot whose manifest is the baseline + * @param {Array} members - Member set the release would seal + */ +async function previewRelationshipChanges(snapshot, members) { + const graph = await resolveClosedGraph(members); + const previousEntries = snapshot.content_manifest_id + ? (await loadEntries(snapshot.content_manifest_id)).filter( + (entry) => entry.kind === 'relationship', + ) + : []; + const previousByKey = new Map(previousEntries.map((entry) => [entry.revision_key, entry])); + const nextByKey = new Map( + graph.relationships.map((candidate) => [ + revisionKey(candidate.relationship.stix.id, candidate.relationship.stix.modified), + candidate, + ]), + ); + + const added = []; + const staleEndpoints = []; + for (const [key, candidate] of nextByKey) { + if (!previousByKey.has(key)) { + added.push(relationshipSummary(candidate.relationship, candidate.source, candidate.target)); + } + if (candidate.stale_endpoints.length > 0) { + staleEndpoints.push( + relationshipSummary(candidate.relationship, candidate.source, candidate.target, { + stale_endpoints: candidate.stale_endpoints, + }), + ); + } + } + const removed = []; + for (const [key, entry] of previousByKey) { + if (nextByKey.has(key)) continue; + removed.push({ + object_ref: entry.object_ref, + object_modified: new Date(entry.object_modified).toISOString(), + source_ref: entry.source?.object_ref, + target_ref: entry.target?.object_ref, + }); + } + + return { + selected_count: nextByKey.size, + added_count: added.length, + removed_count: removed.length, + unchanged_count: nextByKey.size - added.length, + added, + removed, + stale_endpoints: staleEndpoints, + }; +} + +// ============================================================================= +// Statistics and protection lookups +// ============================================================================= + +function emptyStatistics() { + return { + primary_count: 0, + secondary_count: 0, + relationship_count: 0, + supporting_count: 0, + link_target_count: 0, + total_count: 0, + }; +} + +/** + * Count manifest entries by semantic role for a page of snapshot summaries. + * One aggregate covers every requested manifest to avoid a per-snapshot query. + */ +async function getStatisticsByManifestIds(manifestIds) { + const uniqueManifestIds = [...new Set(manifestIds.filter(Boolean))]; + const statisticsByManifestId = new Map( + uniqueManifestIds.map((manifestId) => [manifestId, emptyStatistics()]), + ); + if (uniqueManifestIds.length === 0) return statisticsByManifestId; + + const counts = await ReleaseTrackContentManifestEntry.aggregate([ + { $match: { manifest_id: { $in: uniqueManifestIds } } }, + { + $group: { + _id: { manifest_id: '$manifest_id', kind: '$kind' }, + count: { $sum: 1 }, + }, + }, + ]).exec(); + + for (const result of counts) { + const statistics = statisticsByManifestId.get(result._id.manifest_id); + const field = STATISTIC_FIELDS_BY_KIND[result._id.kind]; + if (!statistics || !field) continue; + statistics[field] = result.count; + statistics.total_count += result.count; + } + return statisticsByManifestId; +} + +async function protectedEntries(query, projection) { + const entries = await ReleaseTrackContentManifestEntry.find(query) + .select({ ...projection, _id: 0 }) + .lean() + .exec(); + if (entries.length === 0) return []; + + const protectedManifestIds = new Set( + ( + await ReleaseTrackContentManifest.find({ + manifest_id: { $in: entries.map((entry) => entry.manifest_id) }, + state: { $in: ['pending', 'active'] }, + }) + .select({ manifest_id: 1, _id: 0 }) + .lean() + .exec() + ).map((manifest) => manifest.manifest_id), + ); + return entries.filter((entry) => protectedManifestIds.has(entry.manifest_id)); +} + +async function findPinsForRevision(objectRef, objectModified) { + return protectedEntries( + { object_ref: objectRef, object_modified: objectModified, ...MUTATION_PROTECTED_ENTRY_FILTER }, + { manifest_id: 1, track_id: 1, snapshot_modified: 1, kind: 1, tier: 1 }, + ); +} + +async function findPinsForObject(objectRef) { + return protectedEntries( + { object_ref: objectRef, object_modified: { $ne: null }, ...MUTATION_PROTECTED_ENTRY_FILTER }, + { manifest_id: 1, track_id: 1, snapshot_modified: 1, object_modified: 1, kind: 1, tier: 1 }, + ); +} + +module.exports = { + seal, + activate, + discard, + discardUnreferenced, + discardOrphans, + discardTrack, + replay, + resolveLive, + resolveClosedGraph, + ensurePublicationSupport, + previewRelationshipChanges, + prepareSourceReconstruction, + isSameSourceReconstruction, + findManifest, + getStatisticsByManifestIds, + findPinsForRevision, + findPinsForObject, + manifestUuid, + MANIFEST_ID_PREFIX, + MANIFEST_SCHEMA_VERSION, +}; diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index 0695c463..f76037ca 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -3,12 +3,16 @@ // ============================================================================= // Export Service // -// Hydrates STIX object refs (from snapshot members/staged/candidates tiers) -// into full STIX documents, then formats the output as one of: +// Renders a snapshot as one of: // - bundle: Standard STIX 2.0 or 2.1 bundle // - workbench: Custom format with workflow metadata // - filesystemstore: Directory structure organized by STIX type // +// Bundle export has exactly one content path: replay the snapshot's sealed +// content manifest. Two preview variants resolve the same closed-member graph +// live instead of replaying: release previews of an unsaved planned snapshot, +// and draft exports that add workflow tiers through `include`. +// // This service performs cross-service READS (permitted by the event-driven // architecture — see docs/CROSS_SERVICE_READS_PATTERN.md) by querying STIX // repositories directly. It does NOT write to any external repository. @@ -17,18 +21,23 @@ // app/lib/release-tracks/export-schemas.js for schema definitions. // ============================================================================= -const config = require('../../config/config'); +const { v5: uuidv5 } = require('uuid'); const logger = require('../../lib/logger'); const linkById = require('../../lib/linkById'); +const revisionReference = require('../../lib/release-tracks/revision-reference'); const primaryRevisionService = require('./primary-revision-service'); -const graphManifestService = require('./graph-manifest-service'); -const systemConfigurationService = require('../system/system-configuration-service'); +const contentManifestService = require('./content-manifest-service'); +const publicationService = require('./publication-service'); +const { BadRequestError } = require('../../exceptions'); const { bundleTransformSchema, workbenchTransformSchema, filesystemStoreTransformSchema, } = require('../../lib/release-tracks/export-schemas'); +// Namespace for deterministic draft bundle identifiers. +const DRAFT_BUNDLE_NAMESPACE = 'c1d8c0a6-6a3d-4f0a-9c9b-4d9d0c8a5f21'; + // ============================================================================= // Hydration // ============================================================================= @@ -36,9 +45,6 @@ const { /** * Hydrate an array of tier entries into full STIX documents. * - * Groups entries by STIX type (extracted from the `object_ref` prefix) and - * batch-queries each repository in parallel via `findManyByIdAndModified`. - * * @param {Array<{object_ref: string, object_modified: string|Date}>} entries * @returns {Promise>} Full Mongoose lean documents ({ stix, workspace, ... }) */ @@ -52,8 +58,8 @@ exports.hydrateMembers = async function hydrateMembers(entries) { /** * Convert LinkById tags (e.g. "(LinkById: T1234)") in descriptions to - * markdown citations using only object revisions supplied by the resolved - * live or persisted graph. + * markdown citations using only object revisions supplied by the replayed or + * resolved graph. * * @param {Array} documents - Hydrated lean documents ({ stix, ... }) */ @@ -71,17 +77,8 @@ async function convertLinkByIdTags(documents, linkTargetDocuments) { } } -function requiresLiveGraph(snapshot, options) { - return ( - options.captureGraph || - snapshot.version == null || - !snapshot.graph_manifest_id || - (options.include || []).some((tier) => ['staged', 'candidates'].includes(tier)) - ); -} - function normalizeSourceBundleDefaults(documents, graph) { - if (graph.manifest?.resolver_version !== 'source-bundle-pointer-v2') return documents; + if (!graph.sourceOmittedDefaults?.size) return documents; return documents.map((document) => { const normalized = { ...document, stix: { ...document.stix } }; @@ -94,9 +91,35 @@ function normalizeSourceBundleDefaults(documents, graph) { }); } -function bundleIdForManifest(manifest) { - const uuid = manifest?.manifest_id?.split('--')[1]; - return uuid ? `bundle--${uuid}` : undefined; +/** + * Select the draft workflow-tier entries requested through `include`, + * narrowed by `state`, and resolve dynamic selectors to exact revisions. + */ +async function includedTierEntries(snapshot, options) { + const include = options.include || []; + const entries = []; + for (const tier of ['staged', 'candidates']) { + if (!include.includes(tier)) continue; + for (const entry of snapshot[tier] || []) { + if ( + options.state && + entry.object_status !== 'reviewed' && + !options.state.includes(entry.object_status) + ) { + continue; + } + entries.push({ object_ref: entry.object_ref, object_modified: entry.object_modified }); + } + } + return revisionReference.resolveEntries(entries); +} + +function bundleIdFor(snapshot) { + if (snapshot.bundle_id) return snapshot.bundle_id; + return `bundle--${uuidv5( + `${snapshot.id}|${new Date(snapshot.modified).toISOString()}`, + DRAFT_BUNDLE_NAMESPACE, + )}`; } // ============================================================================= @@ -111,7 +134,7 @@ function bundleIdForManifest(manifest) { * * @param {Object} snapshot - The raw snapshot document * @param {Array} hydratedObjects - Hydrated lean documents - * @param {Object} [options] - { stixVersion?, includeToc?, attackSpecVersion? } + * @param {Object} [options] - { stixVersion?, publication?, bundleId? } */ exports.formatAsBundle = function formatAsBundle(snapshot, hydratedObjects, options) { return bundleTransformSchema.parse({ snapshot, hydratedObjects, options }); @@ -149,49 +172,52 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd * returns the release-track snapshot shape with UI-friendly tier entry details. * * Bundle exports (see docs/developer/release-tracks/bundle-export.md): - * - The same pipeline applies to standard snapshots and materialized virtual - * snapshots because both persist exact member revisions. - * 1. Select tier entries — members always; staged/candidates via - * options.include, narrowed by options.state - * 2. Hydrate entries into full documents - * 3. Resolve live relationships or replay exact persisted graph pointers - * 4. Append referenced identities and marking definitions - * 5. Convert LinkById tags to markdown citations - * 6. Assemble the bundle (STIX version conformance + optional TOC) via the - * Zod transform schema + * 1. Replay the sealed content manifest (members, closed relationships, + * supporting objects, LinkById targets). A release preview or a draft + * export with `include` resolves the same closed graph live instead. + * 2. Convert LinkById tags to markdown citations + * 3. Assemble the bundle (STIX version conformance + collection object for + * STIX 2.1) via the Zod transform schema * * @param {Object} snapshot - The raw snapshot document from the dynamic repo * @param {string} format - One of: 'bundle', 'filesystemstore' * @param {Object} [options] - Additional options - * @param {Array} [options.include] - Extra tiers to include in bundles ('staged', 'candidates') - * @param {Array} [options.state] - Workflow status filter for included staged/candidates + * @param {Array} [options.include] - Draft-only extra tiers ('staged', 'candidates') + * @param {Array} [options.state] - Workflow status filter for included tiers * @param {string} [options.stixVersion] - '2.0' or '2.1' (default '2.1') - * @param {boolean} [options.includeToc] - Include the x-mitre-collection TOC object (default true) + * @param {boolean} [options.resolveLive] - Resolve the graph live (release previews) * @returns {Promise} The formatted export */ exports.exportSnapshot = async function exportSnapshot(snapshot, format, options = {}) { if (format === 'bundle') { - // A persisted graph is an opt-in guarantee for members only. Graphless - // snapshots and exports that add mutable draft tiers resolve the current - // relationship frontier instead of implying determinism they do not have. - const graph = requiresLiveGraph(snapshot, options) - ? await graphManifestService.replayPlannedSnapshot(snapshot, options) - : await graphManifestService.replay(snapshot, options); - const allObjects = normalizeSourceBundleDefaults(graph.documents, graph); - await convertLinkByIdTags(allObjects, graph.linkTargetDocuments); - let createdByRef; - if (options.stixVersion !== '2.0' && options.includeToc !== false && !graph.collectionObject) { - const organizationIdentity = await systemConfigurationService.retrieveOrganizationIdentity(); - createdByRef = organizationIdentity.stix.id; + const include = options.include || []; + if (include.length > 0 && snapshot.version != null) { + throw new BadRequestError({ + message: + 'Tagged snapshots export members only. The include parameter is a draft preview option.', + details: { include }, + }); } + let graph; + if (options.resolveLive || include.length > 0) { + const extraEntries = include.length > 0 ? await includedTierEntries(snapshot, options) : []; + graph = await contentManifestService.resolveLive(snapshot, extraEntries); + } else { + graph = await contentManifestService.replay(snapshot); + } + + const publication = await publicationService.publicationForExport(snapshot); + const allObjects = [ + ...normalizeSourceBundleDefaults(graph.documents, graph), + ...(await contentManifestService.ensurePublicationSupport(graph.documents, publication)), + ]; + await convertLinkByIdTags(allObjects, graph.linkTargetDocuments); + return exports.formatAsBundle(snapshot, allObjects, { stixVersion: options.stixVersion, - includeToc: options.includeToc, - attackSpecVersion: config.app.attackSpecVersion, - collectionObject: graph.collectionObject, - createdByRef, - bundleId: bundleIdForManifest(graph.manifest), + publication, + bundleId: bundleIdFor(snapshot), }); } diff --git a/app/services/release-tracks/graph-manifest-service.js b/app/services/release-tracks/graph-manifest-service.js deleted file mode 100644 index d8c2d7c3..00000000 --- a/app/services/release-tracks/graph-manifest-service.js +++ /dev/null @@ -1,1262 +0,0 @@ -'use strict'; - -const { isDeepStrictEqual } = require('node:util'); -const { v4: uuidv4 } = require('uuid'); -const config = require('../../config/config'); -const linkById = require('../../lib/linkById'); -const bundleRelationships = require('../../lib/stix-bundle-relationships'); -const attackObjectsRepository = require('../../repository/attack-objects-repository'); -const relationshipsRepository = require('../../repository/relationships-repository'); -const detectionStrategiesRepository = require('../../repository/detection-strategies-repository'); -const BundleGraphResolver = require('../stix/bundle-graph-resolver'); -const { - ReleaseTrackGraphManifest, - ReleaseTrackGraphManifestEntry, -} = require('../../models/release-tracks/release-track-graph-manifest-model'); -const { ReleaseContentIntegrityError } = require('../../exceptions'); -const { buildTocObject } = require('../../lib/release-tracks/export-schemas'); -const systemConfigurationService = require('../system/system-configuration-service'); -const primaryRevisionService = require('./primary-revision-service'); - -const MANIFEST_SCHEMA_VERSION = 2; -const RESOLVER_VERSION = 'closed-member-graph-v3'; -const SOURCE_BUNDLE_RESOLVER_VERSION = 'source-bundle-pointer-v2'; -const TIERS = ['members', 'staged', 'candidates', 'quarantine']; -const STATISTIC_FIELDS_BY_KIND = { - root: 'primary_count', - secondary: 'secondary_count', - relationship: 'relationship_count', - supporting: 'supporting_count', - link_target: 'link_target_count', -}; -const MUTATION_PROTECTED_ENTRY_FILTER = { - $or: [ - { kind: { $ne: 'root' } }, - { kind: 'root', tier: { $in: ['members', 'quarantine'] } }, - { kind: 'root', 'discovered_from.0': { $exists: true } }, - ], -}; - -function normalizeDomain(domain) { - return domain.endsWith('-attack') ? domain : `${domain}-attack`; -} - -function virtualSnapshotDomains(snapshot) { - if (snapshot.type !== 'virtual') return null; - - const domains = (snapshot.composition?.component_tracks || []).flatMap( - (component) => component.filters?.domains || [], - ); - if (domains.length === 0) return null; - return new Set(domains.map(normalizeDomain)); -} - -function objectDomains(stixObject) { - if (Array.isArray(stixObject.x_mitre_domains)) { - return stixObject.x_mitre_domains; - } - if (stixObject.type === 'x-mitre-matrix') { - return (stixObject.external_references || []) - .map((reference) => reference.external_id) - .filter((externalId) => typeof externalId === 'string' && externalId.endsWith('-attack')); - } - return []; -} - -function secondaryObjectIsValid(document, allowedDomains) { - if (!document) return false; - if (!allowedDomains) return true; - - const domains = objectDomains(document.stix); - return ( - domains.length === 0 || domains.some((domain) => allowedDomains.has(normalizeDomain(domain))) - ); -} - -function revisionKey(objectRef, objectModified) { - return `${objectRef}::${new Date(objectModified).getTime()}`; -} - -async function getFirstCollectionCreated(trackId, fallback) { - const firstCollection = await ReleaseTrackGraphManifestEntry.findOne({ - track_id: trackId, - kind: 'collection', - }) - .sort({ 'frozen_stix.created': 1, _id: 1 }) - .select('frozen_stix.created') - .lean() - .exec(); - return firstCollection?.frozen_stix?.created || fallback; -} - -function collectionIdForTrack(trackId) { - return `x-mitre-collection--${trackId.split('--')[1]}`; -} - -async function organizationIdentityRef() { - const organizationIdentity = await systemConfigurationService.retrieveOrganizationIdentity(); - return organizationIdentity.stix.id; -} - -async function upsertCollectionEntry(snapshot, entries, manifest) { - const graph = await replayEntries(entries, manifest, {}); - const created = await getFirstCollectionCreated(manifest.track_id, manifest.created_at); - const createdByRef = await organizationIdentityRef(); - const collectionId = collectionIdForTrack(manifest.track_id); - const collectionObject = buildTocObject( - snapshot, - graph.documents.map((document) => document.stix), - { - stixVersion: '2.1', - attackSpecVersion: config.app.attackSpecVersion, - collectionId, - createdByRef, - created, - modified: manifest.created_at, - }, - ); - const entry = { - manifest_id: manifest.manifest_id, - track_id: manifest.track_id, - snapshot_modified: snapshot.modified, - revision_key: `${collectionObject.id}::collection`, - kind: 'collection', - object_ref: collectionObject.id, - frozen_stix: collectionObject, - }; - const storedEntry = await ReleaseTrackGraphManifestEntry.findOneAndUpdate( - { manifest_id: manifest.manifest_id, kind: 'collection' }, - { $set: entry }, - { new: true, upsert: true, runValidators: true, lean: true }, - ).exec(); - const existingIndex = entries.findIndex((candidate) => candidate.kind === 'collection'); - if (existingIndex === -1) entries.push(storedEntry); - else entries[existingIndex] = storedEntry; - return storedEntry; -} - -function endpointFor(relationship, side) { - const endpoint = relationship.workspace?.relationship_endpoints?.[side]; - const objectRef = relationship.stix[`${side}_ref`]; - if (!endpoint || endpoint.object_ref !== objectRef || !endpoint.object_modified) { - return null; - } - return { - object_ref: endpoint.object_ref, - object_modified: endpoint.object_modified, - }; -} - -async function resolveBoundedGraph(hydratedRoots, allowedDomains, missing) { - const rootObjectRefs = new Set(hydratedRoots.entries.map((entry) => entry.object_ref)); - let frontierObjectRefs = new Set(rootObjectRefs); - - while (true) { - const relationships = await relationshipsRepository.retrieveLatestTouchingObjectRefs( - [...frontierObjectRefs], - { includeRevoked: false, includeDeprecated: false }, - ); - const pinnedRelationships = []; - for (const relationship of relationships) { - const source = endpointFor(relationship, 'source'); - const target = endpointFor(relationship, 'target'); - if (!source || !target) { - // Legacy relationships outside this snapshot's bounded graph cannot - // affect its replay. Fail closed only when an unpinned relationship - // touches a primary member by STIX ID. - if ( - rootObjectRefs.has(relationship.stix.source_ref) || - rootObjectRefs.has(relationship.stix.target_ref) - ) { - missing.push({ - object_ref: relationship.stix.id, - object_modified: new Date(relationship.stix.modified).toISOString(), - dependency: 'relationship_endpoints', - }); - } - continue; - } - pinnedRelationships.push({ relationship, source, target }); - } - if (missing.length > 0) { - throw new ReleaseContentIntegrityError(missing, { - details: 'Snapshot graph capture found relationships without exact endpoint pins.', - }); - } - - // One batched exact-revision hydration per STIX type replaces the - // resolver's historical one-query-per-secondary behavior. - const hydratedEndpoints = await primaryRevisionService.hydrateEntries( - pinnedRelationships.flatMap(({ source, target }) => [source, target]), - ); - const graphResolver = new BundleGraphResolver({ - attackObjectsRepository, - detectionStrategiesRepository, - repositoryMap: primaryRevisionService.getRepositoryMap(), - policy: { - isDeprecatedPattern: bundleRelationships.isDeprecatedPattern, - relationshipIsActive: bundleRelationships.relationshipIsActive, - secondaryObjectIsValid: (document) => secondaryObjectIsValid(document, allowedDomains), - }, - options: { - inferDomains: false, - includeRevoked: true, - includeDeprecated: true, - includeMissingAttackId: true, - }, - relationships: pinnedRelationships.map((candidate) => candidate.relationship), - prefetchedDocuments: hydratedEndpoints.documents, - onMissingDependency(reference) { - missing.push({ - ...reference, - object_modified: new Date(reference.object_modified).toISOString(), - }); - }, - }); - const resolvedGraph = await graphResolver.resolve(hydratedRoots.documents); - if (missing.length > 0) { - const uniqueMissing = [ - ...new Map( - missing.map((reference) => [ - `${reference.object_ref}::${reference.object_modified}`, - reference, - ]), - ).values(), - ]; - throw new ReleaseContentIntegrityError(uniqueMissing, { - details: 'Snapshot graph capture could not hydrate every exact dependency.', - }); - } - - const resolvedObjectRefs = new Set(resolvedGraph.documents.map((document) => document.stix.id)); - const expanded = [...resolvedObjectRefs].some( - (objectRef) => !frontierObjectRefs.has(objectRef), - ); - if (!expanded) { - return { graphResolver, resolvedGraph }; - } - frontierObjectRefs = new Set([...frontierObjectRefs, ...resolvedObjectRefs]); - } -} - -function endpointIsSelected(endpoint, membersByObjectRef) { - const member = endpoint && membersByObjectRef.get(endpoint.object_ref); - return ( - member && - revisionKey(member.object_ref, member.object_modified) === - revisionKey(endpoint.object_ref, endpoint.object_modified) - ); -} - -function exactMemberMap(entries) { - const membersByObjectRef = new Map(); - for (const entry of entries) { - const existing = membersByObjectRef.get(entry.object_ref); - if ( - existing && - revisionKey(existing.object_ref, existing.object_modified) !== - revisionKey(entry.object_ref, entry.object_modified) - ) { - throw new ReleaseContentIntegrityError( - [ - { - object_ref: entry.object_ref, - object_modified: new Date(entry.object_modified).toISOString(), - dependency: 'unique_member_revision', - }, - ], - { details: 'A deterministic snapshot cannot select two revisions of one STIX object.' }, - ); - } - membersByObjectRef.set(entry.object_ref, entry); - } - return membersByObjectRef; -} - -async function loadPredecessorRelationshipCandidates( - snapshot, - predecessorManifestId, - membersByObjectRef, -) { - if (!predecessorManifestId) return []; - - const predecessorManifest = await ReleaseTrackGraphManifest.findOne({ - manifest_id: predecessorManifestId, - track_id: snapshot.id, - snapshot_modified: { $lt: snapshot.modified }, - state: { $in: ['pending', 'active'] }, - }) - .lean() - .exec(); - if (!predecessorManifest) { - throw new ReleaseContentIntegrityError( - [{ manifest_id: predecessorManifestId, dependency: 'predecessor_graph_manifest' }], - { details: 'The preceding tagged snapshot references a missing graph manifest.' }, - ); - } - - const entries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: predecessorManifestId, - kind: 'relationship', - }) - .lean() - .exec(); - const selectedEntries = entries.filter( - (entry) => - endpointIsSelected(entry.source, membersByObjectRef) && - endpointIsSelected(entry.target, membersByObjectRef), - ); - if (selectedEntries.length === 0) return []; - - const hydrated = await primaryRevisionService.assertStoredEntries(selectedEntries); - const documentsByRevision = new Map( - hydrated.documents.map((document) => [ - revisionKey(document.stix.id, document.stix.modified), - document, - ]), - ); - const candidates = []; - for (const entry of selectedEntries) { - const relationship = documentsByRevision.get(entry.revision_key); - if ( - relationship?.stix.type !== 'relationship' || - relationship.stix.source_ref !== entry.source.object_ref || - relationship.stix.target_ref !== entry.target.object_ref - ) { - throw new ReleaseContentIntegrityError( - [{ object_ref: entry.object_ref, dependency: 'predecessor_relationship_pointer' }], - { details: 'A predecessor graph relationship no longer matches its stored endpoints.' }, - ); - } - candidates.push({ relationship, source: entry.source, target: entry.target }); - } - return candidates; -} - -async function resolveClosedMemberRelationships(snapshot, hydratedRoots, predecessorManifestId) { - const membersByObjectRef = exactMemberMap(hydratedRoots.entries); - const storedRelationships = await relationshipsRepository.retrieveRevisionsTouchingExactEndpoints( - hydratedRoots.entries, - ); - const candidatesByRevision = new Map(); - - for (const relationship of storedRelationships) { - const source = endpointFor(relationship, 'source'); - const target = endpointFor(relationship, 'target'); - if ( - !endpointIsSelected(source, membersByObjectRef) || - !endpointIsSelected(target, membersByObjectRef) - ) { - continue; - } - candidatesByRevision.set(revisionKey(relationship.stix.id, relationship.stix.modified), { - relationship, - source, - target, - }); - } - - const predecessorCandidates = await loadPredecessorRelationshipCandidates( - snapshot, - predecessorManifestId, - membersByObjectRef, - ); - for (const candidate of predecessorCandidates) { - const key = revisionKey(candidate.relationship.stix.id, candidate.relationship.stix.modified); - if (!candidatesByRevision.has(key)) candidatesByRevision.set(key, candidate); - } - - const candidatesByRelationship = new Map(); - for (const candidate of candidatesByRevision.values()) { - const entries = candidatesByRelationship.get(candidate.relationship.stix.id) || []; - entries.push(candidate); - candidatesByRelationship.set(candidate.relationship.stix.id, entries); - } - - const selected = []; - for (const [relationshipId, candidates] of candidatesByRelationship) { - const endpointPairs = new Set( - candidates.map( - ({ source, target }) => - `${revisionKey(source.object_ref, source.object_modified)}->${revisionKey( - target.object_ref, - target.object_modified, - )}`, - ), - ); - if (endpointPairs.size > 1) { - throw new ReleaseContentIntegrityError( - [{ object_ref: relationshipId, dependency: 'relationship_lineage_endpoints' }], - { - details: - 'One relationship lineage resolves to multiple endpoint pairs in the same member graph.', - }, - ); - } - - candidates.sort( - (left, right) => - new Date(right.relationship.stix.modified).getTime() - - new Date(left.relationship.stix.modified).getTime(), - ); - const newest = candidates[0]; - if ( - bundleRelationships.relationshipIsActive(newest.relationship) && - !bundleRelationships.isDeprecatedPattern(newest.relationship.stix) - ) { - selected.push(newest); - } - } - return selected; -} - -async function buildClosedMemberManifestEntries(snapshot, options) { - const rootRequests = (snapshot.members || []).map((entry) => ({ ...entry, tier: 'members' })); - const hydratedRoots = await primaryRevisionService.assertStoredEntries(rootRequests); - exactMemberMap(hydratedRoots.entries); - - const selectedRelationships = await resolveClosedMemberRelationships( - snapshot, - hydratedRoots, - options.predecessorManifestId, - ); - const relationshipDocuments = selectedRelationships.map((candidate) => candidate.relationship); - const graphResolver = new BundleGraphResolver({ - attackObjectsRepository, - detectionStrategiesRepository, - repositoryMap: primaryRevisionService.getRepositoryMap(), - policy: { - isDeprecatedPattern: bundleRelationships.isDeprecatedPattern, - relationshipIsActive: bundleRelationships.relationshipIsActive, - secondaryObjectIsValid: () => false, - }, - options: { - inferDomains: false, - includeRevoked: true, - includeDeprecated: true, - includeMissingAttackId: true, - }, - relationships: relationshipDocuments, - prefetchedDocuments: hydratedRoots.documents, - }); - const supportingDocuments = await graphResolver.loadSupportingDocuments([ - ...hydratedRoots.documents.map((document) => document.stix), - ...relationshipDocuments.map((document) => document.stix), - ]); - - const selectedObjectRefs = new Set(hydratedRoots.entries.map((entry) => entry.object_ref)); - const rootMetadata = new Map( - hydratedRoots.entries.map((entry) => [ - revisionKey(entry.object_ref, entry.object_modified), - entry, - ]), - ); - const supportingByObjectRef = new Map(); - for (const document of supportingDocuments) { - if (!selectedObjectRefs.has(document.stix.id)) { - supportingByObjectRef.set(document.stix.id, document); - } - } - - const selectedByAttackId = new Map(); - for (const document of hydratedRoots.documents) { - const attackId = linkById.getAttackId(document.stix); - if (attackId) selectedByAttackId.set(attackId, document); - } - const linkTargets = new Map(); - for (const document of [...hydratedRoots.documents, ...relationshipDocuments]) { - for (const attackId of linkById.extractLinkByIds(document.stix)) { - if (selectedByAttackId.has(attackId) || linkTargets.has(attackId)) continue; - const target = await linkById.getAttackObjectFromDatabase(attackId); - if (target) linkTargets.set(attackId, target); - } - } - - const entries = hydratedRoots.documents.map((document) => { - const key = revisionKey(document.stix.id, document.stix.modified); - const root = rootMetadata.get(key); - return { - revision_key: key, - kind: 'root', - tier: 'members', - object_status: root?.object_status, - object_ref: document.stix.id, - object_modified: document.stix.modified, - }; - }); - for (const candidate of selectedRelationships) { - entries.push({ - revision_key: revisionKey( - candidate.relationship.stix.id, - candidate.relationship.stix.modified, - ), - kind: 'relationship', - object_ref: candidate.relationship.stix.id, - object_modified: candidate.relationship.stix.modified, - source: candidate.source, - target: candidate.target, - }); - } - for (const document of supportingByObjectRef.values()) { - const isVersioned = Boolean(document.stix.modified); - entries.push({ - revision_key: isVersioned - ? revisionKey(document.stix.id, document.stix.modified) - : `${document.stix.id}::unversioned`, - kind: 'supporting', - object_ref: document.stix.id, - object_modified: document.stix.modified, - frozen_stix: isVersioned ? undefined : document.stix, - }); - } - for (const document of linkTargets.values()) { - entries.push({ - revision_key: revisionKey(document.stix.id, document.stix.modified), - kind: 'link_target', - object_ref: document.stix.id, - object_modified: document.stix.modified, - }); - } - return entries; -} - -async function buildManifestEntries(snapshot, options = {}) { - if (options.memberOnly) { - return buildClosedMemberManifestEntries(snapshot, options); - } - - const allowedDomains = virtualSnapshotDomains(snapshot); - const rootRequests = []; - const rootTiers = TIERS; - for (const tier of rootTiers) { - for (const entry of snapshot[tier] || []) { - rootRequests.push({ ...entry, tier }); - } - } - - const hydratedRoots = await primaryRevisionService.assertStoredEntries(rootRequests); - const rootMetadata = new Map( - hydratedRoots.entries.map((entry) => [ - revisionKey(entry.object_ref, entry.object_modified), - entry, - ]), - ); - - const missing = []; - const { graphResolver, resolvedGraph } = await resolveBoundedGraph( - hydratedRoots, - allowedDomains, - missing, - ); - const selectedDocuments = new Map( - resolvedGraph.documents.map((document) => [ - revisionKey(document.stix.id, document.stix.modified), - document, - ]), - ); - const selectedRelationships = resolvedGraph.relationships.map((relationship) => ({ - relationship, - source: endpointFor(relationship, 'source'), - target: endpointFor(relationship, 'target'), - })); - const relationshipDocuments = selectedRelationships.map((candidate) => candidate.relationship); - const discoverySources = resolvedGraph.dependencies; - const supportingDocuments = await graphResolver.loadSupportingDocuments(resolvedGraph.objects); - - const linkTargets = new Map(); - for (const document of [...selectedDocuments.values(), ...relationshipDocuments]) { - for (const attackId of linkById.extractLinkByIds(document.stix)) { - if (!linkTargets.has(attackId)) { - const target = await linkById.getAttackObjectFromDatabase(attackId); - if (target) { - linkTargets.set(attackId, target); - } - } - } - } - - const entries = []; - for (const [key, document] of selectedDocuments) { - const root = rootMetadata.get(key); - entries.push({ - revision_key: key, - kind: root ? 'root' : 'secondary', - tier: root?.tier, - object_status: root?.object_status, - object_ref: document.stix.id, - object_modified: document.stix.modified, - discovered_from: discoverySources.get(key) || [], - }); - } - for (const candidate of selectedRelationships) { - entries.push({ - revision_key: revisionKey( - candidate.relationship.stix.id, - candidate.relationship.stix.modified, - ), - kind: 'relationship', - object_ref: candidate.relationship.stix.id, - object_modified: candidate.relationship.stix.modified, - source: candidate.source, - target: candidate.target, - // Live previews reuse the legacy replay selector, which carries the - // request-local relationship payload without persisting it. Persisted - // schema-v2 member manifests deliberately omit this field. - frozen_stix: candidate.relationship.stix, - }); - } - for (const document of supportingDocuments) { - const isVersioned = Boolean(document.stix.modified); - entries.push({ - revision_key: isVersioned - ? revisionKey(document.stix.id, document.stix.modified) - : `${document.stix.id}::unversioned`, - kind: 'supporting', - object_ref: document.stix.id, - object_modified: document.stix.modified, - frozen_stix: isVersioned ? undefined : document.stix, - }); - } - for (const document of linkTargets.values()) { - entries.push({ - revision_key: revisionKey(document.stix.id, document.stix.modified), - kind: 'link_target', - object_ref: document.stix.id, - object_modified: document.stix.modified, - }); - } - - return entries; -} - -async function prepare(snapshot, options = {}) { - const manifestId = `release-track-graph-manifest--${uuidv4()}`; - const schemaVersion = options.schemaVersion ?? MANIFEST_SCHEMA_VERSION; - const memberOnly = schemaVersion >= MANIFEST_SCHEMA_VERSION; - const resolverVersion = memberOnly ? RESOLVER_VERSION : 'bounded-attack-graph-v1'; - const entries = await buildManifestEntries(snapshot, { - memberOnly, - predecessorManifestId: options.predecessorManifestId, - }); - const common = { - manifest_id: manifestId, - track_id: snapshot.id, - snapshot_modified: snapshot.modified, - }; - - const manifest = { - ...common, - state: 'pending', - schema_version: schemaVersion, - resolver_version: resolverVersion, - baseline_reconstruction: options.baselineReconstruction === true, - created_at: new Date(), - }; - await ReleaseTrackGraphManifest.create(manifest); - try { - if (entries.length > 0) { - await ReleaseTrackGraphManifestEntry.insertMany( - entries.map((entry) => ({ ...common, ...entry })), - ); - } - // The pending manifest now protects every inserted pointer from deletion. - // Rehydrate once inside that protection window so a revision deleted - // during graph discovery cannot leave an attachable dangling manifest. - await upsertCollectionEntry(snapshot, entries, manifest); - } catch (err) { - await discard(manifestId); - throw err; - } - return manifestId; -} - -function sourcePlanIntegrityError(details, references = []) { - return new ReleaseContentIntegrityError(references, { details }); -} - -async function buildSourceManifestEntries(snapshot, plan) { - const seenObjectRefs = new Set(); - const planned = []; - - for (const input of plan.entries) { - if (seenObjectRefs.has(input.object_ref)) { - throw sourcePlanIntegrityError( - `Source bundle contains more than one revision for '${input.object_ref}'.`, - [{ object_ref: input.object_ref, dependency: 'unique_source_revision' }], - ); - } - seenObjectRefs.add(input.object_ref); - - const isVersioned = input.object_modified != null; - if (isVersioned && input.frozen_stix) { - throw sourcePlanIntegrityError( - 'Versioned source-bundle entries must be exact database pointers, not frozen payloads.', - [{ object_ref: input.object_ref, dependency: 'pointer_only_manifest' }], - ); - } - if (!isVersioned) { - if ( - input.kind !== 'supporting' || - input.frozen_stix?.type !== 'marking-definition' || - input.frozen_stix?.id !== input.object_ref || - input.frozen_stix?.modified != null - ) { - throw sourcePlanIntegrityError( - 'Only unversioned marking definitions may be frozen in a schema-v2 manifest.', - [{ object_ref: input.object_ref, dependency: 'unversioned_supporting_object' }], - ); - } - } - if (input.kind === 'relationship') { - if (!input.source || !input.target || !isVersioned) { - throw sourcePlanIntegrityError( - 'Relationship entries require an exact relationship pointer and exact endpoint pins.', - [{ object_ref: input.object_ref, dependency: 'relationship_endpoints' }], - ); - } - } else if (input.source || input.target) { - throw sourcePlanIntegrityError( - 'Only relationship entries may declare source and target endpoint pins.', - [{ object_ref: input.object_ref, dependency: 'relationship_endpoints' }], - ); - } - - planned.push({ - ...input, - object_modified: isVersioned ? new Date(input.object_modified) : undefined, - source: input.source - ? { ...input.source, object_modified: new Date(input.source.object_modified) } - : undefined, - target: input.target - ? { ...input.target, object_modified: new Date(input.target.object_modified) } - : undefined, - revision_key: isVersioned - ? revisionKey(input.object_ref, input.object_modified) - : `${input.object_ref}::unversioned`, - }); - } - - const expectedRoots = new Map( - (snapshot.members || []).map((entry) => [ - revisionKey(entry.object_ref, entry.object_modified), - entry, - ]), - ); - const suppliedRoots = planned.filter((entry) => entry.kind === 'root'); - const suppliedRootKeys = new Set(suppliedRoots.map((entry) => entry.revision_key)); - if ( - suppliedRoots.length !== expectedRoots.size || - [...expectedRoots.keys()].some((key) => !suppliedRootKeys.has(key)) - ) { - throw sourcePlanIntegrityError( - 'Source bundle root pointers must exactly equal the tagged snapshot members.', - [{ track_id: snapshot.id, dependency: 'snapshot_members' }], - ); - } - - const versioned = planned.filter((entry) => entry.object_modified); - const hydrated = await primaryRevisionService.assertStoredEntries(versioned); - const documentsByRevision = new Map( - hydrated.documents.map((document) => [ - revisionKey(document.stix.id, document.stix.modified), - document, - ]), - ); - const selectableKeys = new Set( - planned - .filter((entry) => ['root', 'secondary'].includes(entry.kind)) - .map((entry) => entry.revision_key), - ); - - for (const entry of planned) { - if (!entry.object_modified) continue; - const document = documentsByRevision.get(entry.revision_key); - if (entry.kind === 'relationship') { - if (document.stix.type !== 'relationship') { - throw sourcePlanIntegrityError(`'${entry.object_ref}' is not a relationship revision.`, [ - { object_ref: entry.object_ref, dependency: 'relationship_type' }, - ]); - } - for (const side of ['source', 'target']) { - const endpoint = entry[side]; - if (document.stix[`${side}_ref`] !== endpoint.object_ref) { - throw sourcePlanIntegrityError( - `Relationship '${entry.object_ref}' has a mismatched ${side} pointer.`, - [{ object_ref: entry.object_ref, dependency: `${side}_ref` }], - ); - } - if (!selectableKeys.has(revisionKey(endpoint.object_ref, endpoint.object_modified))) { - throw sourcePlanIntegrityError( - `Relationship '${entry.object_ref}' references an endpoint revision absent from the source graph.`, - [{ ...endpoint, dependency: `${side}_revision` }], - ); - } - } - } else if (document.stix.type === 'relationship') { - throw sourcePlanIntegrityError( - `Relationship revision '${entry.object_ref}' must use kind 'relationship'.`, - [{ object_ref: entry.object_ref, dependency: 'entry_kind' }], - ); - } - } - - const includedObjectRefs = new Set(planned.map((entry) => entry.object_ref)); - for (const document of hydrated.documents) { - const supportingRefs = [ - document.stix.created_by_ref, - ...(document.stix.object_marking_refs || []), - ].filter(Boolean); - const missingRef = supportingRefs.find((objectRef) => !includedObjectRefs.has(objectRef)); - if (missingRef) { - throw sourcePlanIntegrityError(`Source graph omits supporting object '${missingRef}'.`, [ - { object_ref: missingRef, dependency: 'supporting_object' }, - ]); - } - } - - return planned.map((entry) => { - if (entry.kind !== 'root') return entry; - const root = expectedRoots.get(entry.revision_key); - return { ...entry, tier: 'members', object_status: root.object_status }; - }); -} - -async function prepareSourceReconstruction(snapshot, plan) { - const manifestId = `release-track-graph-manifest--${uuidv4()}`; - const entries = await buildSourceManifestEntries(snapshot, plan); - const common = { - manifest_id: manifestId, - track_id: snapshot.id, - snapshot_modified: snapshot.modified, - }; - const manifest = { - ...common, - state: 'pending', - schema_version: MANIFEST_SCHEMA_VERSION, - resolver_version: SOURCE_BUNDLE_RESOLVER_VERSION, - baseline_reconstruction: true, - source_attestation: plan.source_attestation, - created_at: new Date(), - }; - - await ReleaseTrackGraphManifest.create(manifest); - try { - await ReleaseTrackGraphManifestEntry.insertMany( - entries.map((entry) => ({ ...common, ...entry })), - ); - await upsertCollectionEntry(snapshot, entries, manifest); - } catch (err) { - await discard(manifestId); - throw err; - } - return manifestId; -} - -async function assertSourceReconstruction(snapshot, sourceAttestation) { - const manifest = await ReleaseTrackGraphManifest.findOne({ - manifest_id: snapshot.graph_manifest_id, - track_id: snapshot.id, - snapshot_modified: snapshot.modified, - state: { $in: ['pending', 'active'] }, - }) - .lean() - .exec(); - if ( - !manifest || - manifest.resolver_version !== SOURCE_BUNDLE_RESOLVER_VERSION || - !isDeepStrictEqual(manifest.source_attestation, sourceAttestation) - ) { - throw sourcePlanIntegrityError( - 'Snapshot already has a graph that was not reconstructed from the same source bundle.', - [{ manifest_id: snapshot.graph_manifest_id, dependency: 'source_attestation' }], - ); - } -} - -async function activate(manifestId) { - await ReleaseTrackGraphManifest.updateOne( - { manifest_id: manifestId, state: 'pending' }, - { $set: { state: 'active' } }, - ).exec(); -} - -async function discard(manifestId) { - await Promise.all([ - ReleaseTrackGraphManifestEntry.deleteMany({ manifest_id: manifestId }).exec(), - ReleaseTrackGraphManifest.deleteOne({ manifest_id: manifestId }).exec(), - ]); -} - -async function discardSnapshot(trackId, snapshotModified) { - const manifests = await ReleaseTrackGraphManifest.find({ - track_id: trackId, - snapshot_modified: snapshotModified, - }) - .select({ manifest_id: 1, _id: 0 }) - .lean() - .exec(); - const manifestIds = manifests.map((manifest) => manifest.manifest_id); - if (manifestIds.length === 0) return; - - await Promise.all([ - ReleaseTrackGraphManifestEntry.deleteMany({ - manifest_id: { $in: manifestIds }, - }).exec(), - ReleaseTrackGraphManifest.deleteMany({ - manifest_id: { $in: manifestIds }, - }).exec(), - ]); -} - -async function discardTrack(trackId) { - const manifests = await ReleaseTrackGraphManifest.find({ track_id: trackId }) - .select({ manifest_id: 1, _id: 0 }) - .lean() - .exec(); - const manifestIds = manifests.map((manifest) => manifest.manifest_id); - - await Promise.all([ - manifestIds.length > 0 - ? ReleaseTrackGraphManifestEntry.deleteMany({ - manifest_id: { $in: manifestIds }, - }).exec() - : Promise.resolve(), - ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }).exec(), - ]); -} - -function emptyStatistics() { - return { - primary_count: 0, - secondary_count: 0, - relationship_count: 0, - supporting_count: 0, - link_target_count: 0, - total_count: 0, - }; -} - -/** - * Count manifest entries by semantic role for a page of snapshot summaries. - * One aggregate covers every requested manifest to avoid a per-snapshot query. - * - * @param {string[]} manifestIds - * @returns {Promise>} - */ -async function getStatisticsByManifestIds(manifestIds) { - const uniqueManifestIds = [...new Set(manifestIds.filter(Boolean))]; - const statisticsByManifestId = new Map( - uniqueManifestIds.map((manifestId) => [manifestId, emptyStatistics()]), - ); - if (uniqueManifestIds.length === 0) return statisticsByManifestId; - - const counts = await ReleaseTrackGraphManifestEntry.aggregate([ - { $match: { manifest_id: { $in: uniqueManifestIds } } }, - { - $group: { - _id: { manifest_id: '$manifest_id', kind: '$kind' }, - count: { $sum: 1 }, - }, - }, - ]).exec(); - - for (const result of counts) { - const statistics = statisticsByManifestId.get(result._id.manifest_id); - const field = STATISTIC_FIELDS_BY_KIND[result._id.kind]; - if (!statistics || !field) continue; - statistics[field] = result.count; - statistics.total_count += result.count; - } - return statisticsByManifestId; -} - -function rootIsSelected(entry, options) { - if (entry.tier === 'members') return true; - if (!['staged', 'candidates'].includes(entry.tier)) return false; - if (!(options.include || []).includes(entry.tier)) return false; - if (!options.state) return true; - return entry.object_status === 'reviewed' || options.state.includes(entry.object_status); -} - -async function replayEntries(entries, manifest, options) { - const pointerOnlyMemberGraph = manifest.schema_version >= MANIFEST_SCHEMA_VERSION; - const versionedEntries = entries.filter( - (entry) => - entry.object_modified && - (entry.kind !== 'relationship' || (pointerOnlyMemberGraph && !entry.frozen_stix)), - ); - const hydrated = await primaryRevisionService.assertStoredEntries( - versionedEntries.map((entry) => ({ - object_ref: entry.object_ref, - object_modified: entry.object_modified, - })), - ); - const documentsByRevision = new Map( - hydrated.documents.map((document) => [ - revisionKey(document.stix.id, document.stix.modified), - document, - ]), - ); - for (const entry of entries) { - if (entry.frozen_stix) { - documentsByRevision.set(entry.revision_key, { - stix: entry.frozen_stix, - }); - } - } - - const selectedRevisionKeys = new Set( - entries - .filter((entry) => - pointerOnlyMemberGraph - ? ['root', 'secondary'].includes(entry.kind) - : entry.kind === 'root' && rootIsSelected(entry, options), - ) - .map((entry) => entry.revision_key), - ); - - // Special embedded-reference dependencies can be chained (for example, a - // detection strategy discovered through an analytic that was itself a - // relationship secondary). Replay only follows edges frozen in the - // manifest; it never asks the live database to expand the graph. - if (!pointerOnlyMemberGraph) { - let added; - do { - added = false; - for (const entry of entries) { - if ( - !['root', 'secondary'].includes(entry.kind) || - selectedRevisionKeys.has(entry.revision_key) - ) { - continue; - } - if ( - (entry.discovered_from || []).some((source) => - selectedRevisionKeys.has(revisionKey(source.object_ref, source.object_modified)), - ) - ) { - selectedRevisionKeys.add(entry.revision_key); - added = true; - } - } - } while (added); - } - - const selectedRelationships = entries.filter( - (entry) => - entry.kind === 'relationship' && - selectedRevisionKeys.has( - revisionKey(entry.source.object_ref, entry.source.object_modified), - ) && - selectedRevisionKeys.has(revisionKey(entry.target.object_ref, entry.target.object_modified)), - ); - for (const entry of selectedRelationships) { - selectedRevisionKeys.add(entry.revision_key); - } - - const selectedDocuments = [...selectedRevisionKeys] - .map((key) => documentsByRevision.get(key)) - .filter(Boolean); - const supportingRefs = new Set(); - for (const document of selectedDocuments) { - if (document.stix.created_by_ref) { - supportingRefs.add(document.stix.created_by_ref); - } - for (const objectRef of document.stix.object_marking_refs || []) { - supportingRefs.add(objectRef); - } - } - - const supportingDocuments = entries - .filter((entry) => entry.kind === 'supporting' && supportingRefs.has(entry.object_ref)) - .map((entry) => - entry.object_modified - ? documentsByRevision.get(entry.revision_key) - : { stix: entry.frozen_stix }, - ) - .filter(Boolean); - const linkTargetDocuments = entries - .filter((entry) => entry.kind === 'link_target') - .map((entry) => documentsByRevision.get(entry.revision_key)) - .filter(Boolean); - const sourceOmittedDefaults = new Map( - entries - .filter((entry) => entry.omitted_optional_defaults?.length) - .map((entry) => [entry.object_ref, entry.omitted_optional_defaults]), - ); - const collectionObject = entries.find((entry) => entry.kind === 'collection')?.frozen_stix; - - const emittedByRevision = new Map(); - for (const document of [...selectedDocuments, ...supportingDocuments]) { - const key = document.stix.modified - ? revisionKey(document.stix.id, document.stix.modified) - : `${document.stix.id}::unversioned`; - emittedByRevision.set(key, document); - } - - return { - documents: [...emittedByRevision.values()], - linkTargetDocuments, - sourceOmittedDefaults, - collectionObject, - manifest, - }; -} - -async function replay(snapshot, options = {}) { - if (!snapshot.graph_manifest_id) { - throw new ReleaseContentIntegrityError( - [ - { - track_id: snapshot.id, - snapshot_modified: new Date(snapshot.modified).toISOString(), - dependency: 'graph_manifest', - }, - ], - { details: 'Snapshot does not reference a deterministic graph manifest.' }, - ); - } - - const manifest = await ReleaseTrackGraphManifest.findOne({ - manifest_id: snapshot.graph_manifest_id, - track_id: snapshot.id, - snapshot_modified: snapshot.modified, - state: { $in: ['pending', 'active'] }, - }) - .lean() - .exec(); - if (!manifest) { - throw new ReleaseContentIntegrityError( - [{ manifest_id: snapshot.graph_manifest_id, dependency: 'graph_manifest' }], - { details: 'Snapshot graph manifest is missing.' }, - ); - } - - // A snapshot link is the durable commit record. If the process stopped - // after linking a complete pending manifest but before activation, replay - // remains deterministic and repairs the visibility marker opportunistically. - if (manifest.state === 'pending') { - await activate(manifest.manifest_id); - manifest.state = 'active'; - } - - const entries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: manifest.manifest_id, - }) - .sort({ _id: 1 }) - .lean() - .exec(); - return replayEntries(entries, manifest, options); -} - -async function replayPlannedSnapshot(snapshot, options = {}) { - const entries = await buildManifestEntries(snapshot); - return replayEntries( - entries, - { - manifest_id: null, - track_id: snapshot.id, - snapshot_modified: snapshot.modified, - state: 'preview', - schema_version: 1, - resolver_version: RESOLVER_VERSION, - }, - options, - ); -} - -async function refreshCollectionEntry(snapshot, manifest) { - const entries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: manifest.manifest_id, - }) - .sort({ _id: 1 }) - .lean() - .exec(); - return upsertCollectionEntry(snapshot, entries, manifest); -} - -async function findPinsForRevision(objectRef, objectModified) { - const entries = await ReleaseTrackGraphManifestEntry.find({ - object_ref: objectRef, - object_modified: objectModified, - ...MUTATION_PROTECTED_ENTRY_FILTER, - }) - .select({ - manifest_id: 1, - track_id: 1, - snapshot_modified: 1, - kind: 1, - tier: 1, - _id: 0, - }) - .lean() - .exec(); - if (entries.length === 0) return []; - - const protectedManifestIds = new Set( - ( - await ReleaseTrackGraphManifest.find({ - manifest_id: { $in: entries.map((entry) => entry.manifest_id) }, - state: { $in: ['pending', 'active'] }, - }) - .select({ manifest_id: 1, _id: 0 }) - .lean() - .exec() - ).map((manifest) => manifest.manifest_id), - ); - return entries.filter((entry) => protectedManifestIds.has(entry.manifest_id)); -} - -async function findPinsForObject(objectRef) { - const entries = await ReleaseTrackGraphManifestEntry.find({ - object_ref: objectRef, - object_modified: { $ne: null }, - ...MUTATION_PROTECTED_ENTRY_FILTER, - }) - .select({ - manifest_id: 1, - track_id: 1, - snapshot_modified: 1, - object_modified: 1, - kind: 1, - tier: 1, - _id: 0, - }) - .lean() - .exec(); - if (entries.length === 0) return []; - - const protectedManifestIds = new Set( - ( - await ReleaseTrackGraphManifest.find({ - manifest_id: { $in: entries.map((entry) => entry.manifest_id) }, - state: { $in: ['pending', 'active'] }, - }) - .select({ manifest_id: 1, _id: 0 }) - .lean() - .exec() - ).map((manifest) => manifest.manifest_id), - ); - return entries.filter((entry) => protectedManifestIds.has(entry.manifest_id)); -} - -module.exports = { - prepare, - prepareSourceReconstruction, - assertSourceReconstruction, - activate, - discard, - discardSnapshot, - discardTrack, - replay, - replayPlannedSnapshot, - refreshCollectionEntry, - collectionIdForTrack, - getStatisticsByManifestIds, - findPinsForRevision, - findPinsForObject, - buildManifestEntries, - MANIFEST_SCHEMA_VERSION, - RESOLVER_VERSION, - SOURCE_BUNDLE_RESOLVER_VERSION, -}; diff --git a/app/services/release-tracks/member-sync-service.js b/app/services/release-tracks/member-sync-service.js index 2caa2940..a08d5d8d 100644 --- a/app/services/release-tracks/member-sync-service.js +++ b/app/services/release-tracks/member-sync-service.js @@ -27,8 +27,9 @@ // Subscribes to BaseService CRUD events ({type}::created, {type}::updated) // via the EventBus. When a STIX object is created or updated, this service // checks whether any release track references it and syncs if configured. -// Relationships are deliberately not subscribed: bundle export pulls -// active relationships dynamically. +// Relationships are deliberately not subscribed: they are not tier entries. +// Sealed content manifests select relationships closed over members when a +// snapshot's members are written. // ============================================================================= const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); diff --git a/app/services/release-tracks/publication-service.js b/app/services/release-tracks/publication-service.js new file mode 100644 index 00000000..23ffca15 --- /dev/null +++ b/app/services/release-tracks/publication-service.js @@ -0,0 +1,180 @@ +'use strict'; + +// ============================================================================= +// Publication Service +// +// Resolves the metadata that appears on a snapshot's emitted +// x-mitre-collection object. Each attribute follows an inheritance rule: the +// track configuration may carry an explicit override, otherwise the value is +// inherited from the global system configuration. Release commit freezes the +// resolved values onto the tagged snapshot so later configuration changes +// cannot alter a published release; drafts resolve the rule at export time. +// +// This service performs cross-service READS only (organization identity and +// default marking definitions). +// ============================================================================= + +const config = require('../../config/config'); +const systemConfigurationService = require('../system/system-configuration-service'); +const { BadRequestError, ReleaseConflictError } = require('../../exceptions'); + +function collectionIdForTrack(trackId) { + return `x-mitre-collection--${trackId.split('--')[1]}`; +} + +function inheritedSetting(setting) { + if (!setting || setting.inherit !== false) return { inherit: true }; + return { inherit: false, value: setting.value }; +} + +/** + * Resolve every publication attribute for a snapshot from its configuration + * and the global scope. + * + * @param {Object} snapshot + * @returns {Promise<{ + * collection_id: string, + * created: Date, + * created_by_ref: string, + * object_marking_refs: string[], + * attack_spec_version: string, + * sources: Object, + * }>} + */ +async function resolvePublication(snapshot) { + const publication = snapshot.config?.publication || {}; + const identitySetting = inheritedSetting(publication.created_by_ref); + const markingSetting = inheritedSetting(publication.object_marking_refs); + + let createdByRef; + if (identitySetting.inherit) { + const organizationIdentity = await systemConfigurationService.retrieveOrganizationIdentity(); + createdByRef = organizationIdentity.stix.id; + } else { + createdByRef = identitySetting.value; + } + + let objectMarkingRefs; + if (markingSetting.inherit) { + objectMarkingRefs = await systemConfigurationService.retrieveDefaultMarkingDefinitions({ + refOnly: true, + }); + } else { + objectMarkingRefs = markingSetting.value || []; + } + + return { + collection_id: publication.collection_id || collectionIdForTrack(snapshot.id), + created: new Date(publication.created || snapshot.created || snapshot.modified), + created_by_ref: createdByRef, + object_marking_refs: [...objectMarkingRefs], + attack_spec_version: config.app.attackSpecVersion, + sources: { + collection_id: publication.collection_id ? 'track' : 'derived', + created: publication.created ? 'track' : 'derived', + created_by_ref: identitySetting.inherit ? 'global' : 'track', + // When neither scope configures markings the exported collection object + // carries the marking definitions referenced by its contents. + object_marking_refs: !markingSetting.inherit + ? 'track' + : objectMarkingRefs.length > 0 + ? 'global' + : 'content', + }, + }; +} + +/** + * Publication values used to render a snapshot's collection object: the + * frozen values for a tagged snapshot, otherwise the currently resolved rule. + */ +async function publicationForExport(snapshot) { + if (snapshot.version != null && snapshot.publication) { + return { + collection_id: snapshot.publication.collection_id, + created: new Date(snapshot.publication.created), + created_by_ref: snapshot.publication.created_by_ref, + object_marking_refs: [...(snapshot.publication.object_marking_refs || [])], + attack_spec_version: snapshot.publication.attack_spec_version, + }; + } + const resolved = await resolvePublication(snapshot); + delete resolved.sources; + return resolved; +} + +/** + * Freeze the resolved publication values for a release commit. + */ +async function freezePublication(snapshot) { + const resolved = await resolvePublication(snapshot); + return { + collection_id: resolved.collection_id, + created: resolved.created, + created_by_ref: resolved.created_by_ref, + object_marking_refs: resolved.object_marking_refs, + attack_spec_version: resolved.attack_spec_version, + }; +} + +/** + * Merge a publication configuration update onto the existing configuration, + * enforcing that collection identity and creation time cannot change once the + * track has a tagged release. + * + * @param {Object} existing - Current config.publication (may be undefined) + * @param {Object} update - Validated request publication object + * @param {boolean} hasReleases - Whether the track has any tagged snapshot + * @returns {Object} Merged publication configuration + */ +function mergePublicationConfig(existing = {}, update = {}, hasReleases = false) { + const merged = { + ...existing, + created_by_ref: inheritedSetting(existing.created_by_ref), + object_marking_refs: inheritedSetting(existing.object_marking_refs), + }; + + for (const field of ['collection_id', 'created']) { + if (!Object.prototype.hasOwnProperty.call(update, field)) continue; + const next = update[field] == null ? undefined : update[field]; + const current = existing[field] == null ? undefined : existing[field]; + const changed = + field === 'created' + ? (next ? new Date(next).getTime() : undefined) !== + (current ? new Date(current).getTime() : undefined) + : next !== current; + if (changed && hasReleases) { + throw new ReleaseConflictError( + `Publication ${field} cannot change after the release track has a tagged release`, + { field }, + ); + } + if (next === undefined) delete merged[field]; + else merged[field] = field === 'created' ? new Date(next) : next; + } + + for (const field of ['created_by_ref', 'object_marking_refs']) { + if (!Object.prototype.hasOwnProperty.call(update, field)) continue; + const setting = update[field]; + if (setting.inherit) { + merged[field] = { inherit: true }; + } else { + if (setting.value === undefined) { + throw new BadRequestError({ + message: `Publication ${field} requires a value when inherit is false`, + }); + } + merged[field] = { inherit: false, value: setting.value }; + } + } + + return merged; +} + +module.exports = { + collectionIdForTrack, + resolvePublication, + publicationForExport, + freezePublication, + mergePublicationConfig, +}; diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index e98a5078..7ec0d15e 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -361,16 +361,12 @@ exports.deleteSnapshot = function deleteSnapshot(trackId, modified) { return snapshotService.deleteSnapshot(trackId, modified); }; -exports.createSnapshotGraph = function createSnapshotGraph(trackId, modified) { - return snapshotService.createGraph(trackId, modified); -}; - -exports.reconstructSnapshotGraph = function reconstructSnapshotGraph(trackId, modified, plan) { - return snapshotService.reconstructGraph(trackId, modified, plan); -}; - -exports.deleteSnapshotGraph = function deleteSnapshotGraph(trackId, modified) { - return snapshotService.deleteGraph(trackId, modified); +exports.reconstructSnapshotManifest = function reconstructSnapshotManifest( + trackId, + modified, + plan, +) { + return snapshotService.reconstructManifest(trackId, modified, plan); }; // ----------------------------------------------------------------------------- @@ -443,9 +439,9 @@ async function renderReleasePlan(plan, options) { if (format === 'bundle') { return exportService.exportSnapshot(plan.plannedSnapshot, format, { ...options, - // Release previews are intentionally live. Determinism begins only if a - // caller explicitly creates a graph after the snapshot is tagged. - captureGraph: true, + // The planned snapshot is unsaved and has no sealed manifest yet, so a + // preview resolves the same closed-member graph the commit would seal. + resolveLive: true, }); } return formatWorkbenchSnapshot(plan.plannedSnapshot, options); diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 4c63ddfa..ad202317 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -6,6 +6,11 @@ // Core snapshot lifecycle operations: track creation, retrieval, cloning, // metadata updates, configuration, and deletion. // +// Every snapshot references a sealed content manifest from birth. Writes that +// change the members tier seal a new manifest; every other clone inherits +// its predecessor's manifest by reference (see +// docs/developer/release-tracks/sealed-content-manifests.md). +// // This is the foundational sub-service consumed by the facade and by other // sub-services (standard-track, versioning, virtual-track) that need to // clone or read snapshots. @@ -21,8 +26,8 @@ const versionUtils = require('../../lib/release-tracks/version-utils'); const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-invariant'); const primaryRevisionService = require('./primary-revision-service'); const reconciliationService = require('./reconciliation-service'); -const graphManifestService = require('./graph-manifest-service'); -const bundleHashService = require('./bundle-hash-service'); +const contentManifestService = require('./content-manifest-service'); +const publicationService = require('./publication-service'); const { TrackNotFoundError, NotFoundError, @@ -89,6 +94,7 @@ async function syncRegistryCounters(trackId) { updated_at: new Date(), }); } +exports.syncRegistryCounters = syncRegistryCounters; /** * Notify listeners that a track's current (latest) snapshot changed so they @@ -106,6 +112,32 @@ async function emitContentsChanged(trackId, snapshot) { } exports.emitContentsChanged = emitContentsChanged; +/** + * Seal a manifest for a snapshot that is about to be saved, then persist the + * snapshot referencing it. The manifest is discarded if the save fails, so a + * snapshot is never observable without its content manifest. + * + * @param {string} trackId + * @param {Object} snapshotData - Snapshot to save (members already final) + * @param {string} reason - seal_reason enum value + * @returns {Promise} The saved snapshot + */ +async function saveSealedSnapshot(trackId, snapshotData, reason) { + const manifestId = await contentManifestService.seal(snapshotData, { reason }); + let saved; + try { + saved = await dynamicRepo.saveSnapshot(trackId, { + ...snapshotData, + content_manifest_id: manifestId, + }); + } catch (err) { + await contentManifestService.discard(manifestId); + throw err; + } + await contentManifestService.activate(manifestId); + return saved; +} + // ============================================================================= // Track management // ============================================================================= @@ -138,7 +170,7 @@ exports.listTracks = async function listTracks(options) { /** * Create a new release track with an initial empty draft snapshot. * - * @param {Object} data - { name, description?, snapshot_description?, type, userAccountId?, object_marking_refs?, composition?, snapshot_schedule?, scheduled_materialization?, config? } + * @param {Object} data - { name, description?, snapshot_description?, type, userAccountId?, composition?, snapshot_schedule?, scheduled_materialization?, config? } * @returns {Promise} The initial snapshot document */ exports.createTrack = async function createTrack(data) { @@ -156,7 +188,6 @@ exports.createTrack = async function createTrack(data) { snapshot_description: data.snapshot_description || undefined, created: now, created_by_ref: data.userAccountId || undefined, - object_marking_refs: data.object_marking_refs, members: [], staged: trackType === 'standard' ? [] : undefined, candidates: trackType === 'standard' ? [] : undefined, @@ -167,9 +198,9 @@ exports.createTrack = async function createTrack(data) { version_history: [], }; - // Create collection + indexes, then persist the initial snapshot + // Create collection + indexes, then persist the initial sealed snapshot await modelFactory.ensureIndexes(trackId); - const snapshot = await dynamicRepo.saveSnapshot(trackId, initialSnapshot); + const snapshot = await saveSealedSnapshot(trackId, initialSnapshot, 'track_creation'); // Register in the central registry await registryRepo.create({ @@ -197,8 +228,8 @@ exports.createTrack = async function createTrack(data) { * List lightweight summaries of a track's snapshots. * * Standard summaries expose members/staged/candidates counts. Virtual - * summaries expose members/quarantine counts. Summaries linked to a graph - * manifest also expose counts by manifest entry role. + * summaries expose members/quarantine counts. Every summary exposes its + * content manifest ID and counts by manifest entry role. * * @param {string} trackId * @param {Object} options - { tagged?, limit, offset } @@ -212,8 +243,8 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { } const result = await dynamicRepo.getSnapshotSummaries(trackId, options); - const graphStatisticsByManifestId = await graphManifestService.getStatisticsByManifestIds( - result.data.map((snapshot) => snapshot.graph_manifest_id), + const statisticsByManifestId = await contentManifestService.getStatisticsByManifestIds( + result.data.map((snapshot) => snapshot.content_manifest_id), ); return { ...result, @@ -223,11 +254,12 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { type: snapshot.type, modified: snapshot.modified, version: snapshot.version, - graph_manifest_id: snapshot.graph_manifest_id, + content_manifest_id: snapshot.content_manifest_id, + bundle_id: snapshot.bundle_id, bundle_hashes: snapshot.bundle_hashes, snapshot_description: snapshot.snapshot_description, - graph_statistics: snapshot.graph_manifest_id - ? graphStatisticsByManifestId.get(snapshot.graph_manifest_id) + content_statistics: snapshot.content_manifest_id + ? statisticsByManifestId.get(snapshot.content_manifest_id) : undefined, name: snapshot.name, description: snapshot.description, @@ -297,19 +329,30 @@ exports.getSnapshotByModified = async function getSnapshotByModified(trackId, mo * * Every mutation (metadata update, contents update, tier change) produces a * new snapshot via this method. Clones are always drafts (version = null). + * A clone that rewrites `members` seals a new content manifest; any other + * clone inherits the source snapshot's manifest by reference. * * @param {string} trackId - The track to save the clone into * @param {Object} sourceSnapshot - The snapshot to clone * @param {Object} [overrides] - Fields to merge into the clone + * @param {Object} [options] + * @param {string} [options.sealReason] - seal_reason when members are rewritten * @returns {Promise} The saved clone */ -exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, overrides) { +exports.cloneSnapshot = async function cloneSnapshot( + trackId, + sourceSnapshot, + overrides, + options = {}, +) { const clone = deepClone(sourceSnapshot); const hasSnapshotDescriptionOverride = Object.prototype.hasOwnProperty.call( overrides || {}, 'snapshot_description', ); - delete clone.graph_manifest_id; + const rewritesMembers = overrides?.members !== undefined; + delete clone.publication; + delete clone.bundle_id; delete clone.bundle_hashes; clone.modified = new Date(); clone.version = null; // clones are always drafts @@ -336,13 +379,22 @@ exports.cloneSnapshot = async function cloneSnapshot(trackId, sourceSnapshot, ov } const normalized = tierRevisionInvariant.normalizeSnapshot(clone); - const saved = await dynamicRepo.saveSnapshot(trackId, normalized.snapshot); + let saved; + if (rewritesMembers || !normalized.snapshot.content_manifest_id) { + saved = await saveSealedSnapshot( + trackId, + normalized.snapshot, + options.sealReason || 'members_written', + ); + } else { + saved = await dynamicRepo.saveSnapshot(trackId, normalized.snapshot); + } + if (saved.type === 'standard') { const prunedDrafts = await dynamicRepo.deleteOlderDrafts(trackId, saved.modified); - await Promise.all( - prunedDrafts - .filter((snapshot) => snapshot.graph_manifest_id) - .map((snapshot) => graphManifestService.discard(snapshot.graph_manifest_id)), + await contentManifestService.discardUnreferenced( + trackId, + prunedDrafts.map((snapshot) => snapshot.content_manifest_id), ); } await syncRegistryCounters(trackId); @@ -397,7 +449,9 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { const now = new Date(); const clone = deepClone(sourceSnapshot); - delete clone.graph_manifest_id; + delete clone.content_manifest_id; + delete clone.publication; + delete clone.bundle_id; delete clone.bundle_hashes; clone.id = newTrackId; clone.modified = now; @@ -408,6 +462,12 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { clone.version_history = []; delete clone.scheduled_materialization; delete clone.snapshot_description; + // Collection identity belongs to the source lineage; the copy derives its + // own and inherits the remaining publication rules. + if (clone.config?.publication) { + delete clone.config.publication.collection_id; + delete clone.config.publication.created; + } const normalized = tierRevisionInvariant.normalizeSnapshot(clone); await primaryRevisionService.assertStoredEntries( @@ -415,7 +475,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { ); await modelFactory.ensureIndexes(newTrackId); - const saved = await dynamicRepo.saveSnapshot(newTrackId, normalized.snapshot); + const saved = await saveSealedSnapshot(newTrackId, normalized.snapshot, 'track_clone'); await registryRepo.create({ track_id: newTrackId, @@ -450,7 +510,7 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { * Update metadata on the latest snapshot (creates a new snapshot clone). * * @param {string} trackId - * @param {Object} updates - { name?, description?, object_marking_refs? } + * @param {Object} updates - { name?, description? } * @param {string} [_userId] * @returns {Promise} The new snapshot */ @@ -460,8 +520,6 @@ exports.updateMetadata = async function updateMetadata(trackId, updates, _userId const overrides = {}; if (updates.name !== undefined) overrides.name = updates.name; if (updates.description !== undefined) overrides.description = updates.description; - if (updates.object_marking_refs !== undefined) - overrides.object_marking_refs = updates.object_marking_refs; // Also update the registry name/description if changed const registryUpdates = {}; @@ -476,12 +534,12 @@ exports.updateMetadata = async function updateMetadata(trackId, updates, _userId }; /** - * Set or clear a snapshot-local description without changing its identity, - * release tag, members, or release-track registry metadata. + * Set or clear a draft snapshot's description without changing its identity + * or contents. * - * Snapshot descriptions are editable workspace annotations until a graph - * manifest freezes the bundle content. Cached snapshots must have their graph - * deleted before their description can change. + * Snapshot descriptions become the emitted collection object's description. + * A tagged snapshot is immutable, notes included, so its description can only + * be set while releasing. * * @param {string} trackId * @param {string|Date} modified @@ -494,11 +552,11 @@ exports.updateSnapshotDescription = async function updateSnapshotDescription( description, ) { const snapshot = await exports.getSnapshotByModified(trackId, modified); - if (snapshot.graph_manifest_id) { - throw new ReleaseConflictError('Delete the bundle cache before editing snapshot notes.', { + if (snapshot.version != null) { + throw new ReleaseConflictError('Snapshot notes are immutable once the snapshot is released.', { track_id: trackId, snapshot_modified: new Date(snapshot.modified).toISOString(), - graph_manifest_id: snapshot.graph_manifest_id, + version: snapshot.version, }); } @@ -520,21 +578,27 @@ exports.updateSnapshotDescription = async function updateSnapshotDescription( // ============================================================================= /** - * Get the configuration from the latest snapshot. + * Get the configuration from the latest snapshot, including the currently + * resolved publication values and where each one comes from. * * @param {string} trackId - * @returns {Promise} The config sub-document + * @returns {Promise} The config sub-document plus publication_resolved */ exports.getConfig = async function getConfig(trackId) { const snapshot = await exports.getLatestSnapshot(trackId); - return snapshot.config || {}; + const config = JSON.parse(JSON.stringify(snapshot.config || {})); + const resolved = await publicationService.resolvePublication(snapshot); + return { + ...config, + publication_resolved: resolved, + }; }; /** * Update configuration on the latest snapshot (creates a new snapshot clone). * * Performs a shallow merge at the top level, and a nested merge for - * the `promotion_conflicts` sub-object. + * the `promotion_conflicts`, `member_sync`, and `publication` sub-objects. * * @param {string} trackId * @param {Object} config - Partial config to merge @@ -571,15 +635,36 @@ exports.updateConfig = async function updateConfig(trackId, config, _userId) { }; } } + if (config.publication !== undefined) { + const hasReleases = (source.version_history || []).length > 0; + mergedConfig.publication = publicationService.mergePublicationConfig( + existing.publication, + config.publication, + hasReleases, + ); + } return exports.cloneSnapshot(trackId, source, { config: mergedConfig }); }; // ============================================================================= -// Optional deterministic member graphs +// Source-attested manifest reconstruction (administrative) // ============================================================================= -async function createGraph(trackId, modified, prepareManifest, validateExisting) { +/** + * Replace a tagged snapshot's content manifest with one reconstructed from an + * externally verified source bundle. + * + * Repeating the same attestation is idempotent. Any other manifest is + * replaced only when the caller names it in `replace_manifest_id`, so a + * concurrent change is never silently overwritten. + * + * @param {string} trackId + * @param {string|Date} modified + * @param {Object} plan - Validated reconstruction request + * @returns {Promise<{ snapshot: Object, created: boolean }>} + */ +exports.reconstructManifest = async function reconstructManifest(trackId, modified, plan) { const snapshot = await dynamicRepo.getSnapshotByModified(trackId, modified); if (!snapshot) { throw new NotFoundError({ @@ -587,104 +672,53 @@ async function createGraph(trackId, modified, prepareManifest, validateExisting) }); } if (snapshot.version == null) { - throw new ReleaseConflictError('Only tagged snapshots can be made deterministic', { + throw new ReleaseConflictError('Only tagged snapshots can be reconstructed from a source', { track_id: trackId, snapshot_modified: new Date(snapshot.modified).toISOString(), }); } - if (snapshot.graph_manifest_id) { - if (validateExisting) await validateExisting(snapshot); - return { snapshot, created: false }; - } - const manifestId = await prepareManifest(snapshot); - const attached = await dynamicRepo.attachGraphManifest(trackId, snapshot.modified, manifestId); - if (!attached) { - await graphManifestService.discard(manifestId); - const current = await dynamicRepo.getSnapshotByModified(trackId, modified); - if (current?.graph_manifest_id) return { snapshot: current, created: false }; - throw new ReleaseConflictError('Snapshot changed while its graph was being created', { - track_id: trackId, - snapshot_modified: new Date(snapshot.modified).toISOString(), - }); - } - - try { - await graphManifestService.activate(manifestId); - } catch (err) { - logger.warn( - `SnapshotService: Deferred activation for graph manifest "${manifestId}": ${err.message}`, - ); + const currentManifestId = snapshot.content_manifest_id; + if ( + await contentManifestService.isSameSourceReconstruction( + currentManifestId, + plan.source_attestation, + ) + ) { + return { snapshot, created: false }; } - try { - const bundleHashes = await bundleHashService.generateBundleHashes(attached); - const hashed = await dynamicRepo.attachBundleHashes( - trackId, - snapshot.modified, - manifestId, - bundleHashes, - ); - if (!hashed) { - throw new ReleaseConflictError('Snapshot graph changed while its hashes were generated', { + if (plan.replace_manifest_id !== currentManifestId) { + throw new ReleaseConflictError( + 'Snapshot already has a content manifest that was not reconstructed from this source. ' + + 'Name it in replace_manifest_id to replace it.', + { track_id: trackId, snapshot_modified: new Date(snapshot.modified).toISOString(), - }); - } - return { snapshot: hashed, created: true }; - } catch (err) { - await dynamicRepo.detachGraphManifest(trackId, snapshot.modified, manifestId); - await graphManifestService.discard(manifestId); - throw err; - } -} - -exports.createGraph = function createLiveGraph(trackId, modified) { - return createGraph(trackId, modified, async (snapshot) => { - const predecessor = await dynamicRepo.getLatestTaggedSnapshotBefore(trackId, snapshot.modified); - return graphManifestService.prepare(snapshot, { - predecessorManifestId: predecessor?.graph_manifest_id, - }); - }); -}; - -exports.reconstructGraph = function reconstructGraph(trackId, modified, plan) { - return createGraph( - trackId, - modified, - (snapshot) => graphManifestService.prepareSourceReconstruction(snapshot, plan), - (snapshot) => - graphManifestService.assertSourceReconstruction(snapshot, plan.source_attestation), - ); -}; - -exports.deleteGraph = async function deleteGraph(trackId, modified) { - const snapshot = await dynamicRepo.getSnapshotByModified(trackId, modified); - if (!snapshot) { - throw new NotFoundError({ - details: `Snapshot with modified '${modified}' not found for track '${trackId}'`, - }); - } - if (snapshot.version == null) { - throw new ReleaseConflictError('Only tagged snapshots can have deterministic graphs', { - track_id: trackId, - snapshot_modified: new Date(snapshot.modified).toISOString(), - }); + content_manifest_id: currentManifestId, + }, + ); } - if (!snapshot.graph_manifest_id) return false; - const detached = await dynamicRepo.detachGraphManifest( + const manifestId = await contentManifestService.prepareSourceReconstruction(snapshot, plan); + const replaced = await dynamicRepo.replaceContentManifest( trackId, snapshot.modified, - snapshot.graph_manifest_id, + currentManifestId, + manifestId, ); - if (!detached) { - throw new ReleaseConflictError('Snapshot graph changed while it was being deleted', { + if (!replaced) { + await contentManifestService.discard(manifestId); + throw new ReleaseConflictError('Snapshot changed while its manifest was being reconstructed', { track_id: trackId, snapshot_modified: new Date(snapshot.modified).toISOString(), }); } - await graphManifestService.discard(snapshot.graph_manifest_id); - return true; + await contentManifestService.activate(manifestId); + await contentManifestService.discardUnreferenced(trackId, [currentManifestId]); + + const versioningService = require('./versioning-service'); + const hashed = await versioningService.refreshReleaseArtifacts(replaced); + return { snapshot: hashed, created: true }; }; // ============================================================================= @@ -702,12 +736,12 @@ exports.deleteTrack = async function deleteTrack(trackId) { if (!registry) { // A previous delete may have removed the registry only after dropping the // dynamic snapshot collection but stopped before manifest cleanup. - await graphManifestService.discardTrack(trackId); + await contentManifestService.discardTrack(trackId); throw new TrackNotFoundError(trackId); } await dynamicRepo.dropCollection(trackId); - await graphManifestService.discardTrack(trackId); + await contentManifestService.discardTrack(trackId); await registryRepo.deleteByTrackId(trackId); // Remove all backrefs to the deleted track @@ -728,7 +762,7 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { if (!snapshot) { // Make a retry after an interrupted delete clean any orphaned manifests // even though the snapshot document is already gone. - await graphManifestService.discardSnapshot(trackId, modified); + await contentManifestService.discardOrphans(trackId); throw new NotFoundError({ details: `Snapshot with modified '${modified}' not found for track '${trackId}'`, }); @@ -752,7 +786,7 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { } await dynamicRepo.deleteSnapshot(trackId, modified); - await graphManifestService.discardSnapshot(trackId, snapshot.modified); + await contentManifestService.discardUnreferenced(trackId, [snapshot.content_manifest_id]); await syncRegistryCounters(trackId); // Deleting the latest snapshot reverts membership to the previous snapshot diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 700d2c4c..1f014fb8 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -12,7 +12,9 @@ const tierRevisionInvariant = require('../../lib/release-tracks/tier-revision-in const revisionReference = require('../../lib/release-tracks/revision-reference'); const releaseHistoryService = require('./release-history-service'); const primaryRevisionService = require('./primary-revision-service'); -const graphManifestService = require('./graph-manifest-service'); +const contentManifestService = require('./content-manifest-service'); +const publicationService = require('./publication-service'); +const bundleHashService = require('./bundle-hash-service'); const registryRepo = require('../../repository/release-tracks/release-track-registry.repository'); const uuid = require('uuid'); const logger = require('../../lib/logger'); @@ -300,7 +302,7 @@ async function planLoadedSnapshot(trackId, snapshot, options) { ...(releaseInput.staged || []), ]); - return planRelease( + const plan = planRelease( trackId, releaseInput, versionHistory, @@ -308,43 +310,101 @@ async function planLoadedSnapshot(trackId, snapshot, options) { new Date(), previousTaggedSnapshot, ); + + // A standard commit seals a fresh manifest over the planned members, so the + // preview reports exactly which relationships that seal would add or drop + // relative to the draft's inherited manifest. Virtual commits publish the + // materialization manifest unchanged. + if (!plan.blockingError && snapshot.type === 'standard') { + plan.summary.relationships = await contentManifestService.previewRelationshipChanges( + snapshot, + plan.plannedSnapshot.members, + ); + } + return plan; } +/** + * Freeze publication metadata, assign a stable bundle ID, and store the + * SHA-256 hashes of both bundle serializations on a tagged snapshot. + * + * @param {Object} tagged - The tagged snapshot (already referencing its manifest) + * @returns {Promise} The updated snapshot + */ +async function refreshReleaseArtifacts(tagged) { + const publication = tagged.publication || (await publicationService.freezePublication(tagged)); + const bundleId = tagged.bundle_id || `bundle--${uuid.v4()}`; + const withArtifacts = await dynamicRepo.updateSnapshot(tagged.id, tagged.modified, { + $set: { publication, bundle_id: bundleId }, + }); + const current = withArtifacts?.toObject ? withArtifacts.toObject() : withArtifacts; + const bundleHashes = await bundleHashService.generateBundleHashes(current); + const hashed = await dynamicRepo.attachBundleHashes( + tagged.id, + tagged.modified, + current.content_manifest_id, + bundleHashes, + ); + if (!hashed) { + throw new ReleaseConflictError('Snapshot changed while its bundle hashes were generated', { + track_id: tagged.id, + snapshot_modified: new Date(tagged.modified).toISOString(), + }); + } + return hashed; +} +exports.refreshReleaseArtifacts = refreshReleaseArtifacts; + async function commitPlan(plan) { if (plan.blockingError) throw plan.blockingError; - const obsoleteManifestId = plan.sourceSnapshot.graph_manifest_id; + const source = plan.sourceSnapshot; + const inheritedManifestId = source.content_manifest_id; const unsetOps = {}; - if (obsoleteManifestId) { - unsetOps.graph_manifest_id = ''; - unsetOps.bundle_hashes = ''; - } if (plan.clearSnapshotDescription) unsetOps.snapshot_description = ''; - const tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, plan.sourceSnapshot.modified, { - version: plan.version, - versionHistoryEntry: plan.versionHistoryEntry, - additionalOps: plan.additionalOps, - // Older deployments attached graphs to drafts. Releasing changes the - // member set, so that legacy draft graph cannot describe the release. - unsetOps: Object.keys(unsetOps).length ? unsetOps : undefined, - }); + + // A standard commit is the moment members are finalized, so it seals a + // fresh manifest over the planned member set (even when nothing was staged, + // so relationships added since the last seal are captured). A virtual + // commit publishes the materialization manifest that was reviewed. + let sealedManifestId; + const setOps = { ...plan.additionalOps }; + if (source.type === 'standard') { + sealedManifestId = await contentManifestService.seal( + { ...source, members: plan.plannedSnapshot.members }, + { reason: 'release' }, + ); + setOps.content_manifest_id = sealedManifestId; + } + setOps.publication = await publicationService.freezePublication(source); + setOps.bundle_id = `bundle--${uuid.v4()}`; + + let tagged; + try { + tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, source.modified, { + version: plan.version, + versionHistoryEntry: plan.versionHistoryEntry, + additionalOps: setOps, + unsetOps: Object.keys(unsetOps).length ? unsetOps : undefined, + }); + } catch (err) { + await contentManifestService.discard(sealedManifestId); + throw err; + } if (!tagged) { + await contentManifestService.discard(sealedManifestId); await releaseHistoryService.reconcileTaggedReleases(plan.trackId); throw new AlreadyReleasedError('(concurrent release)'); } - if (obsoleteManifestId) { - try { - await graphManifestService.discard(obsoleteManifestId); - } catch (err) { - logger.warn( - `VersioningService: Deferred cleanup for obsolete graph manifest ` + - `"${obsoleteManifestId}": ${err.message}`, - ); - } + if (sealedManifestId) { + await contentManifestService.activate(sealedManifestId); + await contentManifestService.discardUnreferenced(plan.trackId, [inheritedManifestId]); } + const withArtifacts = await refreshReleaseArtifacts(tagged); + await releaseHistoryService.reconcileTaggedReleases(plan.trackId); const latest = await dynamicRepo.getLatestSnapshot(plan.trackId); await snapshotService.emitContentsChanged(plan.trackId, latest); @@ -360,7 +420,7 @@ async function commitPlan(plan) { ); } - return tagged; + return withArtifacts; } async function withReleaseLock(trackId, operation) { diff --git a/app/services/stix/relationships-service.js b/app/services/stix/relationships-service.js index 79f9c9e4..887f772a 100644 --- a/app/services/stix/relationships-service.js +++ b/app/services/stix/relationships-service.js @@ -1,6 +1,5 @@ 'use strict'; -const _ = require('lodash'); const { BaseService } = require('../meta-classes'); const relationshipsRepository = require('../../repository/relationships-repository'); const attackObjectsRepository = require('../../repository/attack-objects-repository'); @@ -27,8 +26,14 @@ const objectTypeMap = new Map([ class RelationshipsService extends BaseService { /** - * Resolve STIX ID-only relationship endpoints to the exact object revisions - * they mean when this relationship revision is created. + * Record the exact object revisions the endpoints resolve to when this + * relationship revision is created. + * + * The pins are authoring context: they let release previews warn when a + * relationship is shipped against a different endpoint revision than the + * one its author saw. They do not drive selection (sealed content manifests + * pair each relationship with the member revisions it ships with) and a + * later endpoint revision never clones the relationship. * * The pins are Workbench metadata rather than custom STIX properties. They * are therefore validated by Mongoose, remain server-controlled, and are @@ -104,17 +109,6 @@ class RelationshipsService extends BaseService { * Called once on module load. */ static initializeEventListeners() { - const endpointRevisionEvents = [ - ...new Set([ - ...Object.values(EventConstants).filter((eventName) => eventName.endsWith('::created')), - 'identity::created', - 'note::created', - ]), - ]; - for (const event of endpointRevisionEvents) { - EventBus.on(event, this.handleEndpointRevisionCreated.bind(this)); - } - const revokedEvents = [ EventConstants.ATTACK_PATTERN_REVOKED, EventConstants.TACTIC_REVOKED, @@ -156,80 +150,6 @@ class RelationshipsService extends BaseService { logger.info('RelationshipsService: Event listeners initialized'); } - /** - * Carry active relationship edges forward when one of their exact endpoint - * revisions advances. - * - * The prior SRO revision remains pinned to the prior endpoint revisions. - * A new SRO revision is created for the new endpoint state, preserving STIX - * revision immutability while retaining the current graph. - * - * @param {Object} payload Standard BaseService created-event payload - * @returns {Promise<{created: Array}>} - */ - static async handleEndpointRevisionCreated(payload) { - const document = payload?.document; - if (!document?.stix?.id || !document?.stix?.modified) { - return { created: [] }; - } - - const versions = await attackObjectsRepository.retrieveAllById(document.stix.id); - const createdRevisionIndex = versions.findIndex( - (version) => - new Date(version.stix.modified).getTime() === new Date(document.stix.modified).getTime(), - ); - - // Only the latest revision advances the current graph. Older revisions - // arriving in a bulk import retain their historical position. - if (createdRevisionIndex !== 0 || versions.length < 2) { - return { created: [] }; - } - - const previousRevision = versions[1]; - const relationships = await relationshipsRepository.retrieveAllBySourceOrTarget( - document.stix.id, - ); - const relationshipsToAdvance = relationships.filter((relationship) => { - if (relationship.stix.revoked || relationship.stix.x_mitre_deprecated) { - return false; - } - - const endpoints = relationship.workspace?.relationship_endpoints; - return ['source', 'target'].some( - (side) => - endpoints?.[side]?.object_ref === previousRevision.stix.id && - new Date(endpoints[side].object_modified).getTime() === - new Date(previousRevision.stix.modified).getTime(), - ); - }); - - const created = []; - for (const relationship of relationshipsToAdvance) { - const relationshipData = _.cloneDeep(relationship); - delete relationshipData._id; - delete relationshipData.__v; - delete relationshipData.__t; - if (relationshipData.workspace) { - delete relationshipData.workspace.release_tracks; - delete relationshipData.workspace.relationship_endpoints; - } - - const previousRelationshipModified = new Date(relationship.stix.modified).getTime(); - relationshipData.stix.modified = new Date( - Math.max(Date.now(), previousRelationshipModified + 1), - ).toISOString(); - - created.push( - await module.exports.create(relationshipData, { - userAccountId: payload.options?.userAccountId, - automationContext: payload.options?.automationContext, - }), - ); - } - - return { created }; - } - /** * Return the latest active relationship revisions whose endpoints are both * in the requested bundle object set. diff --git a/app/tests/api/attack-objects/attack-objects.spec.js b/app/tests/api/attack-objects/attack-objects.spec.js index bd4a9184..45d0b285 100644 --- a/app/tests/api/attack-objects/attack-objects.spec.js +++ b/app/tests/api/attack-objects/attack-objects.spec.js @@ -142,9 +142,9 @@ describe('ATT&CK Objects API', function () { expect(markingDefinitions.length).toBe(5); // Placeholder identity, 4 TLP marking definitions, 18 imported collection contents, - // 2 collection objects, and the propagated relationship revision pinned to the - // second bundle's newer target revision. - expect(attackObjects.length).toBe(1 + 4 + 18 + 2 + 1); + // and 2 collection objects. A newer endpoint revision no longer clones the + // relationship that references it. + expect(attackObjects.length).toBe(1 + 4 + 18 + 2); }); it('GET /api/attack-objects returns zero objects with an ATT&CK ID that does not exist', async function () { diff --git a/app/tests/api/relationships/relationship-endpoint-pins.spec.js b/app/tests/api/relationships/relationship-endpoint-pins.spec.js index 672e98a4..d9082b3b 100644 --- a/app/tests/api/relationships/relationship-endpoint-pins.spec.js +++ b/app/tests/api/relationships/relationship-endpoint-pins.spec.js @@ -116,33 +116,23 @@ describe('Relationship endpoint revision pins', function () { expect(relationship.stix.x_mitre_target_ref_modified).toBeUndefined(); }); - it('creates a new SRO revision when an endpoint advances', async function () { + it('does not clone the SRO when an endpoint advances; pins remain authoring context', async function () { const sourceRevision = cloneForCreate(source); sourceRevision.stix.modified = new Date( new Date(source.stix.modified).getTime() + 1000, ).toISOString(); sourceRevision.stix.description = 'A newer source revision.'; - const newSource = await post('/api/software', sourceRevision); + await post('/api/software', sourceRevision); const response = await request(app) .get(`/api/relationships/${relationship.stix.id}?versions=all`) .set('Accept', 'application/json') .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) .expect(200); - expect(response.body).toHaveLength(2); - const [latestRelationship, originalRelationship] = response.body; - expect(latestRelationship.stix.id).toBe(relationship.stix.id); - expect(latestRelationship.stix.modified).not.toBe(originalRelationship.stix.modified); - expect(latestRelationship.workspace.relationship_endpoints.source).toEqual({ - object_ref: source.stix.id, - object_modified: newSource.stix.modified, - }); - expect(latestRelationship.workspace.relationship_endpoints.target).toEqual({ - object_ref: target.stix.id, - object_modified: target.stix.modified, - }); - expect(originalRelationship.workspace.relationship_endpoints.source).toEqual({ + expect(response.body).toHaveLength(1); + expect(response.body[0].stix.modified).toBe(relationship.stix.modified); + expect(response.body[0].workspace.relationship_endpoints.source).toEqual({ object_ref: source.stix.id, object_modified: source.stix.modified, }); diff --git a/app/tests/api/release-tracks/content-manifests.spec.js b/app/tests/api/release-tracks/content-manifests.spec.js new file mode 100644 index 00000000..40066c98 --- /dev/null +++ b/app/tests/api/release-tracks/content-manifests.spec.js @@ -0,0 +1,599 @@ +'use strict'; + +const crypto = require('node:crypto'); +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const { + ReleaseTrackContentManifest, + ReleaseTrackContentManifestEntry, +} = require('../../../models/release-tracks/release-track-content-manifest-model'); +const AttackObject = require('../../../models/attack-object-model'); +const { releaseExactMembers, stageExactMembers } = require('./release-track-test-helpers'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +function sha256(payload) { + return crypto + .createHash('sha256') + .update(JSON.stringify(payload, null, 4), 'utf8') + .digest('hex'); +} + +describe('Sealed release-track content manifests', function () { + let app; + let passportCookie; + let organizationIdentity; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + organizationIdentity = ( + await authenticated(request(app).get('/api/config/organization-identity')).expect(200) + ).body; + }); + + function authenticated(builder) { + return builder + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + } + + function technique(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_domains: ['enterprise-attack'], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + x_mitre_version: '1.0', + }, + }; + } + + function relationship(source, target, previous) { + const modified = previous + ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() + : new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + id: previous?.stix.id, + created: previous?.stix.created || modified, + modified, + spec_version: '2.1', + type: 'relationship', + relationship_type: 'uses', + source_ref: source.stix.id, + target_ref: target.stix.id, + description: previous ? 'New relationship revision' : 'Original relationship revision', + object_marking_refs: [markingDefinitionId], + }, + }; + } + + async function post(path, body, status = 201) { + return (await authenticated(request(app).post(path).send(body)).expect(status)).body; + } + + async function get(path, status = 200) { + return (await authenticated(request(app).get(path)).expect(status)).body; + } + + async function createTrack(name) { + return post('/api/release-tracks/new', { name, type: 'standard' }); + } + + async function bundle(trackId, modified, query = '') { + return get( + `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent(modified)}?format=bundle${query}`, + ); + } + + async function entriesFor(manifestId) { + return ReleaseTrackContentManifestEntry.find({ manifest_id: manifestId }).lean().exec(); + } + + it('seals a manifest at creation, inherits it through workflow clones, and reseals at release', async function () { + const primary = await post('/api/techniques', technique('Sealed Primary')); + const secondary = await post('/api/techniques', technique('Sealed Secondary')); + const originalRelationship = await post('/api/relationships', relationship(primary, secondary)); + + const track = await createTrack('Sealed Manifest Track'); + expect(track.content_manifest_id).toMatch(/^release-track-content-manifest--/); + const initialManifest = await ReleaseTrackContentManifest.findOne({ + manifest_id: track.content_manifest_id, + }) + .lean() + .exec(); + expect(initialManifest).toMatchObject({ + state: 'active', + schema_version: 2, + seal_reason: 'track_creation', + }); + // An empty track still seals the publishing identity as a supporting + // object so its collection object is self-contained. + const initialEntries = await entriesFor(track.content_manifest_id); + expect(initialEntries.filter((entry) => entry.kind === 'root')).toHaveLength(0); + expect(initialEntries).toEqual([ + expect.objectContaining({ kind: 'supporting', object_ref: organizationIdentity.stix.id }), + ]); + + const staged = await stageExactMembers(app, passportCookie, track.id, [primary, secondary]); + expect(staged.content_manifest_id).toBe(track.content_manifest_id); + + const released = await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.0' }, + 200, + ); + expect(released.content_manifest_id).not.toBe(track.content_manifest_id); + expect(released.bundle_id).toMatch(/^bundle--/); + expect(released.publication).toMatchObject({ + collection_id: `x-mitre-collection--${track.id.split('--')[1]}`, + created: track.created, + created_by_ref: organizationIdentity.stix.id, + attack_spec_version: config.app.attackSpecVersion, + }); + expect(released.bundle_hashes).toEqual({ + manifest_id: released.content_manifest_id, + stix_2_0: expect.stringMatching(/^[a-f0-9]{64}$/), + stix_2_1: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + // The initial manifest is no longer referenced by any snapshot. + expect(await ReleaseTrackContentManifest.countDocuments({ track_id: track.id })).toBe(1); + + const sealed = await ReleaseTrackContentManifest.findOne({ + manifest_id: released.content_manifest_id, + }) + .lean() + .exec(); + expect(sealed).toMatchObject({ state: 'active', seal_reason: 'release' }); + const entries = await entriesFor(released.content_manifest_id); + expect(entries.filter((entry) => entry.kind === 'root')).toHaveLength(2); + expect(entries.some((entry) => entry.kind === 'secondary')).toBe(false); + expect(entries.some((entry) => entry.kind === 'collection')).toBe(false); + const relationshipEntry = entries.find((entry) => entry.kind === 'relationship'); + expect(relationshipEntry).toMatchObject({ + object_ref: originalRelationship.stix.id, + object_modified: new Date(originalRelationship.stix.modified), + source: { object_ref: primary.stix.id, object_modified: new Date(primary.stix.modified) }, + target: { + object_ref: secondary.stix.id, + object_modified: new Date(secondary.stix.modified), + }, + }); + expect(relationshipEntry).not.toHaveProperty('frozen_stix'); + expect( + entries.find((entry) => entry.object_ref === markingDefinitionId).frozen_stix, + ).toBeDefined(); + expect( + entries.find((entry) => entry.object_ref === organizationIdentity.stix.id), + ).toMatchObject({ kind: 'supporting' }); + + for (const stixVersion of ['2.0', '2.1']) { + const exported = await bundle(track.id, released.modified, `&stixVersion=${stixVersion}`); + expect(sha256(exported)).toBe(released.bundle_hashes[`stix_2_${stixVersion.split('.')[1]}`]); + expect(exported.id).toBe(released.bundle_id); + if (stixVersion === '2.0') { + expect(exported.objects.some((object) => object.type === 'x-mitre-collection')).toBe(false); + } else { + expect(exported.objects[0]).toMatchObject({ + type: 'x-mitre-collection', + id: released.publication.collection_id, + x_mitre_version: '1.0', + created: new Date(track.created).toISOString(), + modified: new Date(released.modified).toISOString(), + created_by_ref: organizationIdentity.stix.id, + // No scope configures markings in the test environment, so the + // collection carries the markings referenced by its contents. + object_marking_refs: [markingDefinitionId], + }); + } + } + + // A later relationship revision never changes the sealed release. + const corrected = await post( + '/api/relationships', + relationship(primary, secondary, originalRelationship), + ); + expect(corrected.stix.id).toBe(originalRelationship.stix.id); + const replayed = await bundle(track.id, released.modified); + const replayedRelationship = replayed.objects.find( + (object) => object.id === originalRelationship.stix.id, + ); + expect(replayedRelationship.modified).toBe(originalRelationship.stix.modified); + expect(replayedRelationship.description).toBe('Original relationship revision'); + + // The rolling draft inherits the release manifest by reference. + const draft = await post(`/api/release-tracks/${track.id}/meta`, { name: 'Sealed Next' }, 200); + expect(draft.content_manifest_id).toBe(released.content_manifest_id); + expect(draft).not.toHaveProperty('bundle_id'); + expect(draft).not.toHaveProperty('publication'); + const draftBundle = await bundle(track.id, draft.modified); + expect(draftBundle.objects[0].type).toBe('x-mitre-collection'); + expect(draftBundle.objects[0]).not.toHaveProperty('x_mitre_version'); + expect(draftBundle.objects[0].modified).toBe(new Date(draft.modified).toISOString()); + expect( + draftBundle.objects.find((object) => object.id === originalRelationship.stix.id).modified, + ).toBe(originalRelationship.stix.modified); + expect(draftBundle.id).not.toBe(released.bundle_id); + expect((await bundle(track.id, draft.modified)).id).toBe(draftBundle.id); + }); + + it('reseals at commit so relationships added between releases ship and previews report them', async function () { + const source = await post('/api/techniques', technique('Late Relationship Source')); + const target = await post('/api/techniques', technique('Late Relationship Target')); + const track = await createTrack('Late Relationship Track'); + const first = await releaseExactMembers(app, passportCookie, track.id, [source, target], { + version: '1.0', + }); + expect( + (await entriesFor(first.content_manifest_id)).some((e) => e.kind === 'relationship'), + ).toBe(false); + + const late = await post('/api/relationships', relationship(source, target)); + const draft = await post(`/api/release-tracks/${track.id}/meta`, { description: 'v1.1' }, 200); + expect(draft.content_manifest_id).toBe(first.content_manifest_id); + + const preview = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?version=1.1`, + ); + expect(preview.relationships).toMatchObject({ + selected_count: 1, + added_count: 1, + removed_count: 0, + stale_endpoints: [], + }); + expect(preview.relationships.added[0]).toMatchObject({ object_ref: late.stix.id }); + + const second = await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.1' }, + 200, + ); + expect(second.content_manifest_id).not.toBe(first.content_manifest_id); + const secondBundle = await bundle(track.id, second.modified); + expect(secondBundle.objects.some((object) => object.id === late.stix.id)).toBe(true); + const firstBundle = await bundle(track.id, first.modified); + expect(firstBundle.objects.some((object) => object.id === late.stix.id)).toBe(false); + }); + + it('ships relationships against member revisions without cloning them when an endpoint advances', async function () { + const source = await post('/api/techniques', technique('Advancing Source')); + const target = await post('/api/techniques', technique('Advancing Target')); + const edge = await post('/api/relationships', relationship(source, target)); + const track = await createTrack('Advancing Endpoint Track'); + await authenticated( + request(app) + .put(`/api/release-tracks/${track.id}/config`) + .send({ promotion_conflicts: { staged_to_members: 'always_overwrite' } }), + ).expect(200); + const first = await releaseExactMembers(app, passportCookie, track.id, [source, target], { + version: '1.0', + }); + + const revisedPayload = structuredClone(source); + revisedPayload.stix.modified = new Date( + new Date(source.stix.modified).getTime() + 1000, + ).toISOString(); + revisedPayload.stix.description = 'A newer source revision'; + const revised = await post('/api/techniques', revisedPayload); + + const versions = await get(`/api/relationships/${edge.stix.id}?versions=all`); + expect(versions).toHaveLength(1); + expect(versions[0].workspace.relationship_endpoints.source.object_modified).toBe( + source.stix.modified, + ); + + // Member sync enrolled the revision as a candidate; promote it to stage + // the next release against the newer source revision. + await post( + `/api/release-tracks/${track.id}/candidates/promote`, + { object_refs: [source.stix.id] }, + 200, + ); + const preview = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?version=1.1`, + ); + expect(preview.relationships.stale_endpoints).toEqual([ + expect.objectContaining({ + object_ref: edge.stix.id, + source_ref: source.stix.id, + stale_endpoints: ['source'], + }), + ]); + + const second = await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.1' }, + 200, + ); + const entry = (await entriesFor(second.content_manifest_id)).find( + (candidate) => candidate.kind === 'relationship', + ); + expect(entry.source.object_modified).toEqual(new Date(revised.stix.modified)); + expect(entry.object_modified).toEqual(new Date(edge.stix.modified)); + const firstEntry = (await entriesFor(first.content_manifest_id)).find( + (candidate) => candidate.kind === 'relationship', + ); + expect(firstEntry.source.object_modified).toEqual(new Date(source.stix.modified)); + }); + + it('does not resurrect an older active relationship when the newest revision is inactive', async function () { + const source = await post('/api/techniques', technique('Inactive Relationship Source')); + const target = await post('/api/techniques', technique('Inactive Relationship Target')); + const active = await post('/api/relationships', relationship(source, target)); + const inactivePayload = relationship(source, target, active); + inactivePayload.stix.x_mitre_deprecated = true; + const inactive = await post('/api/relationships', inactivePayload); + const track = await createTrack('Inactive Relationship Track'); + const released = await releaseExactMembers(app, passportCookie, track.id, [source, target]); + + expect(inactive.stix.id).toBe(active.stix.id); + const entries = await ReleaseTrackContentManifestEntry.find({ + manifest_id: released.content_manifest_id, + object_ref: active.stix.id, + }) + .lean() + .exec(); + expect(entries).toHaveLength(0); + }); + + it('closes the manifest over exact members without pulling non-member endpoints', async function () { + const member = await post('/api/techniques', technique('Closed Member')); + const outside = await post('/api/techniques', technique('Closed Outside Object')); + const excluded = await post('/api/relationships', relationship(member, outside)); + const track = await createTrack('Closed Member Track'); + const released = await releaseExactMembers(app, passportCookie, track.id, [member]); + + const entries = await entriesFor(released.content_manifest_id); + expect(entries.filter((entry) => entry.kind === 'root')).toHaveLength(1); + expect(entries.some((entry) => entry.object_ref === outside.stix.id)).toBe(false); + expect(entries.some((entry) => entry.object_ref === excluded.stix.id)).toBe(false); + }); + + it('treats include as a draft-only preview and rejects it on released snapshots', async function () { + const member = await post('/api/techniques', technique('Include Member')); + const candidate = await post('/api/techniques', technique('Include Candidate')); + const edge = await post('/api/relationships', relationship(member, candidate)); + const track = await createTrack('Include Preview Track'); + const released = await releaseExactMembers(app, passportCookie, track.id, [member]); + await post( + `/api/release-tracks/${track.id}/candidates`, + { object_refs: [{ id: candidate.stix.id, modified: candidate.stix.modified }] }, + 200, + ); + + await authenticated( + request(app).get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}?format=bundle&include=candidates`, + ), + ).expect(400); + + const withCandidates = await get( + `/api/release-tracks/${track.id}/snapshots/latest?format=bundle&include=candidates`, + ); + const ids = withCandidates.objects.map((object) => object.id); + expect(ids).toContain(candidate.stix.id); + expect(ids).toContain(edge.stix.id); + const membersOnly = await get(`/api/release-tracks/${track.id}/snapshots/latest?format=bundle`); + expect(membersOnly.objects.some((object) => object.id === edge.stix.id)).toBe(false); + }); + + it('replaces a release manifest with a source-attested reconstruction only when named', async function () { + const primary = await post('/api/techniques', technique('Source Graph Primary')); + const secondary = await post('/api/techniques', technique('Source Graph Secondary')); + const linkTarget = await post('/api/techniques', technique('Source Graph Link Target')); + const originalRelationship = await post('/api/relationships', relationship(primary, secondary)); + const track = await createTrack('Source Attested Track'); + const released = await releaseExactMembers(app, passportCookie, track.id, [primary]); + + const revisedSecondaryPayload = structuredClone(secondary); + revisedSecondaryPayload.stix.modified = new Date( + new Date(secondary.stix.modified).getTime() + 1000, + ).toISOString(); + revisedSecondaryPayload.stix.description = 'Post-release secondary revision'; + const revisedSecondary = await post('/api/techniques', revisedSecondaryPayload); + const revisedRelationship = await post( + '/api/relationships', + relationship(primary, revisedSecondary, originalRelationship), + ); + + const supporting = await AttackObject.find({ + 'stix.id': { $in: [primary.stix.created_by_ref, markingDefinitionId] }, + }) + .lean() + .exec(); + const plan = { + source_attestation: { + kind: 'source-bundle', + bundle_sha256: '0'.repeat(64), + collection_id: 'x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019', + release: '19.1', + domain: 'enterprise-attack', + }, + entries: [ + { + kind: 'root', + object_ref: primary.stix.id, + object_modified: primary.stix.modified, + omitted_optional_defaults: ['revoked'], + }, + { + kind: 'secondary', + object_ref: secondary.stix.id, + object_modified: secondary.stix.modified, + }, + { + kind: 'relationship', + object_ref: originalRelationship.stix.id, + object_modified: originalRelationship.stix.modified, + source: { object_ref: primary.stix.id, object_modified: primary.stix.modified }, + target: { object_ref: secondary.stix.id, object_modified: secondary.stix.modified }, + }, + { + kind: 'link_target', + object_ref: linkTarget.stix.id, + object_modified: linkTarget.stix.modified, + }, + ...supporting.map((document) => ({ + kind: 'supporting', + object_ref: document.stix.id, + object_modified: document.stix.modified + ? new Date(document.stix.modified).toISOString() + : null, + ...(document.stix.modified ? {} : { frozen_stix: document.stix }), + })), + ], + }; + const path = `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}/graph/reconstruct`; + + // The sealed release manifest must be named explicitly. + await authenticated(request(app).post(path).send(plan)).expect(409); + await authenticated( + request(app) + .post(path) + .send({ ...plan, replace_manifest_id: 'release-track-content-manifest--wrong' }), + ).expect(409); + + const reconstructed = await post(path, { + ...plan, + replace_manifest_id: released.content_manifest_id, + }); + expect(reconstructed.content_manifest_id).not.toBe(released.content_manifest_id); + expect(reconstructed.bundle_id).toBe(released.bundle_id); + expect(reconstructed.bundle_hashes.manifest_id).toBe(reconstructed.content_manifest_id); + expect(reconstructed.bundle_hashes.stix_2_1).not.toBe(released.bundle_hashes.stix_2_1); + expect(await ReleaseTrackContentManifest.countDocuments({ track_id: track.id })).toBe(1); + const manifest = await ReleaseTrackContentManifest.findOne({ + manifest_id: reconstructed.content_manifest_id, + }) + .lean() + .exec(); + expect(manifest).toMatchObject({ + seal_reason: 'source_reconstruction', + source_attestation: plan.source_attestation, + }); + expect(manifest).not.toHaveProperty('resolver_version'); + expect(manifest).not.toHaveProperty('baseline_reconstruction'); + + const exported = await bundle(track.id, released.modified); + expect(sha256(exported)).toBe(reconstructed.bundle_hashes.stix_2_1); + expect(exported.objects.find((object) => object.id === secondary.stix.id).modified).toBe( + secondary.stix.modified, + ); + expect( + exported.objects.find((object) => object.id === originalRelationship.stix.id).modified, + ).toBe(originalRelationship.stix.modified); + expect( + exported.objects.some((object) => object.modified === revisedRelationship.stix.modified), + ).toBe(false); + expect(exported.objects.some((object) => object.id === linkTarget.stix.id)).toBe(false); + expect(exported.objects.find((object) => object.id === primary.stix.id)).not.toHaveProperty( + 'revoked', + ); + + // Same attestation is idempotent; a different one needs a fresh name. + const idempotent = await post(path, plan, 200); + expect(idempotent.content_manifest_id).toBe(reconstructed.content_manifest_id); + const conflicting = structuredClone(plan); + conflicting.source_attestation.bundle_sha256 = '1'.repeat(64); + await authenticated(request(app).post(path).send(conflicting)).expect(409); + }); + + it('rejects reconstruction of an untagged snapshot', async function () { + const track = await createTrack('Draft Reconstruction Rejection'); + await authenticated( + request(app) + .post( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + track.modified, + )}/graph/reconstruct`, + ) + .send({ + source_attestation: { + kind: 'source-bundle', + bundle_sha256: '0'.repeat(64), + collection_id: 'x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019', + release: '19.1', + domain: 'enterprise-attack', + }, + entries: [ + { + kind: 'root', + object_ref: 'attack-pattern--00000000-0000-4000-8000-000000000000', + object_modified: null, + }, + ], + }), + ).expect(409); + }); + + it('keeps one rolling draft per standard track and releases its inherited manifest', async function () { + const track = await createTrack('Rolling Standard Draft'); + await post(`/api/release-tracks/${track.id}/meta`, { description: 'first replacement' }, 200); + const latest = await post( + `/api/release-tracks/${track.id}/meta`, + { description: 'second replacement' }, + 200, + ); + + const snapshots = await dynamicRepo.getAllSnapshots(track.id); + expect(snapshots.data.filter((snapshot) => snapshot.version == null)).toHaveLength(1); + expect(new Date(snapshots.data[0].modified).getTime()).toBe( + new Date(latest.modified).getTime(), + ); + expect(latest.content_manifest_id).toBe(track.content_manifest_id); + expect(await ReleaseTrackContentManifest.countDocuments({ track_id: track.id })).toBe(1); + }); + + it('treats versioned STIX payloads as immutable while allowing workspace-only PUTs', async function () { + const object = await post('/api/techniques', technique('Immutable STIX Revision')); + const changed = structuredClone(object); + changed.stix.description = 'An illegal in-place STIX correction'; + + const rejected = await authenticated( + request(app) + .put(`/api/techniques/${object.stix.id}/modified/${object.stix.modified}`) + .send(changed), + ).expect(409); + expect(rejected.body.message).toMatch(/immutable/i); + + const workspaceOnly = structuredClone(object); + workspaceOnly.workspace.workflow.state = 'awaiting-review'; + const accepted = await authenticated( + request(app) + .put(`/api/techniques/${object.stix.id}/modified/${object.stix.modified}`) + .send(workspaceOnly), + ).expect(200); + expect(accepted.body.stix.description).toBe(object.stix.description); + expect(accepted.body.workspace.workflow.state).toBe('awaiting-review'); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js index 2bd2ff5b..47678a77 100644 --- a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -1,5 +1,6 @@ 'use strict'; +const crypto = require('node:crypto'); const mongoose = require('mongoose'); const request = require('supertest'); const { expect } = require('expect'); @@ -8,23 +9,34 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); -const migration = require('../../../../migrations/20260730180000-backfill-deterministic-snapshot-graphs'); +const pinMigration = require('../../../../migrations/20260730180000-backfill-deterministic-snapshot-graphs'); +const sealMigration = require('../../../../migrations/20260902120000-seal-release-track-content-manifests'); const Relationship = require('../../../models/relationship-model'); const { - ReleaseTrackGraphManifest, - ReleaseTrackGraphManifestEntry, -} = require('../../../models/release-tracks/release-track-graph-manifest-model'); + ReleaseTrackContentManifest, + ReleaseTrackContentManifestEntry, +} = require('../../../models/release-tracks/release-track-content-manifest-model'); const { releaseExactMembers } = require('./release-track-test-helpers'); const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; -describe('Deterministic snapshot graph migration', function () { +function sha256(payload) { + return crypto + .createHash('sha256') + .update(JSON.stringify(payload, null, 4), 'utf8') + .digest('hex'); +} + +describe('Release-track manifest migrations', function () { let app; let passportCookie; let technique; let group; let relationship; - let trackId; + let legacyTrackId; + let sealedTrackId; + let sealedManifestId; + let orphanTrackId; const deprecatedDanglingRelationshipId = 'relationship--f7a41277-6599-49df-9567-82c9227fb8b5'; const activeDanglingRelationshipId = 'relationship--932fabf0-2868-46ed-9453-41e33dab7f39'; const missingEndpointId = 'campaign--5f4e747c-11d7-49ae-a947-a0f436879d62'; @@ -48,6 +60,19 @@ describe('Deterministic snapshot graph migration', function () { return response.body; } + async function get(path, status = 200) { + const response = await request(app) + .get(path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + return response.body; + } + + function trackCollection(trackId) { + return mongoose.connection.db.collection(trackId); + } + before('create and then downgrade representative legacy data', async function () { const timestamp = new Date().toISOString(); technique = await post('/api/techniques', { @@ -73,8 +98,8 @@ describe('Deterministic snapshot graph migration', function () { spec_version: '2.1', created: timestamp, modified: timestamp, - name: 'Migration graph secondary', - description: 'A secondary migration fixture.', + name: 'Migration graph group', + description: 'A group migration fixture.', object_marking_refs: [markingDefinitionId], }, }); @@ -91,28 +116,139 @@ describe('Deterministic snapshot graph migration', function () { object_marking_refs: [markingDefinitionId], }, }); - const track = await post( + + // Track 1: a fully legacy track — graphless release, a rolling draft with + // the same members, and the retired top-level object_marking_refs field. + const legacyTrack = await post( '/api/release-tracks/new', - { name: 'Legacy migration track', type: 'standard' }, + { name: 'Legacy graphless track', type: 'standard' }, 201, ); - trackId = track.id; - await releaseExactMembers(app, passportCookie, trackId, [technique]); + legacyTrackId = legacyTrack.id; + await releaseExactMembers(app, passportCookie, legacyTrackId, [technique, group]); + await post(`/api/release-tracks/${legacyTrackId}/meta`, { description: 'draft' }, 200); + await trackCollection(legacyTrackId).updateMany( + {}, + { + $unset: { content_manifest_id: '', publication: '', bundle_id: '', bundle_hashes: '' }, + $set: { object_marking_refs: [markingDefinitionId] }, + }, + ); + await Promise.all([ + ReleaseTrackContentManifest.deleteMany({ track_id: legacyTrackId }), + ReleaseTrackContentManifestEntry.deleteMany({ track_id: legacyTrackId }), + ]); - await Relationship.updateOne( + // Track 2: a release that already had an opt-in graph under the old + // field name, with a frozen collection entry and no publication record. + const sealedTrack = await post( + '/api/release-tracks/new', + { name: 'Legacy cached track', type: 'standard' }, + 201, + ); + sealedTrackId = sealedTrack.id; + const sealedRelease = await releaseExactMembers(app, passportCookie, sealedTrackId, [ + technique, + ]); + sealedManifestId = sealedRelease.content_manifest_id.replace( + 'release-track-content-manifest--', + 'release-track-graph-manifest--', + ); + await trackCollection(sealedTrackId).updateMany( + { version: { $type: 'string' } }, { - 'stix.id': relationship.stix.id, - 'stix.modified': relationship.stix.modified, + $set: { graph_manifest_id: sealedManifestId }, + $unset: { content_manifest_id: '', publication: '', bundle_id: '' }, }, - { $unset: { 'workspace.relationship_endpoints': '' } }, ); + // Move the sealed manifest into the legacy collections with the legacy + // header shape so the migration exercises the rename and normalization. + const legacyManifests = mongoose.connection.db.collection('releaseTrackGraphManifests'); + const legacyEntries = mongoose.connection.db.collection('releaseTrackGraphManifestEntries'); + const modernManifest = await ReleaseTrackContentManifest.findOne({ + manifest_id: sealedRelease.content_manifest_id, + }) + .lean() + .exec(); + const modernEntries = await ReleaseTrackContentManifestEntry.find({ + manifest_id: sealedRelease.content_manifest_id, + }) + .lean() + .exec(); + delete modernManifest._id; + delete modernManifest.seal_reason; + await legacyManifests.insertOne({ + ...modernManifest, + manifest_id: sealedManifestId, + resolver_version: 'closed-member-graph-v3', + baseline_reconstruction: false, + }); + await legacyEntries.insertMany([ + ...modernEntries.map((entry) => { + const legacyEntry = { ...entry, manifest_id: sealedManifestId }; + delete legacyEntry._id; + return legacyEntry; + }), + { + manifest_id: sealedManifestId, + track_id: sealedTrackId, + snapshot_modified: new Date(sealedRelease.modified), + revision_key: `x-mitre-collection--${sealedTrackId.split('--')[1]}::collection`, + kind: 'collection', + object_ref: `x-mitre-collection--${sealedTrackId.split('--')[1]}`, + frozen_stix: { type: 'x-mitre-collection' }, + }, + ]); + await Promise.all([ + ReleaseTrackContentManifest.deleteMany({ manifest_id: sealedRelease.content_manifest_id }), + ReleaseTrackContentManifestEntry.deleteMany({ + manifest_id: sealedRelease.content_manifest_id, + }), + ]); + await trackCollection(sealedTrackId).updateMany( + {}, + { $set: { 'config.include_secondary_objects': { enabled: true } } }, + ); + + // Track 3: an orphan collection left behind by an interrupted deletion. + // Its members reference a revision that no longer exists, exactly the + // production shape that must never block startup. + const orphanTrack = await post( + '/api/release-tracks/new', + { name: 'Orphan collection', type: 'standard' }, + 201, + ); + orphanTrackId = orphanTrack.id; + await releaseExactMembers(app, passportCookie, orphanTrackId, [technique]); await mongoose.connection.db - .collection(trackId) - .updateMany({}, { $unset: { graph_manifest_id: '' } }); + .collection('releaseTrackRegistry') + .deleteOne({ track_id: orphanTrackId }); + await trackCollection(orphanTrackId).updateMany( + {}, + { + $unset: { content_manifest_id: '', publication: '', bundle_id: '', bundle_hashes: '' }, + $set: { + 'members.0.object_modified': new Date('2000-01-01T00:00:00.000Z'), + }, + }, + ); await Promise.all([ - ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }), - ReleaseTrackGraphManifestEntry.deleteMany({ track_id: trackId }), + ReleaseTrackContentManifest.deleteMany({ track_id: orphanTrackId }), + ReleaseTrackContentManifestEntry.deleteMany({ track_id: orphanTrackId }), ]); + await ReleaseTrackContentManifest.create({ + manifest_id: 'release-track-content-manifest--orphan-crashed-run', + track_id: orphanTrackId, + snapshot_modified: new Date(), + state: 'active', + schema_version: 2, + seal_reason: 'migration', + }); + + await Relationship.updateOne( + { 'stix.id': relationship.stix.id, 'stix.modified': relationship.stix.modified }, + { $unset: { 'workspace.relationship_endpoints': '' } }, + ); await mongoose.connection.db.collection('relationships').insertOne({ workspace: {}, stix: { @@ -131,7 +267,7 @@ describe('Deterministic snapshot graph migration', function () { }); }); - it('fails closed when an active latest relationship has a dangling endpoint', async function () { + it('fails the pin backfill closed when an active latest relationship has a dangling endpoint', async function () { const timestamp = new Date(); await mongoose.connection.db.collection('relationships').insertOne({ workspace: {}, @@ -142,28 +278,17 @@ describe('Deterministic snapshot graph migration', function () { created: timestamp, modified: timestamp, relationship_type: 'uses', - source_ref: group.stix.id, - target_ref: missingEndpointId, + source_ref: missingEndpointId, + target_ref: technique.stix.id, revoked: false, - x_mitre_deprecated: false, object_marking_refs: [markingDefinitionId], }, }); try { - await expect( - migration._private.run(mongoose.connection.db, { - dryRun: true, - }), - ).rejects.toMatchObject({ - message: expect.stringContaining(activeDanglingRelationshipId), - missing_relationship_endpoints: [ - expect.objectContaining({ - relationship_ref: activeDanglingRelationshipId, - missing_endpoints: [missingEndpointId], - }), - ], - }); + await expect(pinMigration.up(mongoose.connection.db)).rejects.toThrow( + /Cannot pin 1 active latest relationship/, + ); } finally { await mongoose.connection.db .collection('relationships') @@ -171,93 +296,175 @@ describe('Deterministic snapshot graph migration', function () { } }); - it('supports a non-mutating dry run with unrelated deprecated dangling data', async function () { - const report = await migration._private.run(mongoose.connection.db, { - dryRun: true, - }); - - expect(report.dry_run).toBe(true); - expect(report.relationship_pins_written).toBeGreaterThan(0); - expect(report.manifests_created).toBeGreaterThan(0); - const storedRelationship = await Relationship.findOne({ - 'stix.id': relationship.stix.id, - }) - .lean() - .exec(); - expect(storedRelationship.workspace.relationship_endpoints).toBeUndefined(); - expect(await ReleaseTrackGraphManifest.countDocuments({ track_id: trackId })).toBe(0); - }); - - it('pins latest relationships and rerunnably backfills baseline manifests', async function () { - await migration.up(mongoose.connection.db); + it('pins latest relationships without creating manifests (superseded backfill)', async function () { + const dryRun = await pinMigration._private.run(mongoose.connection.db, { dryRun: true }); + expect(dryRun.dry_run).toBe(true); + expect(dryRun.relationship_pins_written).toBeGreaterThan(0); + expect(dryRun.manifests_created).toBe(0); + expect(dryRun.superseded_by).toBe('20260902120000-seal-release-track-content-manifests'); - const storedRelationship = await Relationship.findOne({ - 'stix.id': relationship.stix.id, - }) + await pinMigration.up(mongoose.connection.db); + const storedRelationship = await Relationship.findOne({ 'stix.id': relationship.stix.id }) .lean() .exec(); expect(storedRelationship.workspace.relationship_endpoints.source).toEqual({ object_ref: group.stix.id, object_modified: new Date(group.stix.modified), }); - expect(storedRelationship.workspace.relationship_endpoints.target).toEqual({ - object_ref: technique.stix.id, - object_modified: new Date(technique.stix.modified), + expect(await ReleaseTrackContentManifest.countDocuments({ track_id: legacyTrackId })).toBe(0); + }); + + it('previews the content-manifest migration without writing', async function () { + const report = await sealMigration._private.run(mongoose.connection.db, { dryRun: true }); + + expect(report.dry_run).toBe(true); + // Exactly the graphless release is sealed and its draft shares it; the + // legacy-prefixed manifest of the cached track is recognised in place. + expect(report.manifests_sealed).toBe(1); + expect(report.manifests_shared).toBe(1); + expect(report.renamed_manifest_fields).toBe(1); + expect(report.marking_refs_migrated).toBeGreaterThanOrEqual(2); + expect(report.collection_entries_removed).toBe(1); + expect(report.legacy_manifest_documents_moved).toBeGreaterThanOrEqual(2); + expect(report.manifest_headers_normalized).toBeGreaterThanOrEqual(1); + expect(report.tracks).toBeGreaterThanOrEqual(2); + // Other spec files may leave unregistered collections behind in the shared + // test database, so assert on this spec's orphan rather than the full list. + expect(report.orphan_track_collections).toEqual( + expect.arrayContaining([ + expect.objectContaining({ collection: orphanTrackId, snapshots: 1, manifests: 1 }), + ]), + ); + expect(report.orphan_manifests_discarded).toBeGreaterThanOrEqual(1); + expect(await ReleaseTrackContentManifest.countDocuments({ track_id: orphanTrackId })).toBe(1); + expect(await ReleaseTrackContentManifest.countDocuments({ track_id: legacyTrackId })).toBe(0); + const untouched = await trackCollection(sealedTrackId).findOne({ version: '1.0' }); + expect(untouched.graph_manifest_id).toBe(sealedManifestId); + expect( + await mongoose.connection.db.collection('releaseTrackGraphManifests').countDocuments({}), + ).toBe(1); + }); + + it('seals every snapshot, freezes publication, and is rerunnable', async function () { + await sealMigration.up(mongoose.connection.db); + + const legacySnapshots = await trackCollection(legacyTrackId) + .find({}) + .sort({ modified: 1 }) + .toArray(); + const [legacyRelease, legacyDraft] = legacySnapshots; + expect(legacyRelease.version).toBe('1.0'); + expect(legacyRelease.content_manifest_id).toMatch(/^release-track-content-manifest--/); + expect(legacyRelease).not.toHaveProperty('graph_manifest_id'); + expect(legacyRelease).not.toHaveProperty('object_marking_refs'); + expect(legacyRelease.config.publication.object_marking_refs).toEqual({ + inherit: false, + value: [markingDefinitionId], }); - const deprecatedDanglingRelationship = await mongoose.connection.db - .collection('relationships') - .findOne({ 'stix.id': deprecatedDanglingRelationshipId }); - expect(deprecatedDanglingRelationship.workspace.relationship_endpoints).toBeUndefined(); + expect(legacyRelease.publication).toMatchObject({ + object_marking_refs: [markingDefinitionId], + collection_id: `x-mitre-collection--${legacyTrackId.split('--')[1]}`, + }); + expect(legacyRelease.bundle_id).toBe( + legacyRelease.content_manifest_id.replace('release-track-content-manifest--', 'bundle--'), + ); + expect(legacyRelease.bundle_hashes.manifest_id).toBe(legacyRelease.content_manifest_id); + // The draft has the same members, so it shares the release manifest. + expect(legacyDraft.version).toBeNull(); + expect(legacyDraft.content_manifest_id).toBe(legacyRelease.content_manifest_id); + expect(legacyDraft).not.toHaveProperty('publication'); - const manifests = await ReleaseTrackGraphManifest.find({ - track_id: trackId, + const sealedManifest = await ReleaseTrackContentManifest.findOne({ + manifest_id: legacyRelease.content_manifest_id, }) .lean() .exec(); - expect(manifests.length).toBeGreaterThan(0); - expect(manifests.every((manifest) => manifest.baseline_reconstruction === true)).toBe(true); - expect(manifests.every((manifest) => manifest.schema_version === 1)).toBe(true); - const legacyRelationshipEntry = await ReleaseTrackGraphManifestEntry.findOne({ - manifest_id: { $in: manifests.map((manifest) => manifest.manifest_id) }, - kind: 'relationship', - object_ref: relationship.stix.id, + expect(sealedManifest).toMatchObject({ state: 'active', seal_reason: 'migration' }); + expect(sealedManifest).not.toHaveProperty('baseline_reconstruction'); + const entries = await ReleaseTrackContentManifestEntry.find({ + manifest_id: legacyRelease.content_manifest_id, }) .lean() .exec(); - expect(legacyRelationshipEntry.frozen_stix.description).toBe(relationship.stix.description); - const countAfterFirstRun = manifests.length; + expect(entries.filter((entry) => entry.kind === 'root')).toHaveLength(2); + expect(entries.some((entry) => entry.object_ref === relationship.stix.id)).toBe(true); - await migration.up(mongoose.connection.db); - expect(await ReleaseTrackGraphManifest.countDocuments({ track_id: trackId })).toBe( - countAfterFirstRun, + const cached = await trackCollection(sealedTrackId).findOne({ version: '1.0' }); + const normalizedId = sealedManifestId.replace( + 'release-track-graph-manifest--', + 'release-track-content-manifest--', ); + expect(cached.content_manifest_id).toBe(normalizedId); + expect(cached).not.toHaveProperty('graph_manifest_id'); + expect(cached.config).not.toHaveProperty('include_secondary_objects'); + expect(cached.bundle_id).toBe( + sealedManifestId.replace('release-track-graph-manifest--', 'bundle--'), + ); + expect(cached.bundle_hashes.manifest_id).toBe(normalizedId); + expect(cached.publication).toBeDefined(); + const normalized = await ReleaseTrackContentManifest.findOne({ manifest_id: normalizedId }) + .lean() + .exec(); + expect(normalized).toMatchObject({ seal_reason: 'legacy_graph', schema_version: 2 }); + expect(normalized).not.toHaveProperty('resolver_version'); + expect(normalized).not.toHaveProperty('baseline_reconstruction'); + expect( + await ReleaseTrackContentManifestEntry.countDocuments({ manifest_id: normalizedId }), + ).toBeGreaterThan(0); + const legacyCollections = ( + await mongoose.connection.db.listCollections({}, { nameOnly: true }).toArray() + ).map((collection) => collection.name); + expect(legacyCollections).not.toContain('releaseTrackGraphManifests'); + expect(legacyCollections).not.toContain('releaseTrackGraphManifestEntries'); + expect( + await ReleaseTrackContentManifestEntry.countDocuments({ + track_id: sealedTrackId, + kind: 'collection', + }), + ).toBe(0); - const response = await request(app) - .get(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`) - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200); - const objectIds = response.body.objects.map((object) => object.id); + const bundle = await get( + `/api/release-tracks/${legacyTrackId}/snapshots/${encodeURIComponent( + new Date(legacyRelease.modified).toISOString(), + )}?format=bundle`, + ); + expect(sha256(bundle)).toBe(legacyRelease.bundle_hashes.stix_2_1); + expect(bundle.id).toBe(legacyRelease.bundle_id); + const objectIds = bundle.objects.map((object) => object.id); expect(objectIds).toContain(technique.stix.id); expect(objectIds).toContain(group.stix.id); expect(objectIds).toContain(relationship.stix.id); + expect(bundle.objects[0]).toMatchObject({ + type: 'x-mitre-collection', + x_mitre_version: '1.0', + object_marking_refs: [markingDefinitionId], + }); + + // The orphan collection is skipped, its stale manifest is discarded, and + // its dangling member never blocks the migration. + const orphanSnapshot = await trackCollection(orphanTrackId).findOne({}); + expect(orphanSnapshot).not.toHaveProperty('content_manifest_id'); + expect(await ReleaseTrackContentManifest.countDocuments({ track_id: orphanTrackId })).toBe(0); + + const manifestCount = await ReleaseTrackContentManifest.countDocuments({}); + await sealMigration.up(mongoose.connection.db); + expect(await ReleaseTrackContentManifest.countDocuments({})).toBe(manifestCount); + const rerun = await trackCollection(legacyTrackId).findOne({ version: '1.0' }); + expect(rerun.content_manifest_id).toBe(legacyRelease.content_manifest_id); + expect(rerun.bundle_hashes).toEqual(legacyRelease.bundle_hashes); }); it('replays and activates a complete linked pending manifest after interruption', async function () { - const snapshot = await mongoose.connection.db - .collection(trackId) - .findOne({}, { sort: { modified: -1 } }); - await ReleaseTrackGraphManifest.updateOne( - { manifest_id: snapshot.graph_manifest_id }, + const snapshot = await trackCollection(legacyTrackId).findOne({}, { sort: { modified: -1 } }); + await ReleaseTrackContentManifest.updateOne( + { manifest_id: snapshot.content_manifest_id }, { $set: { state: 'pending' } }, ).exec(); - await request(app) - .get(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`) - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(200); + await get(`/api/release-tracks/${legacyTrackId}/snapshots/latest?format=bundle`); - const manifest = await ReleaseTrackGraphManifest.findOne({ - manifest_id: snapshot.graph_manifest_id, + const manifest = await ReleaseTrackContentManifest.findOne({ + manifest_id: snapshot.content_manifest_id, }) .lean() .exec(); diff --git a/app/tests/api/release-tracks/opt-in-graphs.spec.js b/app/tests/api/release-tracks/opt-in-graphs.spec.js deleted file mode 100644 index 49e36c9a..00000000 --- a/app/tests/api/release-tracks/opt-in-graphs.spec.js +++ /dev/null @@ -1,699 +0,0 @@ -'use strict'; - -const crypto = require('node:crypto'); -const request = require('supertest'); -const { expect } = require('expect'); - -const config = require('../../../config/config'); -const database = require('../../../lib/database-in-memory'); -const databaseConfiguration = require('../../../lib/database-configuration'); -const login = require('../../shared/login'); -const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); -const { - ReleaseTrackGraphManifest, - ReleaseTrackGraphManifestEntry, -} = require('../../../models/release-tracks/release-track-graph-manifest-model'); -const relationshipsRepository = require('../../../repository/relationships-repository'); -const AttackObject = require('../../../models/attack-object-model'); -const Relationship = require('../../../models/relationship-model'); -const { releaseExactMembers } = require('./release-track-test-helpers'); - -const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; - -describe('Opt-in deterministic release-track graphs', function () { - let app; - let passportCookie; - - before(async function () { - await database.initializeConnection(); - await databaseConfiguration.checkSystemConfiguration(); - config.validateRequests.withAttackDataModel = true; - config.validateRequests.withOpenApi = true; - app = await require('../../../index').initializeApp(); - passportCookie = await login.loginAnonymous(app); - }); - - function authenticated(builder) { - return builder - .set('Accept', 'application/json') - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); - } - - function technique(name) { - const timestamp = new Date().toISOString(); - return { - workspace: { workflow: { state: 'work-in-progress' } }, - stix: { - created: timestamp, - modified: timestamp, - name, - description: `${name} description`, - spec_version: '2.1', - type: 'attack-pattern', - object_marking_refs: [markingDefinitionId], - kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], - x_mitre_domains: ['enterprise-attack'], - x_mitre_is_subtechnique: false, - x_mitre_platforms: ['Windows'], - x_mitre_version: '1.0', - }, - }; - } - - function relationship(source, target, previous) { - const modified = previous - ? new Date(new Date(previous.stix.modified).getTime() + 1000).toISOString() - : new Date().toISOString(); - return { - workspace: { workflow: { state: 'work-in-progress' } }, - stix: { - id: previous?.stix.id, - created: previous?.stix.created || modified, - modified, - spec_version: '2.1', - type: 'relationship', - relationship_type: 'uses', - source_ref: source.stix.id, - target_ref: target.stix.id, - description: previous ? 'New relationship revision' : 'Original relationship revision', - object_marking_refs: [markingDefinitionId], - }, - }; - } - - async function post(path, body, status = 201) { - return (await authenticated(request(app).post(path).send(body)).expect(status)).body; - } - - async function createTrack(name) { - return post('/api/release-tracks/new', { name, type: 'standard' }); - } - - async function sourcePlan(primary, secondary, relationshipRevision, secondaryKind = 'secondary') { - const supporting = await AttackObject.find({ - 'stix.id': { - $in: [primary.stix.created_by_ref, markingDefinitionId], - }, - }) - .lean() - .exec(); - return { - source_attestation: { - kind: 'source-bundle', - bundle_sha256: '0'.repeat(64), - collection_id: 'x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019', - release: '19.1', - domain: 'enterprise-attack', - }, - entries: [ - { - kind: 'root', - object_ref: primary.stix.id, - object_modified: primary.stix.modified, - omitted_optional_defaults: ['revoked'], - }, - { - kind: secondaryKind, - object_ref: secondary.stix.id, - object_modified: secondary.stix.modified, - }, - { - kind: 'relationship', - object_ref: relationshipRevision.stix.id, - object_modified: relationshipRevision.stix.modified, - source: { - object_ref: primary.stix.id, - object_modified: primary.stix.modified, - }, - target: { - object_ref: secondary.stix.id, - object_modified: secondary.stix.modified, - }, - }, - ...supporting.map((document) => ({ - kind: 'supporting', - object_ref: document.stix.id, - object_modified: document.stix.modified - ? new Date(document.stix.modified).toISOString() - : null, - ...(document.stix.modified ? {} : { frozen_stix: document.stix }), - })), - ], - }; - } - - it('creates pointer-only member graphs only when a tagged snapshot opts in', async function () { - const primary = await post('/api/techniques', technique('Opt-in Graph Primary')); - const secondary = await post('/api/techniques', technique('Opt-in Graph Secondary')); - const originalRelationship = await post('/api/relationships', relationship(primary, secondary)); - const track = await createTrack('Opt in Graph Track'); - const released = await releaseExactMembers(app, passportCookie, track.id, [primary, secondary]); - - expect(released).not.toHaveProperty('graph_manifest_id'); - expect(await ReleaseTrackGraphManifest.countDocuments({ track_id: track.id })).toBe(0); - - const globalRelationshipScan = relationshipsRepository.retrieveAllForBundle; - relationshipsRepository.retrieveAllForBundle = async () => { - throw new Error('graph capture must not scan every relationship'); - }; - let graphSnapshot; - try { - graphSnapshot = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - {}, - ); - } finally { - relationshipsRepository.retrieveAllForBundle = globalRelationshipScan; - } - expect(graphSnapshot.graph_manifest_id).toBeDefined(); - expect(graphSnapshot.bundle_hashes).toEqual({ - manifest_id: graphSnapshot.graph_manifest_id, - stix_2_0: expect.stringMatching(/^[a-f0-9]{64}$/), - stix_2_1: expect.stringMatching(/^[a-f0-9]{64}$/), - }); - - const manifest = await ReleaseTrackGraphManifest.findOne({ - manifest_id: graphSnapshot.graph_manifest_id, - }) - .lean() - .exec(); - expect(manifest.schema_version).toBe(2); - - const entries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: graphSnapshot.graph_manifest_id, - }) - .lean() - .exec(); - expect(entries).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - kind: 'root', - object_ref: primary.stix.id, - object_modified: expect.any(Date), - }), - expect.objectContaining({ - kind: 'root', - object_ref: secondary.stix.id, - object_modified: expect.any(Date), - }), - expect.objectContaining({ - kind: 'relationship', - object_ref: originalRelationship.stix.id, - object_modified: expect.any(Date), - }), - ]), - ); - const relationshipEntry = entries.find((entry) => entry.kind === 'relationship'); - expect(relationshipEntry).not.toHaveProperty('frozen_stix'); - expect(entries.filter((entry) => entry.kind === 'secondary')).toHaveLength(0); - for (const entry of entries.filter((item) => item.kind === 'root')) { - expect(entry.discovered_from).toBeUndefined(); - } - const markingEntry = entries.find((entry) => entry.object_ref === markingDefinitionId); - expect(markingEntry.frozen_stix).toBeDefined(); - const collectionEntry = entries.find((entry) => entry.kind === 'collection'); - const organizationIdentity = ( - await authenticated(request(app).get('/api/config/organization-identity')).expect(200) - ).body; - expect(collectionEntry).toMatchObject({ - manifest_id: graphSnapshot.graph_manifest_id, - track_id: track.id, - object_ref: `x-mitre-collection--${track.id.split('--')[1]}`, - frozen_stix: { - type: 'x-mitre-collection', - id: `x-mitre-collection--${track.id.split('--')[1]}`, - created_by_ref: organizationIdentity.stix.id, - description: '', - created: manifest.created_at, - modified: manifest.created_at, - }, - }); - - for (const stixVersion of ['2.0', '2.1']) { - const bundle = ( - await authenticated( - request(app).get( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}?format=bundle&stixVersion=${stixVersion}`, - ), - ).expect(200) - ).body; - const hash = crypto - .createHash('sha256') - .update(JSON.stringify(bundle, null, 4), 'utf8') - .digest('hex'); - expect(hash).toBe(graphSnapshot.bundle_hashes[`stix_2_${stixVersion.split('.')[1]}`]); - expect(bundle.id).toBe( - graphSnapshot.graph_manifest_id.replace('release-track-graph-manifest--', 'bundle--'), - ); - if (stixVersion === '2.0') { - expect(bundle.objects.some((object) => object.type === 'x-mitre-collection')).toBe(false); - } else { - expect(bundle.objects[0]).toEqual( - expect.objectContaining({ - id: collectionEntry.frozen_stix.id, - created_by_ref: organizationIdentity.stix.id, - created: collectionEntry.frozen_stix.created.toISOString(), - modified: collectionEntry.frozen_stix.modified.toISOString(), - }), - ); - } - } - - const correctedRelationship = await post( - '/api/relationships', - relationship(primary, secondary, originalRelationship), - ); - expect(correctedRelationship.stix.id).toBe(originalRelationship.stix.id); - - const bundle = ( - await authenticated( - request(app).get( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}?format=bundle`, - ), - ).expect(200) - ).body; - const exportedRelationship = bundle.objects.find( - (object) => object.id === originalRelationship.stix.id, - ); - expect(exportedRelationship.modified).toBe(originalRelationship.stix.modified); - expect(exportedRelationship.description).toBe('Original relationship revision'); - - const idempotent = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - {}, - 200, - ); - expect(idempotent.graph_manifest_id).toBe(graphSnapshot.graph_manifest_id); - expect(idempotent.bundle_hashes).toEqual(graphSnapshot.bundle_hashes); - - await post(`/api/release-tracks/${track.id}/meta`, { name: 'Opt in Graph Track Next' }, 200); - const nextRelease = await post( - `/api/release-tracks/${track.id}/snapshots/latest/release`, - { version: '2.0' }, - 200, - ); - const nextGraph = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(nextRelease.modified)}/graph`, - {}, - ); - const nextManifest = await ReleaseTrackGraphManifest.findOne({ - manifest_id: nextGraph.graph_manifest_id, - }) - .lean() - .exec(); - const nextCollection = await ReleaseTrackGraphManifestEntry.findOne({ - manifest_id: nextGraph.graph_manifest_id, - kind: 'collection', - }) - .lean() - .exec(); - expect(nextCollection.frozen_stix.id).toBe(collectionEntry.frozen_stix.id); - expect(nextCollection.frozen_stix.created).toEqual(collectionEntry.frozen_stix.created); - expect(nextCollection.frozen_stix.modified).toEqual(nextManifest.created_at); - expect(nextCollection.frozen_stix.modified).not.toEqual(collectionEntry.frozen_stix.modified); - - await authenticated( - request(app).delete( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - ), - ).expect(204); - await authenticated( - request(app).delete( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - ), - ).expect(204); - - const liveBundle = ( - await authenticated( - request(app).get( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}?format=bundle`, - ), - ).expect(200) - ).body; - const liveRelationship = liveBundle.objects.find( - (object) => object.id === originalRelationship.stix.id, - ); - expect(liveRelationship.modified).toBe(correctedRelationship.stix.modified); - expect(liveRelationship.description).toBe('New relationship revision'); - }); - - it('closes deterministic graphs over exact members without pulling secondary revisions', async function () { - const member = await post('/api/techniques', technique('Closed Graph Member')); - const outside = await post('/api/techniques', technique('Closed Graph Outside Object')); - const excludedRelationship = await post('/api/relationships', relationship(member, outside)); - const track = await createTrack('Closed Member Graph Track'); - const released = await releaseExactMembers(app, passportCookie, track.id, [member]); - - const graphSnapshot = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - {}, - ); - const entries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: graphSnapshot.graph_manifest_id, - }) - .lean() - .exec(); - - expect(entries.filter((entry) => entry.kind === 'root')).toHaveLength(1); - expect(entries.some((entry) => entry.object_ref === outside.stix.id)).toBe(false); - expect(entries.some((entry) => entry.object_ref === excludedRelationship.stix.id)).toBe(false); - expect(entries.some((entry) => entry.kind === 'secondary')).toBe(false); - }); - - it('does not leak a newer endpoint revision or its remapped relationship', async function () { - const original = await post('/api/techniques', technique('Revision-pinned Graph Member')); - const peer = await post('/api/techniques', technique('Revision-pinned Graph Peer')); - const originalRelationship = await post('/api/relationships', relationship(original, peer)); - const track = await createTrack('Pinned Member Graph'); - const released = await releaseExactMembers(app, passportCookie, track.id, [original, peer]); - - const revisedPayload = structuredClone(original); - revisedPayload.stix.modified = new Date( - new Date(original.stix.modified).getTime() + 1000, - ).toISOString(); - revisedPayload.stix.description = 'A later revision that is not a snapshot member'; - const revised = await post('/api/techniques', revisedPayload); - - const advancedRelationship = await Relationship.findOne({ - 'stix.id': originalRelationship.stix.id, - 'workspace.relationship_endpoints.source.object_modified': revised.stix.modified, - }) - .sort({ 'stix.modified': -1 }) - .lean() - .exec(); - expect(advancedRelationship).toBeTruthy(); - - const graphSnapshot = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - {}, - ); - const entries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: graphSnapshot.graph_manifest_id, - }) - .lean() - .exec(); - const objectEntries = entries.filter((entry) => - [original.stix.id, peer.stix.id].includes(entry.object_ref), - ); - const relationshipEntries = entries.filter( - (entry) => entry.object_ref === originalRelationship.stix.id, - ); - - expect(objectEntries).toHaveLength(2); - expect(objectEntries.every((entry) => entry.kind === 'root')).toBe(true); - expect( - objectEntries.find((entry) => entry.object_ref === original.stix.id).object_modified, - ).toEqual(new Date(original.stix.modified)); - expect(entries.some((entry) => entry.kind === 'secondary')).toBe(false); - expect(relationshipEntries).toHaveLength(1); - expect(relationshipEntries[0].object_modified).toEqual( - new Date(originalRelationship.stix.modified), - ); - - const bundle = ( - await authenticated( - request(app).get( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}?format=bundle`, - ), - ).expect(200) - ).body; - expect(bundle.objects.filter((object) => object.id === original.stix.id)).toEqual([ - expect.objectContaining({ modified: original.stix.modified }), - ]); - expect( - bundle.objects.some( - (object) => - object.id === originalRelationship.stix.id && - object.modified === new Date(advancedRelationship.stix.modified).toISOString(), - ), - ).toBe(false); - }); - - it('does not resurrect an older active relationship when the newest exact revision is inactive', async function () { - const source = await post('/api/techniques', technique('Inactive Relationship Source')); - const target = await post('/api/techniques', technique('Inactive Relationship Target')); - const active = await post('/api/relationships', relationship(source, target)); - const inactivePayload = relationship(source, target, active); - inactivePayload.stix.x_mitre_deprecated = true; - const inactive = await post('/api/relationships', inactivePayload); - const track = await createTrack('Inactive Relationship Graph Track'); - const released = await releaseExactMembers(app, passportCookie, track.id, [source, target]); - - const graphSnapshot = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - {}, - ); - const entries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: graphSnapshot.graph_manifest_id, - object_ref: active.stix.id, - }) - .lean() - .exec(); - - expect(inactive.stix.id).toBe(active.stix.id); - expect(entries).toHaveLength(0); - }); - - it('carries source-attested v19.1 relationship pins into the next member graph', async function () { - const source = await post('/api/techniques', technique('Predecessor Graph Source')); - const target = await post('/api/techniques', technique('Predecessor Graph Target')); - const relationshipRevision = await post('/api/relationships', relationship(source, target)); - const track = await createTrack('Predecessor Manifest Graph Track'); - const baseline = await releaseExactMembers(app, passportCookie, track.id, [source, target], { - version: '1.0', - }); - const plan = await sourcePlan(source, target, relationshipRevision, 'root'); - await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - baseline.modified, - )}/graph/reconstruct`, - plan, - ); - - const storedRelationship = await Relationship.findOne({ - 'stix.id': relationshipRevision.stix.id, - 'stix.modified': relationshipRevision.stix.modified, - }) - .lean() - .exec(); - await Relationship.collection.updateOne( - { _id: storedRelationship._id }, - { $unset: { 'workspace.relationship_endpoints': '' } }, - ); - - try { - await post(`/api/release-tracks/${track.id}/meta`, { description: 'v1.1 draft' }, 200); - const next = await post( - `/api/release-tracks/${track.id}/snapshots/latest/release`, - { version: '1.1' }, - 200, - ); - const graphSnapshot = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(next.modified)}/graph`, - {}, - ); - const carried = await ReleaseTrackGraphManifestEntry.findOne({ - manifest_id: graphSnapshot.graph_manifest_id, - object_ref: relationshipRevision.stix.id, - }) - .lean() - .exec(); - - expect(carried).toMatchObject({ - kind: 'relationship', - source: { - object_ref: source.stix.id, - object_modified: new Date(source.stix.modified), - }, - target: { - object_ref: target.stix.id, - object_modified: new Date(target.stix.modified), - }, - }); - expect(carried.object_modified).toEqual(new Date(relationshipRevision.stix.modified)); - } finally { - await Relationship.collection.updateOne( - { _id: storedRelationship._id }, - { - $set: { - 'workspace.relationship_endpoints': storedRelationship.workspace.relationship_endpoints, - }, - }, - ); - } - }); - - it('rejects graph creation for an untagged snapshot', async function () { - const track = await createTrack('Draft Graph Rejection'); - await authenticated( - request(app) - .post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(track.modified)}/graph`, - ) - .send({}), - ).expect(409); - }); - - it('reconstructs a historical graph from exact source-bundle pointers', async function () { - const primary = await post('/api/techniques', technique('Source Graph Primary')); - const secondary = await post('/api/techniques', technique('Source Graph Secondary')); - const linkTarget = await post('/api/techniques', technique('Source Graph Link Target')); - const originalRelationship = await post('/api/relationships', relationship(primary, secondary)); - const track = await createTrack('Source Attested Graph Track'); - const released = await releaseExactMembers(app, passportCookie, track.id, [primary]); - - const revisedSecondaryPayload = structuredClone(secondary); - revisedSecondaryPayload.stix.modified = new Date( - new Date(secondary.stix.modified).getTime() + 1000, - ).toISOString(); - revisedSecondaryPayload.stix.description = 'Post-release secondary revision'; - const revisedSecondary = await post('/api/techniques', revisedSecondaryPayload); - const revisedRelationship = await post( - '/api/relationships', - relationship(primary, revisedSecondary, originalRelationship), - ); - - const plan = await sourcePlan(primary, secondary, originalRelationship); - plan.entries.push({ - kind: 'link_target', - object_ref: linkTarget.stix.id, - object_modified: linkTarget.stix.modified, - }); - const invalidPlan = structuredClone(plan); - invalidPlan.entries.find((entry) => entry.kind === 'relationship').target.object_modified = - revisedSecondary.stix.modified; - await authenticated( - request(app) - .post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}/graph/reconstruct`, - ) - .send(invalidPlan), - ).expect(409); - - const reconstructed = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}/graph/reconstruct`, - plan, - ); - const manifest = await ReleaseTrackGraphManifest.findOne({ - manifest_id: reconstructed.graph_manifest_id, - }) - .lean() - .exec(); - expect(manifest).toMatchObject({ - schema_version: 2, - resolver_version: 'source-bundle-pointer-v2', - baseline_reconstruction: true, - source_attestation: plan.source_attestation, - }); - - const entries = await ReleaseTrackGraphManifestEntry.find({ - manifest_id: reconstructed.graph_manifest_id, - }) - .lean() - .exec(); - expect( - entries.find((entry) => entry.object_ref === originalRelationship.stix.id), - ).not.toHaveProperty('frozen_stix'); - expect(entries.find((entry) => entry.object_ref === linkTarget.stix.id)).toMatchObject({ - kind: 'link_target', - }); - - const bundle = ( - await authenticated( - request(app).get( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}?format=bundle`, - ), - ).expect(200) - ).body; - expect(bundle.objects.find((object) => object.id === secondary.stix.id).modified).toBe( - secondary.stix.modified, - ); - expect( - bundle.objects.find((object) => object.id === originalRelationship.stix.id).modified, - ).toBe(originalRelationship.stix.modified); - expect( - bundle.objects.some((object) => object.modified === revisedRelationship.stix.modified), - ).toBe(false); - expect(bundle.objects.some((object) => object.id === linkTarget.stix.id)).toBe(false); - expect(bundle.objects.find((object) => object.id === primary.stix.id)).not.toHaveProperty( - 'revoked', - ); - - await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}/graph/reconstruct`, - plan, - 200, - ); - const conflictingAttestation = structuredClone(plan); - conflictingAttestation.source_attestation.bundle_sha256 = '1'.repeat(64); - await authenticated( - request(app) - .post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}/graph/reconstruct`, - ) - .send(conflictingAttestation), - ).expect(409); - }); - - it('keeps one rolling draft per standard track', async function () { - const track = await createTrack('Rolling Standard Draft'); - await post(`/api/release-tracks/${track.id}/meta`, { description: 'first replacement' }, 200); - const latest = await post( - `/api/release-tracks/${track.id}/meta`, - { description: 'second replacement' }, - 200, - ); - - const snapshots = await dynamicRepo.getAllSnapshots(track.id); - expect(snapshots.data.filter((snapshot) => snapshot.version == null)).toHaveLength(1); - expect(new Date(snapshots.data[0].modified).getTime()).toBe( - new Date(latest.modified).getTime(), - ); - expect(latest).not.toHaveProperty('graph_manifest_id'); - }); - - it('treats versioned STIX payloads as immutable while allowing workspace-only PUTs', async function () { - const object = await post('/api/techniques', technique('Immutable STIX Revision')); - const changed = structuredClone(object); - changed.stix.description = 'An illegal in-place STIX correction'; - - const rejected = await authenticated( - request(app) - .put(`/api/techniques/${object.stix.id}/modified/${object.stix.modified}`) - .send(changed), - ).expect(409); - expect(rejected.body.message).toMatch(/immutable/i); - - const workspaceOnly = structuredClone(object); - workspaceOnly.workspace.workflow.state = 'awaiting-review'; - const accepted = await authenticated( - request(app) - .put(`/api/techniques/${object.stix.id}/modified/${object.stix.modified}`) - .send(workspaceOnly), - ).expect(200); - expect(accepted.body.stix.description).toBe(object.stix.description); - expect(accepted.body.workspace.workflow.state).toBe('awaiting-review'); - }); - - after(async function () { - await database.closeConnection(); - }); -}); diff --git a/app/tests/api/release-tracks/publication-config.spec.js b/app/tests/api/release-tracks/publication-config.spec.js new file mode 100644 index 00000000..4225ac5a --- /dev/null +++ b/app/tests/api/release-tracks/publication-config.spec.js @@ -0,0 +1,255 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const { releaseExactMembers } = require('./release-track-test-helpers'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; +const tlpGreenMarkingId = 'marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da'; + +describe('Release-track publication configuration', function () { + let app; + let passportCookie; + let organizationIdentity; + let trackIdentity; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + organizationIdentity = (await get('/api/config/organization-identity')).stix; + const timestamp = new Date().toISOString(); + trackIdentity = ( + await post('/api/identities', { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'identity', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: 'Track Publishing Identity', + identity_class: 'organization', + object_marking_refs: [markingDefinitionId], + }, + }) + ).stix; + }); + + function api(method, path, body, status) { + const call = request(app) + [method](path) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + if (body !== undefined) call.send(body); + return call.expect(status); + } + + async function post(path, body, status = 201) { + return (await api('post', path, body, status)).body; + } + + async function put(path, body, status = 200) { + return (await api('put', path, body, status)).body; + } + + async function get(path, status = 200) { + return (await api('get', path, undefined, status)).body; + } + + function technique(name) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], + x_mitre_domains: ['enterprise-attack'], + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + }, + }; + } + + async function collectionObject(trackId, selector = 'latest') { + const bundle = await get( + `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent(selector)}?format=bundle`, + ); + return bundle.objects[0]; + } + + it('inherits identity and markings from the global scope by default and reports the sources', async function () { + const track = await post('/api/release-tracks/new', { + name: 'Publication Defaults', + type: 'standard', + }); + const trackConfig = await get(`/api/release-tracks/${track.id}/config`); + + expect(trackConfig.publication).toEqual({ + created_by_ref: { inherit: true }, + object_marking_refs: { inherit: true }, + }); + expect(trackConfig.publication_resolved).toMatchObject({ + collection_id: `x-mitre-collection--${track.id.split('--')[1]}`, + created: track.created, + created_by_ref: organizationIdentity.id, + attack_spec_version: config.app.attackSpecVersion, + sources: { + collection_id: 'derived', + created: 'derived', + created_by_ref: 'global', + // The test environment configures no default markings, so the + // collection object falls back to the markings its contents reference. + object_marking_refs: 'content', + }, + }); + const collection = await collectionObject(track.id); + expect(collection.created_by_ref).toBe(organizationIdentity.id); + // An empty draft has no content markings to fall back to, so the + // (empty) array is stripped by STIX conformance. + expect(collection.object_marking_refs).toBeUndefined(); + }); + + it('applies explicit track overrides to draft exports and freezes them at release', async function () { + const member = await post('/api/techniques', technique('Publication Override Member')); + const track = await post('/api/release-tracks/new', { + name: 'Publication Overrides', + type: 'standard', + }); + const canonicalCollectionId = 'x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019'; + const canonicalCreated = '2018-01-17T12:56:55.080Z'; + + const updated = await put(`/api/release-tracks/${track.id}/config`, { + publication: { + collection_id: canonicalCollectionId, + created: canonicalCreated, + created_by_ref: { inherit: false, value: trackIdentity.id }, + object_marking_refs: { inherit: false, value: [tlpGreenMarkingId] }, + }, + }); + expect(updated.config.publication).toMatchObject({ + collection_id: canonicalCollectionId, + created: canonicalCreated, + created_by_ref: { inherit: false, value: trackIdentity.id }, + object_marking_refs: { inherit: false, value: [tlpGreenMarkingId] }, + }); + const resolved = (await get(`/api/release-tracks/${track.id}/config`)).publication_resolved; + expect(resolved.sources).toEqual({ + collection_id: 'track', + created: 'track', + created_by_ref: 'track', + object_marking_refs: 'track', + }); + + const draftCollection = await collectionObject(track.id); + expect(draftCollection).toMatchObject({ + id: canonicalCollectionId, + created: canonicalCreated, + created_by_ref: trackIdentity.id, + object_marking_refs: [tlpGreenMarkingId], + }); + expect(draftCollection).not.toHaveProperty('x_mitre_version'); + + const released = await releaseExactMembers(app, passportCookie, track.id, [member], { + version: '1.0', + }); + expect(released.publication).toMatchObject({ + collection_id: canonicalCollectionId, + created: canonicalCreated, + created_by_ref: trackIdentity.id, + object_marking_refs: [tlpGreenMarkingId], + }); + const releaseBundle = await get( + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + released.modified, + )}?format=bundle`, + ); + expect(releaseBundle.objects[0]).toMatchObject({ + id: canonicalCollectionId, + x_mitre_version: '1.0', + created: canonicalCreated, + created_by_ref: trackIdentity.id, + object_marking_refs: [tlpGreenMarkingId], + }); + // The publishing identity and configured markings ship as supporting objects. + expect(releaseBundle.objects.some((object) => object.id === trackIdentity.id)).toBe(true); + expect(releaseBundle.objects.some((object) => object.id === tlpGreenMarkingId)).toBe(true); + expect(releaseBundle.objects[0].x_mitre_contents.map((entry) => entry.object_ref)).toContain( + trackIdentity.id, + ); + + // Reverting the rule affects future drafts but never the frozen release. + await put(`/api/release-tracks/${track.id}/config`, { + publication: { created_by_ref: { inherit: true } }, + }); + expect((await collectionObject(track.id)).created_by_ref).toBe(organizationIdentity.id); + expect((await collectionObject(track.id, released.modified)).created_by_ref).toBe( + trackIdentity.id, + ); + + // Collection identity is immutable once the track has a release. + const conflict = await put( + `/api/release-tracks/${track.id}/config`, + { publication: { collection_id: null } }, + 409, + ); + expect(conflict.message).toMatch(/collection_id cannot change/); + await put( + `/api/release-tracks/${track.id}/config`, + { publication: { created: '2019-01-01T00:00:00.000Z' } }, + 409, + ); + // Restating the same values is not a change. + await put(`/api/release-tracks/${track.id}/config`, { + publication: { collection_id: canonicalCollectionId, created: canonicalCreated }, + }); + }); + + it('rejects malformed publication rules and the retired top-level marking field', async function () { + const track = await post('/api/release-tracks/new', { + name: 'Publication Validation', + type: 'standard', + }); + await put( + `/api/release-tracks/${track.id}/config`, + { publication: { created_by_ref: { inherit: false } } }, + 400, + ); + await put( + `/api/release-tracks/${track.id}/config`, + { publication: { object_marking_refs: { inherit: true, value: [] } } }, + 400, + ); + await put( + `/api/release-tracks/${track.id}/config`, + { publication: { collection_id: 'attack-pattern--not-a-collection' } }, + 400, + ); + await post( + '/api/release-tracks/new', + { + name: 'Retired Marking Field', + type: 'standard', + object_marking_refs: [markingDefinitionId], + }, + 400, + ); + }); + + after(async function () { + await database.closeConnection(); + }); +}); diff --git a/app/tests/api/release-tracks/reconciliation-durability.spec.js b/app/tests/api/release-tracks/reconciliation-durability.spec.js index ca9ad18d..8f17d397 100644 --- a/app/tests/api/release-tracks/reconciliation-durability.spec.js +++ b/app/tests/api/release-tracks/reconciliation-durability.spec.js @@ -146,14 +146,14 @@ describe('Release-track durable backref reconciliation', function () { status: 'completed', }); + // Only outstanding work is retained: a repaired reconciliation leaves no + // record behind. record = await ReleaseTrackReconciliation.findOne({ reconciliation_id: release.body.reconciliation_id, }) .lean() .exec(); - expect(record.status).toBe('completed'); - expect(record.attempts).toBe(2); - expect(record.completed_at).toBeInstanceOf(Date); + expect(record).toBeNull(); stored = await getTechnique(technique); expect(stored.workspace.release_tracks).toEqual([ diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index a73f9547..19af8532 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -11,15 +11,15 @@ * Covered behavior: * - Default bundle contains members only, plus referenced identities and * marking definitions (self-contained bundle) - * - A deterministic snapshot graph contains active relationships only when - * both exact endpoint revisions are members + * - A sealed content manifest contains active relationships only when both + * endpoint IDs are members; releases replay it and drafts inherit it * - `include` adds staged and/or candidate tiers (comma-separated or * repeated, singular or plural tier names) * - `state` narrows the included staged/candidate entries by workflow * status; entries marked 'reviewed' are always included * - `stixVersion` controls bundle/object STIX version conformance - * - `includeToc` controls the x-mitre-collection table-of-contents object, - * which is derived from the release-track metadata + * - STIX 2.1 bundles always begin with the x-mitre-collection object, which + * is projected from the snapshot and its publication metadata * - LinkById tags are converted to markdown citations * - Invalid `include`/`state` values are rejected with 400 */ @@ -256,11 +256,6 @@ describe('Release Tracks Bundle Export API', function () { secondaryGroup, ]); taggedModified = tagged.modified; - await postAction( - `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent(taggedModified)}/graph`, - {}, - 201, - ); // Candidates (all start as work-in-progress) await postAction(`/api/release-tracks/${trackId}/candidates`, { @@ -323,8 +318,10 @@ describe('Release Tracks Bundle Export API', function () { expect(member.workspace).toBeUndefined(); }); - it('GET /api/release-tracks/:id/snapshots/latest?format=bundle includes a TOC derived from the track metadata', async function () { + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle includes a collection object projected from the snapshot', async function () { const bundle = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle`); + const snapshot = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest`); + const trackConfig = await getBundle(`/api/release-tracks/${trackId}/config`); const toc = bundle.objects[0]; expect(toc.type).toBe('x-mitre-collection'); @@ -333,15 +330,20 @@ describe('Release Tracks Bundle Export API', function () { // This rolling draft belongs to the next release cycle, so it has no // snapshot-local description and falls back to the track description. expect(toc.description).toBe('Release track bundle export test'); - // Draft snapshots (version: null) fall back to '0.1' - expect(toc.x_mitre_version).toBe('0.1'); + // Draft snapshots have no publication version, so the key is omitted. + expect(toc).not.toHaveProperty('x_mitre_version'); expect(toc.x_mitre_attack_spec_version).toBe(config.app.attackSpecVersion); expect(toc.spec_version).toBe('2.1'); expect(toc.created_by_ref).toBe(organizationIdentityId); - - // Marking definitions are tracked in object_marking_refs, everything else - // in x_mitre_contents - expect(toc.object_marking_refs).toContain(staticMarkingDefinitionId); + expect(toc.created).toBe(new Date(snapshot.created).toISOString()); + expect(toc.modified).toBe(new Date(snapshot.modified).toISOString()); + + // Collection markings follow the publication rule. Neither scope + // configures markings here, so the object carries the markings referenced + // by its contents; everything emitted except marking definitions is + // listed in x_mitre_contents + expect(trackConfig.publication_resolved.sources.object_marking_refs).toBe('content'); + expect(toc.object_marking_refs).toEqual([staticMarkingDefinitionId]); const contentRefs = toc.x_mitre_contents.map((entry) => entry.object_ref); expect(contentRefs).toContain(memberObject.stix.id); expect(contentRefs).toContain(includedRelationship.stix.id); @@ -351,9 +353,7 @@ describe('Release Tracks Bundle Export API', function () { }); it('adds only relationships whose endpoints are both selected for the bundle', async function () { - const bundle = await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, - ); + const bundle = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle`); const ids = bundleObjectIds(bundle); expect(ids).toContain(includedRelationship.stix.id); @@ -366,7 +366,7 @@ describe('Release Tracks Bundle Export API', function () { ); }); - it('replays exact relationship pointers and protects graph dependencies', async function () { + it('replays sealed relationship pointers and protects manifest dependencies', async function () { const relationshipUpdate = JSON.parse(JSON.stringify(secondaryRelationship)); delete relationshipUpdate._id; delete relationshipUpdate.__v; @@ -392,7 +392,7 @@ describe('Release Tracks Bundle Export API', function () { const bundle = await getBundle( `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent( taggedModified, - )}?format=bundle&includeToc=false`, + )}?format=bundle`, ); const pinnedRelationship = bundle.objects.find( (object) => object.id === secondaryRelationship.stix.id, @@ -429,7 +429,7 @@ describe('Release Tracks Bundle Export API', function () { .expect(409); }); - it('maps a graph-backed snapshot description onto the collection TOC', async function () { + it('maps the release notes onto the collection object of the released snapshot', async function () { const bundle = await getBundle( `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent( taggedModified, @@ -442,7 +442,7 @@ describe('Release Tracks Bundle Export API', function () { }); }); - it('protects graph dependencies from collection cascade deletion', async function () { + it('protects manifest dependencies from collection cascade deletion', async function () { const timestamp = new Date().toISOString(); const collection = await postObject('/api/collections', { workspace: { @@ -457,7 +457,7 @@ describe('Release Tracks Bundle Export API', function () { created: timestamp, modified: timestamp, name: 'Graph protection cascade fixture', - description: 'Attempts to cascade-delete a protected graph member.', + description: 'Attempts to cascade-delete a protected manifest member.', x_mitre_version: '1.0', x_mitre_contents: [ { @@ -486,12 +486,20 @@ describe('Release Tracks Bundle Export API', function () { .expect(200); }); - it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&includeToc=false omits the TOC', async function () { - const bundle = await getBundle( + it('no longer accepts includeToc: the collection object is always present in STIX 2.1', async function () { + await getBundle( `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&includeToc=false`, + 400, + ); + }); + + it('rejects include on a released snapshot', async function () { + await getBundle( + `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent( + taggedModified, + )}?format=bundle&include=candidates`, + 400, ); - const tocObjects = bundle.objects.filter((o) => o.type === 'x-mitre-collection'); - expect(tocObjects.length).toBe(0); }); it('GET /api/release-tracks/:id/snapshots/latest?format=bundle converts LinkById tags to markdown citations', async function () { diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index 7a492aae..e249d3b2 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -149,10 +149,15 @@ describe('Release-track release planning and commit API', function () { }); const bundle = await get( - `/api/release-tracks/${track.id}/snapshots/latest/release/preview?format=bundle&version=2.4&includeToc=false`, + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?format=bundle&version=2.4`, ); expect(bundle.body.type).toBe('bundle'); - expect(bundle.body.objects).toEqual([]); + // An empty release still ships the collection object and its publishing + // identity so the bundle is self-contained. + expect(bundle.body.objects).toEqual([ + expect.objectContaining({ type: 'x-mitre-collection', x_mitre_version: '2.4' }), + expect.objectContaining({ type: 'identity' }), + ]); const unchanged = await get(`/api/release-tracks/${track.id}/snapshots/latest`); expect(unchanged.body.version).toBeNull(); @@ -242,14 +247,15 @@ describe('Release-track release planning and commit API', function () { }); const draftBundle = await get( - `/api/release-tracks/${track.id}/snapshots/latest` + - '?format=bundle&include=staged&includeToc=false', + `/api/release-tracks/${track.id}/snapshots/latest` + '?format=bundle&include=staged', ); expect(draftBundle.body.objects).toEqual([ + expect.objectContaining({ type: 'x-mitre-collection' }), expect.objectContaining({ id: revisionB.stix.id, modified: revisionB.stix.modified, }), + expect.objectContaining({ type: 'identity' }), ]); const preview = await get( diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index 92819b3f..a29e099b 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -269,10 +269,23 @@ describe('Release Tracks API', function () { .expect(201) .expect('Content-Type', /json/); - expect(response.body.config).toEqual(suppliedConfig); + expect(response.body.config).toEqual({ + ...suppliedConfig, + // Publication rules default to inheriting the global scope. + publication: { + created_by_ref: { inherit: true }, + object_marking_refs: { inherit: true }, + }, + }); const persistedSnapshot = await snapshotService.getLatestSnapshot(response.body.id); - expect(persistedSnapshot.config).toEqual(suppliedConfig); + expect(persistedSnapshot.config).toEqual({ + ...suppliedConfig, + publication: { + created_by_ref: { inherit: true }, + object_marking_refs: { inherit: true }, + }, + }); }); after(async function () { diff --git a/app/tests/api/release-tracks/snapshot-descriptions.spec.js b/app/tests/api/release-tracks/snapshot-descriptions.spec.js index 44ff04f5..50d54466 100644 --- a/app/tests/api/release-tracks/snapshot-descriptions.spec.js +++ b/app/tests/api/release-tracks/snapshot-descriptions.spec.js @@ -8,9 +8,6 @@ const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); -const { - ReleaseTrackGraphManifestEntry, -} = require('../../../models/release-tracks/release-track-graph-manifest-model'); describe('Release-track snapshot descriptions', function () { let app; @@ -92,7 +89,7 @@ describe('Release-track snapshot descriptions', function () { expect(unchanged.modified).toBe(track.modified); }); - it('sets release notes while tagging and permits later annotation edits in place', async function () { + it('sets release notes while tagging and rejects later edits on the released snapshot', async function () { const track = await createTrack('Snapshot Description Release', { description: 'Stable track description', }); @@ -107,63 +104,21 @@ describe('Release-track snapshot descriptions', function () { description: 'Stable track description', snapshot_description: 'What changed in the first publication.', }); - - const edited = await put(descriptionPath(released), { - description: 'Corrected internal release context.', - }); - expect(edited).toMatchObject({ - modified: track.modified, - version: '1.0', - snapshot_description: 'Corrected internal release context.', - }); - - const registry = await get('/api/release-tracks'); - const registryTrack = registry.data.find((entry) => entry.track_id === track.id); - expect(registryTrack.description).toBe('Stable track description'); - }); - - it('rejects cached note edits until the cache is deleted and regenerated', async function () { - const track = await createTrack('Snapshot Description Cached', { - description: 'Stable fallback description', - }); - const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { - version: '1.0', - description: 'Initial cached notes.', - }); - const cached = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - {}, - 201, - ); - const originalHashes = cached.bundle_hashes; - const originalCollection = await ReleaseTrackGraphManifestEntry.findOne({ - manifest_id: cached.graph_manifest_id, - kind: 'collection', - }) - .lean() - .exec(); + const originalHashes = released.bundle_hashes; const conflict = await put( descriptionPath(released), - { description: 'Corrected cached notes.' }, + { description: 'Corrected internal release context.' }, 409, ); - expect(conflict.message).toBe('Delete the bundle cache before editing snapshot notes.'); + expect(conflict.message).toBe('Snapshot notes are immutable once the snapshot is released.'); const unchangedSnapshot = await get( `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}`, ); - expect(unchangedSnapshot.snapshot_description).toBe('Initial cached notes.'); + expect(unchangedSnapshot.snapshot_description).toBe('What changed in the first publication.'); expect(unchangedSnapshot.bundle_hashes).toEqual(originalHashes); - const unchangedCollection = await ReleaseTrackGraphManifestEntry.findOne({ - manifest_id: cached.graph_manifest_id, - kind: 'collection', - }) - .lean() - .exec(); - expect(unchangedCollection.frozen_stix).toEqual(originalCollection.frozen_stix); - for (const stixVersion of ['2.0', '2.1']) { const bundle = await get( `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( @@ -179,40 +134,13 @@ describe('Release-track snapshot descriptions', function () { if (stixVersion === '2.0') { expect(collection).toBeUndefined(); } else { - expect(collection.description).toBe('Initial cached notes.'); + expect(collection.description).toBe('What changed in the first publication.'); } } - await api( - 'delete', - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - undefined, - 204, - ); - const edited = await put(descriptionPath(released), { - description: 'Corrected cached notes.', - }); - expect(edited.snapshot_description).toBe('Corrected cached notes.'); - expect(edited).not.toHaveProperty('graph_manifest_id'); - expect(edited).not.toHaveProperty('bundle_hashes'); - - const recached = await post( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}/graph`, - {}, - 201, - ); - expect(recached.graph_manifest_id).not.toBe(cached.graph_manifest_id); - expect(recached.bundle_hashes.stix_2_0).not.toBe(originalHashes.stix_2_0); - expect(recached.bundle_hashes.stix_2_1).not.toBe(originalHashes.stix_2_1); - - const regeneratedCollection = await ReleaseTrackGraphManifestEntry.findOne({ - manifest_id: recached.graph_manifest_id, - kind: 'collection', - }) - .lean() - .exec(); - expect(regeneratedCollection.frozen_stix.description).toBe('Corrected cached notes.'); - expect(regeneratedCollection.frozen_stix.id).toBe(originalCollection.frozen_stix.id); + const registry = await get('/api/release-tracks'); + const registryTrack = registry.data.find((entry) => entry.track_id === track.id); + expect(registryTrack.description).toBe('Stable track description'); }); it('clears existing draft notes when release explicitly supplies an empty description', async function () { diff --git a/app/tests/api/release-tracks/snapshot-history.spec.js b/app/tests/api/release-tracks/snapshot-history.spec.js index 3553408b..f1e55800 100644 --- a/app/tests/api/release-tracks/snapshot-history.spec.js +++ b/app/tests/api/release-tracks/snapshot-history.spec.js @@ -7,8 +7,8 @@ const databaseConfiguration = require('../../../lib/database-configuration'); const login = require('../../shared/login'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); const { - ReleaseTrackGraphManifestEntry, -} = require('../../../models/release-tracks/release-track-graph-manifest-model'); + ReleaseTrackContentManifestEntry, +} = require('../../../models/release-tracks/release-track-content-manifest-model'); const markingDefinitionId = 'marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168'; const objectRevisions = []; @@ -77,9 +77,10 @@ describe('GET /api/release-tracks/:id/snapshots', function () { ...snapshotBase(standardTrack), modified: standardTaggedModified, version: '1.0', - graph_manifest_id: 'release-track-graph-manifest--snapshot-history', + content_manifest_id: 'release-track-content-manifest--snapshot-history', + bundle_id: 'bundle--snapshot-history', bundle_hashes: { - manifest_id: 'release-track-graph-manifest--snapshot-history', + manifest_id: 'release-track-content-manifest--snapshot-history', stix_2_0: 'a'.repeat(64), stix_2_1: 'b'.repeat(64), }, @@ -92,7 +93,7 @@ describe('GET /api/release-tracks/:id/snapshots', function () { ], }); const manifestCommon = { - manifest_id: 'release-track-graph-manifest--snapshot-history', + manifest_id: 'release-track-content-manifest--snapshot-history', track_id: standardTrack.id, snapshot_modified: standardTaggedModified, }; @@ -106,7 +107,7 @@ describe('GET /api/release-tracks/:id/snapshots', function () { object_modified: objectRevisions[index].modified, ...extra, }); - await ReleaseTrackGraphManifestEntry.insertMany([ + await ReleaseTrackContentManifestEntry.insertMany([ versionedManifestEntry(0, 'root', { tier: 'members' }), versionedManifestEntry(1, 'root', { tier: 'members' }), versionedManifestEntry(2, 'secondary'), @@ -213,20 +214,34 @@ describe('GET /api/release-tracks/:id/snapshots', function () { candidates_count: 1, }); expect(response.body.data[0]).not.toHaveProperty('quarantine_count'); - expect(response.body.data[0]).not.toHaveProperty('graph_statistics'); + // The rolling draft inherits the track-creation manifest, which holds + // only the publishing identity as a supporting object. + expect(response.body.data[0]).toMatchObject({ + content_manifest_id: standardTrack.content_manifest_id, + content_statistics: { + primary_count: 0, + secondary_count: 0, + relationship_count: 0, + supporting_count: 1, + link_target_count: 0, + total_count: 1, + }, + }); + expect(response.body.data[0]).not.toHaveProperty('bundle_id'); expect(response.body.data[1]).toMatchObject({ modified: standardTaggedModified.toISOString(), version: '1.0', - graph_manifest_id: 'release-track-graph-manifest--snapshot-history', + content_manifest_id: 'release-track-content-manifest--snapshot-history', + bundle_id: 'bundle--snapshot-history', bundle_hashes: { - manifest_id: 'release-track-graph-manifest--snapshot-history', + manifest_id: 'release-track-content-manifest--snapshot-history', stix_2_0: 'a'.repeat(64), stix_2_1: 'b'.repeat(64), }, members_count: 2, staged_count: 1, candidates_count: 3, - graph_statistics: { + content_statistics: { primary_count: 2, secondary_count: 2, relationship_count: 1, diff --git a/app/tests/api/release-tracks/virtual-graph-integrity.spec.js b/app/tests/api/release-tracks/virtual-graph-integrity.spec.js index 55f5be91..0867aa32 100644 --- a/app/tests/api/release-tracks/virtual-graph-integrity.spec.js +++ b/app/tests/api/release-tracks/virtual-graph-integrity.spec.js @@ -10,6 +10,9 @@ const login = require('../../shared/login'); const AttackObject = require('../../../models/attack-object-model'); const linkById = require('../../../lib/linkById'); const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const { + ReleaseTrackContentManifest, +} = require('../../../models/release-tracks/release-track-content-manifest-model'); const { releaseExactMembers } = require('./release-track-test-helpers'); const staticMarkingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; @@ -171,18 +174,17 @@ describe('Virtual release-track graph integrity', function () { expect(exportedRoot.description).toBe(`See [Active Link Target](${attackReference.url}).`); }); - it('keeps virtual drafts and releases graphless until a tagged snapshot opts in', async function () { + it('seals virtual materialization and publishes that manifest unchanged at release', async function () { const root = await post('/api/techniques', technique('Frozen Virtual Root')); const virtual = await createVirtual('Virtual Frozen Release Graph', [root]); const draft = await dynamicRepo.getLatestSnapshot(virtual.id); - expect(draft.graph_manifest_id).toBeUndefined(); - await post( - `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent( - new Date(draft.modified).toISOString(), - )}/graph`, - {}, - 409, - ); + expect(draft.content_manifest_id).toBeDefined(); + const materializationManifest = await ReleaseTrackContentManifest.findOne({ + manifest_id: draft.content_manifest_id, + }) + .lean() + .exec(); + expect(materializationManifest.seal_reason).toBe('members_written'); const preview = await get( `/api/release-tracks/${virtual.id}/snapshots/latest/release/preview` + @@ -195,40 +197,22 @@ describe('Virtual release-track graph integrity', function () { { version: '1.0' }, 200, ); - expect(releasedResponse.graph_manifest_id).toBeUndefined(); - - const deterministic = await post( - `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent( - releasedResponse.modified, - )}/graph`, - {}, - 201, - ); - expect(deterministic.graph_manifest_id).toBeDefined(); + expect(releasedResponse.content_manifest_id).toBe(draft.content_manifest_id); + expect(releasedResponse.publication).toBeDefined(); + expect(releasedResponse.bundle_id).toMatch(/^bundle--/); + expect(releasedResponse.bundle_hashes.manifest_id).toBe(draft.content_manifest_id); const releasedBundle = await get( `/api/release-tracks/${virtual.id}/snapshots/latest?format=bundle`, ); + expect(releasedBundle.id).toBe(releasedResponse.bundle_id); expect(releasedBundle.objects.find((object) => object.id === root.stix.id).name).toBe( root.stix.name, ); - - const graphPath = `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent( - releasedResponse.modified, - )}/graph`; - await request(app) - .delete(graphPath) - .set('Accept', 'application/json') - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(204); - await request(app) - .delete(graphPath) - .set('Accept', 'application/json') - .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(204); - - const graphlessRelease = await dynamicRepo.getLatestSnapshot(virtual.id); - expect(graphlessRelease.graph_manifest_id).toBeUndefined(); + expect(releasedBundle.objects[0]).toMatchObject({ + type: 'x-mitre-collection', + x_mitre_version: '1.0', + }); }); after(async function () { diff --git a/docs/README.md b/docs/README.md index 9666db1a..c7e44ff1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -47,6 +47,7 @@ Architecture, patterns, and implementation details for contributors. - [Frontend Handoff](developer/FRONTEND_TODO.md): Backend contract changes requiring downstream Angular updates - [Entities](developer/release-tracks/entities.md): Database schemas and data models +- [Sealed Content Manifests](developer/release-tracks/sealed-content-manifests.md): Why every snapshot seals its bill of materials, how the collection object is projected, and publication inheritance - [Backref Reconciliation](developer/release-tracks/backref-reconciliation.md): How `workspace.release_tracks` backrefs stay in sync with snapshots - [Member Sync Strategies](developer/release-tracks/member-sync-strategies.md): Automatic tracking of member object revisions - [Error Handling](developer/release-tracks/error-handling.md): Error handling patterns diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 4c0fbb8e..0d6641b4 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,168 @@ # Release Track TODOs +## Sealed snapshot content manifests (Problem 1) + +Design: [release-tracks/sealed-content-manifests.md](release-tracks/sealed-content-manifests.md). +Decisions confirmed by the developer on 2026-09-02: drop the relationship +advancement cascade, revert the frontend related-object reset to PUT, keep the +collection object in every STIX 2.1 bundle including drafts, track-scope +publication metadata with inherit-from-global as the default, collection +`modified` = snapshot `modified`, bundle `id` changes per snapshot while the +collection `id` is constant per track, and production-grade migration. + +### Backend (branch `beta`) + +- [x] Model: manifest gains `sealed_at`, `seal_reason`; snapshot gains + `content_manifest_id` (renamed from `graph_manifest_id`), `publication`, + `bundle_id`; `config.publication` with inherit/explicit identity and + markings plus optional `collection_id` and `created`; remove top-level + `object_marking_refs`. +- [x] Content manifest service (renamed from graph-manifest-service): seal + over exact members with ID-closed relationship selection, supporting and + link-target entries, no `secondary`, no predecessor carry-forward; + reference-counted discard; legacy schema-v1 reader retained. +- [x] Snapshot service: seal when `members` is written (create track, clone + with members override), inherit otherwise; prune drafts without + discarding shared manifests; notes editable on drafts only; remove + graph create/delete. +- [x] Versioning service: standard commit seals over planned members inside + the guarded tag update; virtual commit reuses the materialization seal; + freeze `publication`, generate `bundle_id`, store hashes; preview + reports relationship additions, removals, and stale authored endpoints. +- [x] Publication resolution: shared resolver for identity and markings + (track override or global), collection id and created overrides, + immutability after first release, config GET returns resolved values. +- [x] Export: single replay path; drafts may add live `include` tiers through + the same closure rule; tagged + `include` is 400; collection object + always present in 2.1, absent in 2.0; drafts omit `x_mitre_version`; + remove `includeToc`; bundle id stored for tagged, UUIDv5 for drafts. +- [x] Relationships: remove `handleEndpointRevisionCreated` and its + subscriptions; keep create-time endpoint pinning; drop the exact-endpoint + repository query if unused. +- [x] Routes/controller/OpenAPI: remove graph create/delete; reconstruct + accepts `replace_manifest_id`; rename response fields + (`content_manifest_id`, `content_statistics`); publication config + schema; remove `includeToc` and top-level marking refs. +- [x] Migration: rename field, seal unsealed tagged snapshots as baseline + reconstructions, drafts inherit or seal, migrate track marking refs into + `config.publication`, freeze `publication` and `bundle_id` on tagged + snapshots, recompute hashes, drop frozen `collection` entries; make the + 2026-07-30 migration's manifest backfill a no-op; dry-run preview + script; regression spec. +- [x] Tests: content-manifest lifecycle spec (replaces opt-in-graphs), + bundle spec updates, no-cascade relationship spec, publication config + spec, migration spec, virtual/history/description spec updates; run + focused specs then full `npm test`. +- [x] Docs: bundle-export.md, entities.md, implementation-notes.md, user + output-formats/versioning/api-reference/terminology, relationships doc. +- [x] Bruno: remove graph create/delete requests, update reconstruct, + config, snapshot export, and track creation requests. +- [x] Propose conventional commit messages. + +### Frontend (feature branch off `beta`) + +- [x] Revert relationship save to update related objects with PUT. +- [x] Remove bundle cache controls, cache status, and cache statistics; show + content statistics, hashes, and bundle id on tagged snapshots. +- [x] Publication section in track configuration: identity and markings with + inherit toggle and resolved-value display; collection id and created + overrides editable until first release. +- [x] Release preview: relationship inventory summary and stale-endpoint + warnings. +- [x] Snapshot notes read-only after release; field renames in classes, + connector, and specs; docs update; focused and full frontend + verification. + +Verification (2026-09-02): + +- Backend focused specs pass: content manifests 10, publication config 3, + manifest migrations 5, bundle export 19, snapshot descriptions 7, snapshot + history 7, virtual graph integrity 3, release planning 23, endpoint pins 3, + release tracks 3, attack objects, collection bundles, reconciliation. +- Three complete `npm test` runs: 1009/1012/1010 API cases passing with the + documented roaming shared-server failures only (attack-objects count, + collection-bundles 400, reconciliation 400, analytics ECONNRESET, groups + timeout); every affected spec passes in isolation. OpenAPI, config, + middleware, and scheduler suites pass. `npm run lint` is clean for the + changed files (pre-existing findings remain in untouched migrations and + `scripts/loadBundle.js`). +- Restored-production trial (2026-09-02): the first run crashed on + `release-track--4bf296be…`, one of eight unregistered collections left by + the first v19.1 bootstrap, whose members reference six technique revisions + replaced by the platform-ordering repair. The migration now migrates + registered tracks only, reports orphans, discards their manifests, and names + the failing snapshot, step, and missing references on any other error. +- The crashed API container kept restarting (`restart: unless-stopped`, 23 + restarts) on the old image, re-running the unfixed migration and re-creating + the four orphan manifests each time; the rebuilt image's idempotent run + discards them again. Stop or rebuild the container before judging the + database state. +- After the fix, a dry run against the restore reported 6 tracks, 21 + snapshots, 7 seals, 1 shared manifest, 4 renames, and 7 orphan collections; + applying it sealed and froze exactly that, recomputed 7 hash sets, and a + second apply changed nothing. Every released snapshot's stored STIX 2.1 + hash matches a fresh export and every draft exports with the version key + omitted. +- Design record: `docs/developer/release-tracks/sealed-content-manifests.md`. +- Follow-up for the developer's v19.1 bootstrap tooling (`.nocommit/`): read + `content_manifest_id` instead of `graph_manifest_id`, and replace the + delete-then-reconstruct recovery with a single reconstruct request naming + `replace_manifest_id`. +- Frontend (branch `feat/sealed-content-manifests`): focused specs pass + (page 53, connector, relationship, preview dialog); the complete vitest + suite passes (166 files, 379 tests); `tsc --noEmit`, ESLint on changed + files, and the production `ng build` succeed. +- Proposed REST commit: `feat(release-tracks): seal snapshot content manifests`. + Proposed frontend commit: `feat(release-tracks): surface sealed content and + publication settings`. + + +### Review follow-ups (2026-09-02) + +- [x] Fix the virtual-track config editor crash: the connector's identity and + marking getters return functions that must be invoked as methods. +- [x] Rename the History tab to Releases; drop the per-snapshot "Sealed" chip + (every snapshot is sealed, so it carried no information) and label the + statistics section "Content". +- [x] Restore release deletion: administrators may delete the track's most + recent release with a typed version confirmation (`confirm_version`); + the ledger entry is retracted from every remaining snapshot, the manifest + is discarded when unreferenced, the registry is reconciled, and a + `delete_release` audit event is recorded. Frontend button on tagged + release cards for administrators. +- [x] Data-model review (KISS): rename manifest storage to + `releaseTrackContentManifest*` with `release-track-content-manifest--` + ids; drop `resolver_version` and `baseline_reconstruction` in favour of a + required `seal_reason`; remove the dead `config.include_secondary_objects` + block and its frontend section; keep `releaseTrackReconciliations` as an + outstanding-work queue (completed records are deleted, so it is normally + empty); keep `releaseTrackAuditEvents` (now used by both destructive + actions) and document every collection in `entities.md`. +- [x] Migration extended in place (unreleased): collection rename, id rewrite, + header normalization, dead-config removal, completed-reconciliation + cleanup; dry run stays accurate before the rename. +Verification (2026-09-02, review follow-ups): + +- Backend focused specs pass: manifest migrations 5, destructive authorization + 3, content manifests 10, snapshot history 7, virtual graph integrity 3, + reconciliation durability 2, releases by object 8, snapshot immutability 2, + backrefs 24; OpenAPI validation passes. +- Restored production database: dry run reports 131,534 legacy manifest + documents to move and 17 headers to normalize; apply completes, all 21 + snapshots export with matching hashes, storage now lists only + `releaseTrackContentManifests`, `releaseTrackContentManifestEntries`, + `releaseTrackRegistry`, `releaseTrackReconciliations` (empty), and + `releaseTrackAuditEvents`; a second dry run reports nothing left to do. +- Frontend: focused page and connector specs pass (71); complete suite + 381 tests with one unrelated save-dialog flake that passes alone; `tsc`, + ESLint, Prettier, and the production build are clean. +- [ ] Recommendation, not implemented: `version_history` is copied into every + snapshot document although only the tagged snapshot's own entry is read + (`tagMetadataForSnapshot`) and the registry's `tagged_releases` is the + catalogue. Storing the entry only on the tagged snapshot would remove the + duplication but touches release planning, cloning, and the frontend + history view; defer to a dedicated slice. + ## Remove nightly-only migration compatibility - [x] Remove regression code that imports the retired beta bundle-integrity @@ -52,6 +215,35 @@ Verification (2026-08-07): - Proposed REST API commit: `feat(config): expose REST API build information`. Proposed frontend commit: `feat(shell): display component build versions`. +## Targeted v19.1 source-graph recovery + +- [x] Add a read-only preflight that targets one exact tagged virtual snapshot, + reconstructs its canonical v19.1 pointer plan, and requires every prior + immutable baseline repair to already exist. +- [x] Add a separately confirmed apply mode that replaces only the target + snapshot's graph manifest and refuses track replacement or STIX writes. +- [x] Verify the snapshot members, source-pointer hydration, manifest + attestation, and final emitted bundle against the canonical source. +- [x] Tag future public v19.1 virtual baselines as `19.1` while retaining the + internal standard-track baseline tag. +- [x] Add operator documentation and regression coverage for safety, + idempotence, and the incorrect ordinary-manifest recovery scenario. +- [x] Run focused Python checks followed by the complete `npm test` suite. +- [x] Propose a conventional commit message without committing unless asked. + +Verification (2026-08-05): + +- Bootstrap regressions pass: 42 cases covering graph-only no-write preflight, + guarded replacement, missing-repair fail-closed behavior, manifest races, + and the public `19.1` virtual tag. Ruff, Python compilation, and diff + whitespace checks pass. +- The clean complete REST suite passes under repository-pinned Node 22.14.0: + OpenAPI 2, config 21, API 1012, middleware 29, and scheduler 10. +- Earlier complete runs encountered the documented roaming shared-server + failures; each affected bundle, pagination, backref, virtual-graph, and notes + spec passed independently before the clean run. +- Proposed commit: `fix(release-tracks): add targeted v19.1 graph recovery`. + ## Deterministic graph collection identity repair - [x] Reproduce the incorrect graph collection creator, STIX 2.0 TOC @@ -80,6 +272,64 @@ Verification (2026-08-05): - The developer subsequently confirmed a complete all-green test run. - Proposed commit: `fix(release-tracks): repair deterministic bundle integrity`. +## Stateful snapshot collection objects and bundle hashes + +- [x] Persist one frozen `x-mitre-collection` entry in every graph manifest, + with a track-stable ID, first-manifest `created`, and current-manifest + `modified` timestamp. +- [x] Replay the frozen collection entry and a manifest-stable bundle envelope + ID for deterministic STIX 2.0 and STIX 2.1 downloads. +- [x] Generate SHA-256 hashes from the exact pretty-printed download bytes and + store both hashes on graph-backed snapshots with their manifest ID. +- [x] Reject snapshot-note edits while a graph exists so the frozen collection + and hashes remain immutable; require deletion and regeneration to edit. +- [x] Expose hashes through snapshot responses/OpenAPI and update REST docs, + Bruno coverage, and regression tests. +- [x] Replace frontend bundle prefetch/hashing with server-provided hashes, + preserving copy controls and exact download serialization. +- [x] Run focused and complete REST/frontend verification and propose + conventional commit messages without committing unless asked. + +Verification (2026-08-04): + +- Focused REST graph, description, history, and bundle specs pass, including + exact SHA-256 comparisons against both downloaded bundle serializations. +- The complete REST `npm test` suite passes after the repository's documented + roaming harness failures were confirmed in isolation and rerun. +- The complete frontend suite passes: 165 files and 381 tests. Targeted + TypeScript, ESLint, and Prettier checks also pass. +- Cached-note regression coverage proves the API returns 409 without changing + either bundle hash, then permits editing after graph deletion and freezes the + revised notes when the graph is recreated. +- Repeated complete REST runs encountered the documented roaming harness + failures in unrelated specs (transient 400/404/ECONNRESET responses); every + affected spec passes when rerun in isolation. +- Proposed REST commit: `feat(release-tracks): persist snapshot bundle hashes`. + Proposed frontend commit: `feat(release-tracks): display snapshot bundle hashes`. + +## Relationship review-state revision safety + +- [x] Reproduce the relationship-save source/target workflow reset and prove it + currently uses in-place PUT updates. +- [x] Reset related SDO workflow state through POST-created revisions so graph- + pinned revisions remain immutable. +- [x] Add frontend regression coverage for request method, WIP transition, and + sequential relationship/source/target saves. +- [x] Update frontend workflow documentation and run focused plus complete + frontend verification. +- [x] Propose a conventional commit message without committing unless asked. + +Verification (2026-08-04): + +- The focused relationship revision regression passes: 1 case proving ordered + relationship/source/target POSTs, WIP resets, and no related-object PUTs. +- The complete frontend suite passes: 164 files and 377 tests. The production + Angular build succeeds with existing bundle/style budget warnings. +- Prettier and the new regression's ESLint check pass. The legacy relationship + class retains its existing unrelated lint findings; the changed transport + line introduces none. +- Proposed frontend commit: `fix(relationships): revise related objects on save`. + ## Snapshot collection descriptions and bounded release versions - [x] Map each snapshot's user-authored description onto emitted @@ -132,7 +382,7 @@ Verification (2026-08-03): with the local persistent cache temporarily disabled to avoid the documented environment-specific native crash; `angular.json` was restored afterward. - Proposed frontend commit: `feat(release-tracks): manage snapshot bundle - caches`. +caches`. ## Source-attested v19.1 graph reconstruction @@ -200,8 +450,63 @@ Verification (2026-08-03): placing it in a graph URL; its isolated virtual-graph-integrity spec passes: 3 cases. - Proposed implementation commit: `fix(release-tracks): hydrate historical - relationship graphs`. Proposed test-only commit: `test(release-tracks): - serialize snapshot timestamps in graph URLs`. +relationship graphs`. Proposed test-only commit: `test(release-tracks): +serialize snapshot timestamps in graph URLs`. + +### Production bootstrap replacement recovery + +- [x] Reproduce the production preflight failure against the restored database. +- [x] Keep dynamic `latest` workflow selectors out of exact historical-pin + compatibility comparisons. +- [x] Add an explicit, confirmed option to replace only the six exact-name + bootstrap tracks, deleting virtual tracks before their standard inputs. +- [x] Repair six persisted v19.1 technique revisions whose platform arrays + contain the correct values in a different order from the source bundles. +- [x] Run the corrected preflight against the restored production database and + complete focused script verification. + +Verification (2026-08-04): + +- The restored-production preflight completes without treating nine dynamic + `latest` candidates as timestamps and inventories all six existing tracks for + replacement. +- It identifies 297 immutable baseline repairs: 291 canonical-domain revisions + and six `x_mitre_platforms` ordering revisions. +- Two replacement applies complete successfully. The second reuses all 297 + semantic repair revisions, proving restart safety, and all three final bundle + comparisons report `identical_excluding_collection: true` with member counts + Enterprise 4,815, ICS 503, and Mobile 743. +- Bootstrap regressions pass: 36 cases. Python Ruff and the read-only production + preflight pass. + +### Closed-member deterministic relationship graphs + +- [x] Reproduce exact-revision leakage when a relationship endpoint pins a + different revision of an existing snapshot member. +- [x] Replace ID-frontier secondary expansion with indexed exact-endpoint + relationship selection requiring both endpoint revisions in `members`. +- [x] Select only the latest relationship revision for each exact endpoint + pair, with a later revoked or deprecated revision suppressing older + active history. +- [x] Seed the v19.1 transition from the preceding source-attested manifest so + historical relationships without truthful stored endpoint pins remain + available while their exact member endpoints survive. +- [x] Keep schema-v2 relationships pointer-only and reject duplicate emitted + STIX revisions or inconsistent relationship lineages. +- [x] Update release-track documentation and add regression coverage for + closed membership, predecessor carry-forward, relationship advancement, + and mutation protection. +- [x] Run focused release-track specs followed by the complete `npm test` + suite. + +Verification (2026-08-04): + +- Closed-member graph regressions pass: 9 opt-in graph cases and 17 bundle + cases. +- The complete release-track directory passes with 187 cases after two roaming + harness failures were rerun successfully in isolation (27 cases). +- Lint and the complete `npm test` suite pass, including OpenAPI, + configuration, API, middleware, and scheduler suites. ## Snapshot-history graph cache statistics @@ -309,7 +614,7 @@ Verification (2026-08-03): errors; the shared release-track API type retains one pre-existing index-signature violation. - Proposed frontend commit: `feat(release-tracks): add deterministic bundle - cache controls`. +cache controls`. ## Frontend canonical-domain preservation @@ -332,7 +637,7 @@ Verification (2026-08-03): - Repository-wide lint remains red on 256 pre-existing errors outside this change; no new lint errors remain in the hotfix files. - Proposed frontend commit: `fix(stix): preserve canonical domains in - editors`. +editors`. ## C0028 campaign revision / released virtual-snapshot investigation @@ -372,7 +677,7 @@ Investigation (2026-08-03): references; backend campaign regression proving a missing cited source is rejected and the corrected revision succeeds with ADM validation enabled. - Proposed implementation commit: `fix(campaigns): preserve domains and cited - references in revisions`. +references in revisions`. Verification (2026-08-03): @@ -2292,6 +2597,7 @@ Links/references between notes and snapshot objects will be one-to-many. A singl "stix": "StixObject" } ``` + ## Deterministic v19.1 virtual-track bootstrap graph - [x] Preserve the materialized virtual snapshot graph when previewing and committing a release. diff --git a/docs/developer/release-tracks/backref-reconciliation.md b/docs/developer/release-tracks/backref-reconciliation.md index a30b0ce7..b6fe5c54 100644 --- a/docs/developer/release-tracks/backref-reconciliation.md +++ b/docs/developer/release-tracks/backref-reconciliation.md @@ -73,10 +73,13 @@ the durable `reconciliation_id`; the release-track mutation may already be persisted and must not be retried blindly. Every attempt is written to `releaseTrackReconciliations` before listeners -run. Records move through `pending`, `completed`, or `failed` and retain the -requested snapshot, attempt count, timestamps, and last error. If recording -completion fails after the listeners succeeded, the record remains pending; -replaying it is safe because reconciliation is idempotent. +run and deleted when they succeed, so the collection holds outstanding work +only: every document is a `pending` or `failed` attempt that still needs +repair, with the requested snapshot, attempt count, timestamps, and last +error. If deleting the record fails after the listeners succeeded, it remains +pending; replaying it is safe because reconciliation is idempotent. The +repair command (`repairOutstanding`) and the full scan (`reconcileAll`) +return completed summaries without persisting them. ## Reconciliation algorithm diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index ca547126..dab9b6f7 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -53,10 +53,10 @@ changes what bundle emission needs to be: `GET /api/stix-bundles` is therefore **deprecated** (marked in the OpenAPI spec) and will be removed in a future release. Its replacements: -| Legacy usage | Replacement | -|--------------|-------------| -| Domain-scoped ad hoc bundle | `GET /api/release-tracks/ephemeral/:domain` | -| Release/publication bundle | `GET /api/release-tracks/:id/snapshots/latest?format=bundle` (or `/snapshots/:modified?format=bundle`) | +| Legacy usage | Replacement | +| --------------------------- | ------------------------------------------------------------------------------------------------------ | +| Domain-scoped ad hoc bundle | `GET /api/release-tracks/ephemeral/:domain` | +| Release/publication bundle | `GET /api/release-tracks/:id/snapshots/latest?format=bundle` (or `/snapshots/:modified?format=bundle`) | ### Ephemeral endpoint parameter mapping @@ -66,19 +66,19 @@ object-selection logic above is preserved verbatim. The query-parameter surface was simplified (see [ephemeral-service.js](../../../app/services/release-tracks/ephemeral-service.js)): -| Legacy parameter | Disposition | -|------------------|-------------| -| `stixVersion` | **Preserved** (default changed to `2.1`) | -| `includeRevoked` / `includeDeprecated` | **Preserved** (default `false`) | -| `includeMissingAttackId` | **Renamed** to `includeObjectsWithMissingAttackId` (default `false`) | -| `includeCollectionObject` | **Renamed** to `includeToc` (default `true`). "TOC" (table of contents) describes what the `x-mitre-collection` object actually is, and avoids overloading the term "collection". It applies only to STIX 2.1; STIX 2.0 always omits the object. | -| `collectionObjectVersion` | **Removed** — fixed at `0.1`, signifying an ephemerally generated collection not connected to a release track | -| `collectionObjectModified` | **Removed** — fixed at the current timestamp | -| `collectionAttackSpecVersion` | **Removed** — fixed at the global default (`config.app.attackSpecVersion`) | -| `includeNotes` | **Removed** — notes are Workbench-native objects, not STIX objects, and are never emitted in bundles | -| `includeDataSources` | **Removed** — data sources are deprecated (ATT&CK Spec v3.3.0) and were marked deprecated/revoked in ATT&CK v18, so their inclusion is governed entirely by `includeDeprecated`/`includeRevoked`. Internally the delegation passes `includeDataSources: true` and lets those flags filter. | -| `useLegacyMethod` | **Removed** — the pre-v17 code path (`stix-bundles-service-old.js`) is not supported by the new endpoints | -| `state` | **Removed** — workflow status is now scoped to release tracks; a domain-scoped endpoint has no workflow-status concept | +| Legacy parameter | Disposition | +| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `stixVersion` | **Preserved** (default changed to `2.1`) | +| `includeRevoked` / `includeDeprecated` | **Preserved** (default `false`) | +| `includeMissingAttackId` | **Renamed** to `includeObjectsWithMissingAttackId` (default `false`) | +| `includeCollectionObject` | **Renamed** to `includeToc` (default `true`). "TOC" describes what the `x-mitre-collection` object is and avoids overloading "collection". It applies only to STIX 2.1; STIX 2.0 always omits the object. | +| `collectionObjectVersion` | **Removed** — fixed at `0.1`, signifying an ephemerally generated collection not connected to a release track | +| `collectionObjectModified` | **Removed** — fixed at the current timestamp | +| `collectionAttackSpecVersion` | **Removed** — fixed at the global default (`config.app.attackSpecVersion`) | +| `includeNotes` | **Removed** — notes are Workbench-native objects, not STIX objects, and are never emitted in bundles | +| `includeDataSources` | **Removed** — data sources are deprecated (ATT&CK Spec v3.3.0) and were marked deprecated/revoked in ATT&CK v18, so their inclusion is governed entirely by `includeDeprecated`/`includeRevoked`. Internally the delegation passes `includeDataSources: true` and lets those flags filter. | +| `useLegacyMethod` | **Removed** — the pre-v17 code path (`stix-bundles-service-old.js`) is not supported by the new endpoints | +| `state` | **Removed** — workflow status is now scoped to release tracks; a domain-scoped endpoint has no workflow-status concept | Note on the bundle envelope: STIX 2.0 requires `spec_version` on the bundle object, while STIX 2.1 removed it (objects declare their own `spec_version` @@ -93,60 +93,158 @@ Implemented in [export-schemas.js](../../../app/lib/release-tracks/export-schemas.js) (`bundleTransformSchema`). Standard snapshots and materialized virtual snapshots use this same pipeline; virtual composition metadata does not alter -STIX version serialization. The pipeline: - -1. **Tier selection** — members are always exported. `include` (values - `staged` and/or `candidates`; singular forms accepted) adds tiers. - `state` (values `work-in-progress` and/or `awaiting-review`) narrows the - added tiers; entries whose `object_status` is `reviewed` always pass the - filter, mirroring the fact that members are inherently reviewed. `state` - never affects members. `reviewed` is intentionally not a valid `state` - value for this reason. -2. **Graph selection** — a member-only export replays the schema-v2 graph when - the tagged snapshot has explicitly opted in. Graphless snapshots resolve a - live bounded graph. Any request that includes `staged` or `candidates` is - also live; determinism is promised for `members` only. -3. **Closed member graph** — persisted deterministic graphs emit only exact - `members` revisions as graph objects. A relationship is selected only when - both of its stored exact endpoint revisions are members; relationships do - not pull additional SDOs into the graph. Persisted schema-v2 manifests store - exact-revision pointers, not cloned STIX payloads. -4. **Supporting objects** — referenced identities and marking definitions are - appended. Versioned supporting objects use pointers; unversioned marking - definitions retain a frozen payload in persisted graphs. -5. **LinkById conversion** — deterministic replay uses the exact render target - pointer captured in the graph. Live resolution uses the current eligible - target. -6. **Assembly** (Zod transform) — notes are dropped, objects are conformed to +STIX version serialization. The design is recorded in +[sealed-content-manifests.md](sealed-content-manifests.md). The pipeline: + +1. **Replay the sealed content manifest** — every snapshot references a + manifest from birth. The manifest holds exact-revision pointers for the + member roots, the relationships closed over those members, supporting + identities and marking definitions, and non-emitted LinkById render + targets. Export hydrates those pointers and nothing else: no relationship + query, no domain inference, no "latest" lookup. +2. **Draft previews** — `include` (values `staged` and/or `candidates`; + singular forms accepted) adds workflow tiers to a draft export and `state` + (values `work-in-progress` and/or `awaiting-review`) narrows them; entries + whose `object_status` is `reviewed` always pass. Because those tiers may + hold dynamic `latest` selectors, an `include` export resolves the same + closed-member graph live over members plus the included entries instead of + replaying. Tagged snapshots reject `include` with `400`. Release previews + of an unsaved planned snapshot resolve live the same way. +3. **Supporting objects** — identities and marking definitions referenced by + emitted objects, plus the identity and markings the collection object + itself references, are appended so the bundle is self-contained. +4. **LinkById conversion** — uses the exact render target pointers captured in + the manifest. +5. **Assembly** (Zod transform) — notes are dropped, objects are conformed to `stixVersion` via the shared `lib/stix-conformance.js` helpers, and the bundle envelope is emitted (with `spec_version: "2.0"` only when `stixVersion=2.0` — STIX 2.1 removed `spec_version` from the bundle object). -7. **TOC** — for STIX 2.1, unless `includeToc=false`, an - `x-mitre-collection` object is prepended. STIX 2.0 always omits this ATT&CK - extension object. Graphless 2.1 exports derive it from live snapshot - metadata. Graph creation freezes it as a `collection` manifest entry and - every member-only 2.1 replay uses that stored value: - - `id`: `x-mitre-collection--` — stable across exports of the - same track - - `created_by_ref`: the configured organization identity's STIX ID - - `name`/`object_marking_refs`: from the snapshot metadata - - `description`: from `snapshot_description` when present, otherwise the - snapshot's long-lived track `description` - - `x_mitre_version`: the snapshot's tagged version, or `0.1` for drafts - - `created`: the first cached collection object's creation timestamp for - the release track - - `modified`: the current graph manifest's creation timestamp - - `x_mitre_contents`: every bundle object except marking definitions - (which are recorded in `object_marking_refs`), sorted by `object_ref` -8. **Deterministic file identity** — graph-backed member-only bundles use the - graph manifest UUID for the bundle envelope ID. After graph creation, the - server serializes each STIX version with `JSON.stringify(bundle, null, 4)`, - hashes those exact UTF-8 bytes with SHA-256, and stores both digests on the - snapshot as `bundle_hashes`. The graph, collection object, notes, and hashes - form one immutable cache boundary. Snapshot-note edits return `409 Conflict` - until the graph is deleted; callers then edit the notes and regenerate the - graph and hashes. +6. **Collection object** — every STIX 2.1 bundle begins with an + `x-mitre-collection` object; STIX 2.0 bundles never contain this ATT&CK + extension object. It is a projection, never a stored object: + - `id`: `config.publication.collection_id`, defaulting to + `x-mitre-collection--`; constant across every snapshot of the + track + - `created`: `config.publication.created`, defaulting to the track's + `created` + - `modified`: the snapshot's `modified` + - `x_mitre_version`: the tagged version; drafts omit the key + - `created_by_ref` and `object_marking_refs`: the publication inheritance + rule (track override, else global system configuration). When neither + scope configures markings, the object carries the marking definitions + referenced by its contents so it never ships unmarked + - `name`: the snapshot's track name + - `description`: `snapshot_description`, falling back to the track + `description` + - `x_mitre_attack_spec_version`: the deployment's ATT&CK spec version + - `x_mitre_contents`: every bundle object except marking definitions, + sorted by `object_ref` + Drafts resolve the inheritance rule at export so they preview the current + configuration; release commit freezes the resolved values onto the tagged + snapshot as `publication`. +7. **Bundle identity and hashes** — a released snapshot stores a stable + `bundle_id` assigned at commit; drafts derive a UUIDv5 from the track ID + and snapshot `modified`. The bundle ID therefore changes across snapshots + while the collection ID stays constant per track. Release commit serializes + each STIX version with `JSON.stringify(bundle, null, 4)`, hashes the exact + UTF-8 bytes with SHA-256, and stores both digests on the snapshot as + `bundle_hashes` bound to the manifest ID. + +### Sealed content manifests + +`content-manifest-service.js` owns the one graph algorithm +(`resolveClosedGraph`): + +- Roots are the exact `members` revisions; a member set naming two revisions + of one STIX ID is rejected. +- A relationship lineage is a candidate when its `source_ref` and + `target_ref` are both member IDs (indexed `$in` queries on the two ref + fields, batched). The newest revision of each lineage is chosen first, then + discarded if it is revoked, deprecated, or a deprecated pattern, so an older + active revision is never resurrected by a newer inactive one. The member + revisions become the manifest entry's `source` and `target` pins. +- No SDO is ever discovered through a relationship. The former `secondary` + role survives only in legacy manifests. +- Supporting identities and marking definitions are pointers (versioned) or + frozen payloads (unversioned marking definitions). LinkById targets outside + the bundle are non-emitted `link_target` entries. + +A manifest is sealed whenever a snapshot's `members` tier is written: track +creation, release commit, virtual materialization, bundle import, quarantine +promotion, and track clone. Candidate, staged, config, and metadata clones +inherit the predecessor's manifest by reference, so manifest storage is +bounded by member-changing writes rather than by snapshot count. Sealing +writes a pending manifest and its entries, re-verifies every pointer inside +that protection window, then saves the snapshot referencing the manifest and +activates it; a failed save discards the manifest. A manifest is discarded only +when no snapshot in its track references it. + +A standard release commit seals a fresh manifest over the planned member set +inside the guarded tag update, even when nothing was staged, so relationships +created since the last seal are captured. The release preview reports exactly +what that seal would change: `relationships.added`, `relationships.removed`, +and `relationships.stale_endpoints` (relationships whose authoring-time +endpoint revision differs from the member revision being shipped). A virtual +commit publishes the materialization manifest unchanged, because the +materialized draft is the artifact that was reviewed. + +Every relationship revision still records server-controlled exact endpoint +pins under `workspace.relationship_endpoints` at creation. They are authoring +context for the stale-endpoint warning and are not emitted. A new endpoint +revision no longer clones the relationship: exact pairing for a release lives +in the sealed manifest, so editing an object creates no relationship +revisions and editing a relationship creates exactly one. + +Active and pending manifests protect every exact versioned dependency from +hard deletion. Persisted STIX content is globally immutable through PUT; +corrections are new POSTed revisions. Tagged snapshots are immutable including +their notes. + +#### Historical baselines + +Baselines whose relationships predate endpoint-pin capture use the admin-only +`POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct`. Its body +contains a source-bundle attestation and a decoupled pointer plan, not the +bundle payload. The server verifies that roots exactly equal tagged members, +every exact revision exists, each relationship's STIX refs agree with the +supplied endpoint IDs, the endpoint revisions are included, and required +supporting objects are present. Because every tagged snapshot already +references a sealed manifest, the request must name that manifest in +`replace_manifest_id`; the same attestation is idempotent and any other +current manifest is rejected. Replacement recomputes the bundle hashes. The +resulting manifest uses resolver version `source-bundle-pointer-v2`, records +the attestation, and sets `baseline_reconstruction: true`. Source plans may +carry `link_target` pointers and narrow `omitted_optional_defaults` +serialization hints exactly as before. + +#### Migration + +`20260902120000-seal-release-track-content-manifests.js` upgrades existing +databases in place: it renames `graph_manifest_id` to `content_manifest_id`, +seals a `baseline_reconstruction` manifest for every tagged snapshot that had +none, lets drafts share the manifest of a preceding tagged snapshot with an +identical member set (or seals them), moves the retired top-level +`object_marking_refs` into `config.publication.object_marking_refs`, freezes +`publication` and a `bundle_id` (preserving the manifest-derived envelope ID +those snapshots exported before) onto tagged snapshots, recomputes +`bundle_hashes`, and removes frozen `collection` entries. Only tracks in +`releaseTrackRegistry` are migrated: a dynamic `release-track--*` collection +without a registry document is an orphan of an interrupted or pre-registry +deletion whose snapshots routinely point at revisions that no longer exist. +The migration reports each orphan, discards any manifests it owns so they +cannot protect stale revisions, and leaves the collection for an operator to +drop. It also renames the manifest collections from `releaseTrackGraphManifest*` +to `releaseTrackContentManifest*`, moves manifest ids to the +`release-track-content-manifest--` prefix, replaces `resolver_version` and +`baseline_reconstruction` with a required `seal_reason`, removes the retired +`config.include_secondary_objects` block, and deletes completed +`releaseTrackReconciliations` records. Preview it with +`npm run preview:content-manifests`; a failure names the track, snapshot, +step, and missing references. The earlier +`20260730180000` migration keeps its relationship-pin backfill but no longer +creates manifests. Legacy schema-v1 manifests (frozen relationship payloads, +`secondary` entries) remain replayable. ### Canonical domains and the legacy graph renderer @@ -155,157 +253,14 @@ object has one revision whose `x_mitre_domains` contains the complete domain union. That same revision may appear in multiple domain bundles; its array is not narrowed to the domain requested by a particular export. -The legacy and ephemeral graph renderer now preserves every nonempty +The legacy and ephemeral graph renderer preserves every nonempty `x_mitre_domains` array it hydrates. Export-time inference remains only as a compatibility fallback for exact historical domainless revisions pinned before canonical-domain enforcement, including historical matrix revisions. The fallback affects the rendered copy and does not update the stored -revision. The release-agnostic startup migration creates a canonical -replacement only when an exact collection TOC entry proves the object's -domain. Unmapped legacy objects remain unchanged, are reported for follow-up, -and keep the temporary validation bypasses active. All subsequent content must -persist canonical domains so virtual composition, snapshot export, and -ephemeral export observe the same membership. - -Because snapshot contents are explicitly curated, primary entries do **not** -receive the legacy attack-id / deprecated / revoked filters. Graphless and -candidate/staged exports retain the established live bounded ATT&CK expansion -rules. A persisted deterministic member graph instead closes over `members` -and never discovers additional SDO revisions through relationships. - -#### Closed-member relationship consistency boundary - -Release-track exports distinguish persisted deterministic content from live -compatibility expansion: - -- Primary objects are explicit snapshot tier entries. Members and quarantine - record exact `(object_ref, object_modified)` revisions. Standard candidates - and staged entries may instead store `"latest"` and are resolved just in - time when a draft export includes those tiers. -- A persisted deterministic graph contains only `members` as graph objects. - Relationships, supporting identities/marking definitions, and non-emitted - LinkById targets are dependencies, not implicit membership. A relationship - endpoint outside `members` causes that relationship to be omitted. -- Graphless and candidate/staged exports remain live and may use the legacy - secondary-object expansion rules. They carry no determinism guarantee. - -Tagged standard membership is deterministic because release planning resolves -staged selectors before promoting them to members. Virtual materialization -likewise copies exact member revisions from tagged component snapshots and -never follows a component's later `track_latest` candidate movement. -When a virtual component declares `filters.domains`, virtual materialization -uses those filters to choose exact primary members. Deterministic graph capture -does not perform a second domain-inference pass: the materialized member set is -the complete SDO boundary. Domainless supporting metadata remains eligible. - -Every relationship revision stores server-controlled exact source and target -pins under `workspace.relationship_endpoints`. These fields identify the -precise `(object_ref, object_modified)` pair represented by each side of the -SRO. They are not emitted because bundle output includes only the `stix` -object. When an endpoint advances, Workbench creates a new SRO revision with -updated pins rather than rewriting the older SRO. - -Snapshots are graphless by default. After tagging, an editor may call -`POST /api/release-tracks/:id/snapshots/:modified/graph`. The service builds a -schema-v2 closed-member graph. It rejects duplicate member revisions for one -STIX ID, selects relationship revisions only when both exact endpoint pins are -members, writes a pending manifest and decoupled entry rows, rehydrates every -pointer while those pending rows already protect deletion, then atomically -attaches the manifest ID to the still-tagged snapshot. Replay can self-activate -a complete linked pending manifest after an interrupted activation. `DELETE` -on the same graph resource detaches and removes it. Each manifest also owns one -frozen `x-mitre-collection` entry. The attached snapshot records SHA-256 values -for the exact STIX 2.0 and STIX 2.1 browser-download serialization, bound to the -same manifest ID. - -Historical baselines whose relationships predate endpoint-pin capture require -a different, admin-only path: -`POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct`. Its body -contains a source-bundle attestation and a decoupled pointer plan, not the -bundle payload. The caller must independently verify the named bundle and its -SHA-256 digest. The server then verifies that roots exactly equal tagged -members, every exact revision exists, each relationship's STIX refs agree with -the supplied endpoint IDs, the endpoint revisions are included, and required -supporting objects are present. Versioned entries are always pointers; only an -unversioned marking definition may be frozen by value. The resulting manifest -uses resolver version `source-bundle-pointer-v2`, records the attestation, and -sets `baseline_reconstruction: true`. - -Source plans may contain `link_target` pointers for objects outside the emitted -domain bundle. They are hydrated for LinkById conversion but are not emitted. -Active ATT&CK-ID targets are preferred; a unique inactive historical target is -accepted only when no active v19.1 target exists. - -The v19.1 production bootstrap uses this path without importing the published -bundles. Because each official domain bundle contains one revision per STIX -ID, it can infer legacy SRO endpoint revisions by joining `source_ref` and -`target_ref` to those unique objects. Before tagging, the script batch-hydrates -the entire pointer plan from Workbench and compares its STIX object set with -the source bundle. This is the missing provenance that live database traversal -cannot recover after endpoint lineages have advanced. The bootstrap routes -entity pointers to `attackObjects` and relationship pointers to the dedicated -`relationships` collection. Its pre-tag comparison mirrors export-time -LinkById rendering. A pointer may carry a narrow serialization hint when the -attested source omitted a persisted optional `revoked: false` or -`x_mitre_remote_support: false` default. Most source objects explicitly emit -those false values and retain them. True values and every other payload -difference remain significant. Ordinary release-track exports retain their -existing serialization. - -Ordinary graph creation uses the compound indexes on -`workspace.relationship_endpoints.{source,target}` rather than scanning all -relationships. Exact member revisions are queried in bounded batches. A -candidate survives only when both exact endpoint pairs occur in `members`. -Candidates are then grouped by relationship lineage and exact endpoint pair; -the newest revision wins before revoked, deprecated, and obsolete-pattern -filters run, so an older active revision cannot be resurrected by a newer -inactive revision. - -The immediately preceding tagged graph also seeds relationship candidates -whose exact endpoints remain members. This creates a provenance chain from a -source-attested v19.1 baseline, including legacy relationships whose current -`workspace.relationship_endpoints` metadata cannot be reconstructed -truthfully. The indexed database query is still performed on every graph so a -new relationship connecting unchanged members is discovered. Current exact -relationship revisions override carried history; removed or revised member -endpoints naturally drop predecessor edges. - -Ordinary manifests created by this algorithm use resolver version -`closed-member-graph-v3`. Existing `bounded-member-graph-v2` manifests are not -rewritten in place. To repair an affected post-v19.1 graph, preserve the -source-attested v1.0 baseline, DELETE only the affected later snapshot's graph, -then POST that graph again. If the tagged snapshot's member pins are already -correct, deleting the snapshot itself is unnecessary; the recreated graph uses -v1.0 (or the immediately preceding tagged graph) as its predecessor. Published -artifacts produced from the removed graph must be regenerated. - -Active and pending manifests protect every exact versioned dependency from -hard deletion. Persisted STIX content is globally immutable through PUT, -whether or not it is graph-pinned; corrections are new POSTed revisions. -Schema-v2 relationships therefore need no frozen payload or mutation -exemption. Legacy schema-v1 manifests still replay their frozen relationship -payloads. Deleting a graph or track releases protection that no other graph or -tagged membership needs. - -Existing data is upgraded by an idempotent migration. Only the latest -revision of each legacy relationship can be endpoint-pinned truthfully. -Pre-existing snapshot manifests are labeled `baseline_reconstruction` -because they describe the graph visible during migration rather than an -unknowable historical graph. They must not be represented as historical truth. -A verified external bundle can reconstruct a historical graph through the -admin operation above; without such an artifact, exact legacy endpoint -selection remains unknowable. - -Drafts and tagged snapshots without graphs resolve live. Candidate/staged -exports also resolve live even when the snapshot has a graph, because those -tiers are expected to move. Release preview is live and release commit does -not create a graph. Determinism begins only with the explicit tagged-snapshot -graph operation and applies only to member exports. - -The graph and object payload are reproducible, but the bundle is not promised -to be byte-for-byte identical: the bundle envelope receives a newly generated -bundle ID. Consumers should compare the emitted STIX object set and revisions, -not the envelope UUID. +revision. Release-track exports never infer domains: the sealed member set is +the complete SDO boundary, and virtual materialization applies component +`filters.domains` when it selects members. ### Where validation happens @@ -324,9 +279,17 @@ release planning also enter through non-controller paths. ### Regression tests +- [content-manifests.spec.js](../../../app/tests/api/release-tracks/content-manifests.spec.js) + — sealing at creation, inheritance through clones, resealing at release, + preview inventories, source-attested replacement, draft-only `include` +- [publication-config.spec.js](../../../app/tests/api/release-tracks/publication-config.spec.js) + — publication inheritance, overrides, freezing, and immutability +- [deterministic-graph-migration.spec.js](../../../app/tests/api/release-tracks/deterministic-graph-migration.spec.js) + — the relationship-pin and content-manifest migrations - [release-tracks-bundle.spec.js](../../../app/tests/api/release-tracks/release-tracks-bundle.spec.js) — snapshot bundle exports (tier selection, state filtering, STIX version - conformance, TOC, LinkById, supporting objects, validation errors) + conformance, collection object, LinkById, supporting objects, validation + errors) - [ephemeral-bundle.spec.js](../../../app/tests/api/release-tracks/ephemeral-bundle.spec.js) — ephemeral bundles (legacy-parity object selection, parameter mapping, TOC defaults, workbench format) diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index a41044b9..2a13ed35 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -2,6 +2,23 @@ This document tracks new database schemas, interfaces, etc.; as well as changes to any such existing entities. +### Collections at a glance + +| Collection | Purpose | Written by | Growth and retention | +| --- | --- | --- | --- | +| `releaseTrackRegistry` | One document per track: name, type, denormalized counters, the tagged-release catalogue (`tagged_releases`), the release lock, and virtual schedules. The index that maps a track to its own snapshot collection. | Track create/delete, every snapshot write (counters), release commit and release deletion (catalogue). | One document per track. | +| `release-track--` | The track's snapshots: at most one rolling draft plus every tagged release for a standard track; every materialized draft plus releases for a virtual track. | Snapshot service and release commit. | Bounded by releases plus one draft (standard) or by materializations (virtual). | +| `releaseTrackContentManifests` | The sealed bill of materials each snapshot references (`content_manifest_id`). Several snapshots share one manifest when their member sets are identical. | Sealed whenever members are written; discarded when no snapshot references it. | Bounded by member-changing writes, not by snapshot count. | +| `releaseTrackContentManifestEntries` | One exact-revision pointer per object a manifest emits or depends on. The `(object_ref, object_modified)` index is what protects referenced revisions from deletion. | With its manifest. | Roughly members + relationships + a few supporting objects per manifest. | +| `releaseTrackReconciliations` | Outstanding backref reconciliation work only: a record is created before the `workspace.release_tracks` listeners run and deleted when they succeed, so anything present is pending or failed and needs repair. | Every snapshot write. | Normally empty. | +| `releaseTrackAuditEvents` | Audit trail for administrator-only destructive operations: `delete_track` and `delete_release`, with actor, confirmation, and outcome. | Those two operations. | Empty until an administrator deletes a track or release. | +| `virtualTrackScheduleOccurrences` | Durable claims for scheduled virtual materialization (cron or dated schedules) so restarts and duplicate delivery execute each occurrence once. | The scheduler. | One record per scheduled occurrence; empty when no virtual track has a schedule. | + +Removed by the sealed-manifest work: the former `releaseTrackGraphManifests` +and `releaseTrackGraphManifestEntries` collections (renamed in place by the +2026-09-02 migration), the frozen `collection` manifest entries, and the +`config.include_secondary_objects` block (secondary objects no longer exist). + ### Release Track `ReleaseTrack` instances will be tracked as independent MongoDB Collections. The reason for this is because the volume of snapshot permutations is expected to be very high given the frequency of changes that typically occur between releases. @@ -106,12 +123,27 @@ Each release track snapshot will be tracked as an individual MongoDB Document in version: "18.0", // null if draft release snapshot_description: "Why this snapshot matters to our team", + // Sealed content manifest (every snapshot references one; see + // sealed-content-manifests.md). Member-changing writes seal a new + // manifest; other clones inherit the predecessor's by reference. + content_manifest_id: "release-track-content-manifest--uuid", + + // Release-only fields frozen at commit + publication: { + collection_id: "x-mitre-collection--uuid", + created: "2024-01-01T10:00:00.000Z", + created_by_ref: "identity--uuid", + object_marking_refs: ["marking-definition--uuid"], + attack_spec_version: "3.3.0" + }, + bundle_id: "bundle--uuid", + bundle_hashes: { manifest_id: "release-track-content-manifest--uuid", stix_2_0: "…", stix_2_1: "…" }, + // Release track metadata name: "ATT&CK Enterprise", description: "...", created: "2024-01-01T10:00:00.000Z", // when the release track was created - created_by_ref: "identity--uuid", - object_marking_refs: ["marking-definition--uuid"], + created_by_ref: "identity--uuid", // the user account that created the track // Objects in this snapshot members: [ @@ -167,10 +199,6 @@ Each release track snapshot will be tracked as an individual MongoDB Document in config: { candidacy_threshold: "awaiting-review", // "work-in-progress" | "awaiting-review" | "reviewed" auto_promote: true, // Auto-promote reviewed objects to staged - include_secondary_objects: { - enabled: true, - status_threshold: "reviewed" - }, promotion_conflicts: { into_candidates: "prefer_latest", // "always_overwrite" | "always_reject" | "prefer_latest" | "abort" candidates_to_staged: "prefer_latest", // "always_overwrite" | "always_reject" | "prefer_latest" @@ -184,6 +212,16 @@ Each release track snapshot will be tracked as an individual MongoDB Document in behavior: "replace", // "replace" | "queue" | "ignore" status_policy: "reset" // "reset" | "preserve" } + }, + // Publication metadata for the emitted x-mitre-collection object. Each + // attribute inherits the global system configuration unless overridden. + // collection_id and created default to track-derived values and become + // immutable once the track has a tagged release. + publication: { + collection_id: "x-mitre-collection--uuid", // optional override + created: "2018-01-17T12:56:55.080Z", // optional override + created_by_ref: { inherit: true }, // or { inherit: false, value: "identity--uuid" } + object_marking_refs: { inherit: true } // or { inherit: false, value: ["marking-definition--uuid"] } } }, @@ -205,11 +243,12 @@ Each release track snapshot will be tracked as an individual MongoDB Document in } ``` -`snapshot_description` is mutable workspace metadata stored directly on the -snapshot document. It is deliberately separate from the release track's -long-lived `description`. Editing it does not change `modified`, `version`, -tier contents, or an attached graph manifest. Rolling edits to the same draft -preserve its description; the first draft of a new release cycle starts blank. +`snapshot_description` is stored directly on the snapshot document and is +deliberately separate from the release track's long-lived `description`. It +becomes the emitted collection object's `description`. Editing it on a draft +does not change `modified`, tier contents, or the content manifest; once the +snapshot is released it is immutable. Rolling edits to the same draft preserve +its description; the first draft of a new release cycle starts blank. ### Version History @@ -304,7 +343,6 @@ Virtual release tracks compute their contents by aggregating objects from compon description: "Virtual aggregation of Enterprise content across multiple source tracks", created: "2024-01-01T10:00:00.000Z", created_by_ref: "identity--uuid", - object_marking_refs: ["marking-definition--uuid"], // Objects in this snapshot (Virtual tracks use 2-tier system) members: [ @@ -452,16 +490,61 @@ copies the exact member revisions from the selected tagged component snapshots, and later component activity cannot change the persisted virtual snapshot. -A tagged snapshot may optionally reference an internal schema-v2 member graph -manifest. `POST /api/release-tracks/:id/snapshots/:modified/graph` closes the -graph over exact `members` and stores exact-revision pointers for those roots, -relationships whose two endpoint revisions are members, versioned supporting -objects, and LinkById targets. Ordinary graphs contain no relationship-added -secondary SDOs. Only unversioned supporting objects such as marking definitions -retain a frozen payload. Drafts are always graphless. A tagged snapshot without -a manifest is exportable, but graph relationships and secondary objects are -resolved live. Exports that include `candidates` or `staged` are also live even -when the tagged snapshot has a member manifest. +Every snapshot references a sealed content manifest (`content_manifest_id`) +from birth. The manifest closes over exact `members` and stores exact-revision +pointers for those roots, relationships whose source and target IDs are both +members (pinned to the member revisions), versioned supporting objects, and +LinkById targets; only unversioned marking definitions retain a frozen +payload. No relationship-discovered secondary SDO is ever added. Writes that +change `members` seal a new manifest; other clones inherit their +predecessor's. A standard release commit reseals over the planned members; +a virtual commit publishes the materialization manifest unchanged. See +[sealed-content-manifests.md](sealed-content-manifests.md). + +### Content Manifest Schema + +```javascript +// releaseTrackContentManifests +{ + manifest_id: "release-track-content-manifest--uuid", + track_id: "release-track--uuid", + snapshot_modified: "2024-01-15T16:20:00.000Z", // the write that sealed it + state: "active", // "pending" while entries are written; both protect pointers + schema_version: 2, // 2 = pointer-only; 1 = legacy July 2026 frozen-relationship backfill + seal_reason: "release", // track_creation | members_written | release | materialization | + // track_clone | source_reconstruction | migration | legacy_graph + source_attestation: { // source_reconstruction only: the verified bundle the pointers came from + kind: "source-bundle", bundle_sha256: "…", collection_id: "x-mitre-collection--…", + release: "19.1", domain: "enterprise-attack" + }, + created_at: "2024-01-15T16:20:00.100Z" +} + +// releaseTrackContentManifestEntries (one per exact revision) +{ + manifest_id: "release-track-content-manifest--uuid", + track_id: "release-track--uuid", + snapshot_modified: "2024-01-15T16:20:00.000Z", + revision_key: "attack-pattern--aaa::1704880800000", + kind: "root", // root | relationship | supporting | link_target | secondary (legacy) + tier: "members", // root entries only + object_ref: "attack-pattern--aaa", + object_modified: "2024-01-10T10:00:00.000Z", + source: { object_ref, object_modified }, // relationship entries: the member revisions shipped + target: { object_ref, object_modified }, + omitted_optional_defaults: ["revoked"], // source_reconstruction serialization hints only + frozen_stix: { ... } // unversioned marking definitions only +} +``` + +`seal_reason` records which write produced the manifest. `migration` marks a +manifest sealed from the current database by the 2026-09-02 migration rather +than at the time of the original write, and `legacy_graph` marks a manifest +created by the retired opt-in graph endpoint; neither is a historically exact +capture. `source_reconstruction` manifests carry the administrator's +attestation. `state` exists for crash safety: entries are written and verified +under a `pending` manifest before the snapshot references it, and replay +activates a linked pending manifest opportunistically. The three valid `snapshot_schedule` shapes are: diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 7e6ab153..62e21bac 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -54,11 +54,12 @@ reserved for upgrade paths between stable releases. uniqueness remains track-wide. Commits acquire a per-track registry lock so separate API processes cannot validate and write incompatible tags from the same stale bounds; abandoned locks become reclaimable after 15 minutes. -- Snapshot descriptions are bounded to 4000 characters and are the narrow - mutable-metadata exception to snapshot content immutability. They are stored - as `snapshot_description` on the selected document and never update the - registry or the track-level `description`. Bundle exports map the local value - to `x-mitre-collection.description`, falling back to the track description. +- Snapshot descriptions are bounded to 4000 characters and are editable only + on drafts; a released snapshot is immutable including its notes. They are + stored as `snapshot_description` on the selected document and never update + the registry or the track-level `description`. Bundle exports map the local + value to `x-mitre-collection.description`, falling back to the track + description. ### ATT&CK canonical-domain migration @@ -240,15 +241,11 @@ a standard component track. Snapshot retrieval never re-runs composition, so there is no `resolve` query parameter or `resolved_content` response wrapper. Workbench retrieval returns -the persisted primary membership. Bundle export replays a graph only after a -tagged snapshot explicitly opts in; otherwise it resolves the current bounded -graph. Persisted graphs close over exact `members`: relationship revisions -carry server-controlled exact endpoint pins in -`workspace.relationship_endpoints` and are included only when both pinned -revisions are members. Schema-v2 manifests reference those exact revisions -without emitting the internal fields in STIX output. The preceding tagged -graph seeds still-valid relationship pointers so source-attested legacy -provenance can continue into later releases. +the persisted primary membership. Bundle export always replays the snapshot's sealed content manifest. A +materialized virtual draft is sealed at materialization and that manifest is +published unchanged at release; relationship revisions are selected by member +ID closure and pinned to the member revisions. See +[sealed-content-manifests.md](sealed-content-manifests.md). Snapshot schedules use the same strict, mode-discriminated Zod schema at the controller and service boundaries. `manual` has no selector field, `cron` @@ -376,13 +373,13 @@ ambiguous. An omitted `tagged` parameter adds no version predicate; `tagged=true` matches string versions and `tagged=false` matches null draft versions. -For summaries with `graph_manifest_id`, the snapshot service collects all -manifest IDs from the paginated result and performs one aggregation against -`releaseTrackGraphManifestEntries`, grouped by `manifest_id` and `kind`. The -existing `{ manifest_id: 1, kind: 1, tier: 1 }` index supports the match. The -service fills zero-valued categories for empty graphs and attaches -`graph_statistics` only to cached snapshots. This keeps history latency to one -additional bounded query rather than one query per snapshot. +For `content_statistics`, the snapshot service collects every +`content_manifest_id` from the paginated result and performs one aggregation +against `releaseTrackContentManifestEntries`, grouped by `manifest_id` and +`kind`. The existing `{ manifest_id: 1, kind: 1, tier: 1 }` index supports the +match, and shared manifests are counted once. The service fills zero-valued +categories for empty manifests. This keeps history latency to one additional +bounded query rather than one query per snapshot. ## Integrating with the Event-Driven Architecture diff --git a/docs/developer/release-tracks/member-sync-strategies.md b/docs/developer/release-tracks/member-sync-strategies.md index ed601a37..720d4cf8 100644 --- a/docs/developer/release-tracks/member-sync-strategies.md +++ b/docs/developer/release-tracks/member-sync-strategies.md @@ -97,7 +97,7 @@ sync strategy determines what workflow action (if any) to take: > manually created `"latest"` candidate/staged selector still follows the > object by definition. > Relationships are deliberately excluded from sync — bundle export pulls -> active relationships dynamically. +> relationships: sealed content manifests select them by member closure whenever a snapshot's members are written. > > **Behavior evolution (2026-07-13):** further change-capture rules, all > placement decisions now centralized in the **workflow gate** diff --git a/docs/developer/release-tracks/sealed-content-manifests.md b/docs/developer/release-tracks/sealed-content-manifests.md new file mode 100644 index 00000000..03b5f9df --- /dev/null +++ b/docs/developer/release-tracks/sealed-content-manifests.md @@ -0,0 +1,126 @@ +# ADR: Sealed snapshot content manifests + +Status: accepted 2026-09-02. Supersedes the opt-in "deterministic graph" +cache described in earlier revisions of [bundle-export.md](bundle-export.md). + +## Context + +Release-track bundle export had two paths. A tagged snapshot that had opted +into a graph manifest replayed pointers; every other export resolved a +"bounded" ATT&CK graph live, including relationship-discovered secondary +objects. Editors could delete and recreate the manifest, which regenerated the +frozen `x-mitre-collection` object with a new `modified` timestamp and new +bundle hashes. The collection object's `created`, `modified`, +`created_by_ref`, and `object_marking_refs` were each derived differently on +the two paths, and the track-level `object_marking_refs` field was never read +by either. + +Separately, every new SDO revision cloned every active relationship pinned to +the previous revision so that relationship revisions stayed paired 1:1 with +endpoint revisions. The frontend also created new WIP revisions of both +endpoint objects on every relationship save. Editing one relationship +description therefore produced three revisions of that relationship, two +endpoint revisions, and a clone of every other relationship touching either +endpoint. + +## Decisions + +1. **Every snapshot owns a sealed content manifest from birth.** A manifest + is computed whenever a snapshot's `members` tier is written: release + commit, virtual materialization, bundle import, quarantine promotion, track + clone, and track creation. Snapshots produced by candidate, staged, config, + and metadata clones inherit the predecessor's manifest by reference. A + manifest is deleted only when no snapshot in its track references it. +2. **One graph algorithm.** Roots are the exact `members` revisions. A + relationship is selected when its `source_ref` and `target_ref` are both + member IDs; the newest revision of each relationship lineage is chosen and + discarded if it is revoked, deprecated, or a deprecated pattern. The member + revisions are recorded as the manifest entry's exact endpoint pins. + Identities and marking definitions referenced by emitted objects are + supporting entries; LinkById targets outside the bundle are non-emitted + `link_target` entries. No secondary SDO is ever discovered through a + relationship. The bounded resolver survives only behind the deprecated + ephemeral and legacy endpoints. +3. **Relationship revisions are no longer cloned when an endpoint advances.** + `workspace.relationship_endpoints` remains as authoring context recorded on + create, and the release preview flags relationships whose authored endpoint + revision differs from the member revision being shipped. Exact pairing for + a release lives in the sealed manifest. +4. **The `x-mitre-collection` object is a projection, not a stored object.** + It is rendered at export from the snapshot and its manifest. The TAXII + server and other consumers read it from emitted STIX 2.1 bundles, so it is + always present in STIX 2.1 output and never present in STIX 2.0 output. + - `id`: `config.publication.collection_id`, defaulting to + `x-mitre-collection--`; immutable once the track has a release. + - `created`: `config.publication.created`, defaulting to the track's + `created`; immutable once the track has a release. + - `modified`: the snapshot's `modified`. + - `x_mitre_version`: the tagged version. Drafts omit the key. ATT&CK + requires the field, so draft bundles are previews that do not conform to + the ATT&CK specification; a placeholder such as `0.1` collides with a + legitimate first release and is a lie about publication state. + - `created_by_ref` and `object_marking_refs`: resolved through the + publication inheritance rule below. + - `description`: the snapshot's `snapshot_description`, falling back to + the track description. + - `x_mitre_contents`: every emitted object except marking definitions. +5. **Publication metadata inherits from the global scope unless overridden + at the track scope.** `config.publication.created_by_ref` and + `config.publication.object_marking_refs` each take the shape + `{ inherit: true }` (default) or `{ inherit: false, value }`. Inherited + values come from the organization identity and the default marking + definitions in system configuration. Drafts resolve the rule at export so + they preview the current configuration. Release commit freezes the + resolved values into the tagged snapshot's `publication` field, so a later + change to global or track configuration cannot alter a published release. + The former top-level track `object_marking_refs` field is migrated into + this rule and removed. +6. **Release commit seals.** A standard commit computes the manifest over the + planned member set at commit time, so relationships added since the last + seal are captured even when nothing was staged. A virtual commit publishes + the materialization manifest unchanged, because the materialized draft is + the artifact that was reviewed. Commit also stores a stable `bundle_id` + and SHA-256 hashes of both serializations on the tagged snapshot. Draft + bundles use a deterministic UUIDv5 derived from the track ID and snapshot + `modified`; the bundle ID changes across snapshots while the collection ID + stays constant per track. +7. **Tagged snapshots are immutable including notes.** `snapshot_description` + is editable on drafts only. The graph create and delete endpoints are + removed. The admin-only source-attested reconstruction endpoint remains + and can replace an existing manifest when the caller names the manifest it + expects to replace. The correction path for a mistaken release is + deletion: an administrator may delete the track's most recent release with + a typed version confirmation, which retracts its ledger entry, discards its + manifest when unreferenced, and is audited as `delete_release`. +8. **Storage is named for what it holds.** Manifests live in + `releaseTrackContentManifests` and `releaseTrackContentManifestEntries` + with `release-track-content-manifest--` ids. A manifest header carries + `seal_reason` (which write produced it), `schema_version` (2 for pointer + manifests, 1 for the legacy frozen-relationship backfill), `state`, and + the optional `source_attestation`; the former `resolver_version` and + `baseline_reconstruction` fields are gone. `releaseTrackReconciliations` + holds outstanding backref work only and is normally empty. See the + collections table in [entities.md](entities.md). +8. **`include=staged,candidates` is a draft-only preview.** Included tier + entries are resolved live and the same closure rule runs over members plus + the included entries. Requesting `include` on a tagged snapshot is a `400`. + +## Consequences + +- Determinism is unconditional: exporting a tagged snapshot replays pointers + and never queries relationships, and a draft replays its inherited members + graph. +- Manifest storage is bounded by the number of member-changing writes, not + the number of snapshots. Standard tracks already keep only one draft. +- Editing an object creates no relationship revisions; editing a relationship + creates exactly one relationship revision. +- Existing databases are migrated in place: the `graph_manifest_id` field is + renamed, tagged snapshots without a manifest are sealed and labeled as + baseline reconstructions, drafts inherit or seal, publication values are + frozen onto tagged snapshots, and bundle hashes are recomputed. Release-track + exports in production had not been published externally (publication still + used the legacy `GET /api/stix-bundles` endpoint), so recomputing hashes + does not invalidate a distributed artifact. +- Tracks intended to replace a legacy domain bundle set + `config.publication.collection_id` and `config.publication.created` to the + canonical ATT&CK values before their first release. diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 3686193b..620e4d60 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -142,14 +142,14 @@ self-contained. **Query Parameters:** -| Parameter | Values | Default | Description | -| ----------------------------------- | -------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `format` | `bundle` \| `workbench` \| `filesystemstore` | `bundle` | Output format (`filesystemstore` is not yet implemented) | -| `stixVersion` | `2.0` \| `2.1` | `2.1` | STIX version the emitted bundle conforms to (bundle format only) | +| Parameter | Values | Default | Description | +| ----------------------------------- | -------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `format` | `bundle` \| `workbench` \| `filesystemstore` | `bundle` | Output format (`filesystemstore` is not yet implemented) | +| `stixVersion` | `2.0` \| `2.1` | `2.1` | STIX version the emitted bundle conforms to (bundle format only) | | `includeToc` | `true` \| `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) in STIX 2.1. STIX 2.0 always omits it. The TOC uses `x_mitre_version: "0.1"`, the current timestamp, and the deployment's default ATT&CK spec version. | -| `includeObjectsWithMissingAttackId` | `true` \| `false` | `false` | Include objects that should have an ATT&CK ID set but do not | -| `includeDeprecated` | `true` \| `false` | `false` | Include objects with `x_mitre_deprecated: true` (this also governs deprecated Data Sources) | -| `includeRevoked` | `true` \| `false` | `false` | Include objects with `revoked: true` | +| `includeObjectsWithMissingAttackId` | `true` \| `false` | `false` | Include objects that should have an ATT&CK ID set but do not | +| `includeDeprecated` | `true` \| `false` | `false` | Include objects with `x_mitre_deprecated: true` (this also governs deprecated Data Sources) | +| `includeRevoked` | `true` \| `false` | `false` | Include objects with `revoked: true` | > [!Note] > The ephemeral endpoint does not support the `include` or `state` tier @@ -363,12 +363,11 @@ Workbench responses return the release-track snapshot shape. Entries in the `mem **Additional query parameters for `format=bundle`:** -| Parameter | Values | Description | -| ------------- | ------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `include` | `staged` and/or `candidates` (comma-separated or repeated) | Additional tiers to include in the bundle alongside members. If omitted, only members are included. (Note the different semantics from `workbench` responses.) | -| `state` | `work-in-progress` and/or `awaiting-review` (comma-separated or repeated) | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included. Members are unaffected. | -| `stixVersion` | `2.0` \| `2.1` | STIX version the emitted bundle conforms to (default: `2.1`) | -| `includeToc` | `true` \| `false` | Include a table-of-contents object (of type `x-mitre-collection`) in STIX 2.1, derived from release-track metadata (default: `true`). STIX 2.0 always omits it. | +| Parameter | Values | Description | +| ------------- | ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `include` | `staged` and/or `candidates` (comma-separated or repeated) | Draft-only preview: additional tiers to include alongside members, resolved live. Released snapshots reject it with `400`. (Different semantics from `workbench`.) | +| `state` | `work-in-progress` and/or `awaiting-review` (comma-separated or repeated) | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included. Members are unaffected. | +| `stixVersion` | `2.0` \| `2.1` | STIX version the emitted bundle conforms to (default: `2.1`). STIX 2.1 bundles always begin with the `x-mitre-collection` object; STIX 2.0 omits it. | See [Output Formats](output-formats.md) for details on the bundle structure. @@ -420,23 +419,21 @@ of snapshots matching `tagged`, not the total number in the track. Every summary contains `id`, `type`, `modified`, `version`, `name`, the track-level `description` (when set), `snapshot_description` (when the snapshot -has user-authored notes), and `members_count`. A tagged snapshot whose -deterministic member graph has been materialized also contains the opaque -`graph_manifest_id`, `graph_statistics`, and `bundle_hashes`; graphless -snapshots omit all three. `bundle_hashes` contains the manifest ID plus the -SHA-256 digests in `stix_2_0` and `stix_2_1` for the exact four-space-indented -UTF-8 JSON files downloaded by the browser. -Graph statistics describe the cached graph at a glance: +has user-authored notes), `members_count`, the opaque `content_manifest_id` +of the snapshot's sealed content manifest, and `content_statistics`. Released +snapshots also contain `bundle_id` and `bundle_hashes`; `bundle_hashes` +contains the manifest ID plus the SHA-256 digests in `stix_2_0` and +`stix_2_1` for the exact four-space-indented UTF-8 JSON files downloaded by +the browser. Content statistics describe the sealed manifest at a glance: - `primary_count`: member objects deliberately selected for the snapshot. -- `secondary_count`: source-attested historical non-member objects. Ordinary - deterministic member graphs report zero because relationships do not expand - SDO membership. -- `relationship_count`: relationships connecting cached graph objects. +- `secondary_count`: legacy source-attested historical non-member objects. + Sealed manifests report zero because relationships never expand SDO + membership. +- `relationship_count`: relationships whose source and target are both members. - `supporting_count`: supporting identities and marking definitions. - `link_target_count`: objects pinned for deterministic LinkById expansion. -- `total_count`: all emitted dependency entries across those manifest roles; - the collection metadata entry is excluded. +- `total_count`: all entries across those manifest roles. The UI groups supporting and LinkById targets together as **Dependencies**. Snapshot tier count keys continue to reflect the track type: @@ -454,12 +451,13 @@ Inapplicable count keys are omitted rather than returned as zero. "type": "standard", "modified": "2024-01-15T16:20:00.000Z", "version": "14.1", - "graph_manifest_id": "release-track-graph-manifest--01234567-89ab-4cde-8f01-23456789abcd", + "content_manifest_id": "release-track-content-manifest--01234567-89ab-4cde-8f01-23456789abcd", + "bundle_id": "bundle--0f9d2a4e-1c3b-4b7e-9a6d-8e5f4c3b2a10", "name": "Enterprise ATT&CK", "description": "Enterprise domain release track", "snapshot_description": "Reviewed publication for the Q1 threat model.", "members_count": 3247, - "graph_statistics": { + "content_statistics": { "primary_count": 3247, "secondary_count": 0, "relationship_count": 6841, @@ -508,18 +506,18 @@ PUT /api/release-tracks/:id/snapshots/:modified/description The value is trimmed and limited to 4000 characters. Send an empty string to clear it. The API returns the updated snapshot as `snapshot_description` and -does not change the snapshot's `modified` timestamp, semantic version, tier -contents, or the release track's long-lived description. Cached snapshots are -immutable: this endpoint returns `409 Conflict` while a graph manifest exists. -Delete the bundle cache, edit the notes, and cache the bundle again to generate -a new frozen collection object and matching hashes. +does not change the snapshot's `modified` timestamp, tier contents, or the +release track's long-lived description. Notes become the emitted collection +object's `description`, so a released snapshot is immutable and this endpoint +returns `409 Conflict`; set release notes through the release request's +`description` instead. ### Update Metadata A user or team may wish to: - rename a release (e.g., fix a typo like `"Entrprise"` to `"Enterprise"`) or shift the scope/purpose of an existing release track without losing its history (though [cloning](#clone-latest-snapshot) is preferred in this scenario) -- update metadata (which at present consists of a `description` field, `object_marking_references` (typically only includes the global marking definition) and the author (`created_by_ref`). +- update the long-lived `description`. Publication metadata for the emitted collection object (identity, markings, collection ID, creation time) lives in the track configuration; see [Publication configuration](#publication-configuration). ``` POST /api/release-tracks/:id/meta @@ -532,9 +530,7 @@ Creates new snapshot with updated metadata. ```json { "name": "Updated Name", - "description": "Updated description", - "external_references": [], - "object_marking_refs": [] + "description": "Updated description" } ``` @@ -667,7 +663,7 @@ GET /api/release-tracks/:id/snapshots/:modified For `format=bundle`, the same additional parameters as [Get Latest Snapshot](#get-latest-snapshot) apply: `include` (bundle -semantics), `state`, `stixVersion`, and `includeToc`. +semantics, drafts only), `state`, and `stixVersion`. **Example:** @@ -700,54 +696,42 @@ Bootstraps a new release track from the specified snapshot. POST /api/release-tracks/:id/snapshots/:modified/clone ``` -### Create or Delete a Deterministic Member Graph - -``` -POST /api/release-tracks/:id/snapshots/:modified/graph -POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct -DELETE /api/release-tracks/:id/snapshots/:modified/graph -``` - -Only tagged snapshots may have graphs. POST resolves the snapshot's `members` -into a pointer-only exact-revision manifest and returns `201`; repeating it is -idempotent and returns `200`. DELETE removes the manifest and returns `204` -even when no graph exists. Ordinary graph creation emits only member SDO -revisions and relationships whose two exact stored endpoint revisions are both -members. It never follows a relationship to add a secondary SDO or a newer -revision of an existing member. Graphless bundles resolve relationships and -secondary objects live. Requests that include candidates or staged objects -remain live even if the tagged snapshot has a graph. - -When the immediately preceding tagged snapshot has a graph, its still-valid -relationship pointers seed the new graph. Current exact relationship revisions -are selected through indexed endpoint lookups and take precedence. This lets a -source-attested historical baseline anchor later releases without preventing -new relationships between unchanged members from being discovered. - -User interfaces may present this operation as **caching the bundle**: a cached -indicator means member-only bundle exports reuse the exact object and -relationship revisions selected when the cache was created. This is not a -general response cache and does not make candidate or staged exports -deterministic. - -Graph creation also stores one stateful `x-mitre-collection` manifest entry. -Its ID is stable for the release track, `created` comes from the track's first -cached collection object, `created_by_ref` is the configured organization -identity's STIX ID, and `modified` is the current manifest creation time. The -collection object is emitted only in STIX 2.1. The graph-backed bundle envelope -uses the manifest UUID, so repeated STIX 2.0 or STIX 2.1 downloads are -byte-for-byte stable. The graph-creation response and snapshot history expose -SHA-256 hashes for both exact download files. - -Administrators may use the separate `/graph/reconstruct` POST for a historical -baseline backed by an independently verified source bundle. The request sends -the bundle's SHA-256/collection/release/domain attestation plus exact graph -pointers; it does not import source STIX payloads. The server rejects plans -whose roots differ from `members`, whose revisions are missing, or whose -relationship endpoints are inconsistent. This recovery endpoint exists for -controlled bootstrap tooling and is not a replacement for ordinary graph -creation. A retry is idempotent only when the attached graph has the same -source attestation. +### Sealed content manifests + +Every snapshot references a sealed content manifest (`content_manifest_id`) +from the moment it is created. The manifest is the bill of materials that +bundle export replays: exact member revisions, relationships whose source and +target are both members (pinned to those member revisions), supporting +identities and marking definitions, and non-emitted LinkById targets. It never +adds a secondary SDO through a relationship and never follows a relationship +to a newer revision of a member. + +A new manifest is sealed whenever a snapshot's members are written: release, +virtual materialization, bundle import, quarantine promotion, and track +cloning. Candidate, staged, configuration, and metadata changes inherit the +previous manifest by reference. Releasing a standard track reseals over the +final member set, so relationships created since the previous release ship; +the release preview reports them under `relationships` (`added`, `removed`, +`stale_endpoints`, and counts). A stale endpoint means the relationship was +authored against a different revision of an endpoint than the member revision +being shipped. Releasing a virtual track publishes the materialization +manifest unchanged. + +Released snapshots also carry `publication` (the frozen collection metadata), +a stable `bundle_id`, and `bundle_hashes` with SHA-256 values for both exact +download files. There is no operation to delete or regenerate a manifest; +a correction is a new release. + +Administrators may use `POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct` +for a historical baseline backed by an independently verified source bundle. +The request sends the bundle's SHA-256/collection/release/domain attestation +plus exact graph pointers; it does not import source STIX payloads. Because +the snapshot already references a sealed manifest, the request must name it in +`replace_manifest_id`; repeating the same attestation is idempotent and any +other current manifest is rejected with `409`. The server rejects plans whose +roots differ from `members`, whose revisions are missing, or whose relationship +endpoints are inconsistent, and recomputes the bundle hashes after replacement. +This recovery endpoint exists for controlled bootstrap tooling. Pointer roles may also include `link_target`: an exact, non-emitted dependency used only to render historical `(LinkById: ...)` fields deterministically. @@ -756,6 +740,45 @@ An entry may carry `omitted_optional_defaults` containing `revoked` and/or false-valued defaults. This is a serialization-shape hint, not frozen STIX content; all other fields still come from the exact persisted revision. +### Publication configuration + +The emitted `x-mitre-collection` object's metadata follows an inheritance +rule configured under `config.publication` (see +[Get Config](#get-configuration) and [Update Config](#update-configuration)): + +```json +{ + "publication": { + "collection_id": "x-mitre-collection--1f5f1533-f617-4ca8-9ab4-6a02367fa019", + "created": "2018-01-17T12:56:55.080Z", + "created_by_ref": { + "inherit": false, + "value": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5" + }, + "object_marking_refs": { "inherit": true } + } +} +``` + +- `created_by_ref` and `object_marking_refs` each take `{ "inherit": true }` + (the default) to use the organization identity or default marking + definitions from the global system configuration, or + `{ "inherit": false, "value": ... }` for a track-scoped override. When + neither scope configures markings, the collection object carries the + marking definitions referenced by its contents. +- `collection_id` and `created` are optional overrides. They default to a + collection ID derived from the track UUID and the track creation time. + Tracks that replace a legacy ATT&CK domain bundle set them to the canonical + values before the first release. Once the track has a tagged release, + changing either returns `409 Conflict`; `null` clears an override. +- `GET /config` also returns `publication_resolved`: the values currently in + effect for the latest snapshot and, under `sources`, whether each came from + the `track`, the `global` scope, was `derived`, or (for markings) falls back + to `content`. +- Drafts resolve the rule at export so they preview the current + configuration. Release freezes the resolved values onto the tagged snapshot + as `publication`, so later changes never alter a published release. + ### Delete Specific Snapshot ``` @@ -1035,7 +1058,8 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview one selector over the other - `include` - for `workbench`, selects returned tiers; for `bundle`, selects additional non-member tiers -- `state`, `stixVersion`, `includeToc` - bundle representation options +- `state`, `stixVersion` - bundle representation options; summary previews of + standard tracks add `relationships` describing what the release would seal **Response Example:** diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index 3641015e..e611c8ba 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -102,32 +102,32 @@ Standard STIX bundle format: - Self-contained: identities and marking definitions referenced by the exported objects are included automatically - `LinkById` tags in descriptions are converted to markdown citations -- Drafts, graphless tagged snapshots, and every export that includes candidate - or staged tiers resolve the bounded graph live. A tagged member-only export - is deterministic only after its snapshot opts into a graph manifest. That - manifest is closed over exact members: relationships are included only when - both exact endpoint revisions are members, and do not add secondary SDOs. -- Frontends may describe manifest creation as **caching the bundle**. The - cache pins the exact member graph for repeatable export; it is not a general - performance cache, and candidate or staged additions remain live. +- Every export replays the snapshot's sealed content manifest: exact member + revisions, relationships whose source and target are both members (pinned + to those member revisions), supporting objects, and LinkById targets. No + secondary SDO is discovered through a relationship. A draft that adds + candidate or staged tiers through `include` is a preview that resolves the + same closed graph live; released snapshots reject `include`. +- Released snapshots carry a stable `bundle_id` and SHA-256 `bundle_hashes` + for both serializations; repeated downloads are byte-for-byte identical. + Draft bundles use a deterministic identifier derived from the snapshot. - Bundle export is fail-closed for primary content. If any selected exact revision no longer exists, the server returns HTTP `409` with every missing `(object_ref, object_modified)` pair in `missing_references`; it never emits a partial bundle. A repository/database failure is returned as a server error rather than being mistaken for missing content. - Workbench note objects are never included. The snapshot's own - `snapshot_description` is publication metadata and becomes the TOC - `description`. + `snapshot_description` is publication metadata and becomes the collection + object's `description`. - Suitable for external publication **Bundle query parameters** (apply only when `format=bundle`): | Parameter | Values | Default | Description | | ------------- | ------------------------------------------------------------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `include` | `staged`, `candidates` (comma-separated or repeated) | _(members only)_ | Additional tiers to include in the bundle alongside members | +| `include` | `staged`, `candidates` (comma-separated or repeated) | _(members only)_ | Draft-only preview: additional tiers to include alongside members, resolved live. Released snapshots reject it with `400`. | | `state` | `work-in-progress`, `awaiting-review` (comma-separated or repeated) | _(no filter)_ | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included, irrespective of this parameter. Members are unaffected. | -| `stixVersion` | `2.0`, `2.1` | `2.1` | STIX version the emitted bundle conforms to | -| `includeToc` | `true`, `false` | `true` | Include a table-of-contents object (of type `x-mitre-collection`) as the first object in STIX 2.1 bundles. STIX 2.0 bundles never include it. | +| `stixVersion` | `2.0`, `2.1` | `2.1` | STIX version the emitted bundle conforms to. STIX 2.1 bundles always begin with the `x-mitre-collection` object; STIX 2.0 bundles never include it. | Examples: @@ -145,23 +145,40 @@ GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,st GET /api/release-tracks/:id/snapshots/latest?format=bundle&stixVersion=2.0 ``` -**The table of contents (TOC) object** - -By default, STIX 2.1 bundles begin with an `x-mitre-collection` object that -acts as a table of contents. STIX 2.0 bundles omit this ATT&CK extension object -regardless of `includeToc`. The STIX 2.1 object is derived from the -release-track metadata: - -- `id` — stable per track (reuses the track UUID) -- `created_by_ref` — the deployment's configured organization identity -- `name` — from the release track snapshot -- `description` — from the snapshot's `snapshot_description`; falls back to - the long-lived track `description` when no snapshot-local value is set -- `x_mitre_version` — the snapshot's tagged version, or `0.1` for draft snapshots -- `modified` — the snapshot's modified timestamp +**The collection object** + +Every STIX 2.1 bundle begins with an `x-mitre-collection` object, the bundle's +bill of materials. Downstream consumers (the TAXII server among them) read it +from the emitted bundle, so it is always present in STIX 2.1 output. STIX 2.0 +bundles omit this ATT&CK extension object. The object is projected from the +snapshot and the track's publication configuration: + +- `id` — the track's configured `publication.collection_id`, defaulting to a + value derived from the track UUID; constant across every snapshot of the + track +- `created` — the track's configured `publication.created`, defaulting to the + track creation time +- `modified` — the snapshot's `modified` timestamp +- `x_mitre_version` — the tagged version. Draft bundles omit the key: a draft + has no publication version, and a placeholder would collide with a real + first release. Draft bundles are therefore previews that do not conform to + the ATT&CK specification's required-field rule. +- `created_by_ref` — the track's publication identity, inheriting the + deployment's organization identity unless the track overrides it +- `object_marking_refs` — the track's publication markings, inheriting the + deployment's default marking definitions unless the track overrides them. + When neither scope configures markings, the object carries the marking + definitions referenced by its contents. +- `name` — the release track name +- `description` — the snapshot's `snapshot_description`; falls back to the + long-lived track `description` when no snapshot-local value is set - `x_mitre_attack_spec_version` — the deployment's default ATT&CK spec version -- `x_mitre_contents` — every object in the bundle (marking definitions are - recorded in `object_marking_refs` instead) +- `x_mitre_contents` — every object in the bundle except marking definitions + +Release commit freezes the resolved identity, markings, collection ID, +creation time, and spec version onto the released snapshot, so later +configuration changes never alter a published release. See +[Publication configuration](api-reference.md#publication-configuration). ### Format: `filesystemstore` (Not Implemented) diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index 06d4df9c..ad8f3b2d 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -92,16 +92,14 @@ the preview or commit request is handled. Only exact revisions are promoted into `members`, so the tagged release never contains a dynamic member reference. -Releasing does not automatically create a graph manifest. A tagged snapshot -may subsequently opt into deterministic member-graph retrieval with: - -```http -POST /api/release-tracks/:id/snapshots/:modified/graph -``` - -Deleting that manifest with the corresponding `DELETE` operation returns the -snapshot to live graph resolution. Candidate and staged export additions are -always resolved live; the determinism guarantee applies only to `members`. +Releasing a standard track seals a fresh content manifest over the final +member set, so the relationships shipped are exactly those connecting members +at the moment of release; the release preview lists the relationships that +seal would add or remove. Releasing a virtual track publishes the manifest +sealed at materialization. Release also freezes the collection object's +publication metadata, assigns a stable bundle identifier, and records SHA-256 +hashes of both bundle serializations. A released snapshot is immutable, +including its notes. ### In-Place Tagging Strategy @@ -111,6 +109,8 @@ When you release a snapshot: 2. `version` is set to the new version 3. An entry is added to `version_history` for audit trail 4. The `modified` timestamp **does not change** +5. For standard tracks, staged objects are promoted into `members` and a + content manifest is sealed over the result in the same atomic update **Why in-place?** diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index b57dd78f..34127c42 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -231,12 +231,11 @@ filter and a Mobile filter, while `["mobile-attack"]` is excluded by an Enterprise filter. Objects without `x_mitre_domains` are excluded when a domain filter is set. -The domain constraint determines the virtual snapshot's exact member set. An -opt-in deterministic graph is closed over that set, so no relationship can -pull any secondary SDO into the virtual bundle. Graphless live exports retain -the compatibility domain check for relationship-discovered secondaries. -Domainless identities, marking definitions, and other supporting metadata may -still be included when referenced by an included object. +The domain constraint determines the virtual snapshot's exact member set. The +content manifest sealed at materialization is closed over that set, so no +relationship can pull any secondary SDO into the virtual bundle. Domainless +identities, marking definitions, and other supporting metadata may still be +included when referenced by an included object. `x_mitre_domains` is canonical object data. A cross-domain object has one revision containing the complete domain union; Workbench does not create or @@ -987,9 +986,9 @@ quarantined object counts. Use `format=workbench` or `format=bundle` to inspect the literal snapshot or publication artifact that would be tagged. The draft must have a non-null `composition_resolution`, proving that its members and quarantine tiers were materialized from its current composition. -Bundle preview resolves the live graph. Tagging does not implicitly create a -manifest; determinism is a separate opt-in operation on the tagged snapshot: -`POST /api/release-tracks/:id/snapshots/:modified/graph`. +Bundle preview resolves the same closed-member graph live. Tagging publishes +the content manifest sealed at materialization unchanged and freezes the +collection object's publication metadata. ### Retrieve a Materialized Virtual Snapshot @@ -1024,14 +1023,12 @@ Consequently, while the track does not acquire a newer snapshot, `latest` path segment selects the most recent snapshot; it is not a dynamic object-revision selector. -This guarantee also covers `format=bundle` after the tagged snapshot opts into -a graph manifest. The manifest emits only exact members plus relationships -whose two exact endpoint revisions are members; supporting objects and LinkById -render targets are pinned as dependencies. Graphless snapshots resolve the -legacy bounded graph live. -Repeated exports may use a different bundle-envelope UUID, but replay the same -snapshot object graph. See -[Bundle Export](../../developer/release-tracks/bundle-export.md#closed-member-relationship-consistency-boundary). +This guarantee also covers `format=bundle`: materialization seals a content +manifest that emits only exact members plus relationships whose source and +target are both members; supporting objects and LinkById render targets are +pinned as dependencies. Released snapshots also carry a stable bundle +identifier and hashes. See +[Bundle Export](../../developer/release-tracks/bundle-export.md#sealed-content-manifests). ## Quarantine Management diff --git a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js index dc969a24..53f48cf1 100644 --- a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js +++ b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js @@ -2,13 +2,15 @@ /** * Backfill exact endpoint revision pins on the latest revision of each - * relationship, then reconstruct a baseline graph manifest for every - * pre-existing release-track snapshot. + * relationship and establish the manifest indexes. * - * Historical relationships cannot be reconstructed truthfully because their - * endpoint revision was not recorded when they were created. Snapshot - * manifests produced here are therefore explicitly marked as baseline - * reconstructions of the graph visible at migration time. + * This migration originally also reconstructed a baseline graph manifest for + * every pre-existing release-track snapshot using the since-retired bounded + * graph resolver. That step is superseded by + * 20260902120000-seal-release-track-content-manifests.js, which seals a + * content manifest for every snapshot that lacks one. The manifest backfill + * here is therefore a no-op so databases upgrading from an older stable + * release run one algorithm only. */ const TRACK_COLLECTION_PATTERN = @@ -155,83 +157,43 @@ async function findTrackIds(db) { async function ensureManifestIndexes(db) { await Promise.all([ db - .collection('releaseTrackGraphManifests') + .collection('releaseTrackContentManifests') .createIndex({ manifest_id: 1 }, { name: 'manifest_id_1', unique: true }), db - .collection('releaseTrackGraphManifests') + .collection('releaseTrackContentManifests') .createIndex( { track_id: 1, snapshot_modified: 1, state: 1 }, { name: 'manifest_by_snapshot' }, ), db - .collection('releaseTrackGraphManifestEntries') + .collection('releaseTrackContentManifestEntries') .createIndex( { manifest_id: 1, revision_key: 1, kind: 1, tier: 1 }, { name: 'unique_manifest_entry', unique: true }, ), db - .collection('releaseTrackGraphManifestEntries') + .collection('releaseTrackContentManifestEntries') .createIndex( { object_ref: 1, object_modified: 1, manifest_id: 1 }, { name: 'manifest_revision_protection' }, ), db - .collection('releaseTrackGraphManifestEntries') + .collection('releaseTrackContentManifestEntries') .createIndex({ manifest_id: 1, kind: 1, tier: 1 }, { name: 'manifest_id_1_kind_1_tier_1' }), ]); } async function backfillSnapshotManifests(db, options) { - const graphManifestService = require('../app/services/release-tracks/graph-manifest-service'); + // Superseded: sealing every snapshot's content manifest is performed by + // 20260902120000-seal-release-track-content-manifests.js. const trackIds = await findTrackIds(db); - const report = { tracks: trackIds.length, snapshots: 0, manifests_created: 0 }; - - await mapWithConcurrency(trackIds, async (trackId) => { - const collectionExists = await db - .listCollections({ name: trackId }, { nameOnly: true }) - .hasNext(); - if (!collectionExists) return; - - const snapshots = await db.collection(trackId).find({}).toArray(); - report.snapshots += snapshots.length; - for (const snapshot of snapshots) { - if (snapshot.graph_manifest_id) { - const linkedManifest = await db.collection('releaseTrackGraphManifests').findOne({ - manifest_id: snapshot.graph_manifest_id, - state: { $in: ['pending', 'active'] }, - }); - if (linkedManifest) { - if (!options.dryRun && linkedManifest.state === 'pending') { - await graphManifestService.activate(linkedManifest.manifest_id); - } - continue; - } - } - if (options.dryRun) { - report.manifests_created++; - continue; - } - - const manifestId = await graphManifestService.prepare(snapshot, { - baselineReconstruction: true, - // Preserve the historical migration's schema-v1 frozen relationship - // contract. New opt-in graphs use pointer-only schema v2. - schemaVersion: 1, - }); - try { - await db - .collection(trackId) - .updateOne({ _id: snapshot._id }, { $set: { graph_manifest_id: manifestId } }); - await graphManifestService.activate(manifestId); - report.manifests_created++; - } catch (err) { - await graphManifestService.discard(manifestId); - throw err; - } - } - }); - - return report; + return { + tracks: trackIds.length, + snapshots: 0, + manifests_created: 0, + superseded_by: '20260902120000-seal-release-track-content-manifests', + dry_run: options.dryRun === true, + }; } async function run(db, options = {}) { @@ -262,14 +224,14 @@ module.exports = { async up(db) { const report = await run(db); console.log( - `Pinned ${report.relationship_pins_written} active latest relationship revision(s) and ` + - `created ${report.manifests_created} baseline snapshot manifest(s)`, + `Pinned ${report.relationship_pins_written} active latest relationship revision(s); ` + + 'snapshot manifest sealing is performed by the content-manifest migration', ); }, async down(db) { const baselineManifests = await db - .collection('releaseTrackGraphManifests') + .collection('releaseTrackContentManifests') .find({ baseline_reconstruction: true }) .project({ manifest_id: 1, track_id: 1, snapshot_modified: 1, _id: 0 }) .toArray(); @@ -288,10 +250,10 @@ module.exports = { const manifestIds = baselineManifests.map((manifest) => manifest.manifest_id); if (manifestIds.length > 0) { await db - .collection('releaseTrackGraphManifestEntries') + .collection('releaseTrackContentManifestEntries') .deleteMany({ manifest_id: { $in: manifestIds } }); await db - .collection('releaseTrackGraphManifests') + .collection('releaseTrackContentManifests') .deleteMany({ manifest_id: { $in: manifestIds } }); } }, diff --git a/migrations/20260902120000-seal-release-track-content-manifests.js b/migrations/20260902120000-seal-release-track-content-manifests.js new file mode 100644 index 00000000..fe0d543f --- /dev/null +++ b/migrations/20260902120000-seal-release-track-content-manifests.js @@ -0,0 +1,542 @@ +'use strict'; + +/** + * Seal a content manifest for every release-track snapshot and freeze + * publication metadata onto tagged snapshots. + * + * Before this migration a manifest ("graph cache") was an optional, deletable + * attachment on tagged snapshots. Afterwards every snapshot references a + * sealed manifest from birth (see + * docs/developer/release-tracks/sealed-content-manifests.md). Per snapshot: + * + * - `graph_manifest_id` is renamed to `content_manifest_id`. + * - A tagged snapshot without a manifest is sealed from the current + * database and labeled `baseline_reconstruction` because it describes the + * graph visible at migration time, not an unknowable historical graph. + * - A draft without a manifest shares the manifest of the nearest preceding + * tagged snapshot with an identical member set, otherwise it is sealed. + * - The former top-level `object_marking_refs` moves into + * `config.publication.object_marking_refs` as an explicit override. + * - Tagged snapshots receive frozen `publication` values, a `bundle_id` + * (preserving the manifest-derived envelope ID they exported before), and + * recomputed `bundle_hashes`. + * - Frozen `collection` manifest entries are removed; the collection object + * is now a projection rendered at export. + * - Manifest storage moves from `releaseTrackGraphManifest*` to + * `releaseTrackContentManifest*`, manifest ids adopt the + * `release-track-content-manifest--` prefix, `resolver_version` and + * `baseline_reconstruction` are dropped, and `seal_reason` is backfilled. + * - The retired `config.include_secondary_objects` block is removed and + * completed `releaseTrackReconciliations` records are deleted. + * + * Only tracks present in `releaseTrackRegistry` are migrated. A dynamic + * `release-track--*` collection without a registry document is an orphan + * left behind by an interrupted or pre-registry deletion: the API cannot + * list it, its snapshots routinely reference revisions that no longer exist, + * and sealing it would protect stale revisions from deletion. Orphans are + * reported, any manifests they own are discarded, and the collections are + * left in place for an operator to drop. + * + * The migration is idempotent and supports a read-only dry run through + * `_private.run(db, { dryRun: true })`. + */ + +const TRACK_COLLECTION_PATTERN = + /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const LEGACY_MANIFEST_ID_PREFIX = 'release-track-graph-manifest--'; +const MANIFEST_ID_PREFIX = 'release-track-content-manifest--'; +const MANIFESTS = 'releaseTrackContentManifests'; +const ENTRIES = 'releaseTrackContentManifestEntries'; +const LEGACY_COLLECTIONS = { + releaseTrackGraphManifests: MANIFESTS, + releaseTrackGraphManifestEntries: ENTRIES, +}; + +function contentManifestId(manifestId) { + return manifestId?.startsWith(LEGACY_MANIFEST_ID_PREFIX) + ? `${MANIFEST_ID_PREFIX}${manifestId.slice(LEGACY_MANIFEST_ID_PREFIX.length)}` + : manifestId; +} + +async function collectionExists(db, name) { + return db.listCollections({ name }, { nameOnly: true }).hasNext(); +} + +/** + * Move manifests and entries from the former `releaseTrackGraphManifest*` + * collections into their `releaseTrackContentManifest*` successors and + * normalize the manifest header: + * - ids adopt the `release-track-content-manifest--` prefix (bundle ids + * were already frozen onto tagged snapshots, so hashes are unaffected) + * - `resolver_version` and `baseline_reconstruction` are dropped + * - `seal_reason` is backfilled: attested manifests are + * `source_reconstruction`, any other manifest that predates this + * migration is `legacy_graph` + */ +async function normalizeManifestStorage(db, options, report) { + for (const [legacy, target] of Object.entries(LEGACY_COLLECTIONS)) { + if (!(await collectionExists(db, legacy))) continue; + const legacyCount = await db.collection(legacy).countDocuments({}); + report.legacy_manifest_documents_moved += legacyCount; + if (options.dryRun) continue; + if (!(await collectionExists(db, target))) { + if (legacyCount > 0) { + await db.collection(legacy).rename(target); + } else { + await db.collection(legacy).drop(); + } + continue; + } + if (legacyCount > 0) { + const documents = await db.collection(legacy).find({}).toArray(); + try { + await db.collection(target).insertMany(documents, { ordered: false }); + } catch (err) { + if (err.code !== 11000 && !err.writeErrors) throw err; + } + } + await db.collection(legacy).drop(); + } + + const headerFilter = { + $or: [ + { manifest_id: { $regex: `^${LEGACY_MANIFEST_ID_PREFIX}` } }, + { seal_reason: { $exists: false } }, + { resolver_version: { $exists: true } }, + { baseline_reconstruction: { $exists: true } }, + ], + }; + if (options.dryRun) { + // Legacy documents are still in the old collections during a dry run. + report.manifest_headers_normalized += await countAcross(db, 'manifests', headerFilter); + return; + } + if (!(await collectionExists(db, MANIFESTS))) return; + const manifests = await db + .collection(MANIFESTS) + .find(headerFilter) + .project({ manifest_id: 1, seal_reason: 1, source_attestation: 1 }) + .toArray(); + report.manifest_headers_normalized += manifests.length; + + for (const manifest of manifests) { + const newId = contentManifestId(manifest.manifest_id); + const sealReason = + manifest.seal_reason || + (manifest.source_attestation ? 'source_reconstruction' : 'legacy_graph'); + await db.collection(MANIFESTS).updateOne( + { _id: manifest._id }, + { + $set: { manifest_id: newId, seal_reason: sealReason }, + $unset: { resolver_version: '', baseline_reconstruction: '' }, + }, + ); + if (newId !== manifest.manifest_id) { + await db + .collection(ENTRIES) + .updateMany({ manifest_id: manifest.manifest_id }, { $set: { manifest_id: newId } }); + } + } +} + +async function normalizeSnapshotReferences(db, collection, snapshot, options, setOps, unsetOps) { + if (snapshot.content_manifest_id?.startsWith(LEGACY_MANIFEST_ID_PREFIX)) { + setOps.content_manifest_id = contentManifestId(snapshot.content_manifest_id); + } + if (snapshot.bundle_hashes?.manifest_id?.startsWith(LEGACY_MANIFEST_ID_PREFIX)) { + setOps['bundle_hashes.manifest_id'] = contentManifestId(snapshot.bundle_hashes.manifest_id); + } + if (snapshot.config?.include_secondary_objects !== undefined) { + unsetOps['config.include_secondary_objects'] = ''; + } +} + +function memberSetKey(snapshot) { + return (snapshot.members || []) + .map((entry) => `${entry.object_ref}::${new Date(entry.object_modified).getTime()}`) + .sort() + .join('|'); +} + +async function findTrackIds(db) { + const registeredTracks = await db + .collection('releaseTrackRegistry') + .find({}) + .project({ track_id: 1, _id: 0 }) + .toArray(); + return [...new Set(registeredTracks.map((track) => track.track_id))].sort(); +} + +/** + * Dynamic release-track collections that no registry document references. + */ +async function findOrphanTrackCollections(db, registeredTrackIds) { + const registered = new Set(registeredTrackIds); + const collections = await db.listCollections({}, { nameOnly: true }).toArray(); + return collections + .map((collection) => collection.name) + .filter((name) => TRACK_COLLECTION_PATTERN.test(name) && !registered.has(name)) + .sort(); +} + +/** + * Manifest collections to consult. Before the rename step runs (and during a + * dry run, which never renames) legacy documents still live in the old + * collections, so lookups and counts cover both. + */ +async function manifestCollections(db, kind) { + const names = + kind === 'entries' + ? [ENTRIES, 'releaseTrackGraphManifestEntries'] + : [MANIFESTS, 'releaseTrackGraphManifests']; + const present = []; + for (const name of names) { + if (await collectionExists(db, name)) present.push(name); + } + return present; +} + +async function countAcross(db, kind, filter) { + let total = 0; + for (const name of await manifestCollections(db, kind)) { + total += await db.collection(name).countDocuments(filter); + } + return total; +} + +async function activeManifest(db, manifestId) { + if (!manifestId) return null; + const ids = [...new Set([manifestId, contentManifestId(manifestId)])]; + for (const name of await manifestCollections(db, 'manifests')) { + const manifest = await db.collection(name).findOne({ + manifest_id: { $in: ids }, + state: { $in: ['pending', 'active'] }, + }); + if (manifest) return manifest; + } + return null; +} + +function emptyReport(dryRun) { + return { + tracks: 0, + snapshots: 0, + renamed_manifest_fields: 0, + marking_refs_migrated: 0, + manifests_sealed: 0, + manifests_shared: 0, + publications_frozen: 0, + bundle_hashes_recomputed: 0, + collection_entries_removed: 0, + legacy_manifest_documents_moved: 0, + manifest_headers_normalized: 0, + completed_reconciliations_removed: 0, + orphan_track_collections: [], + orphan_manifests_discarded: 0, + warnings: [], + dry_run: dryRun === true, + }; +} + +async function reportOrphanTrackCollections(db, trackIds, options, report) { + const contentManifestService = require('../app/services/release-tracks/content-manifest-service'); + for (const name of await findOrphanTrackCollections(db, trackIds)) { + const snapshots = await db.collection(name).countDocuments({}); + const manifests = await countAcross(db, 'manifests', { track_id: name }); + report.orphan_track_collections.push({ + collection: name, + snapshots, + manifests, + message: + 'Not present in releaseTrackRegistry; skipped. Drop the collection once you have ' + + 'confirmed it is a remnant of a deleted track.', + }); + report.orphan_manifests_discarded += manifests; + if (!options.dryRun && manifests > 0) { + await contentManifestService.discardTrack(name); + } + } +} + +async function migrateTrack(db, trackId, options, report) { + const collection = db.collection(trackId); + const snapshots = await collection.find({}).sort({ modified: 1 }).toArray(); + report.snapshots += snapshots.length; + const sealedTaggedByMembers = new Map(); + + for (const snapshot of snapshots) { + const context = { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }; + try { + await migrateSnapshot(db, collection, snapshot, options, report, sealedTaggedByMembers); + } catch (err) { + throw contextualize(err, { ...context, step: err.step }); + } + } + + const collectionEntries = await countAcross(db, 'entries', { + track_id: trackId, + kind: 'collection', + }); + report.collection_entries_removed += collectionEntries; + if (!options.dryRun && collectionEntries > 0) { + await db.collection(ENTRIES).deleteMany({ track_id: trackId, kind: 'collection' }); + } +} + +/** + * releaseTrackReconciliations now holds outstanding work only; completed + * records written by earlier releases are removed. + */ +async function removeCompletedReconciliations(db, options, report) { + if (!(await collectionExists(db, 'releaseTrackReconciliations'))) return; + const filter = { status: 'completed' }; + report.completed_reconciliations_removed += await db + .collection('releaseTrackReconciliations') + .countDocuments(filter); + if (!options.dryRun) { + await db.collection('releaseTrackReconciliations').deleteMany(filter); + } +} + +function step(name, promise) { + return promise.catch((err) => { + err.step = err.step || name; + throw err; + }); +} + +async function migrateSnapshot(db, collection, snapshot, options, report, sealedTaggedByMembers) { + const contentManifestService = require('../app/services/release-tracks/content-manifest-service'); + const publicationService = require('../app/services/release-tracks/publication-service'); + const bundleHashService = require('../app/services/release-tracks/bundle-hash-service'); + const trackId = snapshot.id; + { + const setOps = {}; + const unsetOps = {}; + // Look the manifest up by the id the snapshot actually carries (possibly + // the legacy prefix during a dry run), then continue with the normalized id. + const storedManifestId = snapshot.content_manifest_id || snapshot.graph_manifest_id; + let manifestId = (await activeManifest(db, storedManifestId)) + ? contentManifestId(storedManifestId) + : null; + + if (!snapshot.content_manifest_id && snapshot.graph_manifest_id) { + report.renamed_manifest_fields++; + unsetOps.graph_manifest_id = ''; + } + normalizeSnapshotReferences(db, collection, snapshot, options, setOps, unsetOps); + + if (Array.isArray(snapshot.object_marking_refs)) { + unsetOps.object_marking_refs = ''; + if (snapshot.object_marking_refs.length > 0) { + report.marking_refs_migrated++; + setOps['config.publication.object_marking_refs'] = { + inherit: false, + value: snapshot.object_marking_refs, + }; + } + } + const working = { ...snapshot }; + if (setOps['config.publication.object_marking_refs']) { + working.config = { + ...(snapshot.config || {}), + publication: { + ...(snapshot.config?.publication || {}), + object_marking_refs: setOps['config.publication.object_marking_refs'], + }, + }; + } + + const tagged = snapshot.version != null; + if (!manifestId) { + const shared = tagged ? null : sealedTaggedByMembers.get(memberSetKey(snapshot)); + if (shared) { + manifestId = shared; + report.manifests_shared++; + } else { + report.manifests_sealed++; + manifestId = options.dryRun + ? `dry-run:${snapshot._id}` + : await step( + 'seal', + contentManifestService.seal(working, { + reason: 'migration', + baselineReconstruction: true, + }), + ); + } + } + if (manifestId && manifestId !== snapshot.content_manifest_id) { + setOps.content_manifest_id = manifestId; + } + if (tagged && manifestId) sealedTaggedByMembers.set(memberSetKey(snapshot), manifestId); + + if (tagged) { + if (!snapshot.publication) { + report.publications_frozen++; + if (!options.dryRun) { + setOps.publication = await step( + 'freeze_publication', + publicationService.freezePublication(working), + ); + } + } + if (!snapshot.bundle_id && manifestId) { + // Preserve the envelope id these releases exported before the + // migration: the manifest uuid, whichever prefix it carried. + setOps.bundle_id = `bundle--${manifestId.split('--')[1]}`; + } + } + + if (options.dryRun) return; + + const update = {}; + if (Object.keys(setOps).length > 0) update.$set = setOps; + if (Object.keys(unsetOps).length > 0) update.$unset = unsetOps; + if (Object.keys(update).length > 0) { + await collection.updateOne({ _id: snapshot._id }, update); + } + if (setOps.content_manifest_id) { + await contentManifestService.activate(setOps.content_manifest_id); + } + + if (tagged) { + const current = await collection.findOne({ _id: snapshot._id }); + const expectedManifest = current.bundle_hashes?.manifest_id; + if (!current.bundle_hashes || expectedManifest !== current.content_manifest_id) { + try { + const bundleHashes = await step( + 'bundle_hashes', + bundleHashService.generateBundleHashes(current), + ); + await collection.updateOne( + { _id: snapshot._id }, + { $set: { bundle_hashes: bundleHashes } }, + ); + report.bundle_hashes_recomputed++; + } catch (err) { + report.warnings.push({ + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + message: `Bundle hashes not recomputed: ${err.message}`, + }); + } + } else if (setOps.publication) { + // Publication values changed the rendered collection object. + const bundleHashes = await step( + 'bundle_hashes', + bundleHashService.generateBundleHashes(current), + ); + await collection.updateOne( + { _id: snapshot._id }, + { $set: { bundle_hashes: bundleHashes } }, + ); + report.bundle_hashes_recomputed++; + } + } + } +} + +async function run(db, options = {}) { + const report = emptyReport(options.dryRun); + const trackIds = await findTrackIds(db); + report.tracks = trackIds.length; + await normalizeManifestStorage(db, options, report); + await reportOrphanTrackCollections(db, trackIds, options, report); + await removeCompletedReconciliations(db, options, report); + + for (const trackId of trackIds) { + const exists = await db.listCollections({ name: trackId }, { nameOnly: true }).hasNext(); + if (!exists) continue; + await migrateTrack(db, trackId, options, report); + } + return report; +} + +/** + * Attach the failing snapshot and the integrity details to an error so the + * startup log names what must be repaired. + */ +function contextualize(err, context) { + const details = [ + `track ${context.track_id}`, + context.snapshot_modified ? `snapshot ${context.snapshot_modified}` : null, + context.step, + err.details, + err.missing_references?.length + ? `missing_references=${JSON.stringify(err.missing_references.slice(0, 10))}` + : null, + ] + .filter(Boolean) + .join('; '); + const wrapped = new Error(`${err.message} (${details})`); + wrapped.cause = err; + wrapped.context = { ...context, missing_references: err.missing_references }; + return wrapped; +} + +module.exports = { + async up(db) { + const report = await run(db); + console.log( + `Normalized ${report.manifest_headers_normalized} manifest header(s), ` + + `sealed ${report.manifests_sealed} content manifest(s), shared ${report.manifests_shared}, ` + + `froze ${report.publications_frozen} publication record(s), recomputed ` + + `${report.bundle_hashes_recomputed} bundle hash set(s)` + + (report.warnings.length ? `; ${report.warnings.length} warning(s)` : '') + + (report.orphan_track_collections.length + ? `; skipped ${report.orphan_track_collections.length} unregistered track collection(s)` + : ''), + ); + for (const warning of report.warnings) { + console.warn(JSON.stringify(warning)); + } + for (const orphan of report.orphan_track_collections) { + console.warn(JSON.stringify(orphan)); + } + }, + + async down(db) { + const trackIds = await findTrackIds(db); + for (const trackId of trackIds) { + const exists = await db.listCollections({ name: trackId }, { nameOnly: true }).hasNext(); + if (!exists) continue; + const collection = db.collection(trackId); + const snapshots = await collection.find({}).toArray(); + for (const snapshot of snapshots) { + const update = { $unset: { content_manifest_id: '', publication: '', bundle_id: '' } }; + const setOps = {}; + const manifest = await activeManifest(db, snapshot.content_manifest_id); + if (manifest && manifest.seal_reason !== 'migration' && snapshot.version != null) { + setOps.graph_manifest_id = snapshot.content_manifest_id; + } + const markings = snapshot.config?.publication?.object_marking_refs; + if (markings && markings.inherit === false) { + setOps.object_marking_refs = markings.value || []; + } + if (Object.keys(setOps).length > 0) update.$set = setOps; + await collection.updateOne({ _id: snapshot._id }, update); + } + } + const migrated = await db + .collection(MANIFESTS) + .find({ seal_reason: 'migration' }) + .project({ manifest_id: 1, _id: 0 }) + .toArray(); + const manifestIds = migrated.map((manifest) => manifest.manifest_id); + if (manifestIds.length > 0) { + await db.collection(ENTRIES).deleteMany({ manifest_id: { $in: manifestIds } }); + await db.collection(MANIFESTS).deleteMany({ manifest_id: { $in: manifestIds } }); + } + }, + + _private: { + run, + findTrackIds, + findOrphanTrackCollections, + memberSetKey, + }, +}; diff --git a/package.json b/package.json index bec0dca8..d3a814d9 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "test:file": "mocha --timeout 10000 --exit", "repair:release-track-backrefs": "node scripts/reconcileReleaseTrackBackrefs.js", "preview:deterministic-snapshot-graphs": "node scripts/previewDeterministicSnapshotGraphMigration.js", + "preview:content-manifests": "node scripts/previewContentManifestMigration.js", "check:lockfile": "bash scripts/check-package-lock.sh" }, "dependencies": { diff --git a/scripts/previewContentManifestMigration.js b/scripts/previewContentManifestMigration.js new file mode 100644 index 00000000..b6e9234e --- /dev/null +++ b/scripts/previewContentManifestMigration.js @@ -0,0 +1,22 @@ +'use strict'; + +const mongoose = require('mongoose'); +const database = require('../app/lib/database-connection'); +const migration = require('../migrations/20260902120000-seal-release-track-content-manifests'); + +async function main() { + await database.initializeConnection(); + const report = await migration._private.run(mongoose.connection.db, { + dryRun: true, + }); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); +} + +main() + .catch((err) => { + process.stderr.write(`${err.stack || err.message}\n`); + process.exitCode = 1; + }) + .finally(async () => { + await mongoose.disconnect(); + }); From ef57b10ba758931937041c9a82ee6b522a9f9239 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:33:34 -0400 Subject: [PATCH 04/14] feat(release-tracks): allow administrators to delete the latest release Restore the ability to remove a tagged snapshot, which the sealed-manifest work had closed off entirely. Only the track's most recent release may be deleted, so the version order of the remaining releases and the provenance of any later release are never disturbed. Deleting a release requires the administrator role (403 otherwise, via the new InsufficientRoleError) and a `confirm_version` query parameter equal to the release version (400 otherwise). The release's ledger entry is retracted from every remaining snapshot so the version becomes available again, its content manifest is discarded when nothing else references it, the registry catalogue is reconciled, later drafts are kept, and a `delete_release` audit event is recorded. Deleting an older release, or one followed by a later release, returns 409. Drafts keep the ordinary editor deletion path; the release branch shares the snapshot deletion route, so the service checks the role itself. Co-Authored-By: Claude Fable 5.1 --- .../paths/release-tracks-paths.yml | 28 ++++- app/controllers/release-tracks-controller.js | 5 +- app/exceptions/index.js | 7 ++ app/lib/error-handler.js | 7 ++ .../release-track-audit-event-model.js | 2 +- .../release-track-dynamic.repository.js | 18 +++ .../release-tracks/release-tracks-service.js | 41 ++++++- .../release-tracks/snapshot-service.js | 54 +++++++++ .../destructive-authorization.spec.js | 110 ++++++++++++++++++ .../release-tracks/releases-by-object.spec.js | 4 +- .../snapshot-immutability.spec.js | 6 +- docs/admin/release-track-audit.md | 6 +- .../developer/release-tracks/authorization.md | 12 +- docs/user/release-tracks/api-reference.md | 19 +-- 14 files changed, 292 insertions(+), 27 deletions(-) diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index f63c96fb..736a49e7 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -1215,10 +1215,17 @@ paths: summary: 'Delete a specific snapshot' operationId: 'release-tracks-snapshot-delete' description: | - Delete the latest untagged draft snapshot by its modified timestamp. - Tagged snapshots and historical drafts are immutable and cannot be - deleted. Deleting the latest draft reverts the track to its immediately - preceding snapshot. + Delete the latest untagged draft snapshot by its modified timestamp + (editor or higher); the track reverts to its immediately preceding + snapshot. Historical drafts have already been pruned. + + An administrator may also delete the track's most recent release by + supplying `confirm_version` equal to that snapshot's version. The + release's ledger entry is retracted from every remaining snapshot, its + content manifest is discarded when nothing else references it, the + registry catalogue is reconciled, and a `delete_release` audit event + is recorded. A release that is followed by a later release cannot be + deleted until the later one is removed. tags: - 'Release Tracks' parameters: @@ -1232,11 +1239,22 @@ paths: required: true schema: type: string + - name: confirm_version + in: query + description: | + Required to delete a release: must equal the release version of + the selected snapshot. + schema: + type: string responses: '204': description: 'Snapshot deleted successfully' + '400': + description: 'Release confirmation missing or incorrect' + '403': + description: 'Deleting a release requires an administrator' '409': - description: 'Cannot delete a tagged snapshot or a historical draft' + description: 'The release is not the most recent one, or the draft is not the latest snapshot' '404': description: 'Snapshot not found' diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 3988e994..96aa1cd8 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -661,7 +661,10 @@ exports.reconstructSnapshotManifest = async function reconstructSnapshotManifest /** DELETE /api/release-tracks/:id/snapshots/:modified */ exports.deleteSnapshotByModified = async function deleteSnapshotByModified(req, res, next) { try { - await releaseTracksService.deleteSnapshot(req.params.id, req.params.modified); + await releaseTracksService.deleteSnapshot(req.params.id, req.params.modified, { + actor: destructiveActor(req), + confirmation: req.query.confirm_version, + }); logger.debug(`Success: Deleted snapshot ${req.params.modified} from track ${req.params.id}`); return res.status(204).end(); } catch (err) { diff --git a/app/exceptions/index.js b/app/exceptions/index.js index 0137f2ef..3360d26b 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -400,6 +400,12 @@ class InvalidVersionError extends CustomError { } } +class InsufficientRoleError extends CustomError { + constructor(requiredRole, options) { + super(`This operation requires the ${requiredRole} role`, options); + } +} + class ReleaseConflictError extends CustomError { constructor(message, options) { super(message || 'Release conflict: promotion aborted due to conflicting objects', options); @@ -474,6 +480,7 @@ module.exports = { //** Release track errors */ ReleaseConflictError, + InsufficientRoleError, ReleaseContentIntegrityError, ReleaseTrackReconciliationError, ReleaseTrackAuditError, diff --git a/app/lib/error-handler.js b/app/lib/error-handler.js index 4e486e31..0d644433 100644 --- a/app/lib/error-handler.js +++ b/app/lib/error-handler.js @@ -42,6 +42,7 @@ const { HistoricalSnapshotDeletionError, InvalidVersionError, ReleaseConflictError, + InsufficientRoleError, ReleaseContentIntegrityError, ReleaseTrackReconciliationError, ReleaseTrackAuditError, @@ -134,6 +135,12 @@ exports.serviceExceptions = function (err, req, res, next) { return res.status(404).send(buildErrorResponse(err)); } + // Handle 403 Forbidden errors (authenticated but insufficient role) + if (err instanceof InsufficientRoleError) { + logger.warn('Forbidden: %s', JSON.stringify(buildErrorResponse(err))); + return res.status(403).send(buildErrorResponse(err)); + } + // Handle 409 Conflict errors (duplicate resources) if ( err instanceof DuplicateIdError || diff --git a/app/models/release-tracks/release-track-audit-event-model.js b/app/models/release-tracks/release-track-audit-event-model.js index 2218f47f..85b1fea0 100644 --- a/app/models/release-tracks/release-track-audit-event-model.js +++ b/app/models/release-tracks/release-track-audit-event-model.js @@ -9,7 +9,7 @@ const releaseTrackAuditEventSchema = new mongoose.Schema( action: { type: String, required: true, - enum: ['delete_track'], + enum: ['delete_track', 'delete_release'], }, track_id: { type: String, required: true, validate: validateTrackId }, status: { diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 600c7d4a..7d750d19 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -457,6 +457,24 @@ class ReleaseTrackDynamicRepository { } } + /** + * Remove one release's ledger entry from every snapshot of the track. The + * ledger is copied forward into each clone, so deleting a release must + * retract it everywhere or the version would stay reserved. + */ + async pullVersionHistory(trackId, version) { + try { + const Model = this._getModel(trackId); + const result = await Model.updateMany( + { id: trackId, 'version_history.version': version }, + { $pull: { version_history: { version } } }, + ).exec(); + return result.modifiedCount; + } catch (err) { + throw new DatabaseError(err); + } + } + async deleteOlderDrafts(trackId, modified) { try { const Model = this._getModel(trackId); diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 7ec0d15e..d166dacb 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -14,7 +14,8 @@ // Phase 6: Export, ephemeral, bundle import → export-service, ephemeral-service, bundle-import-service // ============================================================================= -const { BadRequestError, NotImplementedError } = require('../../exceptions'); +const { BadRequestError, InsufficientRoleError, NotImplementedError } = require('../../exceptions'); +const authz = require('../../lib/authz-middleware'); const { compositionSchema, snapshotScheduleSchema, @@ -357,8 +358,42 @@ exports.deleteTrack = function deleteTrack(trackId, actor, confirmation) { ); }; -exports.deleteSnapshot = function deleteSnapshot(trackId, modified) { - return snapshotService.deleteSnapshot(trackId, modified); +/** + * Delete a snapshot. Drafts follow the ordinary editor rules. A release may + * only be deleted by an administrator who confirms its version, and the + * deletion is recorded as a `delete_release` audit event. + */ +exports.deleteSnapshot = async function deleteSnapshot(trackId, modified, options = {}) { + const snapshot = await snapshotService.getSnapshotByModified(trackId, modified); + if (snapshot.version == null) { + return snapshotService.deleteSnapshot(trackId, modified); + } + + if (options.actor?.role !== authz.userRoles.admin) { + throw new InsufficientRoleError('administrator', { + details: 'Deleting a release requires an administrator.', + track_id: trackId, + version: snapshot.version, + }); + } + if (options.confirmation !== snapshot.version) { + throw new BadRequestError({ + message: 'Destructive release confirmation is required', + details: `Set confirm_version to the exact release version '${snapshot.version}'.`, + parameter_name: 'confirm_version', + expected_version: snapshot.version, + }); + } + + return destructiveAuditService.execute( + { + action: 'delete_release', + trackId, + ...destructiveIdentity(trackId, options.actor, options.confirmation), + request: { snapshot_modified: new Date(snapshot.modified).toISOString() }, + }, + () => snapshotService.deleteRelease(trackId, modified), + ); }; exports.reconstructSnapshotManifest = function reconstructSnapshotManifest( diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index ad202317..43ea9ab8 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -750,6 +750,60 @@ exports.deleteTrack = async function deleteTrack(trackId) { logger.verbose(`SnapshotService: Deleted track "${trackId}"`); }; +/** + * Delete the track's most recent release. + * + * Only the newest tagged snapshot may be deleted, so the version order of the + * remaining releases and the provenance of any later release are never + * disturbed. The release's ledger entry is retracted from every remaining + * snapshot (the ledger is copied forward into clones), its manifest is + * discarded when nothing else references it, and the registry catalogue is + * reconciled. Later drafts survive. + * + * @param {string} trackId + * @param {string|Date} modified + * @returns {Promise} The deleted snapshot + */ +exports.deleteRelease = async function deleteRelease(trackId, modified) { + const snapshot = await exports.getSnapshotByModified(trackId, modified); + if (snapshot.version == null) { + throw new ReleaseConflictError('The selected snapshot is not a release', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }); + } + const latestTagged = await dynamicRepo.getLatestTaggedSnapshot(trackId); + if ( + !latestTagged || + new Date(latestTagged.modified).getTime() !== new Date(snapshot.modified).getTime() + ) { + throw new ReleaseConflictError( + 'Only the most recent release of a track can be deleted; delete later releases first.', + { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + version: snapshot.version, + latest_version: latestTagged?.version ?? null, + }, + ); + } + + await dynamicRepo.deleteSnapshot(trackId, snapshot.modified); + await dynamicRepo.pullVersionHistory(trackId, snapshot.version); + await contentManifestService.discardUnreferenced(trackId, [snapshot.content_manifest_id]); + const releaseHistoryService = require('./release-history-service'); + await releaseHistoryService.reconcileTaggedReleases(trackId); + await syncRegistryCounters(trackId); + + const latest = await dynamicRepo.getLatestSnapshot(trackId); + await emitContentsChanged(trackId, latest); + + logger.verbose( + `SnapshotService: Deleted release v${snapshot.version} (${modified}) from track "${trackId}"`, + ); + return snapshot; +}; + /** * Delete a specific snapshot from a track. * diff --git a/app/tests/api/release-tracks/destructive-authorization.spec.js b/app/tests/api/release-tracks/destructive-authorization.spec.js index 682fee1c..09d58959 100644 --- a/app/tests/api/release-tracks/destructive-authorization.spec.js +++ b/app/tests/api/release-tracks/destructive-authorization.spec.js @@ -12,6 +12,12 @@ const UserAccount = require('../../../models/user-account-model'); const ReleaseTrackAuditEvent = require('../../../models/release-tracks/release-track-audit-event-model'); const auditRepository = require('../../../repository/release-tracks/release-track-audit-event.repository'); const systemConfigurationService = require('../../../services/system/system-configuration-service'); +const { + ReleaseTrackContentManifest, +} = require('../../../models/release-tracks/release-track-content-manifest-model'); +const { releaseExactMembers } = require('./release-track-test-helpers'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; describe('Release-track destructive authorization and audit', function () { let app; @@ -98,6 +104,110 @@ describe('Release-track destructive authorization and audit', function () { }); }); + it('lets only administrators delete the most recent release, with confirmation and audit', async function () { + await setRole('admin'); + const timestamp = new Date().toISOString(); + const technique = await post( + '/api/techniques', + { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + type: 'attack-pattern', + spec_version: '2.1', + created: timestamp, + modified: timestamp, + name: 'Release deletion member', + description: 'Member for release deletion tests.', + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'execution' }], + x_mitre_is_subtechnique: false, + x_mitre_domains: ['enterprise-attack'], + x_mitre_platforms: ['Windows'], + object_marking_refs: [markingDefinitionId], + }, + }, + 201, + ); + const track = await post( + '/api/release-tracks/new', + { name: 'Release deletion track', type: 'standard' }, + 201, + ); + const first = await releaseExactMembers(app, passportCookie, track.id, [technique], { + version: '1.0', + }); + await post(`/api/release-tracks/${track.id}/meta`, { description: 'next' }, 200); + const second = await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.1' }, + 200, + ); + const secondPath = `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + second.modified, + )}`; + const firstPath = `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( + first.modified, + )}`; + + // Editors cannot delete a release even with the right confirmation. + await setRole('editor'); + await api('delete', secondPath, undefined, 403, { confirm_version: '1.1' }); + + await setRole('admin'); + await api('delete', secondPath, undefined, 400); + await api('delete', secondPath, undefined, 400, { confirm_version: '9.9' }); + expect(await ReleaseTrackAuditEvent.countDocuments({ action: 'delete_release' })).toBe(0); + // Only the most recent release can be deleted; the rejected attempt is + // audited as failed, like any confirmed destructive request. + await api('delete', firstPath, undefined, 409, { confirm_version: '1.0' }); + expect( + await ReleaseTrackAuditEvent.countDocuments({ action: 'delete_release', status: 'failed' }), + ).toBe(1); + + await api('delete', secondPath, undefined, 204, { confirm_version: '1.1' }); + + await api('get', secondPath, undefined, 404); + const remaining = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/latest`, + undefined, + 200, + ); + expect(remaining.body.version).toBe('1.0'); + expect(remaining.body.version_history.map((entry) => entry.version)).toEqual(['1.0']); + expect( + await ReleaseTrackContentManifest.countDocuments({ + manifest_id: second.content_manifest_id, + }), + ).toBe(0); + const registry = await api('get', '/api/release-tracks', undefined, 200); + const entry = registry.body.data.find((candidate) => candidate.track_id === track.id); + expect(entry.tagged_release_count).toBe(1); + expect(entry.latest_tagged_version).toBe('1.0'); + + const event = await ReleaseTrackAuditEvent.findOne({ + action: 'delete_release', + status: 'completed', + }) + .lean() + .exec(); + expect(event).toMatchObject({ + track_id: track.id, + confirmation: '1.1', + status: 'completed', + request: { snapshot_modified: new Date(second.modified).toISOString() }, + result: { snapshot_modified: expect.any(Date), version: '1.1', members_count: 1 }, + }); + + // The version is free again and the track keeps working. + await post(`/api/release-tracks/${track.id}/meta`, { description: 'again' }, 200); + const again = await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.1' }, + 200, + ); + expect(again.version).toBe('1.1'); + }); + it('reports an audit-finalization failure without hiding the persisted mutation', async function () { await setRole('admin'); const track = await post( diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js index 516e4870..c82bfefd 100644 --- a/app/tests/api/release-tracks/releases-by-object.spec.js +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -257,11 +257,11 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { await get(`/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?limit=0`, 400); }); - it('rejects deletion of a tagged snapshot', async function () { + it('requires a typed version confirmation before a release can be deleted', async function () { await request(app) .delete(`/api/release-tracks/${trackA}/snapshots/${trackATaggedSnapshot.modified}`) .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(409); + .expect(400); }); it('backfills missing registry refs from authoritative tagged snapshots', async function () { diff --git a/app/tests/api/release-tracks/snapshot-immutability.spec.js b/app/tests/api/release-tracks/snapshot-immutability.spec.js index 19310faa..b57b8bc5 100644 --- a/app/tests/api/release-tracks/snapshot-immutability.spec.js +++ b/app/tests/api/release-tracks/snapshot-immutability.spec.js @@ -106,12 +106,14 @@ describe('Release-track snapshot immutability contract', function () { ); expect(reverted.body.modified).toBe(tagged.modified); + // A release is never deleted by the ordinary draft path: it requires an + // administrator's typed version confirmation. const taggedDelete = await api( 'delete', `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(tagged.modified)}`, undefined, - 409, + 400, ); - expect(taggedDelete.text).toContain('Tagged snapshot version 1.0 cannot be deleted'); + expect(taggedDelete.text).toContain('Destructive release confirmation is required'); }); }); diff --git a/docs/admin/release-track-audit.md b/docs/admin/release-track-audit.md index c9634eb7..b36f071b 100644 --- a/docs/admin/release-track-audit.md +++ b/docs/admin/release-track-audit.md @@ -1,7 +1,9 @@ # Release-Track Destructive Audit Events -Workbench stores administrator-initiated full-track deletion attempts in -`releaseTrackAuditEvents`. +Workbench stores administrator-initiated destructive attempts in +`releaseTrackAuditEvents`: full-track deletion (`delete_track`) and deletion +of a track's most recent release (`delete_release`). The collection is empty +until an administrator performs one of those actions. Each record contains: diff --git a/docs/developer/release-tracks/authorization.md b/docs/developer/release-tracks/authorization.md index 0d6f62d1..70f6df53 100644 --- a/docs/developer/release-tracks/authorization.md +++ b/docs/developer/release-tracks/authorization.md @@ -14,16 +14,20 @@ history requires an administrator. | Create tracks and drafts; manage candidates/staged/config/composition | No | Yes | Yes | | Tag a standard or virtual snapshot | No | Yes | Yes | | Delete the latest untagged draft snapshot | No | Yes | Yes | +| Delete the track's most recent release | No | No | Yes | | Delete an entire track and all snapshot history | No | No | Yes | Full-track deletion also requires `confirm_track_id` to equal the `:id` path -parameter. Authorization runs before the controller, and confirmation runs -before persistence. +parameter, and release deletion requires `confirm_version` to equal the +release version. Track deletion is authorized by route middleware; release +deletion shares the snapshot deletion route, so the service checks the +administrator role itself and answers `403` otherwise. Confirmation runs +before persistence in both cases. ## Audited destructive actions -The `delete_track` action creates a `releaseTrackAuditEvents` record before -the business operation begins. +The `delete_track` and `delete_release` actions create a +`releaseTrackAuditEvents` record before the business operation begins. Each event records the authenticated actor, confirmation value, target track, request summary, timestamps, and a `pending`, `completed`, or `failed` status. diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 620e4d60..62c8acbe 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -783,13 +783,18 @@ rule configured under `config.publication` (see ``` DELETE /api/release-tracks/:id/snapshots/:modified -``` - -Deletes the selected snapshot only when it is both the latest snapshot and an -untagged draft with a predecessor. Deletion reverts the track to that -predecessor. Standard tracks retain only one rolling draft, so replaced -untagged timestamps return `404`. Tagged releases and a track's sole snapshot -return `409 Conflict`. +DELETE /api/release-tracks/:id/snapshots/:modified?confirm_version=1.1 +``` + +Editors may delete the latest untagged draft; the track reverts to the +preceding snapshot. Administrators may also delete the track's most recent +release by confirming its version. The release's ledger entry is retracted +from every remaining snapshot so the version becomes available again, its +content manifest is discarded when nothing else references it, the registry +catalogue is reconciled, later drafts are kept, and a `delete_release` audit +event is recorded. Deleting an older release, or a release followed by a later +one, returns `409 Conflict`; a missing or wrong confirmation returns `400`; +a non-administrator receives `403`. --- From fd8cf1632a7a0093e3d37d50ec50e1355f223c92 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:53:49 -0400 Subject: [PATCH 05/14] feat(release-tracks): export sealed bundles only and add track aliases Bundle exports replay the sealed content manifest and nothing else. The draft-only `include=staged,candidates` / `state` bundle preview is removed: it produced bundles that matched no manifest, duplicated the release preview (`.../release/preview?format=bundle`, which stays the one live path), and the frontend's Export Latest was sending `include=all` and getting 400. `format=bundle` now rejects `include` explicitly so a caller who asked for workflow tiers never mistakes a members-only bundle for the preview they requested; `state` is gone from the OpenAPI spec. Workbench tier entries now carry `type` and `x_mitre_version`, and the relationship-change preview resolves relationships only (no supporting objects or LinkById targets). This lets the frontend stop downloading the entire object catalogue on every release preview. Tracks gain an optional alias: a URL-safe slug stored on the registry under a partial unique index and accepted on every `:id` route. An Express `router.param('id')` resolver rewrites an alias to the canonical track ID before any handler runs, so services only ever see canonical IDs and an unknown alias is a 404 rather than a stray collection name. Aliases are set at creation or through `POST /:id/meta` (`null` clears); an alias-only update does not clone a snapshot. Workbench snapshot responses and registry entries carry `alias`. Co-Authored-By: Claude Fable 5.1 --- .../definitions/components/release-tracks.yml | 19 +++ .../paths/release-tracks-paths.yml | 122 +++++---------- app/controllers/release-tracks-controller.js | 76 +++++++--- .../release-tracks/release-track-schemas.js | 64 ++++---- .../release-track-validators.js | 7 + .../release-track-registry-model.js | 8 + .../release-track-registry.repository.js | 40 +++++ app/routes/release-tracks-routes.js | 3 + .../content-manifest-service.js | 20 ++- app/services/release-tracks/export-service.js | 56 +------ .../release-tracks/release-tracks-service.js | 8 + .../release-tracks/snapshot-service.js | 53 ++++++- .../release-tracks/content-manifests.spec.js | 32 ++-- .../release-tracks-bundle.spec.js | 114 ++++---------- .../release-tracks-release.spec.js | 10 +- .../api/release-tracks/release-tracks.spec.js | 3 + .../api/release-tracks/track-aliases.spec.js | 142 ++++++++++++++++++ docs/developer/TODO.md | 32 +++- .../developer/release-tracks/bundle-export.md | 32 ++-- docs/developer/release-tracks/entities.md | 30 ++-- .../sealed-content-manifests.md | 10 +- docs/user/release-tracks/api-reference.md | 100 ++++++------ docs/user/release-tracks/output-formats.md | 29 ++-- 23 files changed, 605 insertions(+), 405 deletions(-) create mode 100644 app/tests/api/release-tracks/track-aliases.spec.js diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index ecde61f5..ceb2bc22 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -4,6 +4,10 @@ components: type: object description: 'A snapshot document for a release track, containing versioned member objects and workflow tiers' properties: + alias: + type: string + nullable: true + description: 'The track alias from the registry (workbench responses), or null' id: type: string description: 'The release track ID (STIX identifier format)' @@ -284,6 +288,14 @@ components: name: type: string description: 'Object name, if found' + type: + type: string + description: 'STIX object type, if found' + example: 'attack-pattern' + x_mitre_version: + type: string + description: 'ATT&CK object version of the selected revision, if found' + example: '1.2' description: type: string description: 'Object description, if found' @@ -630,6 +642,13 @@ components: description: type: string description: 'Track description' + alias: + type: string + description: | + Optional URL-safe slug (2-64 lowercase letters, digits, and + hyphens) accepted wherever the track ID is, on every /api/release-tracks/{id}... path. + Unique across tracks; absent when unset. + example: 'enterprise-attack' latest_snapshot_modified: type: string format: date-time diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 736a49e7..6c127661 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -238,7 +238,9 @@ paths: the initial snapshot and returned by snapshot and track-list GETs. `description` is long-lived track metadata. The optional `snapshot_description` is a user-authored annotation on the initial - draft snapshot and is limited to 4000 characters. + draft snapshot and is limited to 4000 characters. `alias` is an + optional URL-safe slug (2-64 lowercase letters, digits, and hyphens, + unique across tracks) accepted wherever the track ID is. tags: - 'Release Tracks' # Request body validation moved to Zod in controller @@ -300,7 +302,7 @@ paths: - name: id in: path required: true - description: 'Release track ID' + description: 'Release track ID or alias' schema: type: string - name: confirm_track_id @@ -335,10 +337,12 @@ paths: summary: 'Update metadata on the latest snapshot' operationId: 'release-tracks-update-meta-latest' description: | - Update name or description on the latest snapshot. Publication - markings are configured through the track configuration. - Creates a new snapshot clone with updated metadata. - Request body validated via Zod in controller. + Update name, description, or alias. Name and description live on + the snapshot, so changing either creates a new snapshot clone. The + alias is registry-only routing metadata (a string sets it, null + clears it) and an alias-only update returns the latest snapshot + unchanged. Publication markings are configured through the track + configuration. Request body validated via Zod in controller. tags: - 'Release Tracks' parameters: @@ -478,22 +482,12 @@ paths: type: string - name: include in: query - allowReserved: true - schema: - oneOf: - - type: string - - type: array - items: - type: string - - name: state - in: query - allowReserved: true + description: | + Workbench previews only: which tier arrays to return + (members | staged | candidates | quarantine | all). Bundle + previews reject it with 400. schema: - oneOf: - - type: string - - type: array - items: - type: string + type: string - name: stixVersion in: query schema: @@ -978,7 +972,7 @@ paths: - name: id in: path required: true - description: 'Release track ID' + description: 'Release track ID or alias' schema: type: string - name: tagged @@ -1054,24 +1048,18 @@ paths: - name: id in: path required: true - description: 'Release track ID' + description: 'Release track ID or alias' schema: type: string - name: include in: query description: | - Format-sensitive tier selector. For workbench responses, selects - members, staged, candidates, quarantine, or all. For bundle - responses, selects staged and/or candidates in addition to members; - this is a draft-only preview option that resolves the included - tiers live, and tagged snapshots reject it with 400. - allowReserved: true - schema: - oneOf: - - type: string - - type: array - items: - type: string + Workbench responses only: which tier arrays to return (members, + staged, candidates, quarantine, or all; default all). Bundles + always replay the snapshot's sealed content manifest and reject + include with 400. + schema: + type: string - name: format in: query description: 'Output format. filesystemstore is not yet implemented.' @@ -1082,16 +1070,6 @@ paths: - workbench - filesystemstore default: workbench - - name: state - in: query - description: 'Workflow-status filter for bundle staged/candidate tiers' - allowReserved: true - schema: - oneOf: - - type: string - - type: array - items: - type: string - name: stixVersion in: query description: | @@ -1146,20 +1124,12 @@ paths: - name: include in: query description: | - Format-sensitive tier selector. - For format=workbench (default): a single value controlling which tier arrays - are returned — members | staged | candidates | quarantine | all (default: all). - For format=bundle: a list of additional tiers (staged and/or candidates, - comma-separated or repeated) to include alongside members. If omitted, only - members are included in the bundle. Including tiers is a draft-only preview - option; tagged snapshots reject it with 400. - allowReserved: true - schema: - oneOf: - - type: string - - type: array - items: - type: string + Workbench responses only: a single value controlling which tier + arrays are returned — members | staged | candidates | quarantine | + all (default: all). Bundles always replay the snapshot's sealed + content manifest and reject include with 400. + schema: + type: string - name: format in: query description: 'Output format. filesystemstore is not yet implemented.' @@ -1170,20 +1140,6 @@ paths: - filesystemstore - workbench default: workbench - - name: state - in: query - description: | - Workflow-status filter for the staged/candidate tiers selected via include - (bundle format only). Accepts modified-in-place, work-in-progress and/or - awaiting-review (comma-separated or repeated). Entries marked reviewed are - always included, irrespective of this parameter. Members are unaffected. - allowReserved: true - schema: - oneOf: - - type: string - - type: array - items: - type: string - name: stixVersion in: query description: | @@ -1486,22 +1442,12 @@ paths: type: string - name: include in: query - allowReserved: true - schema: - oneOf: - - type: string - - type: array - items: - type: string - - name: state - in: query - allowReserved: true + description: | + Workbench previews only: which tier arrays to return + (members | staged | candidates | quarantine | all). Bundle + previews reject it with 400. schema: - oneOf: - - type: string - - type: array - items: - type: string + type: string - name: stixVersion in: query schema: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 96aa1cd8..62842693 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -18,14 +18,15 @@ const { InvalidQueryStringParameterError, BadRequestError, NotImplementedError, + NotFoundError, } = require('../exceptions'); const { domainParamSchema, formatQuerySchema, releasePreviewFormatSchema, includeQuerySchema, - bundleIncludeQuerySchema, - bundleStateQuerySchema, + releaseTrackIdSchema, + trackAliasSchema, stixVersionQuerySchema, booleanQuerySchema, snapshotTaggedQuerySchema, @@ -107,18 +108,55 @@ function rejectFilesystemStoreFormat(format, methodName) { }); } +/** + * Route parameter resolver for `:id`. A canonical track ID passes through; + * any other value is treated as an alias and rewritten to the track ID it + * names, so handlers and services only ever see canonical IDs. An unknown + * alias is a 404 here rather than falling through, because the model factory + * would otherwise bind a collection to the raw value. Resolution runs before + * authentication (Express param callbacks precede route handlers), so an + * alias's existence is observable without a session; aliases are public + * slugs, not secrets. + */ +exports.resolveTrackId = async function resolveTrackId(req, res, next, value) { + try { + if (releaseTrackIdSchema.safeParse(value).success) return next(); + const trackId = trackAliasSchema.safeParse(value).success + ? await releaseTracksService.resolveTrackAlias(value) + : null; + if (!trackId) { + return next(new NotFoundError({ details: `Release track '${value}' not found` })); + } + req.params.id = trackId; + req.releaseTrackAlias = value; + return next(); + } catch (err) { + return next(err); + } +}; + +/** + * `include` selects which tier arrays a workbench response returns. A bundle + * always replays the snapshot's sealed content manifest, so `include` has no + * meaning there and is rejected rather than silently ignored: a caller who + * asked for staged or candidate objects must not mistake the members-only + * bundle for the preview they requested. + */ +function rejectBundleInclude(query) { + if (query.include === undefined) return; + throw new InvalidQueryStringParameterError({ + parameterName: 'include', + message: + 'The include parameter applies to format=workbench only; bundles always replay the sealed content manifest.', + }); +} + /** * Parse common query parameters shared across GET snapshot endpoints. * - * The `include` parameter is format-sensitive: - * - format=workbench: single tier name ('members' | 'staged' | 'candidates' - * | 'quarantine' | 'all') controlling which tier arrays are returned - * - format=bundle: list of additional tiers ('staged' and/or 'candidates') - * to hydrate into the bundle alongside members. Omitted → members only. - * - * The `state` and `stixVersion` parameters only apply to format=bundle. - * `include` for bundles is a draft-only preview option; the service rejects it - * for tagged snapshots. + * `include` (workbench only) is a single tier name ('members' | 'staged' | + * 'candidates' | 'quarantine' | 'all') controlling which tier arrays are + * returned. `stixVersion` applies only to format=bundle. */ function parseSnapshotQueryParams(query) { const format = parseOptionalQueryStrict(query.format, formatQuerySchema, 'workbench', 'format'); @@ -133,15 +171,9 @@ function parseSnapshotQueryParams(query) { }; if (format === 'bundle') { + rejectBundleInclude(query); return { ...common, - include: parseOptionalQueryStrict( - query.include, - bundleIncludeQuerySchema, - undefined, - 'include', - ), - state: parseOptionalQueryStrict(query.state, bundleStateQuerySchema, undefined, 'state'), stixVersion: parseOptionalQueryStrict( query.stixVersion, stixVersionQuerySchema, @@ -178,15 +210,9 @@ function parseReleasePreviewQueryParams(query) { const options = { format, ...versionSelection.data }; if (format === 'bundle') { + rejectBundleInclude(query); return { ...options, - include: parseOptionalQueryStrict( - query.include, - bundleIncludeQuerySchema, - undefined, - 'include', - ), - state: parseOptionalQueryStrict(query.state, bundleStateQuerySchema, undefined, 'state'), stixVersion: parseOptionalQueryStrict( query.stixVersion, stixVersionQuerySchema, diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index a44fa698..ef897365 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -75,6 +75,34 @@ const trackNameSchema = z const snapshotDescriptionSchema = z.string().trim().max(4000); +// ----------------------------------------------------------------------------- +// Track alias: an optional URL-safe slug accepted wherever a track ID is +// ----------------------------------------------------------------------------- + +// Static path segments under /api/release-tracks that an alias must never +// shadow, plus the canonical ID prefix. +const RESERVED_TRACK_ALIASES = Object.freeze([ + 'new', + 'new-from-bundle', + 'import', + 'objects', + 'ephemeral', + 'latest', +]); + +const trackAliasSchema = z + .string() + .min(2, { message: 'Release track alias must be at least 2 characters' }) + .max(64, { message: 'Release track alias must be at most 64 characters' }) + .regex(/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])$/, { + message: + 'Release track alias may only contain lowercase letters, digits, and hyphens, and must start and end with a letter or digit', + }) + .refine( + (alias) => !RESERVED_TRACK_ALIASES.includes(alias) && !alias.startsWith('release-track'), + { message: 'Release track alias is reserved' }, + ); + // ----------------------------------------------------------------------------- // Cron expression // See: https://github.com/colinhacks/zod/issues/4239#issuecomment-3161393771 @@ -158,35 +186,6 @@ const releasePreviewFormatSchema = z.enum(['summary', 'bundle', 'filesystemstore const includeQuerySchema = z.enum(['members', 'staged', 'candidates', 'quarantine', 'all']); -/** - * Normalize a query-string value that represents a list. Accepts a repeated - * parameter (array), a comma-separated string, or a single value, and returns - * an array of trimmed strings. - */ -function normalizeQueryArray(value) { - const rawValues = Array.isArray(value) ? value : [value]; - return rawValues - .flatMap((entry) => String(entry).split(',')) - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0); -} - -// `include` for format=bundle: which non-member tiers to add to the bundle. -// Accepts singular or plural tier names; normalized to the plural tier names. -const bundleIncludeQuerySchema = z.preprocess( - (value) => - normalizeQueryArray(value).map((entry) => (entry === 'candidate' ? 'candidates' : entry)), - z.array(z.enum(['candidates', 'staged'])).min(1), -); - -// `state` for format=bundle: workflow-status filter applied to the tiers -// selected via `include`. 'reviewed' is intentionally not a valid filter -// value — reviewed objects are always included. -const bundleStateQuerySchema = z.preprocess( - (value) => normalizeQueryArray(value), - z.array(z.enum(['modified-in-place', 'work-in-progress', 'awaiting-review'])).min(1), -); - const stixVersionQuerySchema = z.enum(['2.0', '2.1']); // Boolean query parameters arrive as strings ('true'/'false') unless the @@ -425,6 +424,7 @@ const compositionSchema = z const createTrackBodySchema = z .object({ name: trackNameSchema, + alias: trackAliasSchema.optional(), description: z.string().optional(), snapshot_description: snapshotDescriptionSchema.optional(), type: trackTypeQuerySchema.default('standard'), @@ -462,6 +462,8 @@ const createFromBundleBodySchema = z.object({ const updateMetadataBodySchema = z.object({ name: trackNameSchema.optional(), description: z.string().optional(), + // A string sets the alias; null clears it. + alias: trackAliasSchema.nullable().optional(), }); /** PUT /release-tracks/:id/snapshots/:modified/description */ @@ -647,8 +649,6 @@ module.exports = { formatQuerySchema, releasePreviewFormatSchema, includeQuerySchema, - bundleIncludeQuerySchema, - bundleStateQuerySchema, stixVersionQuerySchema, booleanQuerySchema, snapshotTaggedQuerySchema, @@ -670,6 +670,8 @@ module.exports = { // Request body schemas createTrackBodySchema, + trackAliasSchema, + RESERVED_TRACK_ALIASES, createFromBundleBodySchema, updateMetadataBodySchema, updateSnapshotDescriptionBodySchema, diff --git a/app/lib/release-tracks/release-track-validators.js b/app/lib/release-tracks/release-track-validators.js index 164c3492..b8345856 100644 --- a/app/lib/release-tracks/release-track-validators.js +++ b/app/lib/release-tracks/release-track-validators.js @@ -19,6 +19,7 @@ const { stixIdentifierSchema, xMitreVersionSchema, createStixIdValidator, + trackAliasSchema, } = require('./release-track-schemas'); // ----------------------------------------------------------------------------- @@ -31,6 +32,11 @@ const validateTrackId = { `"${props.value}" is not a valid release track ID (expected "release-track--")`, }; +const validateTrackAlias = { + validator: (v) => v === undefined || trackAliasSchema.safeParse(v).success, + message: (props) => `"${props.value}" is not a valid release track alias`, +}; + const validateTrackName = { validator: (v) => trackNameSchema.safeParse(v).success, message: (props) => @@ -104,6 +110,7 @@ const validateObjectTypesFilter = { module.exports = { validateTrackId, + validateTrackAlias, validateTrackName, validateStixId, validateIdentityRef, diff --git a/app/models/release-tracks/release-track-registry-model.js b/app/models/release-tracks/release-track-registry-model.js index 8fade81d..fd74b71d 100644 --- a/app/models/release-tracks/release-track-registry-model.js +++ b/app/models/release-tracks/release-track-registry-model.js @@ -3,6 +3,7 @@ const mongoose = require('mongoose'); const { validateTrackId, + validateTrackAlias, validateTrackName, validateVersion, validateCron, @@ -64,6 +65,9 @@ const releaseTrackRegistryDefinition = { validate: validateTrackName, }, description: { type: String }, + // Optional URL-safe slug accepted wherever the track ID is. Absent (not + // null) when unset so the partial unique index ignores the document. + alias: { type: String, validate: validateTrackAlias }, // Denormalized for fast listing (updated on each snapshot/tag) latest_snapshot_modified: { type: Date }, @@ -107,6 +111,10 @@ const releaseTrackRegistrySchema = new mongoose.Schema(releaseTrackRegistryDefin // --- Indexes --- releaseTrackRegistrySchema.index({ type: 1 }); +releaseTrackRegistrySchema.index( + { alias: 1 }, + { unique: true, partialFilterExpression: { alias: { $type: 'string' } } }, +); // --- Model creation --- diff --git a/app/repository/release-tracks/release-track-registry.repository.js b/app/repository/release-tracks/release-track-registry.repository.js index 1c2d65a3..80e2bc72 100644 --- a/app/repository/release-tracks/release-track-registry.repository.js +++ b/app/repository/release-tracks/release-track-registry.repository.js @@ -20,6 +20,11 @@ class ReleaseTrackRegistryRepository { return saved.toObject(); } catch (err) { if (err.name === 'MongoServerError' && err.code === 11000) { + if (err.keyPattern?.alias) { + throw new DuplicateIdError(`Release track alias '${data.alias}' is already in use`, { + details: { alias: data.alias }, + }); + } throw new DuplicateIdError({ details: `Release track with id '${data.track_id}' already exists.`, }); @@ -28,6 +33,41 @@ class ReleaseTrackRegistryRepository { } } + async findByAlias(alias) { + try { + return await this.model.findOne({ alias }).lean().exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + + /** + * Set (string) or clear (null) a track's alias. Clearing unsets the field so + * the partial unique index ignores the document. + */ + async setAlias(trackId, alias) { + const updated_at = new Date(); + const update = alias + ? { $set: { alias, updated_at } } + : { $set: { updated_at }, $unset: { alias: '' } }; + try { + return await this.model + .findOneAndUpdate({ track_id: trackId }, update, { + new: true, + runValidators: true, + lean: true, + }) + .exec(); + } catch (err) { + if (err.name === 'MongoServerError' && err.code === 11000) { + throw new DuplicateIdError(`Release track alias '${alias}' is already in use`, { + details: { alias }, + }); + } + throw new DatabaseError(err); + } + } + async findByTrackId(trackId) { try { return await this.model.findOne({ track_id: trackId }).lean().exec(); diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index c44ff750..832bcd41 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -8,6 +8,9 @@ const authz = require('../lib/authz-middleware'); const router = express.Router(); +// Every `:id` route accepts either a canonical track ID or a track alias. +router.param('id', releaseTracksController.resolveTrackId); + // ============================================================================= // Ephemeral (stateless) bundles // ============================================================================= diff --git a/app/services/release-tracks/content-manifest-service.js b/app/services/release-tracks/content-manifest-service.js index c1d3fe81..7deae832 100644 --- a/app/services/release-tracks/content-manifest-service.js +++ b/app/services/release-tracks/content-manifest-service.js @@ -105,6 +105,8 @@ function authoredPin(relationship, side) { * @param {Array} [options.extraSupportingRefs] - Identity and marking * definition IDs the collection object itself references, so the bundle * stays self-contained + * @param {boolean} [options.relationshipsOnly] - Skip supporting objects and + * LinkById targets; for callers that only compare relationship selection * @returns {Promise<{ * roots: { entries: Array, documents: Array }, * relationships: Array<{ relationship: Object, source: Object, target: Object, @@ -153,6 +155,10 @@ async function resolveClosedGraph(memberEntries, options = {}) { left.relationship.stix.id.localeCompare(right.relationship.stix.id), ); + if (options.relationshipsOnly) { + return { roots, relationships, supportingDocuments: [], linkTargetDocuments: [] }; + } + const emitted = [...roots.documents, ...relationships.map((candidate) => candidate.relationship)]; const supportingDocuments = await loadSupportingDocuments( emitted, @@ -751,12 +757,12 @@ async function replay(snapshot) { } /** - * Resolve the graph live for a member set plus optional extra tier entries. - * Used by release previews (unsaved planned snapshots) and by draft exports - * that add workflow tiers. Not deterministic by design. + * Resolve the graph live for a snapshot's member set. Used by release previews + * of an unsaved planned snapshot, which has nothing sealed yet. Not + * deterministic by design. */ -async function resolveLive(snapshot, extraEntries = []) { - const graph = await resolveClosedGraph([...(snapshot.members || []), ...extraEntries], { +async function resolveLive(snapshot) { + const graph = await resolveClosedGraph(snapshot.members || [], { extraSupportingRefs: await publicationSupportingRefs(snapshot), }); return graphFromResolution(graph); @@ -810,7 +816,9 @@ function relationshipSummary(relationship, source, target, extra = {}) { * @param {Array} members - Member set the release would seal */ async function previewRelationshipChanges(snapshot, members) { - const graph = await resolveClosedGraph(members); + // Only the relationship selection is compared, so the supporting objects + // and LinkById targets a full seal would load are skipped. + const graph = await resolveClosedGraph(members, { relationshipsOnly: true }); const previousEntries = snapshot.content_manifest_id ? (await loadEntries(snapshot.content_manifest_id)).filter( (entry) => entry.kind === 'relationship', diff --git a/app/services/release-tracks/export-service.js b/app/services/release-tracks/export-service.js index f76037ca..8f8d58c9 100644 --- a/app/services/release-tracks/export-service.js +++ b/app/services/release-tracks/export-service.js @@ -9,9 +9,9 @@ // - filesystemstore: Directory structure organized by STIX type // // Bundle export has exactly one content path: replay the snapshot's sealed -// content manifest. Two preview variants resolve the same closed-member graph -// live instead of replaying: release previews of an unsaved planned snapshot, -// and draft exports that add workflow tiers through `include`. +// content manifest. The only exception is a release preview of an unsaved +// planned snapshot, which resolves the same closed-member graph live because +// nothing has been sealed yet. Workflow tiers are never added to a bundle. // // This service performs cross-service READS (permitted by the event-driven // architecture — see docs/CROSS_SERVICE_READS_PATTERN.md) by querying STIX @@ -24,11 +24,9 @@ const { v5: uuidv5 } = require('uuid'); const logger = require('../../lib/logger'); const linkById = require('../../lib/linkById'); -const revisionReference = require('../../lib/release-tracks/revision-reference'); const primaryRevisionService = require('./primary-revision-service'); const contentManifestService = require('./content-manifest-service'); const publicationService = require('./publication-service'); -const { BadRequestError } = require('../../exceptions'); const { bundleTransformSchema, workbenchTransformSchema, @@ -91,29 +89,6 @@ function normalizeSourceBundleDefaults(documents, graph) { }); } -/** - * Select the draft workflow-tier entries requested through `include`, - * narrowed by `state`, and resolve dynamic selectors to exact revisions. - */ -async function includedTierEntries(snapshot, options) { - const include = options.include || []; - const entries = []; - for (const tier of ['staged', 'candidates']) { - if (!include.includes(tier)) continue; - for (const entry of snapshot[tier] || []) { - if ( - options.state && - entry.object_status !== 'reviewed' && - !options.state.includes(entry.object_status) - ) { - continue; - } - entries.push({ object_ref: entry.object_ref, object_modified: entry.object_modified }); - } - } - return revisionReference.resolveEntries(entries); -} - function bundleIdFor(snapshot) { if (snapshot.bundle_id) return snapshot.bundle_id; return `bundle--${uuidv5( @@ -173,8 +148,8 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd * * Bundle exports (see docs/developer/release-tracks/bundle-export.md): * 1. Replay the sealed content manifest (members, closed relationships, - * supporting objects, LinkById targets). A release preview or a draft - * export with `include` resolves the same closed graph live instead. + * supporting objects, LinkById targets). A release preview of an unsaved + * planned snapshot resolves the same closed graph live instead. * 2. Convert LinkById tags to markdown citations * 3. Assemble the bundle (STIX version conformance + collection object for * STIX 2.1) via the Zod transform schema @@ -182,30 +157,15 @@ exports.formatAsFilesystemStore = function formatAsFilesystemStore(snapshot, hyd * @param {Object} snapshot - The raw snapshot document from the dynamic repo * @param {string} format - One of: 'bundle', 'filesystemstore' * @param {Object} [options] - Additional options - * @param {Array} [options.include] - Draft-only extra tiers ('staged', 'candidates') - * @param {Array} [options.state] - Workflow status filter for included tiers * @param {string} [options.stixVersion] - '2.0' or '2.1' (default '2.1') * @param {boolean} [options.resolveLive] - Resolve the graph live (release previews) * @returns {Promise} The formatted export */ exports.exportSnapshot = async function exportSnapshot(snapshot, format, options = {}) { if (format === 'bundle') { - const include = options.include || []; - if (include.length > 0 && snapshot.version != null) { - throw new BadRequestError({ - message: - 'Tagged snapshots export members only. The include parameter is a draft preview option.', - details: { include }, - }); - } - - let graph; - if (options.resolveLive || include.length > 0) { - const extraEntries = include.length > 0 ? await includedTierEntries(snapshot, options) : []; - graph = await contentManifestService.resolveLive(snapshot, extraEntries); - } else { - graph = await contentManifestService.replay(snapshot); - } + const graph = options.resolveLive + ? await contentManifestService.resolveLive(snapshot) + : await contentManifestService.replay(snapshot); const publication = await publicationService.publicationForExport(snapshot); const allObjects = [ diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index d166dacb..6b8e531d 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -132,6 +132,8 @@ function addObjectInfo(entry, resolvedModifiedBySelector, objectsByVersion, user if (object) { entryWithObjectInfo.attack_id = object.workspace?.attack_id; entryWithObjectInfo.name = object.stix?.name; + entryWithObjectInfo.type = object.stix?.type; + entryWithObjectInfo.x_mitre_version = object.stix?.x_mitre_version; } if (object?.stix?.description !== undefined) { @@ -210,9 +212,15 @@ async function formatWorkbenchSnapshot(snapshot, options) { selectedTiers.flatMap((tierName) => snapshot[tierName] || []), ); const enriched = await addObjectInfoToSnapshot(snapshot); + // Registry-derived, read-only: lets clients build alias URLs for the track. + enriched.alias = await snapshotService.getTrackAlias(snapshot.id); return filterSnapshotTiers(enriched, options?.include); } +exports.resolveTrackAlias = function resolveTrackAlias(alias) { + return snapshotService.resolveTrackAlias(alias); +}; + // ----------------------------------------------------------------------------- // Track management (Phase 1 → snapshot-service) // ----------------------------------------------------------------------------- diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 43ea9ab8..102abdda 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -29,11 +29,12 @@ const reconciliationService = require('./reconciliation-service'); const contentManifestService = require('./content-manifest-service'); const publicationService = require('./publication-service'); const { - TrackNotFoundError, - NotFoundError, - TaggedSnapshotDeletionError, + DuplicateIdError, HistoricalSnapshotDeletionError, + NotFoundError, ReleaseConflictError, + TaggedSnapshotDeletionError, + TrackNotFoundError, } = require('../../exceptions'); // ============================================================================= @@ -177,6 +178,7 @@ exports.createTrack = async function createTrack(data) { const trackId = `release-track--${uuidv4()}`; const now = new Date(); const trackType = data.type || 'standard'; + if (data.alias) await assertAliasAvailable(data.alias); const initialSnapshot = { id: trackId, @@ -207,6 +209,7 @@ exports.createTrack = async function createTrack(data) { track_id: trackId, type: trackType, name: data.name, + alias: data.alias || undefined, description: data.description, latest_snapshot_modified: now, snapshot_count: 1, @@ -220,6 +223,35 @@ exports.createTrack = async function createTrack(data) { return snapshot; }; +/** + * An alias must name at most one track. The partial unique index is the + * backstop; this check turns the common case into a descriptive 409. + */ +async function assertAliasAvailable(alias, trackId) { + const existing = await registryRepo.findByAlias(alias); + if (existing && existing.track_id !== trackId) { + throw new DuplicateIdError(`Release track alias '${alias}' is already in use`, { + details: { alias, track_id: existing.track_id }, + }); + } +} + +/** + * Resolve an alias to the track ID it names, or null. + */ +exports.resolveTrackAlias = async function resolveTrackAlias(alias) { + const entry = await registryRepo.findByAlias(alias); + return entry?.track_id ?? null; +}; + +/** + * The alias registered for a track, or null. + */ +exports.getTrackAlias = async function getTrackAlias(trackId) { + const entry = await registryRepo.findByTrackId(trackId); + return entry?.alias ?? null; +}; + // ============================================================================= // Snapshot retrieval // ============================================================================= @@ -509,10 +541,15 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { /** * Update metadata on the latest snapshot (creates a new snapshot clone). * + * Name and description live on the snapshot and in the registry, so changing + * either clones a new draft. The alias is registry-only routing metadata: an + * alias-only update leaves the snapshot history untouched and returns the + * latest snapshot unchanged. + * * @param {string} trackId - * @param {Object} updates - { name?, description? } + * @param {Object} updates - { name?, description?, alias? } (alias null clears) * @param {string} [_userId] - * @returns {Promise} The new snapshot + * @returns {Promise} The new (or, for alias-only updates, latest) snapshot */ // eslint-disable-next-line no-unused-vars exports.updateMetadata = async function updateMetadata(trackId, updates, _userId) { @@ -521,6 +558,11 @@ exports.updateMetadata = async function updateMetadata(trackId, updates, _userId if (updates.name !== undefined) overrides.name = updates.name; if (updates.description !== undefined) overrides.description = updates.description; + if (updates.alias !== undefined) { + if (updates.alias) await assertAliasAvailable(updates.alias, trackId); + await registryRepo.setAlias(trackId, updates.alias); + } + // Also update the registry name/description if changed const registryUpdates = {}; if (updates.name !== undefined) registryUpdates.name = updates.name; @@ -530,6 +572,7 @@ exports.updateMetadata = async function updateMetadata(trackId, updates, _userId await registryRepo.updateByTrackId(trackId, registryUpdates); } + if (Object.keys(overrides).length === 0) return source; return exports.cloneSnapshot(trackId, source, overrides); }; diff --git a/app/tests/api/release-tracks/content-manifests.spec.js b/app/tests/api/release-tracks/content-manifests.spec.js index 40066c98..9edc8db0 100644 --- a/app/tests/api/release-tracks/content-manifests.spec.js +++ b/app/tests/api/release-tracks/content-manifests.spec.js @@ -371,7 +371,7 @@ describe('Sealed release-track content manifests', function () { expect(entries.some((entry) => entry.object_ref === excluded.stix.id)).toBe(false); }); - it('treats include as a draft-only preview and rejects it on released snapshots', async function () { + it('exports the sealed manifest for drafts and releases alike and rejects include', async function () { const member = await post('/api/techniques', technique('Include Member')); const candidate = await post('/api/techniques', technique('Include Candidate')); const edge = await post('/api/relationships', relationship(member, candidate)); @@ -383,22 +383,20 @@ describe('Sealed release-track content manifests', function () { 200, ); - await authenticated( - request(app).get( - `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent( - released.modified, - )}?format=bundle&include=candidates`, - ), - ).expect(400); - - const withCandidates = await get( - `/api/release-tracks/${track.id}/snapshots/latest?format=bundle&include=candidates`, - ); - const ids = withCandidates.objects.map((object) => object.id); - expect(ids).toContain(candidate.stix.id); - expect(ids).toContain(edge.stix.id); - const membersOnly = await get(`/api/release-tracks/${track.id}/snapshots/latest?format=bundle`); - expect(membersOnly.objects.some((object) => object.id === edge.stix.id)).toBe(false); + for (const path of [ + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(released.modified)}`, + `/api/release-tracks/${track.id}/snapshots/latest`, + ]) { + await authenticated(request(app).get(`${path}?format=bundle&include=candidates`)).expect(400); + } + + // The draft inherits the release's manifest: the candidate and the edge + // that would close over it are absent until the members change. + const draft = await get(`/api/release-tracks/${track.id}/snapshots/latest?format=bundle`); + const ids = draft.objects.map((object) => object.id); + expect(ids).toContain(member.stix.id); + expect(ids).not.toContain(candidate.stix.id); + expect(ids).not.toContain(edge.stix.id); }); it('replaces a release manifest with a source-attested reconstruction only when named', async function () { diff --git a/app/tests/api/release-tracks/release-tracks-bundle.spec.js b/app/tests/api/release-tracks/release-tracks-bundle.spec.js index 19af8532..1c0f5e96 100644 --- a/app/tests/api/release-tracks/release-tracks-bundle.spec.js +++ b/app/tests/api/release-tracks/release-tracks-bundle.spec.js @@ -493,95 +493,43 @@ describe('Release Tracks Bundle Export API', function () { ); }); - it('rejects include on a released snapshot', async function () { - await getBundle( - `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent( - taggedModified, - )}?format=bundle&include=candidates`, - 400, - ); - }); - - it('GET /api/release-tracks/:id/snapshots/latest?format=bundle converts LinkById tags to markdown citations', async function () { + it('bundles replay the sealed manifest and never add workflow tiers', async function () { const bundle = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle`); - const member = bundle.objects.find((o) => o.id === memberObject.stix.id); - expect(member.description).toBe(`See [Linked Technique](${linkedAttackUrl}) for details.`); - }); - - it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates adds the candidates tier', async function () { - const bundle = await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidates`, - ); const ids = bundleObjectIds(bundle); expect(ids).toContain(memberObject.stix.id); - expect(ids).toContain(candidateWip.stix.id); - expect(ids).toContain(candidateAwaitingReview.stix.id); - expect(ids).toContain(candidateReviewed.stix.id); expect(ids).not.toContain(stagedObject.stix.id); - }); - - it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=staged adds the staged tier', async function () { - const bundle = await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=staged`, - ); - - const ids = bundleObjectIds(bundle); - expect(ids).toContain(memberObject.stix.id); - expect(ids).toContain(stagedObject.stix.id); expect(ids).not.toContain(candidateWip.stix.id); + expect(ids).not.toContain(candidateAwaitingReview.stix.id); + expect(ids).not.toContain(candidateReviewed.stix.id); }); - it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged adds both tiers', async function () { - const bundle = await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidates,staged`, - ); - - const ids = bundleObjectIds(bundle); - expect(ids).toContain(memberObject.stix.id); - expect(ids).toContain(candidateWip.stix.id); - expect(ids).toContain(candidateAwaitingReview.stix.id); - expect(ids).toContain(candidateReviewed.stix.id); - expect(ids).toContain(stagedObject.stix.id); - }); - - it('GET /api/release-tracks/:id/snapshots/latest?format=bundle accepts singular tier names and repeated params', async function () { - const bundle = await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidate&include=staged`, + it('rejects include for bundles on drafts and releases alike', async function () { + for (const include of ['staged', 'candidates', 'candidates,staged', 'all', 'members']) { + await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=${include}`, + 400, + ); + } + await getBundle( + `/api/release-tracks/${trackId}/snapshots/${encodeURIComponent( + taggedModified, + )}?format=bundle&include=candidates`, + 400, ); - - const ids = bundleObjectIds(bundle); - expect(ids).toContain(candidateWip.stix.id); - expect(ids).toContain(stagedObject.stix.id); }); - it('state narrows included candidates but reviewed entries are always included', async function () { - const bundle = await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidates&state=work-in-progress`, + it('no longer accepts state: bundles carry no workflow tiers to filter', async function () { + await getBundle( + `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&state=work-in-progress`, + 400, ); - - const ids = bundleObjectIds(bundle); - // Members are unaffected by state - expect(ids).toContain(memberObject.stix.id); - // Matching workflow status - expect(ids).toContain(candidateWip.stix.id); - // Reviewed entries are always included, irrespective of state - expect(ids).toContain(candidateReviewed.stix.id); - // Non-matching, non-reviewed status is excluded - expect(ids).not.toContain(candidateAwaitingReview.stix.id); }); - it('state applies to the staged tier as well', async function () { - // The staged object retained its work-in-progress status through promotion - const withMatchingState = await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=staged&state=work-in-progress`, - ); - expect(bundleObjectIds(withMatchingState)).toContain(stagedObject.stix.id); - - const withoutMatchingState = await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=staged&state=awaiting-review`, - ); - expect(bundleObjectIds(withoutMatchingState)).not.toContain(stagedObject.stix.id); + it('GET /api/release-tracks/:id/snapshots/latest?format=bundle converts LinkById tags to markdown citations', async function () { + const bundle = await getBundle(`/api/release-tracks/${trackId}/snapshots/latest?format=bundle`); + const member = bundle.objects.find((o) => o.id === memberObject.stix.id); + expect(member.description).toBe(`See [Linked Technique](${linkedAttackUrl}) for details.`); }); it('GET /api/release-tracks/:id/snapshots/latest?format=bundle&stixVersion=2.0 conforms the bundle to STIX 2.0', async function () { @@ -595,15 +543,7 @@ describe('Release Tracks Bundle Export API', function () { expect(member.spec_version).toBeUndefined(); }); - it('rejects invalid include, state, and stixVersion values for bundle exports', async function () { - await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=quarantine`, - 400, - ); - await getBundle( - `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&include=candidates&state=reviewed`, - 400, - ); + it('rejects invalid stixVersion values for bundle exports', async function () { await getBundle( `/api/release-tracks/${trackId}/snapshots/latest?format=bundle&stixVersion=3.0`, 400, @@ -612,7 +552,7 @@ describe('Release Tracks Bundle Export API', function () { it('GET /api/release-tracks/:id/snapshots/:modified?format=bundle exports a historical snapshot', async function () { const bundle = await getBundle( - `/api/release-tracks/${trackId}/snapshots/${snapshotModified}?format=bundle&include=candidates,staged`, + `/api/release-tracks/${trackId}/snapshots/${snapshotModified}?format=bundle`, ); expect(bundle.type).toBe('bundle'); @@ -621,8 +561,8 @@ describe('Release Tracks Bundle Export API', function () { const ids = bundleObjectIds(bundle); expect(ids).toContain(memberObject.stix.id); - expect(ids).toContain(candidateWip.stix.id); - expect(ids).toContain(stagedObject.stix.id); + expect(ids).not.toContain(candidateWip.stix.id); + expect(ids).not.toContain(stagedObject.stix.id); }); it('GET /api/release-tracks/:id/snapshots/latest (workbench default) is unaffected by bundle parameters', async function () { diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index e249d3b2..e912e079 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -246,10 +246,14 @@ describe('Release-track release planning and commit API', function () { name: revisionB.stix.name, }); - const draftBundle = await get( - `/api/release-tracks/${track.id}/snapshots/latest` + '?format=bundle&include=staged', + // The draft bundle replays the sealed manifest (no members yet); the + // release preview bundle resolves the planned members live. + const draftBundle = await get(`/api/release-tracks/${track.id}/snapshots/latest?format=bundle`); + expect(draftBundle.body.objects.some((object) => object.id === revisionB.stix.id)).toBe(false); + const previewBundle = await get( + `/api/release-tracks/${track.id}/snapshots/latest/release/preview?format=bundle`, ); - expect(draftBundle.body.objects).toEqual([ + expect(previewBundle.body.objects).toEqual([ expect.objectContaining({ type: 'x-mitre-collection' }), expect.objectContaining({ id: revisionB.stix.id, diff --git a/app/tests/api/release-tracks/release-tracks.spec.js b/app/tests/api/release-tracks/release-tracks.spec.js index a29e099b..b56ce27a 100644 --- a/app/tests/api/release-tracks/release-tracks.spec.js +++ b/app/tests/api/release-tracks/release-tracks.spec.js @@ -32,6 +32,7 @@ function buildTechnique(name, description) { kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'persistence' }], x_mitre_is_subtechnique: false, x_mitre_platforms: ['Windows'], + x_mitre_version: '1.0', }, }; } @@ -74,6 +75,8 @@ describe('Release Tracks API', function () { expect(entry).toMatchObject({ attack_id: object.workspace.attack_id, name: object.stix.name, + type: object.stix.type, + x_mitre_version: object.stix.x_mitre_version, description: object.stix.description, modified_by_user: { username: 'anonymous', diff --git a/app/tests/api/release-tracks/track-aliases.spec.js b/app/tests/api/release-tracks/track-aliases.spec.js new file mode 100644 index 00000000..34c1b1eb --- /dev/null +++ b/app/tests/api/release-tracks/track-aliases.spec.js @@ -0,0 +1,142 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const ReleaseTrackRegistry = require('../../../models/release-tracks/release-track-registry-model'); + +describe('Release-track aliases', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + function authenticated(builder) { + return builder + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + } + + async function post(path, body, status = 201) { + return (await authenticated(request(app).post(path).send(body)).expect(status)).body; + } + + async function get(path, status = 200) { + return (await authenticated(request(app).get(path)).expect(status)).body; + } + + async function createTrack(name, alias) { + return post('/api/release-tracks/new', { name, type: 'standard', alias }); + } + + it('creates a track with an alias and resolves it on every :id route', async function () { + const track = await createTrack('Aliased Track', 'aliased-track'); + + const registry = await ReleaseTrackRegistry.findOne({ track_id: track.id }).lean().exec(); + expect(registry.alias).toBe('aliased-track'); + + const viaAlias = await get('/api/release-tracks/aliased-track/snapshots/latest'); + const viaId = await get(`/api/release-tracks/${track.id}/snapshots/latest`); + expect(viaAlias.id).toBe(track.id); + expect(viaAlias.alias).toBe('aliased-track'); + expect(viaAlias.modified).toBe(viaId.modified); + + const snapshots = await get('/api/release-tracks/aliased-track/snapshots'); + expect(snapshots.data[0].id).toBe(track.id); + + const configuration = await get('/api/release-tracks/aliased-track/config'); + expect(configuration).toBeDefined(); + + const listed = await get('/api/release-tracks'); + const entry = listed.data.find((candidate) => candidate.track_id === track.id); + expect(entry.alias).toBe('aliased-track'); + }); + + it('sets, changes, and clears an alias through metadata without cloning a snapshot', async function () { + const track = await createTrack('Renamed Alias Track'); + const before = await get(`/api/release-tracks/${track.id}/snapshots`); + expect(before.data).toHaveLength(1); + + const set = await post(`/api/release-tracks/${track.id}/meta`, { alias: 'renamed-one' }, 200); + expect(set.modified).toBe(before.data[0].modified); + expect((await get('/api/release-tracks/renamed-one/snapshots/latest')).id).toBe(track.id); + + await post('/api/release-tracks/renamed-one/meta', { alias: 'renamed-two' }, 200); + expect((await get('/api/release-tracks/renamed-two/snapshots/latest')).id).toBe(track.id); + await get('/api/release-tracks/renamed-one/snapshots/latest', 404); + + const cleared = await post(`/api/release-tracks/${track.id}/meta`, { alias: null }, 200); + expect(cleared.alias).toBeUndefined(); + expect((await get(`/api/release-tracks/${track.id}/snapshots/latest`)).alias).toBeNull(); + await get('/api/release-tracks/renamed-two/snapshots/latest', 404); + + const after = await get(`/api/release-tracks/${track.id}/snapshots`); + expect(after.data).toHaveLength(1); + + const registry = await ReleaseTrackRegistry.findOne({ track_id: track.id }).lean().exec(); + expect(registry).not.toHaveProperty('alias'); + }); + + it('keeps aliases unique across tracks', async function () { + const first = await createTrack('Unique Alias A', 'shared-alias'); + const second = await createTrack('Unique Alias B'); + + await post( + '/api/release-tracks/new', + { name: 'Unique Alias C', type: 'standard', alias: 'shared-alias' }, + 409, + ); + await post(`/api/release-tracks/${second.id}/meta`, { alias: 'shared-alias' }, 409); + // Re-asserting a track's own alias is not a conflict. + await post(`/api/release-tracks/${first.id}/meta`, { alias: 'shared-alias' }, 200); + }); + + it('rejects malformed and reserved aliases', async function () { + for (const alias of [ + 'Upper-Case', + 'has space', + '-leading', + 'trailing-', + 'a', + 'x'.repeat(65), + 'new', + 'new-from-bundle', + 'import', + 'objects', + 'ephemeral', + 'latest', + 'release-track', + 'release-track--1234', + ]) { + await post('/api/release-tracks/new', { name: 'Bad Alias', type: 'standard', alias }, 400); + } + }); + + it('returns 404 for an unknown alias and 400 for a malformed canonical id', async function () { + await get('/api/release-tracks/no-such-alias/snapshots/latest', 404); + await get('/api/release-tracks/Not-An-Alias/snapshots/latest', 404); + }); + + it('requires the canonical id as the deletion confirmation when addressed by alias', async function () { + const track = await createTrack('Delete By Alias', 'delete-by-alias'); + + await authenticated( + request(app).delete('/api/release-tracks/delete-by-alias?confirm_track_id=delete-by-alias'), + ).expect(400); + await authenticated( + request(app).delete(`/api/release-tracks/delete-by-alias?confirm_track_id=${track.id}`), + ).expect(204); + await get('/api/release-tracks/delete-by-alias/snapshots/latest', 404); + }); +}); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 0d6641b4..e6ff635f 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -114,8 +114,7 @@ Verification (2026-09-02): files, and the production `ng build` succeed. - Proposed REST commit: `feat(release-tracks): seal snapshot content manifests`. Proposed frontend commit: `feat(release-tracks): surface sealed content and - publication settings`. - +publication settings`. ### Review follow-ups (2026-09-02) @@ -141,7 +140,7 @@ Verification (2026-09-02): - [x] Migration extended in place (unreleased): collection rename, id rewrite, header normalization, dead-config removal, completed-reconciliation cleanup; dry run stays accurate before the rename. -Verification (2026-09-02, review follow-ups): + Verification (2026-09-02, review follow-ups): - Backend focused specs pass: manifest migrations 5, destructive authorization 3, content manifests 10, snapshot history 7, virtual graph integrity 3, @@ -362,8 +361,8 @@ Verification (2026-08-04): Prettier checks pass, and the production build succeeds with existing budget warnings. - Proposed backend commit: `feat(release-tracks): bound snapshot publication - versions`. Proposed frontend commit: `feat(release-tracks): tag snapshots - with exact versions`. +versions`. Proposed frontend commit: `feat(release-tracks): tag snapshots +with exact versions`. ## Frontend graph cache lifecycle controls @@ -2634,3 +2633,26 @@ Verification (2026-07-30): - [x] Add frontend creation, display, edit, clear, and feedback flows. - [x] Add backend and frontend regression coverage. - [x] Run focused tests and the complete backend and frontend verification suites. + +## Release-track UX and API follow-ups (2026-09-03) + +Raised after testing the sealed-manifest work on a restored production +database. + +- [x] Remove bundle `include`/`state`: bundles always replay the sealed + manifest; `format=bundle` rejects `include` (400). Frontend Export + Latest no longer sends `include=all` for bundles. +- [x] Workbench tier entries carry `type` and `x_mitre_version` so the + release preview no longer downloads the whole object catalogue. +- [x] Release preview computes relationship changes without loading supporting + objects and LinkById targets. +- [x] Track URL aliases (unique slug per track resolving to the track ID on + every `:id` route; set at creation or via `/meta`). +- [x] Inter-domain relationship report: `GET /api/reports/domain-consistency` + (active SROs whose endpoints share no domain; domain-bearing SDOs + lacking domains) and a Data Quality page section. +- [x] Frontend: Delete release only on the most recent release; Preview & + Release in-progress state. +- [x] Frontend: draft-then-tag flow (header keeps only Create Draft; tagging + from draft cards), DETAILS tab renamed Board, track deletion in a CONFIG + danger zone. diff --git a/docs/developer/release-tracks/bundle-export.md b/docs/developer/release-tracks/bundle-export.md index dab9b6f7..de75a245 100644 --- a/docs/developer/release-tracks/bundle-export.md +++ b/docs/developer/release-tracks/bundle-export.md @@ -71,7 +71,7 @@ surface was simplified | `stixVersion` | **Preserved** (default changed to `2.1`) | | `includeRevoked` / `includeDeprecated` | **Preserved** (default `false`) | | `includeMissingAttackId` | **Renamed** to `includeObjectsWithMissingAttackId` (default `false`) | -| `includeCollectionObject` | **Renamed** to `includeToc` (default `true`). "TOC" describes what the `x-mitre-collection` object is and avoids overloading "collection". It applies only to STIX 2.1; STIX 2.0 always omits the object. | +| `includeCollectionObject` | **Renamed** to `includeToc` (default `true`). "TOC" describes what the `x-mitre-collection` object is and avoids overloading "collection". It applies only to STIX 2.1; STIX 2.0 always omits the object. | | `collectionObjectVersion` | **Removed** — fixed at `0.1`, signifying an ephemerally generated collection not connected to a release track | | `collectionObjectModified` | **Removed** — fixed at the current timestamp | | `collectionAttackSpecVersion` | **Removed** — fixed at the global default (`config.app.attackSpecVersion`) | @@ -102,14 +102,13 @@ STIX version serialization. The design is recorded in identities and marking definitions, and non-emitted LinkById render targets. Export hydrates those pointers and nothing else: no relationship query, no domain inference, no "latest" lookup. -2. **Draft previews** — `include` (values `staged` and/or `candidates`; - singular forms accepted) adds workflow tiers to a draft export and `state` - (values `work-in-progress` and/or `awaiting-review`) narrows them; entries - whose `object_status` is `reviewed` always pass. Because those tiers may - hold dynamic `latest` selectors, an `include` export resolves the same - closed-member graph live over members plus the included entries instead of - replaying. Tagged snapshots reject `include` with `400`. Release previews - of an unsaved planned snapshot resolve live the same way. +2. **Release previews** — a preview of an unsaved planned snapshot has + nothing sealed yet, so it resolves the same closed-member graph live over + the planned members. This is the only live path. Bundles never add + workflow tiers: `include` is a workbench tier selector and `format=bundle` + rejects it with `400` (the former draft-only `include`/`state` preview was + removed on 2026-09-03 because it produced bundles matching no manifest and + duplicated the release preview). 3. **Supporting objects** — identities and marking definitions referenced by emitted objects, plus the identity and markings the collection object itself references, are appended so the bundle is self-contained. @@ -140,9 +139,9 @@ STIX version serialization. The design is recorded in - `x_mitre_attack_spec_version`: the deployment's ATT&CK spec version - `x_mitre_contents`: every bundle object except marking definitions, sorted by `object_ref` - Drafts resolve the inheritance rule at export so they preview the current - configuration; release commit freezes the resolved values onto the tagged - snapshot as `publication`. + Drafts resolve the inheritance rule at export so they preview the current + configuration; release commit freezes the resolved values onto the tagged + snapshot as `publication`. 7. **Bundle identity and hashes** — a released snapshot stores a stable `bundle_id` assigned at commit; drafts derive a UUIDv5 from the track ID and snapshot `modified`. The bundle ID therefore changes across snapshots @@ -266,11 +265,10 @@ the complete SDO boundary, and virtual materialization applies component Query parameters are validated in the controller with Zod ([release-track-schemas.js](../../../app/lib/release-tracks/release-track-schemas.js)). -The OpenAPI spec declares the parameters loosely (`oneOf` string/array with -`allowReserved` for the list-valued `include`/`state`) so that both -comma-separated and repeated-parameter forms reach the Zod layer, which -normalizes and enforces the enums. Invalid values produce a 400 -`InvalidQueryStringParameterError`. +The OpenAPI spec declares the parameters loosely so the Zod layer enforces the +enums. Invalid values, and `include` on a bundle request, produce a 400 +`InvalidQueryStringParameterError`; parameters absent from the OpenAPI spec +(such as the removed `state`) are rejected by the OpenAPI validator. Primary revision existence is validated separately in `primary-revision-service.js`. This is intentionally a service-layer diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 2a13ed35..53105871 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -4,15 +4,15 @@ This document tracks new database schemas, interfaces, etc.; as well as changes ### Collections at a glance -| Collection | Purpose | Written by | Growth and retention | -| --- | --- | --- | --- | -| `releaseTrackRegistry` | One document per track: name, type, denormalized counters, the tagged-release catalogue (`tagged_releases`), the release lock, and virtual schedules. The index that maps a track to its own snapshot collection. | Track create/delete, every snapshot write (counters), release commit and release deletion (catalogue). | One document per track. | -| `release-track--` | The track's snapshots: at most one rolling draft plus every tagged release for a standard track; every materialized draft plus releases for a virtual track. | Snapshot service and release commit. | Bounded by releases plus one draft (standard) or by materializations (virtual). | -| `releaseTrackContentManifests` | The sealed bill of materials each snapshot references (`content_manifest_id`). Several snapshots share one manifest when their member sets are identical. | Sealed whenever members are written; discarded when no snapshot references it. | Bounded by member-changing writes, not by snapshot count. | -| `releaseTrackContentManifestEntries` | One exact-revision pointer per object a manifest emits or depends on. The `(object_ref, object_modified)` index is what protects referenced revisions from deletion. | With its manifest. | Roughly members + relationships + a few supporting objects per manifest. | -| `releaseTrackReconciliations` | Outstanding backref reconciliation work only: a record is created before the `workspace.release_tracks` listeners run and deleted when they succeed, so anything present is pending or failed and needs repair. | Every snapshot write. | Normally empty. | -| `releaseTrackAuditEvents` | Audit trail for administrator-only destructive operations: `delete_track` and `delete_release`, with actor, confirmation, and outcome. | Those two operations. | Empty until an administrator deletes a track or release. | -| `virtualTrackScheduleOccurrences` | Durable claims for scheduled virtual materialization (cron or dated schedules) so restarts and duplicate delivery execute each occurrence once. | The scheduler. | One record per scheduled occurrence; empty when no virtual track has a schedule. | +| Collection | Purpose | Written by | Growth and retention | +| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | +| `releaseTrackRegistry` | One document per track: name, type, denormalized counters, the tagged-release catalogue (`tagged_releases`), the release lock, and virtual schedules. The index that maps a track to its own snapshot collection. | Track create/delete, every snapshot write (counters), release commit and release deletion (catalogue). | One document per track. | +| `release-track--` | The track's snapshots: at most one rolling draft plus every tagged release for a standard track; every materialized draft plus releases for a virtual track. | Snapshot service and release commit. | Bounded by releases plus one draft (standard) or by materializations (virtual). | +| `releaseTrackContentManifests` | The sealed bill of materials each snapshot references (`content_manifest_id`). Several snapshots share one manifest when their member sets are identical. | Sealed whenever members are written; discarded when no snapshot references it. | Bounded by member-changing writes, not by snapshot count. | +| `releaseTrackContentManifestEntries` | One exact-revision pointer per object a manifest emits or depends on. The `(object_ref, object_modified)` index is what protects referenced revisions from deletion. | With its manifest. | Roughly members + relationships + a few supporting objects per manifest. | +| `releaseTrackReconciliations` | Outstanding backref reconciliation work only: a record is created before the `workspace.release_tracks` listeners run and deleted when they succeed, so anything present is pending or failed and needs repair. | Every snapshot write. | Normally empty. | +| `releaseTrackAuditEvents` | Audit trail for administrator-only destructive operations: `delete_track` and `delete_release`, with actor, confirmation, and outcome. | Those two operations. | Empty until an administrator deletes a track or release. | +| `virtualTrackScheduleOccurrences` | Durable claims for scheduled virtual materialization (cron or dated schedules) so restarts and duplicate delivery execute each occurrence once. | The scheduler. | One record per scheduled occurrence; empty when no virtual track has a schedule. | Removed by the sealed-manifest work: the former `releaseTrackGraphManifests` and `releaseTrackGraphManifestEntries` collections (renamed in place by the @@ -69,6 +69,7 @@ collections. track_id: "release-track--123", type: "standard", name: "ATT&CK Enterprise", + alias: "enterprise-attack", // optional; absent when unset latest_snapshot_modified: "2024-02-01T10:00:00.000Z", latest_tagged_version: "2.0", snapshot_count: 47, @@ -90,6 +91,17 @@ collections. } ``` +`alias` is an optional URL-safe slug that every `:id` route accepts in place +of the track ID. It is unique under a partial unique index +(`{ alias: 1 }`, `alias` of type string), so clearing an alias unsets the +field rather than writing `null`. Resolution happens once per request in an +Express `router.param('id')` callback +([release-tracks-controller.js](../../../app/controllers/release-tracks-controller.js) +`resolveTrackId`), which rewrites `req.params.id` to the canonical ID before +any handler runs; services never see aliases. The alias is registry-only: +snapshots do not store it, and workbench snapshot responses attach it from the +registry at read time. + `tagged_release_count` is derived from `tagged_releases.length`, and `latest_tagged_version` is the highest semantic MAJOR.MINOR version rather than the tag on the chronologically newest snapshot. See diff --git a/docs/developer/release-tracks/sealed-content-manifests.md b/docs/developer/release-tracks/sealed-content-manifests.md index 03b5f9df..510d4d9b 100644 --- a/docs/developer/release-tracks/sealed-content-manifests.md +++ b/docs/developer/release-tracks/sealed-content-manifests.md @@ -101,9 +101,13 @@ endpoint. `baseline_reconstruction` fields are gone. `releaseTrackReconciliations` holds outstanding backref work only and is normally empty. See the collections table in [entities.md](entities.md). -8. **`include=staged,candidates` is a draft-only preview.** Included tier - entries are resolved live and the same closure rule runs over members plus - the included entries. Requesting `include` on a tagged snapshot is a `400`. +9. **Bundles never add workflow tiers.** `include` is a workbench tier + selector; `format=bundle` rejects it with `400`. A draft bundle is exactly + its inherited manifest, and the release preview (`format=bundle`) is the + one live path, resolving the planned members. (An earlier revision of this + design kept a draft-only `include=staged,candidates` preview; it was + removed on 2026-09-03 because it produced bundles that matched no manifest + and duplicated the release preview.) ## Consequences diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 62c8acbe..bc4ad84e 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -152,14 +152,34 @@ self-contained. | `includeRevoked` | `true` \| `false` | `false` | Include objects with `revoked: true` | > [!Note] -> The ephemeral endpoint does not support the `include` or `state` tier -> filters because it does not read from a persisted release-track snapshot — -> it includes all objects in the domain. +> The ephemeral endpoint does not support the `include` tier selector because +> it does not read from a persisted release-track snapshot — it includes all +> objects in the domain. --- ## Release Track Management +### Track identifiers and aliases + +Every track has a canonical ID of the form `release-track--`. A track +may also carry an **alias**: a URL-safe slug (2–64 lowercase letters, digits, +and hyphens, starting and ending with a letter or digit) that is unique across +tracks. Every `/api/release-tracks/:id/...` path accepts either form, so +`/api/release-tracks/enterprise-attack/snapshots/latest` and +`/api/release-tracks/release-track--/snapshots/latest` are the same +request. Responses always report the canonical `id`; workbench snapshot +responses and registry entries also carry `alias` (or `null`). + +Aliases are set at creation (`alias` in the create body) or later through +[Update Metadata](#update-metadata) (`alias: null` clears one). A slug that +would shadow a static path segment (`new`, `new-from-bundle`, `import`, +`objects`, `ephemeral`, `latest`) or begins with `release-track` is rejected +with `400`; an alias already used by another track returns `409`; an unknown +alias in a path returns `404`. Values that expect a track ID — such as +`confirm_track_id` on track deletion and `component_tracks[].track_id` in a +virtual composition — take the canonical ID only. + ### List All Release Tracks Retrieves a list of all release tracks (both standard and virtual) with summary information. @@ -236,6 +256,7 @@ POST /api/release-tracks/new ```json { "name": "Release Track Name", + "alias": "release-track-name", "description": "Description", "snapshot_description": "Context for the initial draft", "type": "standard", @@ -349,6 +370,8 @@ Workbench responses return the release-track snapshot shape. Entries in the `mem - `attack_id` - `name` +- `type` (STIX object type of the selected revision) +- `x_mitre_version` (ATT&CK version of the selected revision) - `description` (when available) - `modified_by_user.name` (display name, or username if display name is missing) @@ -363,13 +386,15 @@ Workbench responses return the release-track snapshot shape. Entries in the `mem **Additional query parameters for `format=bundle`:** -| Parameter | Values | Description | -| ------------- | ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `include` | `staged` and/or `candidates` (comma-separated or repeated) | Draft-only preview: additional tiers to include alongside members, resolved live. Released snapshots reject it with `400`. (Different semantics from `workbench`.) | -| `state` | `work-in-progress` and/or `awaiting-review` (comma-separated or repeated) | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included. Members are unaffected. | -| `stixVersion` | `2.0` \| `2.1` | STIX version the emitted bundle conforms to (default: `2.1`). STIX 2.1 bundles always begin with the `x-mitre-collection` object; STIX 2.0 omits it. | +| Parameter | Values | Description | +| ------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `stixVersion` | `2.0` \| `2.1` | STIX version the emitted bundle conforms to (default: `2.1`). STIX 2.1 bundles always begin with the `x-mitre-collection` object; STIX 2.0 omits it. | -See [Output Formats](output-formats.md) for details on the bundle structure. +A bundle always replays the snapshot's sealed content manifest; `include` is +rejected with `400` for `format=bundle`. Use the +[release preview](#preview-latest-release) with `format=bundle` to see what +the next release would ship. See [Output Formats](output-formats.md) for +details on the bundle structure. **Examples:** @@ -377,15 +402,9 @@ See [Output Formats](output-formats.md) for details on the bundle structure. # Get latest snapshot for the Workbench UI GET /api/release-tracks/:id/snapshots/latest -# Get latest snapshot as STIX bundle (members only) +# Get latest snapshot as STIX bundle (sealed content) GET /api/release-tracks/:id/snapshots/latest?format=bundle -# Get latest snapshot as STIX bundle with staged and candidate objects -GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged - -# Get latest snapshot as STIX bundle with candidates awaiting review -GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates&state=awaiting-review - # Get latest snapshot with members and quarantine only GET /api/release-tracks/:id/snapshots/latest?include=quarantine @@ -518,22 +537,29 @@ A user or team may wish to: - rename a release (e.g., fix a typo like `"Entrprise"` to `"Enterprise"`) or shift the scope/purpose of an existing release track without losing its history (though [cloning](#clone-latest-snapshot) is preferred in this scenario) - update the long-lived `description`. Publication metadata for the emitted collection object (identity, markings, collection ID, creation time) lives in the track configuration; see [Publication configuration](#publication-configuration). +- set or clear the track's [alias](#track-identifiers-and-aliases). ``` POST /api/release-tracks/:id/meta ``` -Creates new snapshot with updated metadata. +Name and description live on the snapshot, so changing either creates a new +snapshot with the updated metadata. The alias is registry-only routing +metadata: an alias-only update changes no snapshot and returns the latest +snapshot unchanged. **Request Body:** ```json { "name": "Updated Name", - "description": "Updated description" + "description": "Updated description", + "alias": "updated-name" } ``` +Send `"alias": null` to remove an alias. + ### Snapshot content is append-only There is no endpoint for replacing a persisted snapshot's `members` tier. @@ -661,9 +687,8 @@ GET /api/release-tracks/:id/snapshots/:modified - `format` - `workbench` | `bundle` | `filesystemstore` (default: `workbench`; `filesystemstore` is not yet implemented) - `include` - `members` | `staged` | `candidates` | `quarantine` | `all` (default: all tiers) -For `format=bundle`, the same additional parameters as -[Get Latest Snapshot](#get-latest-snapshot) apply: `include` (bundle -semantics, drafts only), `state`, and `stixVersion`. +For `format=bundle`, `stixVersion` applies as for +[Get Latest Snapshot](#get-latest-snapshot); `include` is rejected. **Example:** @@ -673,9 +698,6 @@ GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z # Get snapshot from January 15, 2024 as STIX bundle GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z?format=bundle - -# Historical snapshot as a bundle including staged objects -GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z?format=bundle&include=staged ``` ### Release/Tag Specific Snapshot @@ -1061,10 +1083,10 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview `increment` - Supplying both selectors returns `400 Bad Request`; the server never chooses one selector over the other -- `include` - for `workbench`, selects returned tiers; for `bundle`, selects - additional non-member tiers -- `state`, `stixVersion` - bundle representation options; summary previews of - standard tracks add `relationships` describing what the release would seal +- `include` - for `workbench`, selects returned tiers; bundle previews reject + it +- `stixVersion` - bundle representation option; summary previews of standard + tracks add `relationships` describing what the release would seal **Response Example:** @@ -1575,7 +1597,9 @@ The following release-track snapshot retrieval endpoints support `include` and - `GET /api/release-tracks/:id/snapshots/:modified` (get specific snapshot) The ephemeral bundle endpoint supports `format`, but not tier `include`, because -it does not read from a persisted release-track snapshot. +it does not read from a persisted release-track snapshot. `include` applies to +`workbench` responses only; `format=bundle` rejects it because a bundle always +replays the sealed content manifest. **Include Parameter** (workbench format — controls which tiers are returned): @@ -1588,24 +1612,6 @@ GET /api/release-tracks/:id/snapshots/latest?include=quarantine # Member GET /api/release-tracks/:id/snapshots/latest?include=all # All tiers ``` -**Include Parameter** (bundle format — controls which tiers are hydrated into -the bundle; members are always included): - -``` -GET /api/release-tracks/:id/snapshots/latest?format=bundle # Members only -GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=staged # Members + staged -GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates # Members + candidates -GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged # Members + both -``` - -**State Parameter** (bundle format only — narrows the tiers selected via -`include` by workflow status; `reviewed` entries are always included): - -``` -GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates&state=work-in-progress -GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged&state=work-in-progress,awaiting-review -``` - **Format Parameter** (controls output format): ``` diff --git a/docs/user/release-tracks/output-formats.md b/docs/user/release-tracks/output-formats.md index e611c8ba..438f3f9b 100644 --- a/docs/user/release-tracks/output-formats.md +++ b/docs/user/release-tracks/output-formats.md @@ -105,9 +105,10 @@ Standard STIX bundle format: - Every export replays the snapshot's sealed content manifest: exact member revisions, relationships whose source and target are both members (pinned to those member revisions), supporting objects, and LinkById targets. No - secondary SDO is discovered through a relationship. A draft that adds - candidate or staged tiers through `include` is a preview that resolves the - same closed graph live; released snapshots reject `include`. + secondary SDO is discovered through a relationship, and no workflow tier is + ever added: a draft bundle is exactly the manifest it inherited. To see what + a release would ship, use the release preview (`.../release/preview?format=bundle`), + which resolves the planned members live. - Released snapshots carry a stable `bundle_id` and SHA-256 `bundle_hashes` for both serializations; repeated downloads are byte-for-byte identical. Draft bundles use a deterministic identifier derived from the snapshot. @@ -123,23 +124,23 @@ Standard STIX bundle format: **Bundle query parameters** (apply only when `format=bundle`): -| Parameter | Values | Default | Description | -| ------------- | ------------------------------------------------------------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `include` | `staged`, `candidates` (comma-separated or repeated) | _(members only)_ | Draft-only preview: additional tiers to include alongside members, resolved live. Released snapshots reject it with `400`. | -| `state` | `work-in-progress`, `awaiting-review` (comma-separated or repeated) | _(no filter)_ | Narrows the staged/candidate entries selected via `include` by workflow status. Entries marked `reviewed` are always included, irrespective of this parameter. Members are unaffected. | -| `stixVersion` | `2.0`, `2.1` | `2.1` | STIX version the emitted bundle conforms to. STIX 2.1 bundles always begin with the `x-mitre-collection` object; STIX 2.0 bundles never include it. | +| Parameter | Values | Default | Description | +| ------------- | ------------ | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `stixVersion` | `2.0`, `2.1` | `2.1` | STIX version the emitted bundle conforms to. STIX 2.1 bundles always begin with the `x-mitre-collection` object; STIX 2.0 bundles never include it. | + +`include` is a `workbench` tier selector. Sending it with `format=bundle` +returns `400`: a bundle always replays the sealed content manifest, so a +request for staged or candidate objects is refused rather than silently +answered with members only. Examples: ```bash -# Members only (default) +# Sealed content (STIX 2.1) GET /api/release-tracks/:id/snapshots/latest?format=bundle -# Members + staged objects -GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=staged - -# Members + candidates and staged objects that are work-in-progress or reviewed -GET /api/release-tracks/:id/snapshots/latest?format=bundle&include=candidates,staged&state=work-in-progress +# What the next release would ship, resolved live over the planned members +GET /api/release-tracks/:id/snapshots/latest/release/preview?format=bundle # STIX 2.0 bundle (the table of contents is always omitted) GET /api/release-tracks/:id/snapshots/latest?format=bundle&stixVersion=2.0 From eb3711a588f0b7bd480ce64b3e4d46512b6d4cbb Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:53:58 -0400 Subject: [PATCH 06/14] feat(reports): add domain consistency report Release-track bundles no longer discover objects through relationships: a relationship ships only when both of its endpoints are members of the same track. Content whose endpoints can never share a domain track is therefore unpublishable, and nothing surfaced it. `GET /api/reports/domain-consistency` lists the latest revisions of active relationships whose source and target objects share no `x_mitre_domains` value (with the latest endpoint objects and their domains) and the latest revisions of active domain-bearing objects that declare no domain, plus counts. Endpoints are evaluated at their latest revision, so adding the missing domain in a new revision clears the finding. Co-Authored-By: Claude Fable 5.1 --- app/api/definitions/openapi.yml | 3 + app/api/definitions/paths/reports-paths.yml | 47 +++++ app/controllers/reports-controller.js | 20 +++ app/routes/reports-routes.js | 8 + app/services/reports-service.js | 98 +++++++++++ .../api/reports/domain-consistency.spec.js | 161 ++++++++++++++++++ docs/README.md | 1 + docs/user/data-quality-reports.md | 72 ++++++++ 8 files changed, 410 insertions(+) create mode 100644 app/tests/api/reports/domain-consistency.spec.js create mode 100644 docs/user/data-quality-reports.md diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index 4d0ecd1f..1fd73ccb 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -509,6 +509,9 @@ paths: /api/reports/parallel-relationships: $ref: 'paths/reports-paths.yml#/paths/~1api~1reports~1parallel-relationships' + /api/reports/domain-consistency: + $ref: 'paths/reports-paths.yml#/paths/~1api~1reports~1domain-consistency' + # Health Checks /api/health/ping: $ref: 'paths/health-paths.yml#/paths/~1api~1health~1ping' diff --git a/app/api/definitions/paths/reports-paths.yml b/app/api/definitions/paths/reports-paths.yml index bff1f68d..bdcd7cfa 100644 --- a/app/api/definitions/paths/reports-paths.yml +++ b/app/api/definitions/paths/reports-paths.yml @@ -67,6 +67,53 @@ paths: type: string example: 'Unable to get objects with missing LinkById. Server error.' + /api/reports/domain-consistency: + get: + summary: 'Get the domain consistency report' + operationId: 'reports-get-domain-consistency' + description: | + Release-track bundles never discover objects through relationships: a + relationship ships only when both of its endpoints are members of the + same track. This report lists the content that can never be published + together as a result. `cross_domain_relationships` are the latest + revisions of active relationships whose source and target objects + share no `x_mitre_domains` value (each carries the latest + `source_object` and `target_object` plus `source_domains` and + `target_domains`). `objects_without_domains` are the latest revisions + of active domain-bearing ATT&CK objects that declare no domain. + tags: + - 'Reports' + responses: + '200': + description: 'Cross-domain relationships and objects without domains.' + content: + application/json: + schema: + type: object + properties: + cross_domain_relationships: + type: array + items: + type: object + objects_without_domains: + type: array + items: + type: object + summary: + type: object + properties: + cross_domain_relationship_count: + type: integer + objects_without_domains_count: + type: integer + '500': + description: 'Server error' + content: + text/plain: + schema: + type: string + example: 'Unable to get the domain consistency report. Server error.' + /api/reports/parallel-relationships: get: summary: 'Get parallel relationships' diff --git a/app/controllers/reports-controller.js b/app/controllers/reports-controller.js index b930dae9..2fc8a44b 100644 --- a/app/controllers/reports-controller.js +++ b/app/controllers/reports-controller.js @@ -24,6 +24,26 @@ exports.getMissingLinkById = async function (req, res) { } }; +/** + * Handler for GET /api/reports/domain-consistency + * Retrieves active relationships whose endpoints share no domain and + * domain-bearing objects that declare no domain. + * @param {Object} req - Express request object + * @param {Object} res - Express response object + */ +exports.getDomainConsistency = async function (req, res) { + try { + const results = await reportsService.getDomainConsistency(); + logger.debug( + `Success: Retrieved ${results.summary.cross_domain_relationship_count} cross-domain relationship(s) and ${results.summary.objects_without_domains_count} object(s) without domains`, + ); + return res.status(200).send(results); + } catch (err) { + logger.error('Failed with error: ' + err); + return res.status(500).send('Unable to get the domain consistency report. Server error.'); + } +}; + /** * Handler for GET /api/reports/parallel-relationships * Retrieves parallel relationships (same source_ref, target_ref, and relationship_type). diff --git a/app/routes/reports-routes.js b/app/routes/reports-routes.js index 2478652b..5740ba43 100644 --- a/app/routes/reports-routes.js +++ b/app/routes/reports-routes.js @@ -24,4 +24,12 @@ router reportsController.getParallelRelationships, ); +router + .route('/reports/domain-consistency') + .get( + authn.authenticate, + authz.requireRole(authz.visitorOrHigher, authz.readOnlyService), + reportsController.getDomainConsistency, + ); + module.exports = router; diff --git a/app/services/reports-service.js b/app/services/reports-service.js index 3bdce9c1..753c3042 100644 --- a/app/services/reports-service.js +++ b/app/services/reports-service.js @@ -4,6 +4,33 @@ const attackObjectsRepository = require('../repository/attack-objects-repository const relationshipsRepository = require('../repository/relationships-repository'); const identitiesService = require('./stix/identities-service'); +// ATT&CK SDO types whose ADM schema carries x_mitre_domains. Relationships, +// identities, marking definitions, and notes are not domain-bearing. +const DOMAIN_BEARING_TYPES = Object.freeze([ + 'attack-pattern', + 'campaign', + 'course-of-action', + 'intrusion-set', + 'malware', + 'tool', + 'x-mitre-analytic', + 'x-mitre-asset', + 'x-mitre-data-component', + 'x-mitre-data-source', + 'x-mitre-detection-strategy', + 'x-mitre-matrix', + 'x-mitre-tactic', +]); + +function domainsOf(document) { + const domains = document?.stix?.x_mitre_domains; + return Array.isArray(domains) ? domains : []; +} + +function isActive(document) { + return !document.stix?.revoked && !document.stix?.x_mitre_deprecated; +} + /** * Service for generating reports on ATT&CK objects and relationships. * These are read-only analytical queries that identify potential data quality issues. @@ -40,6 +67,76 @@ class ReportsService { return results; } + /** + * Domain consistency: release-track bundles never discover objects through + * relationships, so a relationship only ships when both endpoints are members + * of the same track. Endpoints that share no x_mitre_domains value, and + * domain-bearing objects with no domains at all, are therefore content that + * can never be published together and should be fixed at the source. + * + * Evaluates the latest revision of every active relationship against the + * latest revision of each endpoint. A relationship whose endpoint is missing + * or has no domains is not reported as cross-domain (the missing-domain + * object is listed separately). + * + * @returns {Promise<{ + * cross_domain_relationships: Array, + * objects_without_domains: Array, + * summary: { cross_domain_relationship_count: number, objects_without_domains_count: number }, + * }>} + */ + async getDomainConsistency() { + const [relationships, objectsResult] = await Promise.all([ + relationshipsRepository.retrieveAll({ versions: 'latest' }), + attackObjectsRepository.retrieveAll({ + versions: 'latest', + includeRevoked: true, + includeDeprecated: true, + }), + ]); + const objects = objectsResult[0]?.documents || []; + const latestById = new Map(objects.map((document) => [document.stix.id, document])); + + const crossDomainRelationships = []; + for (const relationship of relationships) { + const source = latestById.get(relationship.stix.source_ref); + const target = latestById.get(relationship.stix.target_ref); + if (!source || !target) continue; + const sourceDomains = domainsOf(source); + const targetDomains = domainsOf(target); + if (sourceDomains.length === 0 || targetDomains.length === 0) continue; + if (sourceDomains.some((domain) => targetDomains.includes(domain))) continue; + crossDomainRelationships.push({ + ...relationship, + source_object: source, + target_object: target, + source_domains: sourceDomains, + target_domains: targetDomains, + }); + } + + const objectsWithoutDomains = objects.filter( + (document) => + DOMAIN_BEARING_TYPES.includes(document.stix.type) && + isActive(document) && + domainsOf(document).length === 0, + ); + + await identitiesService.addCreatedByAndModifiedByIdentitiesToAll([ + ...crossDomainRelationships, + ...objectsWithoutDomains, + ]); + + return { + cross_domain_relationships: crossDomainRelationships, + objects_without_domains: objectsWithoutDomains, + summary: { + cross_domain_relationship_count: crossDomainRelationships.length, + objects_without_domains_count: objectsWithoutDomains.length, + }, + }; + } + /** * Retrieves parallel relationships - relationships that share the same source_ref, * target_ref, and relationship_type. @@ -81,3 +178,4 @@ class ReportsService { } module.exports = new ReportsService(); +module.exports.DOMAIN_BEARING_TYPES = DOMAIN_BEARING_TYPES; diff --git a/app/tests/api/reports/domain-consistency.spec.js b/app/tests/api/reports/domain-consistency.spec.js new file mode 100644 index 00000000..770967f1 --- /dev/null +++ b/app/tests/api/reports/domain-consistency.spec.js @@ -0,0 +1,161 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const Technique = require('../../../models/technique-model'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +function technique(name, domains) { + const timestamp = new Date().toISOString(); + const killChains = { + 'enterprise-attack': 'mitre-attack', + 'mobile-attack': 'mitre-mobile-attack', + 'ics-attack': 'mitre-ics-attack', + }; + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: domains.map((domain) => ({ + kill_chain_name: killChains[domain], + phase_name: 'persistence', + })), + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + x_mitre_domains: domains, + x_mitre_version: '1.0', + }, + }; +} + +function relationship(source, target, extra = {}) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + spec_version: '2.1', + type: 'relationship', + relationship_type: 'subtechnique-of', + source_ref: source.stix.id, + target_ref: target.stix.id, + object_marking_refs: [markingDefinitionId], + ...extra, + }, + }; +} + +describe('GET /api/reports/domain-consistency', function () { + let app; + let passportCookie; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + function authenticated(builder) { + return builder + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + } + + async function post(path, body, status = 201) { + return (await authenticated(request(app).post(path).send(body)).expect(status)).body; + } + + it('reports relationships whose endpoints share no domain and objects lacking domains', async function () { + const enterprise = await post( + '/api/techniques', + technique('Enterprise Only', ['enterprise-attack']), + ); + const mobile = await post('/api/techniques', technique('Mobile Only', ['mobile-attack'])); + const both = await post( + '/api/techniques', + technique('Enterprise And Mobile', ['enterprise-attack', 'mobile-attack']), + ); + const cross = await post('/api/relationships', relationship(enterprise, mobile)); + const shared = await post('/api/relationships', relationship(mobile, both)); + const deprecatedCross = await post( + '/api/relationships', + relationship(mobile, enterprise, { x_mitre_deprecated: true }), + ); + + // A domain-bearing object with no domains, inserted directly because the + // API's work-in-progress schemas are the only route that tolerates it. + const domainless = technique('No Domains', []); + domainless.stix.id = 'attack-pattern--6c2f0f0e-1c0e-4d7c-9d3f-2b1b6f4b7a10'; + domainless.stix.kill_chain_phases = [ + { kill_chain_name: 'mitre-attack', phase_name: 'persistence' }, + ]; + delete domainless.stix.x_mitre_domains; + await new Technique(domainless).save(); + + const report = ( + await authenticated(request(app).get('/api/reports/domain-consistency')).expect(200) + ).body; + + const crossIds = report.cross_domain_relationships.map((entry) => entry.stix.id); + expect(crossIds).toContain(cross.stix.id); + expect(crossIds).not.toContain(shared.stix.id); + expect(crossIds).not.toContain(deprecatedCross.stix.id); + + const entry = report.cross_domain_relationships.find((item) => item.stix.id === cross.stix.id); + expect(entry.source_object.stix.id).toBe(enterprise.stix.id); + expect(entry.target_object.stix.id).toBe(mobile.stix.id); + expect(entry.source_domains).toEqual(['enterprise-attack']); + expect(entry.target_domains).toEqual(['mobile-attack']); + + const missingIds = report.objects_without_domains.map((item) => item.stix.id); + expect(missingIds).toContain(domainless.stix.id); + expect(missingIds).not.toContain(enterprise.stix.id); + + expect(report.summary).toEqual({ + cross_domain_relationship_count: report.cross_domain_relationships.length, + objects_without_domains_count: report.objects_without_domains.length, + }); + }); + + it('evaluates the latest revision of each endpoint', async function () { + const enterprise = await post( + '/api/techniques', + technique('Moves To Mobile', ['enterprise-attack']), + ); + const mobile = await post('/api/techniques', technique('Stays Mobile', ['mobile-attack'])); + const edge = await post('/api/relationships', relationship(enterprise, mobile)); + + let report = ( + await authenticated(request(app).get('/api/reports/domain-consistency')).expect(200) + ).body; + expect(report.cross_domain_relationships.map((entry) => entry.stix.id)).toContain(edge.stix.id); + + // A newer revision that adds the shared domain resolves the finding. + const revised = technique('Moves To Mobile', ['enterprise-attack', 'mobile-attack']); + revised.stix.id = enterprise.stix.id; + revised.stix.created = enterprise.stix.created; + await post('/api/techniques', revised); + + report = (await authenticated(request(app).get('/api/reports/domain-consistency')).expect(200)) + .body; + expect(report.cross_domain_relationships.map((entry) => entry.stix.id)).not.toContain( + edge.stix.id, + ); + }); +}); diff --git a/docs/README.md b/docs/README.md index c7e44ff1..ae622cec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ Guides for consumers of the REST API — endpoints, workflows, and terminology. - [Build Information](user/build-information.md): Inspect the running REST API release and build provenance - [Revoke Workflow](user/revoke-workflow.md): How to revoke ATT&CK objects via the API +- [Data Quality Reports](user/data-quality-reports.md): Missing LinkById, parallel relationship, and domain consistency reports under `/api/reports` ### Release Tracks diff --git a/docs/user/data-quality-reports.md b/docs/user/data-quality-reports.md new file mode 100644 index 00000000..3505e842 --- /dev/null +++ b/docs/user/data-quality-reports.md @@ -0,0 +1,72 @@ +# Data Quality Reports + +Read-only analytical endpoints under `/api/reports` that surface content +problems for editors to fix at the source. All require the visitor role or +higher. The Workbench frontend renders them on the dashboard's **Data +Quality** page. + +## Missing LinkById references + +``` +GET /api/reports/link-by-id/missing?type=attack-pattern +``` + +Objects and relationships whose description mentions `attack.mitre.org` +directly instead of using a `(LinkById: ID)` reference. `type` narrows the +result to one STIX type (`relationship` for relationships only). + +## Parallel relationships + +``` +GET /api/reports/parallel-relationships +``` + +Latest relationship revisions grouped by `source_ref--relationship_type--target_ref` +where more than one relationship shares the key — likely duplicates. The +response is a map from that key to the array of relationships, each carrying +its latest `source_object` and `target_object`. + +## Domain consistency + +``` +GET /api/reports/domain-consistency +``` + +Release-track bundles never discover objects through relationships: a +relationship ships only when both of its endpoints are members of the same +track (see [Output Formats](release-tracks/output-formats.md)). Content whose +endpoints can never be members of the same domain track is therefore +unpublishable, and this report lists it: + +- `cross_domain_relationships` — latest revisions of active (not revoked, not + deprecated) relationships whose source and target objects share no + `x_mitre_domains` value. Each entry is the relationship document plus the + latest `source_object` and `target_object` and their `source_domains` and + `target_domains`. Endpoints are evaluated at their latest revision, so + adding the missing domain in a new revision of the object clears the + finding. A relationship whose endpoint is missing or declares no domain is + not listed here. +- `objects_without_domains` — latest revisions of active domain-bearing ATT&CK + objects (techniques, tactics, groups, software, mitigations, campaigns, + data sources, data components, assets, matrices, detection strategies, + analytics) that declare no domain. +- `summary` — counts of both lists. + +```json +{ + "cross_domain_relationships": [ + { + "stix": { "id": "relationship--...", "relationship_type": "uses", "...": "..." }, + "source_object": { "stix": { "id": "intrusion-set--...", "...": "..." } }, + "target_object": { "stix": { "id": "attack-pattern--...", "...": "..." } }, + "source_domains": ["enterprise-attack"], + "target_domains": ["mobile-attack"] + } + ], + "objects_without_domains": [{ "stix": { "id": "attack-pattern--...", "...": "..." } }], + "summary": { "cross_domain_relationship_count": 1, "objects_without_domains_count": 1 } +} +``` + +Fix a finding by revising the object with the domain it belongs to, or by +deprecating the relationship. From 7aca0df9fc86f0c057cbf19b54157e841e006fe1 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:25:32 -0400 Subject: [PATCH 07/14] feat(release-tracks): preserve drafts for safe release rollback and retagging Protect virtual dependencies, publish retag hashes atomically, recover interrupted updates, and validate destructive confirmation under the release lock. --- .../definitions/components/release-tracks.yml | 12 + .../paths/release-tracks-paths.yml | 119 ++++- app/controllers/release-tracks-controller.js | 28 ++ .../release-tracks/release-track-schemas.js | 4 + .../release-track-audit-event-model.js | 2 +- .../release-track-snapshot-schema.js | 21 + .../release-track-dynamic.repository.js | 118 ++++- app/routes/release-tracks-routes.js | 5 + .../release-tracks/release-tracks-service.js | 78 +++- .../release-tracks/snapshot-service.js | 71 +++ .../release-tracks/versioning-service.js | 102 ++++- .../release-tracks/virtual-track-service.js | 84 ++-- .../release-tracks/content-manifests.spec.js | 7 +- .../destructive-authorization.spec.js | 426 +++++++++++++++++- .../deterministic-graph-migration.spec.js | 23 +- .../release-tracks-release.spec.js | 6 +- .../snapshot-descriptions.spec.js | 5 +- docs/admin/release-track-audit.md | 10 +- docs/developer/TODO.md | 39 ++ .../developer/release-tracks/authorization.md | 13 +- docs/developer/release-tracks/entities.md | 16 +- .../release-tracks/implementation-notes.md | 16 +- .../release-tracks/releases-by-object.md | 22 +- .../sealed-content-manifests.md | 33 +- docs/user/release-tracks/api-reference.md | 59 ++- docs/user/release-tracks/release-workflow.md | 2 + docs/user/release-tracks/summary.md | 9 +- docs/user/release-tracks/terminology.md | 16 +- docs/user/release-tracks/versioning.md | 79 ++-- docs/user/release-tracks/workflow-examples.md | 25 +- 30 files changed, 1254 insertions(+), 196 deletions(-) diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index ceb2bc22..642d92d6 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -27,6 +27,13 @@ components: nullable: true description: 'Semantic version (e.g., "1.0", "2.1") if tagged, null for draft snapshots' example: '1.0' + release_source_modified: + type: string + format: date-time + readOnly: true + description: | + Standard releases only: the exact preserved draft snapshot from + which this release snapshot was created. content_manifest_id: type: string readOnly: true @@ -206,6 +213,11 @@ components: type: string nullable: true description: 'Tagged version, or null for an untagged draft' + release_source_modified: + type: string + format: date-time + readOnly: true + description: 'Standard release source draft timestamp' content_manifest_id: type: string readOnly: true diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 6c127661..99d9011a 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -389,18 +389,19 @@ paths: summary: 'Release the latest snapshot' operationId: 'release-tracks-release-latest' description: | - Immutably tag the latest snapshot with a version. Standard tracks - promote staged entries to members. Any staged `object_modified: + Publish the latest snapshot with a version. Standard tracks retain the + exact draft and create a new tagged snapshot; virtual tracks tag the + materialized draft in place. Standard tracks promote staged entries + to members. Any staged `object_modified: "latest"` selector is resolved to the object's actual latest `stix.modified` timestamp during release planning; tagged members always contain exact revision timestamps. Supply either `increment` (`major` or `minor`) or an explicit `version` in `MAJOR.MINOR` form, but never both. Omitting both defaults to a minor increment. An optional `description` is stored as snapshot-local release notes. - Relative increments use the nearest earlier tagged snapshot. The - selected version must be strictly between the nearest earlier and - later tagged snapshots; the later bound is relevant to retroactive - releases. + Relative increments use the latest tagged release. A historical + standard draft is published as a new release at the current time and + must follow the current version lineage. tags: - 'Release Tracks' parameters: @@ -878,6 +879,9 @@ paths: optional strict scheduled_materialization object to the resulting virtual draft. `description` becomes the new snapshot's local notes; it does not replace the release track description. + Component release locks are held from resolution through persistence. + A concurrent release, rollback, retag, or materialization sharing a + component may return 409; retry after the competing operation finishes. tags: - 'Release Tracks' parameters: @@ -905,11 +909,19 @@ paths: '400': description: 'Track is not virtual or cannot resolve its composition' '409': - description: 'A resolved component snapshot references missing primary revisions' + description: 'A component release lock is busy or a resolved snapshot references missing primary revisions' content: application/json: schema: - $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' + anyOf: + - $ref: '../components/release-tracks.yml#/components/schemas/object-revision-error' + - type: object + required: [message, track_id] + properties: + message: + type: string + track_id: + type: string /api/release-tracks/{id}/virtual/quarantine/promote: post: @@ -966,6 +978,8 @@ paths: content_manifest_id together with content_statistics counts for primary, relationship, supporting, and LinkById entries. Tagged summaries also expose bundle_id and bundle_hashes. + Standard releases with a preserved source draft also expose + release_source_modified, allowing clients to hide that retained draft. tags: - 'Release Tracks' parameters: @@ -1175,13 +1189,16 @@ paths: (editor or higher); the track reverts to its immediately preceding snapshot. Historical drafts have already been pruned. - An administrator may also delete the track's most recent release by - supplying `confirm_version` equal to that snapshot's version. The - release's ledger entry is retracted from every remaining snapshot, its - content manifest is discarded when nothing else references it, the - registry catalogue is reconciled, and a `delete_release` audit event - is recorded. A release that is followed by a later release cannot be - deleted until the later one is removed. + An administrator may also roll back the track's most recent standard + release by supplying `confirm_version` equal to that snapshot's + version. The tagged clone is deleted and its exact preserved source + draft becomes available again. The ledger and registry catalogue are + reconciled and a `delete_release` audit event is recorded. Rollback is + blocked if any persisted virtual snapshot resolved the exact release, + if it is followed by a later release, or if it predates preserved + source drafts. Virtual releases retain the existing irreversible + newest-release deletion behavior because virtual tagging remains + in-place. tags: - 'Release Tracks' parameters: @@ -1210,7 +1227,7 @@ paths: '403': description: 'Deleting a release requires an administrator' '409': - description: 'The release is not the most recent one, or the draft is not the latest snapshot' + description: 'The release cannot be rolled back, a virtual snapshot depends on it, or the draft is not latest' '404': description: 'Snapshot not found' @@ -1349,17 +1366,17 @@ paths: summary: 'Release a specific snapshot' operationId: 'release-tracks-release-by-modified' description: | - Immutably tag the snapshot selected by the modified timestamp using - the same version-selection contract as the latest release operation: + Publish the snapshot selected by the modified timestamp using the same + version-selection contract as the latest release operation: supply `increment` or `version`, never both; omit both for a minor increment. Virtual drafts must have composition_resolution from a successful materialization. For standard tracks, dynamic staged references are resolved to exact object revisions when this release - request is handled, including when the selected snapshot is historical. - An optional `description` is stored as snapshot-local release notes. - Relative increments use the nearest earlier tagged snapshot, and the - selected version must be strictly below the nearest later tagged - snapshot when one exists. + request is handled. Standard tracks retain the selected source draft + and create a tagged clone at the current time, including when the + selected draft is historical. An optional `description` is stored as + snapshot-local release notes. Relative increments use the latest + tagged release. tags: - 'Release Tracks' parameters: @@ -1398,6 +1415,62 @@ paths: application/json: schema: $ref: '../components/release-tracks.yml#/components/schemas/release-track-reconciliation-error' + put: + summary: 'Change a release version' + operationId: 'release-tracks-release-retag' + description: | + Administratively replace the semantic version assigned to a tagged + snapshot without changing its identity or contents. The new version + must remain strictly between the chronologically adjacent releases. + Stored bundle hashes and the release catalogue are regenerated. + The STIX 2.1 digest changes; STIX 2.0 omits the collection object and + its digest is unchanged by a version-only correction. Hashes are + prepared before writing and published atomically with the version. + Retry the same version after a failure to finish history/catalogue + reconciliation; a same-version request repairs derived state. + Existing virtual snapshot provenance remains an immutable record of + the version that was resolved at materialization time. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: modified + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - version + additionalProperties: false + properties: + version: + type: string + pattern: '^\d+\.\d+$' + responses: + '200': + description: 'Release version changed successfully' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' + '400': + description: 'Invalid version body or release-lineage violation' + '403': + description: 'Changing a release version requires an administrator' + '404': + description: 'Snapshot not found' + '409': + description: 'Snapshot is a draft or another release operation is in progress' /api/release-tracks/{id}/snapshots/{modified}/release/preview: get: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 62842693..bbdfcb4b 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -41,6 +41,7 @@ const { updateMetadataBodySchema, updateSnapshotDescriptionBodySchema, releaseBodySchema, + retagReleaseBodySchema, releaseVersionSelectionSchema, cloneBodySchema, addCandidatesBodySchema, @@ -630,6 +631,33 @@ exports.releaseByModified = async function releaseByModified(req, res, next) { } }; +/** PUT /api/release-tracks/:id/snapshots/:modified/release */ +exports.retagRelease = async function retagRelease(req, res, next) { + try { + const bodyResult = retagReleaseBodySchema.safeParse(req.body || {}); + if (!bodyResult.success) { + return next( + new BadRequestError({ + message: 'Invalid release version update', + details: bodyResult.error.errors, + }), + ); + } + + const result = await releaseTracksService.retagRelease( + req.params.id, + req.params.modified, + bodyResult.data.version, + destructiveActor(req), + ); + logger.debug(`Success: Changed release version for snapshot ${req.params.modified}`); + return res.status(200).send(result); + } catch (err) { + logger.error('Failed to change release version: ' + err); + return next(err); + } +}; + /** POST /api/release-tracks/:id/snapshots/:modified/clone */ exports.cloneByModified = async function cloneByModified(req, res, next) { try { diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index ef897365..bd777abf 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -495,6 +495,9 @@ const releaseBodySchema = z message: 'increment and version are mutually exclusive', }); +/** PUT /release-tracks/:id/snapshots/:modified/release */ +const retagReleaseBodySchema = z.object({ version: xMitreVersionSchema }).strict(); + /** POST /release-tracks/:id/clone */ const cloneBodySchema = z .object({ @@ -676,6 +679,7 @@ module.exports = { updateMetadataBodySchema, updateSnapshotDescriptionBodySchema, releaseBodySchema, + retagReleaseBodySchema, publicationConfigSchema, cloneBodySchema, addCandidatesBodySchema, diff --git a/app/models/release-tracks/release-track-audit-event-model.js b/app/models/release-tracks/release-track-audit-event-model.js index 85b1fea0..48853cea 100644 --- a/app/models/release-tracks/release-track-audit-event-model.js +++ b/app/models/release-tracks/release-track-audit-event-model.js @@ -9,7 +9,7 @@ const releaseTrackAuditEventSchema = new mongoose.Schema( action: { type: String, required: true, - enum: ['delete_track', 'delete_release'], + enum: ['delete_track', 'delete_release', 'retag_release'], }, track_id: { type: String, required: true, validate: validateTrackId }, status: { diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 5f3acebd..1fa7f312 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -415,6 +415,10 @@ const releaseTrackSnapshotDefinition = { default: null, validate: validateVersion, }, + // Standard releases are new snapshots. This pointer keeps the exact draft + // that was released reachable so deleting the release rolls back to that + // preserved state instead of attempting to reconstruct it. + release_source_modified: { type: Date, default: undefined }, // Every snapshot references the sealed content manifest that describes its // exact member graph. Member-changing writes seal a new manifest; other // clones inherit their predecessor's manifest by reference. @@ -488,6 +492,23 @@ releaseTrackSnapshotSchema.index( }, ); +releaseTrackSnapshotSchema.index( + { id: 1, release_source_modified: 1 }, + { + name: 'unique_standard_release_source', + unique: true, + partialFilterExpression: { + version: { $type: 'string' }, + release_source_modified: { $type: 'date' }, + }, + }, +); + +releaseTrackSnapshotSchema.index({ + 'composition_resolution.component_snapshots.track_id': 1, + 'composition_resolution.component_snapshots.resolved_snapshot_id': 1, +}); + // A scheduled occurrence may materialize at most one snapshot, including // after restart recovery or duplicate delivery by multiple scheduler nodes. releaseTrackSnapshotSchema.index( diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 7d750d19..8a209433 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -126,6 +126,21 @@ class ReleaseTrackDynamicRepository { } } + async getReleaseBySourceModified(trackId, sourceModified) { + try { + const Model = this._getModel(trackId); + return await Model.findOne({ + id: trackId, + version: { $type: 'string' }, + release_source_modified: sourceModified, + }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async getSnapshotByScheduledMaterialization(trackId, scheduledFor) { try { const Model = this._getModel(trackId); @@ -291,6 +306,7 @@ class ReleaseTrackDynamicRepository { content_manifest_id: 1, bundle_id: 1, bundle_hashes: 1, + release_source_modified: 1, snapshot_description: 1, name: 1, description: 1, @@ -329,7 +345,9 @@ class ReleaseTrackDynamicRepository { throw new DuplicateReleaseVersionError(trackId, snapshotData.version, { cause: err }); } throw new DuplicateIdError({ - details: `Snapshot with modified '${snapshotData.modified}' already exists for track '${trackId}'.`, + details: + `Snapshot uniqueness conflict for track '${trackId}' at modified ` + + `'${new Date(snapshotData.modified).toISOString()}': ${JSON.stringify(err.keyValue)}`, cause: err, }); } @@ -379,6 +397,70 @@ class ReleaseTrackDynamicRepository { } } + async retagSnapshotInPlace(trackId, modified, currentVersion, nextVersion, artifacts) { + try { + const Model = this._getModel(trackId); + return await Model.findOneAndUpdate( + { + id: trackId, + modified, + version: currentVersion, + content_manifest_id: artifacts.bundle_hashes.manifest_id, + }, + { + $set: { + version: nextVersion, + 'version_history.$[entry].version': nextVersion, + ...artifacts, + }, + }, + { + arrayFilters: [ + { + 'entry.version': currentVersion, + 'entry.snapshot_id': new Date(modified), + }, + ], + new: true, + runValidators: true, + lean: true, + }, + ).exec(); + } catch (err) { + if (err.name === 'MongoServerError' && err.code === 11000) { + throw new DuplicateReleaseVersionError(trackId, nextVersion, { cause: err }); + } + throw new DatabaseError(err); + } + } + + async replaceVersionHistoryVersion(trackId, snapshotModified, nextVersion) { + try { + const Model = this._getModel(trackId); + const result = await Model.updateMany( + { + id: trackId, + modified: { $ne: snapshotModified }, + version_history: { + $elemMatch: { snapshot_id: snapshotModified }, + }, + }, + { $set: { 'version_history.$[entry].version': nextVersion } }, + { + arrayFilters: [ + { + 'entry.snapshot_id': new Date(snapshotModified), + }, + ], + runValidators: true, + }, + ).exec(); + return result.modifiedCount; + } catch (err) { + throw new DatabaseError(err); + } + } + async updateSnapshot(trackId, modified, updateOps) { try { const Model = this._getModel(trackId); @@ -478,7 +560,16 @@ class ReleaseTrackDynamicRepository { async deleteOlderDrafts(trackId, modified) { try { const Model = this._getModel(trackId); - const query = { id: trackId, version: null, modified: { $lt: modified } }; + const retainedDrafts = await Model.distinct('release_source_modified', { + id: trackId, + version: { $type: 'string' }, + release_source_modified: { $type: 'date' }, + }).exec(); + const query = { + id: trackId, + version: null, + modified: { $lt: modified, ...(retainedDrafts.length ? { $nin: retainedDrafts } : {}) }, + }; const snapshots = await Model.find(query) .select('modified content_manifest_id') .lean() @@ -501,6 +592,29 @@ class ReleaseTrackDynamicRepository { } } + async findSnapshotsResolvingComponent(trackId, componentTrackId, componentSnapshotModified) { + try { + const Model = this._getModel(trackId); + return await Model.find( + { + id: trackId, + 'composition_resolution.component_snapshots': { + $elemMatch: { + track_id: componentTrackId, + resolved_snapshot_id: new Date(componentSnapshotModified), + }, + }, + }, + { id: 1, name: 1, modified: 1, version: 1 }, + ) + .sort({ modified: 1 }) + .lean() + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async deleteAllSnapshots(trackId) { try { const Model = this._getModel(trackId); diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 832bcd41..4e9ecd76 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -266,6 +266,11 @@ router authn.authenticate, authz.requireRole(authz.editorOrHigher), releaseTracksController.releaseByModified, + ) + .put( + authn.authenticate, + authz.requireRole(authz.editorOrHigher), + releaseTracksController.retagRelease, ); router diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 6b8e531d..3af08de2 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -377,31 +377,34 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified, option return snapshotService.deleteSnapshot(trackId, modified); } - if (options.actor?.role !== authz.userRoles.admin) { - throw new InsufficientRoleError('administrator', { - details: 'Deleting a release requires an administrator.', - track_id: trackId, - version: snapshot.version, - }); - } - if (options.confirmation !== snapshot.version) { - throw new BadRequestError({ - message: 'Destructive release confirmation is required', - details: `Set confirm_version to the exact release version '${snapshot.version}'.`, - parameter_name: 'confirm_version', - expected_version: snapshot.version, - }); - } + return versioningService.withReleaseLock(trackId, async () => { + const snapshot = await snapshotService.getSnapshotByModified(trackId, modified); + if (options.actor?.role !== authz.userRoles.admin) { + throw new InsufficientRoleError('administrator', { + details: 'Deleting a release requires an administrator.', + track_id: trackId, + version: snapshot.version, + }); + } + if (options.confirmation !== snapshot.version) { + throw new BadRequestError({ + message: 'Destructive release confirmation is required', + details: `Set confirm_version to the exact release version '${snapshot.version}'.`, + parameter_name: 'confirm_version', + expected_version: snapshot.version, + }); + } - return destructiveAuditService.execute( - { - action: 'delete_release', - trackId, - ...destructiveIdentity(trackId, options.actor, options.confirmation), - request: { snapshot_modified: new Date(snapshot.modified).toISOString() }, - }, - () => snapshotService.deleteRelease(trackId, modified), - ); + return destructiveAuditService.execute( + { + action: 'delete_release', + trackId, + ...destructiveIdentity(trackId, options.actor, options.confirmation), + request: { snapshot_modified: new Date(snapshot.modified).toISOString() }, + }, + () => snapshotService.deleteRelease(trackId, modified), + ); + }); }; exports.reconstructSnapshotManifest = function reconstructSnapshotManifest( @@ -473,6 +476,33 @@ exports.releaseByModified = function releaseByModified(trackId, modified, option return versioningService.releaseByModified(trackId, modified, options); }; +exports.retagRelease = async function retagRelease(trackId, modified, nextVersion, actor) { + return versioningService.withReleaseLock(trackId, async () => { + const snapshot = await snapshotService.getSnapshotByModified(trackId, modified); + if (actor?.role !== authz.userRoles.admin) { + throw new InsufficientRoleError('administrator', { + details: 'Changing a release version requires an administrator.', + track_id: trackId, + version: snapshot.version, + }); + } + + return destructiveAuditService.execute( + { + action: 'retag_release', + trackId, + ...destructiveIdentity(trackId, actor, snapshot.version), + request: { + snapshot_modified: new Date(snapshot.modified).toISOString(), + previous_version: snapshot.version, + next_version: nextVersion, + }, + }, + () => versioningService.retagReleaseLocked(trackId, modified, nextVersion), + ); + }); +}; + async function renderReleasePlan(plan, options) { const format = options.format || 'summary'; rejectFilesystemStoreFormat(format, 'previewRelease'); diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 102abdda..f6f8ddf4 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -62,6 +62,21 @@ function normalizeTierSummary(summary) { }; } +async function mapWithConcurrency(items, concurrency, mapper) { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex++; + results[index] = await mapper(items[index], index); + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker())); + return results; +} + /** * Recompute and persist denormalized registry counters from actual snapshot data. * @@ -113,6 +128,24 @@ async function emitContentsChanged(trackId, snapshot) { } exports.emitContentsChanged = emitContentsChanged; +async function findVirtualSnapshotDependents(componentTrackId, componentSnapshotModified) { + const virtualTracks = (await registryRepo.findAll({ type: 'virtual' })).data; + const matches = await mapWithConcurrency(virtualTracks, 12, async (track) => + dynamicRepo.findSnapshotsResolvingComponent( + track.track_id, + componentTrackId, + componentSnapshotModified, + ), + ); + return matches.flat().map((snapshot) => ({ + track_id: snapshot.id, + track_name: snapshot.name, + snapshot_modified: snapshot.modified, + version: snapshot.version ?? null, + })); +} +exports.findVirtualSnapshotDependents = findVirtualSnapshotDependents; + /** * Seal a manifest for a snapshot that is about to be saved, then persist the * snapshot referencing it. The manifest is discarded if the save fails, so a @@ -289,6 +322,7 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { content_manifest_id: snapshot.content_manifest_id, bundle_id: snapshot.bundle_id, bundle_hashes: snapshot.bundle_hashes, + release_source_modified: snapshot.release_source_modified, snapshot_description: snapshot.snapshot_description, content_statistics: snapshot.content_manifest_id ? statisticsByManifestId.get(snapshot.content_manifest_id) @@ -602,6 +636,18 @@ exports.updateSnapshotDescription = async function updateSnapshotDescription( version: snapshot.version, }); } + const sourceRelease = await dynamicRepo.getReleaseBySourceModified(trackId, snapshot.modified); + if (sourceRelease) { + throw new ReleaseConflictError( + 'Snapshot notes cannot change while the draft is retained as a release rollback point.', + { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + release_version: sourceRelease.version, + release_snapshot_modified: new Date(sourceRelease.modified).toISOString(), + }, + ); + } const update = description ? { $set: { snapshot_description: description } } @@ -831,6 +877,31 @@ exports.deleteRelease = async function deleteRelease(trackId, modified) { ); } + if (snapshot.type === 'standard') { + if (!snapshot.release_source_modified) { + throw new ReleaseConflictError( + 'This release predates preserved source drafts and cannot be rolled back.', + { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + version: snapshot.version, + }, + ); + } + const dependents = await findVirtualSnapshotDependents(trackId, snapshot.modified); + if (dependents.length > 0) { + throw new ReleaseConflictError( + 'This release cannot be deleted because virtual track snapshots depend on it.', + { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + version: snapshot.version, + dependent_snapshots: dependents, + }, + ); + } + } + await dynamicRepo.deleteSnapshot(trackId, snapshot.modified); await dynamicRepo.pullVersionHistory(trackId, snapshot.version); await contentManifestService.discardUnreferenced(trackId, [snapshot.content_manifest_id]); diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 1f014fb8..94ecd22f 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -143,14 +143,18 @@ function planRelease( const normalized = tierRevisionInvariant.normalizeSnapshot(sourceSnapshot); const snapshot = normalized.snapshot; + const releaseModified = + sourceSnapshot.type === 'standard' + ? new Date(Math.max(now.getTime(), new Date(sourceSnapshot.modified).getTime() + 1)) + : sourceSnapshot.modified; const version = versionUtils.calculateNextVersion( versionHistory, options.increment, options.version, - sourceSnapshot.modified, + releaseModified, ); - versionUtils.validateVersionProgression(version, versionHistory, sourceSnapshot.modified); - const versionBounds = versionUtils.findVersionBounds(versionHistory, sourceSnapshot.modified); + versionUtils.validateVersionProgression(version, versionHistory, releaseModified); + const versionBounds = versionUtils.findVersionBounds(versionHistory, releaseModified); const isVirtual = snapshot.type === 'virtual'; const before = isVirtual @@ -197,7 +201,11 @@ function planRelease( const afterSnapshot = { ...snapshot, + modified: releaseModified, version, + ...(sourceSnapshot.type === 'standard' + ? { release_source_modified: sourceSnapshot.modified } + : {}), members: mergedMembers, ...(updatesSnapshotDescription && options.description ? { snapshot_description: options.description } @@ -217,7 +225,7 @@ function planRelease( version, tagged_at: now, tagged_by: options.userAccountId || 'system', - snapshot_id: sourceSnapshot.modified, + snapshot_id: releaseModified, summary: { ...after, promoted_count: blockingError ? 0 : staged.length, @@ -245,6 +253,7 @@ function planRelease( track_id: trackId, type: snapshot.type, source_snapshot_modified: iso(sourceSnapshot.modified), + release_snapshot_modified: iso(releaseModified), version, version_bounds: { lower: versionBounds.lower @@ -280,6 +289,15 @@ function planRelease( } async function planLoadedSnapshot(trackId, snapshot, options) { + if (snapshot.type === 'standard') { + const existingRelease = await dynamicRepo.getReleaseBySourceModified( + trackId, + snapshot.modified, + ); + if (existingRelease) { + throw new AlreadyReleasedError(existingRelease.version); + } + } const [versionHistory, previousTaggedSnapshot, resolvedStaged] = await Promise.all([ releaseHistoryService.getTrackWideVersionHistory(trackId), snapshot.type === 'virtual' @@ -381,12 +399,20 @@ async function commitPlan(plan) { let tagged; try { - tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, source.modified, { - version: plan.version, - versionHistoryEntry: plan.versionHistoryEntry, - additionalOps: setOps, - unsetOps: Object.keys(unsetOps).length ? unsetOps : undefined, - }); + if (source.type === 'standard') { + const releaseSnapshot = { ...plan.plannedSnapshot, ...setOps }; + delete releaseSnapshot._id; + delete releaseSnapshot.__v; + if (plan.clearSnapshotDescription) delete releaseSnapshot.snapshot_description; + tagged = await dynamicRepo.saveSnapshot(plan.trackId, releaseSnapshot); + } else { + tagged = await dynamicRepo.tagSnapshotInPlace(plan.trackId, source.modified, { + version: plan.version, + versionHistoryEntry: plan.versionHistoryEntry, + additionalOps: setOps, + unsetOps: Object.keys(unsetOps).length ? unsetOps : undefined, + }); + } } catch (err) { await contentManifestService.discard(sealedManifestId); throw err; @@ -406,6 +432,9 @@ async function commitPlan(plan) { const withArtifacts = await refreshReleaseArtifacts(tagged); await releaseHistoryService.reconcileTaggedReleases(plan.trackId); + if (source.type === 'standard') { + await snapshotService.syncRegistryCounters(plan.trackId); + } const latest = await dynamicRepo.getLatestSnapshot(plan.trackId); await snapshotService.emitContentsChanged(plan.trackId, latest); @@ -451,6 +480,7 @@ async function withReleaseLock(trackId, operation) { } exports.planRelease = planRelease; +exports.withReleaseLock = withReleaseLock; exports._private = { memberRevisions, sameRevisions, @@ -483,3 +513,55 @@ exports.releaseByModified = async function releaseByModified(trackId, modified, commitPlan(await exports.planReleaseByModified(trackId, modified, options)), ); }; + +// The facade holds the release lock across validation, audit capture, and this operation. +exports.retagReleaseLocked = async function retagReleaseLocked(trackId, modified, nextVersion) { + const snapshot = await snapshotService.getSnapshotByModified(trackId, modified); + if (snapshot.version == null) { + throw new ReleaseConflictError('The selected snapshot is not a release', { + track_id: trackId, + snapshot_modified: iso(snapshot.modified), + }); + } + const currentVersion = snapshot.version; + const versionHistory = (await releaseHistoryService.getTrackWideVersionHistory(trackId)).filter( + (entry) => iso(entry.modified) !== iso(snapshot.modified), + ); + versionUtils.validateVersionProgression(nextVersion, versionHistory, snapshot.modified); + + // Hash the proposed serialization before publishing any change. A failed + // export leaves the old release intact; version and artifacts change in one + // document update. Same-version retries deliberately replay all side effects. + const publication = + snapshot.publication || (await publicationService.freezePublication(snapshot)); + const bundleId = snapshot.bundle_id || `bundle--${uuid.v4()}`; + const bundleHashes = await bundleHashService.generateBundleHashes({ + ...snapshot, + version: nextVersion, + publication, + bundle_id: bundleId, + }); + const retagged = await dynamicRepo.retagSnapshotInPlace( + trackId, + snapshot.modified, + currentVersion, + nextVersion, + { publication, bundle_id: bundleId, bundle_hashes: bundleHashes }, + ); + if (!retagged) { + throw new ReleaseConflictError('The release changed while its version was being updated', { + track_id: trackId, + snapshot_modified: iso(snapshot.modified), + expected_version: currentVersion, + }); + } + + await dynamicRepo.replaceVersionHistoryVersion(trackId, snapshot.modified, nextVersion); + await releaseHistoryService.reconcileTaggedReleases(trackId); + await snapshotService.syncRegistryCounters(trackId); + + logger.verbose( + `VersioningService: Changed release ${currentVersion} to ${nextVersion} on track "${trackId}"`, + ); + return retagged; +}; diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index d7487215..251299a5 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -488,41 +488,59 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, op }); } - // Validate component tracks - const registryMap = await validateComponentTracks(composition.component_tracks); - - // Resolve composition - const { members, quarantined, compositionResolution } = await resolveComposition( - source, - registryMap, - ); - await primaryRevisionService.assertStoredEntries([...members, ...quarantined]); - - // Build overrides for the new snapshot - const overrides = { - members, - quarantine: quarantined, - composition_resolution: compositionResolution, - scheduled_materialization: options.scheduledMaterialization, - snapshot_description: options.description, - }; - - let snapshot; - try { - snapshot = await snapshotService.cloneSnapshot(trackId, source, overrides); - } catch (err) { - if (!scheduledFor || !(err instanceof DuplicateIdError)) throw err; - - const existing = await dynamicRepo.getSnapshotByScheduledMaterialization(trackId, scheduledFor); - if (!existing) throw err; - snapshot = existing; + // Hold component release locks from resolution through persistence. Rollback + // cannot pass its dependency scan while a new dependent is being created. + // Sorted acquisition and fail-fast conflicts also release partial lock sets. + const componentIds = [ + ...new Set(composition.component_tracks.map((entry) => entry.track_id)), + ].sort(); + const { withReleaseLock } = require('./versioning-service'); + async function withComponentLocks(index) { + if (index === componentIds.length) return materialize(); + return withReleaseLock(componentIds[index], () => withComponentLocks(index + 1)); } + return withComponentLocks(0); + + async function materialize() { + // Validate component tracks + const registryMap = await validateComponentTracks(composition.component_tracks); + + // Resolve composition + const { members, quarantined, compositionResolution } = await resolveComposition( + source, + registryMap, + ); + await primaryRevisionService.assertStoredEntries([...members, ...quarantined]); + + // Build overrides for the new snapshot + const overrides = { + members, + quarantine: quarantined, + composition_resolution: compositionResolution, + scheduled_materialization: options.scheduledMaterialization, + snapshot_description: options.description, + }; + + let snapshot; + try { + snapshot = await snapshotService.cloneSnapshot(trackId, source, overrides); + } catch (err) { + if (!scheduledFor || !(err instanceof DuplicateIdError)) throw err; + + const existing = await dynamicRepo.getSnapshotByScheduledMaterialization( + trackId, + scheduledFor, + ); + if (!existing) throw err; + snapshot = existing; + } - logger.verbose( - `VirtualTrackService: Created virtual snapshot for track "${trackId}" ` + - `(${members.length} members, ${quarantined.length} quarantined)`, - ); - return snapshot; + logger.verbose( + `VirtualTrackService: Created virtual snapshot for track "${trackId}" ` + + `(${members.length} members, ${quarantined.length} quarantined)`, + ); + return snapshot; + } }; /** diff --git a/app/tests/api/release-tracks/content-manifests.spec.js b/app/tests/api/release-tracks/content-manifests.spec.js index 9edc8db0..b012f65e 100644 --- a/app/tests/api/release-tracks/content-manifests.spec.js +++ b/app/tests/api/release-tracks/content-manifests.spec.js @@ -158,8 +158,8 @@ describe('Sealed release-track content manifests', function () { stix_2_0: expect.stringMatching(/^[a-f0-9]{64}$/), stix_2_1: expect.stringMatching(/^[a-f0-9]{64}$/), }); - // The initial manifest is no longer referenced by any snapshot. - expect(await ReleaseTrackContentManifest.countDocuments({ track_id: track.id })).toBe(1); + // The preserved source draft still references its inherited manifest. + expect(await ReleaseTrackContentManifest.countDocuments({ track_id: track.id })).toBe(2); const sealed = await ReleaseTrackContentManifest.findOne({ manifest_id: released.content_manifest_id, @@ -485,7 +485,8 @@ describe('Sealed release-track content manifests', function () { expect(reconstructed.bundle_id).toBe(released.bundle_id); expect(reconstructed.bundle_hashes.manifest_id).toBe(reconstructed.content_manifest_id); expect(reconstructed.bundle_hashes.stix_2_1).not.toBe(released.bundle_hashes.stix_2_1); - expect(await ReleaseTrackContentManifest.countDocuments({ track_id: track.id })).toBe(1); + // Reconstruction replaces only the release manifest; the source draft remains intact. + expect(await ReleaseTrackContentManifest.countDocuments({ track_id: track.id })).toBe(2); const manifest = await ReleaseTrackContentManifest.findOne({ manifest_id: reconstructed.content_manifest_id, }) diff --git a/app/tests/api/release-tracks/destructive-authorization.spec.js b/app/tests/api/release-tracks/destructive-authorization.spec.js index 09d58959..8590d920 100644 --- a/app/tests/api/release-tracks/destructive-authorization.spec.js +++ b/app/tests/api/release-tracks/destructive-authorization.spec.js @@ -3,6 +3,13 @@ const request = require('supertest'); const { expect } = require('expect'); const sinon = require('sinon'); +const crypto = require('node:crypto'); +const dynamicRepo = require('../../../repository/release-tracks/release-track-dynamic.repository'); +const registryRepo = require('../../../repository/release-tracks/release-track-registry.repository'); +const snapshotService = require('../../../services/release-tracks/snapshot-service'); +const versioningService = require('../../../services/release-tracks/versioning-service'); +const bundleHashService = require('../../../services/release-tracks/bundle-hash-service'); +const releaseHistoryService = require('../../../services/release-tracks/release-history-service'); const config = require('../../../config/config'); const database = require('../../../lib/database-in-memory'); @@ -135,7 +142,21 @@ describe('Release-track destructive authorization and audit', function () { const first = await releaseExactMembers(app, passportCookie, track.id, [technique], { version: '1.0', }); - await post(`/api/release-tracks/${track.id}/meta`, { description: 'next' }, 200); + const firstSource = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(first.release_source_modified)}`, + undefined, + 200, + ); + expect(firstSource.body).toMatchObject({ + version: null, + staged: [expect.objectContaining({ object_ref: technique.stix.id })], + }); + const secondDraft = await post( + `/api/release-tracks/${track.id}/meta`, + { description: 'next' }, + 200, + ); const second = await post( `/api/release-tracks/${track.id}/snapshots/latest/release`, { version: '1.1' }, @@ -172,7 +193,9 @@ describe('Release-track destructive authorization and audit', function () { undefined, 200, ); - expect(remaining.body.version).toBe('1.0'); + expect(remaining.body.modified).toBe(secondDraft.modified); + expect(remaining.body.version).toBeNull(); + expect(remaining.body.description).toBe('next'); expect(remaining.body.version_history.map((entry) => entry.version)).toEqual(['1.0']); expect( await ReleaseTrackContentManifest.countDocuments({ @@ -208,6 +231,405 @@ describe('Release-track destructive authorization and audit', function () { expect(again.version).toBe('1.1'); }); + it('blocks deletion when implicit or explicit virtual snapshots resolved the release', async function () { + await setRole('admin'); + const component = await post( + '/api/release-tracks/new', + { name: 'Protected component release', type: 'standard' }, + 201, + ); + const released = await post( + `/api/release-tracks/${component.id}/snapshots/latest/release`, + { version: '1.0' }, + 200, + ); + + const virtualSnapshots = []; + for (const [name, rule] of [ + ['Implicit dependent', { resolution_strategy: 'latest_tagged' }], + ['Explicit dependent', { resolution_strategy: 'specific_version', version: '1.0' }], + ]) { + const virtual = await post( + '/api/release-tracks/new', + { + name, + type: 'virtual', + composition: { + component_tracks: [{ track_id: component.id, priority: 1, ...rule }], + }, + }, + 201, + ); + virtualSnapshots.push({ + trackId: virtual.id, + snapshot: await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 201), + }); + } + + const response = await api( + 'delete', + `/api/release-tracks/${component.id}/snapshots/${encodeURIComponent(released.modified)}`, + undefined, + 409, + { confirm_version: '1.0' }, + ); + expect(response.body.dependent_snapshots).toHaveLength(2); + expect(response.body.dependent_snapshots.map((item) => item.track_name).sort()).toEqual([ + 'Explicit dependent', + 'Implicit dependent', + ]); + await api( + 'get', + `/api/release-tracks/${component.id}/snapshots/${encodeURIComponent(released.modified)}`, + undefined, + 200, + ); + + const retagged = await api( + 'put', + `/api/release-tracks/${component.id}/snapshots/${encodeURIComponent(released.modified)}/release`, + { version: '1.1' }, + 200, + ); + expect(retagged.body.version).toBe('1.1'); + for (const virtual of virtualSnapshots) { + const persisted = await api( + 'get', + `/api/release-tracks/${virtual.trackId}/snapshots/${encodeURIComponent(virtual.snapshot.modified)}`, + undefined, + 200, + ); + expect(persisted.body.composition_resolution.component_snapshots[0]).toMatchObject({ + resolved_version: '1.0', + resolved_snapshot_id: released.modified, + }); + } + }); + + it('lets administrators retag a release within its semantic-version bounds', async function () { + await setRole('admin'); + const track = await post( + '/api/release-tracks/new', + { name: 'Retag release track', type: 'standard' }, + 201, + ); + const first = await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '1.0' }, + 200, + ); + await post(`/api/release-tracks/${track.id}/meta`, { description: 'second' }, 200); + const second = await post( + `/api/release-tracks/${track.id}/snapshots/latest/release`, + { version: '2.0' }, + 200, + ); + + await setRole('editor'); + await api( + 'put', + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(second.modified)}/release`, + { version: '1.1' }, + 403, + ); + + await setRole('admin'); + const retagged = await api( + 'put', + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(second.modified)}/release`, + { version: '1.1' }, + 200, + ); + expect(retagged.body).toMatchObject({ + modified: second.modified, + version: '1.1', + bundle_id: second.bundle_id, + }); + expect(retagged.body.bundle_hashes.stix_2_0).toBe(second.bundle_hashes.stix_2_0); + expect(retagged.body.bundle_hashes.stix_2_1).not.toBe(second.bundle_hashes.stix_2_1); + expect(retagged.body.version_history.map((entry) => entry.version)).toEqual(['1.0', '1.1']); + + const history = await api('get', `/api/release-tracks/${track.id}/snapshots`, undefined, 200); + const summary = history.body.data.find((entry) => entry.modified === second.modified); + expect(summary.release_source_modified).toBe(second.release_source_modified); + expect(summary.bundle_hashes).toEqual(retagged.body.bundle_hashes); + for (const stixVersion of ['2.0', '2.1']) { + const download = await api( + 'get', + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(second.modified)}`, + undefined, + 200, + { format: 'bundle', stixVersion }, + ); + const digest = crypto + .createHash('sha256') + .update(JSON.stringify(download.body, null, 4)) + .digest('hex'); + expect(digest).toBe(summary.bundle_hashes[stixVersion === '2.0' ? 'stix_2_0' : 'stix_2_1']); + } + + await api( + 'put', + `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(first.modified)}/release`, + { version: '1.2' }, + 400, + ); + + const event = await ReleaseTrackAuditEvent.findOne({ + action: 'retag_release', + status: 'completed', + }) + .lean() + .exec(); + expect(event).toMatchObject({ + track_id: track.id, + confirmation: '2.0', + request: { previous_version: '2.0', next_version: '1.1' }, + }); + }); + + for (const [label, target, method, versionPublished] of [ + ['hash generation', bundleHashService, 'generateBundleHashes', false], + ['copied history', dynamicRepo, 'replaceVersionHistoryVersion', true], + ['release catalogue', releaseHistoryService, 'reconcileTaggedReleases', true], + ['registry counters', snapshotService, 'syncRegistryCounters', true], + ]) { + it(`recovers a retag interrupted during ${label}`, async function () { + await setRole('admin'); + const track = await post('/api/release-tracks/new', { name: label, type: 'standard' }, 201); + const base = `/api/release-tracks/${track.id}`; + const released = await post(`${base}/snapshots/latest/release`, { version: '1.0' }); + const draft = await post(`${base}/meta`, { description: 'Copied release history' }); + const path = `${base}/snapshots/${encodeURIComponent(released.modified)}`; + const failure = sinon.stub(target, method).rejects(new Error(`Injected ${label} failure`)); + await api('put', `${path}/release`, { version: '1.1' }, 500); + failure.restore(); + const persisted = await dynamicRepo.getSnapshotByModified(track.id, released.modified); + expect(persisted.version).toBe(versionPublished ? '1.1' : '1.0'); + expect(persisted.bundle_hashes.stix_2_0).toBe(released.bundle_hashes.stix_2_0); + if (!versionPublished) expect(persisted.bundle_hashes).toEqual(released.bundle_hashes); + const retried = await api('put', `${path}/release`, { version: '1.1' }, 200); + expect(retried.body.bundle_hashes.stix_2_1).not.toBe(released.bundle_hashes.stix_2_1); + const copied = await dynamicRepo.getSnapshotByModified(track.id, draft.modified); + expect(copied.version_history.map((entry) => entry.version)).toEqual(['1.1']); + const registry = await registryRepo.findByTrackId(track.id); + expect(registry.tagged_releases.map((entry) => entry.version)).toEqual(['1.1']); + const events = await ReleaseTrackAuditEvent.find({ + track_id: track.id, + action: 'retag_release', + }).lean(); + expect(events.map((event) => event.status).sort()).toEqual(['completed', 'failed']); + }); + } + + for (const strategy of ['latest_tagged', 'specific_version']) { + it(`holds component locks until ${strategy} materialization is persisted`, async function () { + await setRole('admin'); + const component = await post( + '/api/release-tracks/new', + { name: strategy.replaceAll('_', ' '), type: 'standard' }, + 201, + ); + const base = `/api/release-tracks/${component.id}`; + const released = await post(`${base}/snapshots/latest/release`, { version: '1.0' }); + const virtual = await post( + '/api/release-tracks/new', + { + name: 'Concurrent dependent', + type: 'virtual', + composition: { + component_tracks: [ + { + track_id: component.id, + priority: 1, + resolution_strategy: strategy, + ...(strategy === 'specific_version' ? { version: '1.0' } : {}), + }, + ], + }, + }, + 201, + ); + const path = `${base}/snapshots/${encodeURIComponent(released.modified)}`; + const clone = snapshotService.cloneSnapshot; + const stub = sinon.stub(snapshotService, 'cloneSnapshot').callsFake(async (...args) => { + await api('delete', path, undefined, 409, { confirm_version: '1.0' }); + return clone(...args); + }); + await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 201); + expect(stub.calledOnce).toBe(true); + stub.restore(); + const blocked = await api('delete', path, undefined, 409, { confirm_version: '1.0' }); + expect(blocked.body.dependent_snapshots).toHaveLength(1); + expect((await registryRepo.findByTrackId(component.id)).release_lock).toBeUndefined(); + }); + } + + it('repairs missing hashes on a same-version retry and holds the lock during audit capture', async function () { + await setRole('admin'); + const track = await post( + '/api/release-tracks/new', + { name: 'Retag repair', type: 'standard' }, + 201, + ); + const base = `/api/release-tracks/${track.id}`; + const released = await post(`${base}/snapshots/latest/release`, { version: '1.0' }); + const path = `${base}/snapshots/${encodeURIComponent(released.modified)}`; + await dynamicRepo.updateSnapshot(track.id, released.modified, { + $unset: { bundle_hashes: '' }, + }); + const create = auditRepository.create; + const stub = sinon.stub(auditRepository, 'create').callsFake(async (...args) => { + await api('put', `${path}/release`, { version: '1.1' }, 409); + return create.apply(auditRepository, args); + }); + const repaired = await api('put', `${path}/release`, { version: '1.0' }, 200); + expect(stub.calledOnce).toBe(true); + expect(repaired.body.version).toBe('1.0'); + expect(repaired.body.bundle_hashes).toEqual(released.bundle_hashes); + const event = await ReleaseTrackAuditEvent.findOne({ + track_id: track.id, + action: 'retag_release', + }).lean(); + expect(event.request.previous_version).toBe('1.0'); + expect(event.request.next_version).toBe('1.0'); + }); + + it('does not publish retag hashes if the content manifest changed during export', async function () { + await setRole('admin'); + const track = await post( + '/api/release-tracks/new', + { name: 'Manifest race', type: 'standard' }, + 201, + ); + const base = `/api/release-tracks/${track.id}`; + const released = await post(`${base}/snapshots/latest/release`, { version: '1.0' }); + const generate = bundleHashService.generateBundleHashes; + sinon.stub(bundleHashService, 'generateBundleHashes').callsFake(async (snapshot) => { + const hashes = await generate(snapshot); + // Simulate administrative manifest replacement after export was read. + await dynamicRepo.replaceContentManifest( + track.id, + released.modified, + released.content_manifest_id, + track.content_manifest_id, + ); + return hashes; + }); + await api( + 'put', + `${base}/snapshots/${encodeURIComponent(released.modified)}/release`, + { version: '1.1' }, + 409, + ); + const current = await dynamicRepo.getSnapshotByModified(track.id, released.modified); + expect(current.version).toBe('1.0'); + expect(current.content_manifest_id).toBe(track.content_manifest_id); + expect(current.bundle_hashes).toBeUndefined(); + }); + + it('blocks materialization while rollback holds the component lock', async function () { + await setRole('admin'); + const component = await post( + '/api/release-tracks/new', + { name: 'Rollback first', type: 'standard' }, + 201, + ); + const base = `/api/release-tracks/${component.id}`; + const released = await post(`${base}/snapshots/latest/release`, { version: '1.0' }); + const virtual = await post( + '/api/release-tracks/new', + { + name: 'Dependent', + type: 'virtual', + composition: { + component_tracks: [ + { track_id: component.id, priority: 1, resolution_strategy: 'latest_tagged' }, + ], + }, + }, + 201, + ); + const find = dynamicRepo.findSnapshotsResolvingComponent; + sinon.stub(dynamicRepo, 'findSnapshotsResolvingComponent').callsFake(async (...args) => { + await api('post', `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 409); + return find.apply(dynamicRepo, args); + }); + await api( + 'delete', + `${base}/snapshots/${encodeURIComponent(released.modified)}`, + undefined, + 204, + { confirm_version: '1.0' }, + ); + expect((await registryRepo.findByTrackId(component.id)).release_lock).toBeUndefined(); + }); + + it('rechecks confirmation after a retag wins the release lock', async function () { + await setRole('admin'); + const track = await post( + '/api/release-tracks/new', + { name: 'Confirmation race', type: 'standard' }, + 201, + ); + const base = `/api/release-tracks/${track.id}`; + const released = await post(`${base}/snapshots/latest/release`, { version: '1.0' }); + const path = `${base}/snapshots/${encodeURIComponent(released.modified)}`; + const acquire = registryRepo.acquireReleaseLock; + const stub = sinon.stub(registryRepo, 'acquireReleaseLock').callsFake(async (...args) => { + stub.restore(); + await api('put', `${path}/release`, { version: '1.1' }, 200); + return acquire.apply(registryRepo, args); + }); + const rejected = await api('delete', path, undefined, 400, { confirm_version: '1.0' }); + expect(rejected.body.expected_version).toBe('1.1'); + expect((await dynamicRepo.getSnapshotByModified(track.id, released.modified)).version).toBe( + '1.1', + ); + const event = await ReleaseTrackAuditEvent.findOne({ + track_id: track.id, + action: 'retag_release', + }).lean(); + expect(event.request.previous_version).toBe('1.0'); + expect(event.request.next_version).toBe('1.1'); + }); + + it('releases partially acquired component locks after a conflict', async function () { + const components = []; + for (let i = 0; i < 2; i++) { + components.push( + await post('/api/release-tracks/new', { name: `Lock ${i}`, type: 'standard' }, 201), + ); + } + components.sort((a, b) => a.id.localeCompare(b.id)); + const virtual = await post( + '/api/release-tracks/new', + { + name: 'Multiple locks', + type: 'virtual', + composition: { + component_tracks: components.map((component, priority) => ({ + track_id: component.id, + priority, + resolution_strategy: 'latest_tagged', + })), + }, + }, + 201, + ); + await versioningService.withReleaseLock(components[1].id, () => + api('post', `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 409), + ); + for (const component of components) { + expect((await registryRepo.findByTrackId(component.id)).release_lock).toBeUndefined(); + } + // Resolution failure must also unwind the complete lock set. + await api('post', `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 400); + for (const component of components) { + expect((await registryRepo.findByTrackId(component.id)).release_lock).toBeUndefined(); + } + }); + it('reports an audit-finalization failure without hiding the persisted mutation', async function () { await setRole('admin'); const track = await post( diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js index 47678a77..b5c075e7 100644 --- a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -73,6 +73,17 @@ describe('Release-track manifest migrations', function () { return mongoose.connection.db.collection(trackId); } + async function removeModernReleaseSource(trackId, release) { + await trackCollection(trackId).deleteOne({ + modified: new Date(release.release_source_modified), + version: null, + }); + await trackCollection(trackId).updateOne( + { modified: new Date(release.modified) }, + { $unset: { release_source_modified: '' } }, + ); + } + before('create and then downgrade representative legacy data', async function () { const timestamp = new Date().toISOString(); technique = await post('/api/techniques', { @@ -125,8 +136,12 @@ describe('Release-track manifest migrations', function () { 201, ); legacyTrackId = legacyTrack.id; - await releaseExactMembers(app, passportCookie, legacyTrackId, [technique, group]); + const legacyRelease = await releaseExactMembers(app, passportCookie, legacyTrackId, [ + technique, + group, + ]); await post(`/api/release-tracks/${legacyTrackId}/meta`, { description: 'draft' }, 200); + await removeModernReleaseSource(legacyTrackId, legacyRelease); await trackCollection(legacyTrackId).updateMany( {}, { @@ -150,6 +165,7 @@ describe('Release-track manifest migrations', function () { const sealedRelease = await releaseExactMembers(app, passportCookie, sealedTrackId, [ technique, ]); + await removeModernReleaseSource(sealedTrackId, sealedRelease); sealedManifestId = sealedRelease.content_manifest_id.replace( 'release-track-content-manifest--', 'release-track-graph-manifest--', @@ -219,7 +235,10 @@ describe('Release-track manifest migrations', function () { 201, ); orphanTrackId = orphanTrack.id; - await releaseExactMembers(app, passportCookie, orphanTrackId, [technique]); + const orphanRelease = await releaseExactMembers(app, passportCookie, orphanTrackId, [ + technique, + ]); + await removeModernReleaseSource(orphanTrackId, orphanRelease); await mongoose.connection.db .collection('releaseTrackRegistry') .deleteOne({ track_id: orphanTrackId }); diff --git a/app/tests/api/release-tracks/release-tracks-release.spec.js b/app/tests/api/release-tracks/release-tracks-release.spec.js index e912e079..b2e9411b 100644 --- a/app/tests/api/release-tracks/release-tracks-release.spec.js +++ b/app/tests/api/release-tracks/release-tracks-release.spec.js @@ -497,7 +497,8 @@ describe('Release-track release planning and commit API', function () { }); const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, {}); - expect(released.body.modified).toBe(updated.body.modified); + expect(released.body.modified).not.toBe(updated.body.modified); + expect(released.body.release_source_modified).toBe(updated.body.modified); expect(released.body.modified).not.toBe(preview.body.source_snapshot_modified); expect(released.body.version).toBe('1.0'); }); @@ -518,7 +519,8 @@ describe('Release-track release planning and commit API', function () { const released = await post(`/api/release-tracks/${track.id}/snapshots/latest/release`, { version: '3.0', }); - expect(released.body.modified).toBe(replacement.body.modified); + expect(released.body.modified).not.toBe(replacement.body.modified); + expect(released.body.release_source_modified).toBe(replacement.body.modified); expect(released.body.version).toBe('3.0'); }); diff --git a/app/tests/api/release-tracks/snapshot-descriptions.spec.js b/app/tests/api/release-tracks/snapshot-descriptions.spec.js index 50d54466..51d4dc3a 100644 --- a/app/tests/api/release-tracks/snapshot-descriptions.spec.js +++ b/app/tests/api/release-tracks/snapshot-descriptions.spec.js @@ -99,11 +99,14 @@ describe('Release-track snapshot descriptions', function () { }); expect(released).toMatchObject({ - modified: track.modified, version: '1.0', description: 'Stable track description', snapshot_description: 'What changed in the first publication.', }); + expect(released.modified).not.toBe(track.modified); + expect(new Date(released.release_source_modified).toISOString()).toBe( + new Date(track.modified).toISOString(), + ); const originalHashes = released.bundle_hashes; const conflict = await put( diff --git a/docs/admin/release-track-audit.md b/docs/admin/release-track-audit.md index b36f071b..e6427d8f 100644 --- a/docs/admin/release-track-audit.md +++ b/docs/admin/release-track-audit.md @@ -1,15 +1,17 @@ # Release-Track Destructive Audit Events Workbench stores administrator-initiated destructive attempts in -`releaseTrackAuditEvents`: full-track deletion (`delete_track`) and deletion -of a track's most recent release (`delete_release`). The collection is empty -until an administrator performs one of those actions. +`releaseTrackAuditEvents`: full-track deletion (`delete_track`), rollback of a +track's most recent release (`delete_release`), and release-version correction +(`retag_release`). The collection is empty until an administrator performs one +of those actions. Each record contains: - `event_id`, `action`, and `track_id` - the authenticated `actor` -- the exact `confirmation` supplied by the caller +- the exact destructive `confirmation` supplied by the caller (or the prior + version for `retag_release`) - a bounded request/result summary - `pending`, `completed`, or `failed` status - start/finish timestamps and failure detail diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index e6ff635f..e52b12d1 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,44 @@ # Release Track TODOs +## Preserve pre-release drafts and protect virtual dependencies + +- [x] Change standard-track release commit from in-place tagging to creation + of a new tagged snapshot while retaining the exact source draft. +- [x] Prevent rolling-draft cleanup from pruning drafts retained as the source + of a tagged standard release. +- [x] Block release deletion when any persisted virtual snapshot resolved the + exact standard release snapshot, for implicit or explicit composition. +- [x] Add a post-hoc release-version update that preserves the snapshot and + validates the replacement against adjacent release versions. +- [x] Reconcile release catalogues, copied version ledgers, bundle hashes, + audit records, and current-snapshot backrefs for both operations. +- [x] Update OpenAPI, user/developer/operator docs, and Bruno requests. +- [x] Add ADM-valid API regressions and run focused specs, then full `npm test`. +- [x] Update the frontend release controls, wording, connector, and tests. +- [x] Propose conventional commit messages without committing. + +## Rollback / retag review follow-up + +- [x] Coordinate component release locks with virtual materialization. +- [x] Publish retag hashes atomically with the version and repair derived state on retry. +- [x] Expose preserved source pointers in snapshot history. +- [x] Validate deletion confirmation and capture audit identity under the release lock. +- [x] Add concurrency, failure-recovery, history, and exact-download hash regressions. +- [x] Update OpenAPI, user/developer docs, and Bruno smoke requests. +- [x] Run focused specs, full npm test, and lint; propose a commit without committing. + +Verification: focused backend group 32 passing, final destructive/retag spec +16 passing; full `npm test` passes (OpenAPI 2, config 22, API 1033, +middleware 29, scheduler 10). Backend lint and frontend page/connector tests +(90) pass. An initial unrelated technique-conversion 404 passed in isolation +(24) and on the final full run; no unrelated source changes were made. + +Proposed commit: `fix(release-tracks): make rollback and retag concurrency-safe` + +Coordinate materialization with component release locks, publish retag hashes +atomically, repair derived state on retry, expose preserved draft pointers, +and validate destructive confirmation under the audit lock. + ## Sealed snapshot content manifests (Problem 1) Design: [release-tracks/sealed-content-manifests.md](release-tracks/sealed-content-manifests.md). diff --git a/docs/developer/release-tracks/authorization.md b/docs/developer/release-tracks/authorization.md index 70f6df53..3254aaef 100644 --- a/docs/developer/release-tracks/authorization.md +++ b/docs/developer/release-tracks/authorization.md @@ -15,6 +15,7 @@ history requires an administrator. | Tag a standard or virtual snapshot | No | Yes | Yes | | Delete the latest untagged draft snapshot | No | Yes | Yes | | Delete the track's most recent release | No | No | Yes | +| Change a tagged release's semantic version | No | No | Yes | | Delete an entire track and all snapshot history | No | No | Yes | Full-track deletion also requires `confirm_track_id` to equal the `:id` path @@ -24,9 +25,19 @@ deletion shares the snapshot deletion route, so the service checks the administrator role itself and answers `403` otherwise. Confirmation runs before persistence in both cases. +Release-version correction uses `PUT /snapshots/:modified/release`, is also +checked in the service, and does not require destructive confirmation because +it preserves the snapshot. It is serialized with release and rollback and is +recorded as `retag_release`. + +Release deletion re-reads the snapshot and checks `confirm_version` under the +release lock. Both deletion and retag capture audit identity under that same +lock, so a competing version correction cannot invalidate confirmation or +change the version between audit capture and mutation. + ## Audited destructive actions -The `delete_track` and `delete_release` actions create a +The `delete_track`, `delete_release`, and `retag_release` actions create a `releaseTrackAuditEvents` record before the business operation begins. Each event records the authenticated actor, confirmation value, target track, diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 53105871..4da5ed91 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -7,11 +7,11 @@ This document tracks new database schemas, interfaces, etc.; as well as changes | Collection | Purpose | Written by | Growth and retention | | ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | | `releaseTrackRegistry` | One document per track: name, type, denormalized counters, the tagged-release catalogue (`tagged_releases`), the release lock, and virtual schedules. The index that maps a track to its own snapshot collection. | Track create/delete, every snapshot write (counters), release commit and release deletion (catalogue). | One document per track. | -| `release-track--` | The track's snapshots: at most one rolling draft plus every tagged release for a standard track; every materialized draft plus releases for a virtual track. | Snapshot service and release commit. | Bounded by releases plus one draft (standard) or by materializations (virtual). | +| `release-track--` | The track's snapshots: one active rolling draft, a preserved source draft per tagged standard release, and every tagged release; every materialized draft plus releases for a virtual track. | Snapshot service and release commit. | Standard tracks grow by two snapshots per release plus one active draft; virtual tracks by materializations. | | `releaseTrackContentManifests` | The sealed bill of materials each snapshot references (`content_manifest_id`). Several snapshots share one manifest when their member sets are identical. | Sealed whenever members are written; discarded when no snapshot references it. | Bounded by member-changing writes, not by snapshot count. | | `releaseTrackContentManifestEntries` | One exact-revision pointer per object a manifest emits or depends on. The `(object_ref, object_modified)` index is what protects referenced revisions from deletion. | With its manifest. | Roughly members + relationships + a few supporting objects per manifest. | | `releaseTrackReconciliations` | Outstanding backref reconciliation work only: a record is created before the `workspace.release_tracks` listeners run and deleted when they succeed, so anything present is pending or failed and needs repair. | Every snapshot write. | Normally empty. | -| `releaseTrackAuditEvents` | Audit trail for administrator-only destructive operations: `delete_track` and `delete_release`, with actor, confirmation, and outcome. | Those two operations. | Empty until an administrator deletes a track or release. | +| `releaseTrackAuditEvents` | Audit trail for administrator-only track deletion, release rollback, and release retagging (`delete_track`, `delete_release`, `retag_release`). | Those operations. | Empty until an administrator performs one of those operations. | | `virtualTrackScheduleOccurrences` | Durable claims for scheduled virtual materialization (cron or dated schedules) so restarts and duplicate delivery execute each occurrence once. | The scheduler. | One record per scheduled occurrence; empty when no virtual track has a schedule. | Removed by the sealed-manifest work: the former `releaseTrackGraphManifests` @@ -262,6 +262,12 @@ does not change `modified`, tier contents, or the content manifest; once the snapshot is released it is immutable. Rolling edits to the same draft preserve its description; the first draft of a new release cycle starts blank. +For a tagged standard snapshot, `release_source_modified` identifies the exact +untagged draft from which it was created. The pair is unique within the track. +Draft pruning excludes these source snapshots, and the UI suppresses them +while the release exists. Removing the newest release therefore exposes the +unchanged source draft without reconstructing state from a ledger or manifest. + ### Version History The `version_history` array tracks all tagged releases in reverse chronological order (newest first): @@ -288,6 +294,12 @@ This provides: - Attribution for each tagged release - Chronological release history +Correcting a release version updates the matching entry identified by +`snapshot_id`, including copies carried forward into later snapshots. Exact +snapshot identity, publication metadata, content, and bundle ID do not change; +the bundle hashes are regenerated because the projected collection version +does change. + ### Object (SDO/SRO/SMO) Document Schema Objects maintain a simple reverse reference to the release tracks that diff --git a/docs/developer/release-tracks/implementation-notes.md b/docs/developer/release-tracks/implementation-notes.md index 62e21bac..2d153600 100644 --- a/docs/developer/release-tracks/implementation-notes.md +++ b/docs/developer/release-tracks/implementation-notes.md @@ -152,8 +152,9 @@ objects; no partial release track points at them. `app/lib/release-tracks/tier-revision-invariant.js` owns selector identity (`object_ref` + normalized `object_modified`) and normalization. Every clone-based mutation passes through `snapshot-service.cloneSnapshot`; -track cloning uses the same normalizer. Tagging is the one in-place mutation, -so `versioning-service` normalizes before the atomic tag update. This covers +track cloning uses the same normalizer. Standard tagging also creates a clone, +while virtual tagging remains an in-place mutation of its materialized draft. +`versioning-service` normalizes before either commit. This covers candidate adds, manual/automatic promotion, demotion, status transitions, candidate pin changes, member sync, direct content replacement, bundle import, standard/virtual snapshot creation, and release commits without @@ -346,6 +347,17 @@ property. Mongoose validates every map value with the shared release-version validator and requires every persisted component resolution to identify its tagged `resolved_version`. +Standard release commit assigns a fresh timestamp, stores +`release_source_modified`, and inserts the tagged clone while retaining the +source. Release, retag, and rollback share the registry release lock. Rollback +queries exact virtual provenance (`track_id` + `resolved_snapshot_id`) across +all virtual snapshot collections and fails closed when any dependent exists; +this catches both implicit `latest_tagged` and explicit resolution rules. + +Retagging preserves `resolved_snapshot_id`. Existing virtual provenance keeps +the `resolved_version` label observed when it materialized; future explicit +rules that name an obsolete label must be updated by the caller. + ### Snapshot history reads Snapshot history is exposed as a nested collection at diff --git a/docs/developer/release-tracks/releases-by-object.md b/docs/developer/release-tracks/releases-by-object.md index 14a58f5e..5bb2d959 100644 --- a/docs/developer/release-tracks/releases-by-object.md +++ b/docs/developer/release-tracks/releases-by-object.md @@ -43,21 +43,19 @@ length. The dynamic snapshot remains authoritative for its contents. ### Reconciliation -Tagging is already a two-document workflow: it mutates the snapshot in its -dynamic collection, then updates the registry. After a successful tag, the +Tagging is already a two-document workflow: it inserts or updates the release +in its dynamic collection, then updates the registry. After a successful tag, the service reads the track's tagged snapshot metadata and replaces the registry -projection. Reconciliation rather than `$push` makes the operation idempotent, -repairs missing entries, and handles retroactive tags. +projection. Reconciliation rather than `$push` makes the operation idempotent +and repairs missing entries. Existing deployments receive the same projection through an idempotent -database migration. Tagged snapshots are immutable and cannot be deleted; -deleting a whole track removes both its dynamic collection and registry -document. Draft-snapshot squashing is orthogonal because it only deletes -snapshots with `version == null`. - -Version calculation and monotonicity validation must use track-wide tagged -release metadata. An older draft's embedded `version_history` can predate -newer tags and is not a safe global ledger for retroactive tagging. +database migration. Tagged snapshot content is immutable. The newest standard +release can be rolled back only when its preserved source draft exists and no +virtual snapshot resolved it. Draft squashing excludes preserved sources. + +Version calculation and monotonicity validation use track-wide tagged release +metadata rather than a draft's copied ledger. ## Query algorithm diff --git a/docs/developer/release-tracks/sealed-content-manifests.md b/docs/developer/release-tracks/sealed-content-manifests.md index 510d4d9b..dc59e079 100644 --- a/docs/developer/release-tracks/sealed-content-manifests.md +++ b/docs/developer/release-tracks/sealed-content-manifests.md @@ -88,10 +88,13 @@ endpoint. is editable on drafts only. The graph create and delete endpoints are removed. The admin-only source-attested reconstruction endpoint remains and can replace an existing manifest when the caller names the manifest it - expects to replace. The correction path for a mistaken release is - deletion: an administrator may delete the track's most recent release with - a typed version confirmation, which retracts its ledger entry, discards its - manifest when unreferenced, and is audited as `delete_release`. + expects to replace. Standard release commit creates a tagged clone and + retains its exact source draft. The correction path for a mistaken latest + release is rollback: an administrator supplies typed version confirmation, + the clone is removed, and the preserved draft becomes active again. + Rollback is blocked while any virtual snapshot resolves the release. + Version-only corrections preserve content and snapshot identity but + regenerate export hashes. 8. **Storage is named for what it holds.** Manifests live in `releaseTrackContentManifests` and `releaseTrackContentManifestEntries` with `release-track-content-manifest--` ids. A manifest header carries @@ -111,6 +114,28 @@ endpoint. ## Consequences +### Rollback and retag concurrency / recovery + +Virtual materialization acquires the existing database-backed release locks +for all component tracks in sorted order, before resolving any release, and +holds them through snapshot persistence. Partial acquisition and failed +materialization unwind the locks. Contention fails fast with 409. The rollback +dependency scan therefore cannot miss an in-flight materialization: either +rollback owns the lock first, or it sees the persisted virtual dependency +after materialization releases the lock. + +Retag prepares both bundle serializations before writing, then atomically +publishes the version, publication metadata, bundle ID, and hashes on the +snapshot document. Export failure leaves the old release unchanged. Copied +version histories are repaired by snapshot identity, not by the previous +version string; this and catalogue/counter reconciliation run even on +same-version retries. This makes an interrupted multi-document update +recoverable without MongoDB transactions. STIX 2.0 bytes do not include the +release tag, so only the STIX 2.1 digest changes on a version-only correction. + +History's repository projection and service summary both expose +`release_source_modified` so clients can identify retained source drafts. + - Determinism is unconditional: exporting a tagged snapshot replays pointers and never queries relationships, and a draft replays its inherited members graph. diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index bc4ad84e..d1669ee8 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -575,7 +575,10 @@ set. ### Release Latest Snapshot -Converts the latest draft snapshot to a tagged release. Tags the snapshot in-place (does not create new snapshot). Dynamically sets `x_mitre_version` based on the request body options. +Creates a tagged snapshot from the latest standard-track draft and retains the +exact source draft as its rollback point. The tagged snapshot has a new +`modified` timestamp and records the source in `release_source_modified`. +Virtual-track releases continue to tag their materialized draft in place. The request may also include an optional `description` (up to 4000 characters) to set the tagged snapshot's notes in the same operation: @@ -593,9 +596,9 @@ to set the tagged snapshot's notes in the same operation: `400 Bad Request` rather than choosing one - If both are omitted, defaults to a minor release - If this is the first release, the version will be `1.0` -- Relative increments use the nearest chronologically earlier tagged snapshot. - The result, or an explicit version, must also be lower than the nearest later - tagged snapshot when retroactively releasing a historical draft. +- Relative increments use the latest tagged release. Releasing a historical + standard draft still creates a new release at the current time, so its + version must follow the current release lineage. ``` POST /api/release-tracks/:id/snapshots/latest/release @@ -702,7 +705,9 @@ GET /api/release-tracks/:id/snapshots/2024-01-15T16:20:00.000Z?format=bundle ### Release/Tag Specific Snapshot -Converts a specific draft snapshot to a tagged release. Tags snapshot in-place (does not create new snapshot). +Releases a specific draft snapshot. Standard tracks create a new tagged +snapshot and preserve the selected draft; virtual tracks tag the selected +materialized draft in place. ``` POST /api/release-tracks/:id/snapshots/:modified/release @@ -710,6 +715,20 @@ POST /api/release-tracks/:id/snapshots/:modified/release **Request Body:** Same as [Release Latest Snapshot](#release-latest-snapshot). +### Change a Release Version + +``` +PUT /api/release-tracks/:id/snapshots/:modified/release +``` + +Administrators may correct the `MAJOR.MINOR` version of a tagged snapshot +without changing its `modified` identity, bundle ID, publication metadata, or +content. The replacement must remain strictly between the preceding and +following release versions. Copied version ledgers, the release catalogue, +bundle hashes, and the audit trail are updated. Virtual snapshots that already +resolved this release retain their exact `resolved_snapshot_id`; their stored +`resolved_version` remains the historical label observed at materialization. + ### Clone Specific Snapshot Bootstraps a new release track from the specified snapshot. @@ -809,14 +828,16 @@ DELETE /api/release-tracks/:id/snapshots/:modified?confirm_version=1.1 ``` Editors may delete the latest untagged draft; the track reverts to the -preceding snapshot. Administrators may also delete the track's most recent -release by confirming its version. The release's ledger entry is retracted -from every remaining snapshot so the version becomes available again, its -content manifest is discarded when nothing else references it, the registry -catalogue is reconciled, later drafts are kept, and a `delete_release` audit -event is recorded. Deleting an older release, or a release followed by a later -one, returns `409 Conflict`; a missing or wrong confirmation returns `400`; -a non-administrator receives `403`. +preceding snapshot. Administrators may roll back the most recent standard +release by confirming its version. The tagged clone is removed, revealing its +exact preserved source draft; the release ledger and catalogue are reconciled +and a `delete_release` audit event is recorded. Rollback returns `409 Conflict` +if any persisted virtual snapshot resolved the exact release (whether through +`latest_tagged` or an explicit rule), or if the release predates preserved +source drafts. Deleting an older release also returns `409`; a missing or wrong +confirmation returns `400`; a non-administrator receives `403`. +The newest virtual release retains the existing irreversible deletion +behavior because virtual materializations are still tagged in place. --- @@ -1095,6 +1116,7 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview "track_id": "release-track--123", "type": "standard", "source_snapshot_modified": "2024-01-15T16:20:00.000Z", + "release_snapshot_modified": "2024-02-01T10:00:00.000Z", "version": "1.2", "version_bounds": { "lower": { "version": "1.1", "modified": "2024-01-01T12:00:00.000Z" }, @@ -1108,9 +1130,12 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview } ``` +`release_snapshot_modified` is the new identity a standard release would +receive; for a virtual release it equals `source_snapshot_modified`. `version_bounds` reports the exclusive adjacent tagged releases used by both -relative and explicit selection. A historical draft can have both a `lower` -and an `upper` bound. +relative and explicit selection. A standard release is created at the current +time, so it ordinarily has no upper bound; a historical virtual draft can have +both bounds. `format=workbench` returns the complete would-be persisted snapshot. `format=bundle` returns its publication-ready STIX bundle. Thus “dry run” is @@ -1277,7 +1302,9 @@ Invalid version format or not greater than previous versions. **Status:** 409 Conflict -Tagged snapshots are immutable and cannot be deleted. +Tagged contents are immutable. Ordinary draft deletion cannot delete a tagged +snapshot; administrators use the guarded newest-release rollback described +above. ### NotFoundError diff --git a/docs/user/release-tracks/release-workflow.md b/docs/user/release-tracks/release-workflow.md index 66fbe35a..2b0967cc 100644 --- a/docs/user/release-tracks/release-workflow.md +++ b/docs/user/release-tracks/release-workflow.md @@ -564,6 +564,7 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview "track_id": "release-track--123", "type": "standard", "source_snapshot_modified": "2024-01-15T16:20:00.000Z", + "release_snapshot_modified": "2024-02-01T10:00:00.000Z", "version": "1.2", "releasable": true, "before": { "members_count": 2, "staged_count": 3, "candidates_count": 1 }, @@ -579,6 +580,7 @@ GET /api/release-tracks/:id/snapshots/latest/release/preview "track_id": "release-track--123", "type": "standard", "source_snapshot_modified": "2024-01-15T16:20:00.000Z", + "release_snapshot_modified": "2024-02-01T10:00:00.000Z", "version": "1.2", "releasable": false, "before": { "members_count": 2, "staged_count": 3, "candidates_count": 1 }, diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index b78499d6..04517060 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -100,16 +100,19 @@ We borrow heavily concepts from git. Snapshots are sort of like commits and tagg - Every supported modification creates a replacement draft snapshot - Identified by `stix.modified` timestamp - Immutable once created -- Standard tracks retain one rolling untagged draft; tagged releases remain historical +- Standard tracks retain one active rolling draft plus the hidden source draft + for each tagged release; tagged releases remain historical - May be a **draft release** (untagged) or **tagged release** (has version number) **Tagged Releases** (like Git tags) - Snapshots are tagged with `version`, which when exported/retrieved as a STIX bundle, will be expressed as `x_mitre_version`. Draft snapshots are denoted by the fact that their `version` key is set to `null`. - Uses MAJOR.MINOR versioning (not MAJOR.MINOR.PATCH), as specified by the [`x_mitre_version` ADM schema](https://github.com/mitre-attack/attack-data-model/blob/f249442b3588de9cca84b819d480306b106d2c1f/src/schemas/common/property-schemas/attack-versioning.ts#L21:L26) -- Snapshots are tagged in-place (no duplicate data) +- Standard releases are tagged clones with exact rollback drafts; virtual + releases are tagged in place - When a snapshot is tagged/released, an event is captured in its `version_history` array -- Once a snapshot is tagged, it cannot be re-tagged. Tagged snapshots are **immutable**. +- Tagged content is **immutable**. Administrators may correct a release label + within semantic-version lineage constraints. ### 3. Three-Tier Workflow Integration with Version Pinning diff --git a/docs/user/release-tracks/terminology.md b/docs/user/release-tracks/terminology.md index 9b90dfb3..7d5d81e2 100644 --- a/docs/user/release-tracks/terminology.md +++ b/docs/user/release-tracks/terminology.md @@ -100,7 +100,7 @@ A **draft release** (or **draft snapshot**) is an untagged snapshot - still in d **Characteristics:** - No version number assigned - Not considered production-ready -- Can transition from draft to tagged state via tagging (in-place) operation +- Standard drafts are preserved when a tagged release clone is created - May contain candidate, staged, and member objects in various states **Examples:** @@ -117,13 +117,15 @@ A **tagged release** (or **tagged snapshot**) is a snapshot that has been marked **Technical Definition:** - A snapshot where `version !== null` - The version follows MAJOR.MINOR format (e.g., "1.0", "2.3", "15.1") -- Created by performing a tagging operation on a draft release -- The `stix.modified` timestamp does not change during tagging (in-place operation) +- Created from a draft release +- Standard releases receive a new `modified` timestamp and retain a link to + their exact source draft; virtual releases are tagged in place **Characteristics:** - Has an explicit version number - Considered production-ready and published -- **Immutable** - cannot be re-tagged or untagged +- Content is immutable; administrators may correct the semantic version or + roll back the newest standard release when no virtual snapshot depends on it - Recorded in `version_history` for audit trail - Analogous to a Git tag @@ -137,11 +139,11 @@ A **tagged release** (or **tagged snapshot**) is a snapshot that has been marked ### Tagging Operation -The **tagging operation** marks an existing snapshot as a tagged release by assigning it a version number. +The **tagging operation** publishes a draft by assigning a version number. **Technical Definition:** -- Sets `version` on an existing snapshot (in-place update) -- Does NOT create a new snapshot (does NOT change `modified`) +- For standard tracks, creates a tagged snapshot and preserves the source draft +- For virtual tracks, sets `version` on the materialized snapshot in place - Adds an entry to `version_history` for audit trail - Can be performed on the latest snapshot or a specific historical snapshot diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index ad8f3b2d..750adbf9 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -101,23 +101,41 @@ publication metadata, assigns a stable bundle identifier, and records SHA-256 hashes of both bundle serializations. A released snapshot is immutable, including its notes. -### In-Place Tagging Strategy +### Preserved Standard-Track Release Strategy When you release a snapshot: -1. The **existing** snapshot is updated in-place -2. `version` is set to the new version -3. An entry is added to `version_history` for audit trail -4. The `modified` timestamp **does not change** -5. For standard tracks, staged objects are promoted into `members` and a - content manifest is sealed over the result in the same atomic update - -**Why in-place?** - -- Avoids duplicate data (no need to copy the entire release track) -- Clear semantics: tagging is metadata, not a content change -- Snapshots remain immutable except for the version tag -- Matches Git's model where tags point to existing commits +1. The selected standard draft remains unchanged as the rollback point. +2. A new tagged snapshot is created with a new `modified` timestamp. +3. `release_source_modified` points to the exact source draft. +4. `version` and a matching `version_history` entry are added to the release. +5. Staged objects are promoted into `members` and a fresh content manifest is + sealed over the release. + +Virtual tracks still tag their already-materialized draft in place. Standard +tracks use a clone because rollback must restore notes, workflow tiers, +dynamic selectors, and manifest identity exactly as they existed immediately +before release. + +The preserved source draft is hidden from the normal Releases timeline while +its tagged clone exists. Rolling-draft cleanup does not prune it. + +Administrators can correct a tagged snapshot's version with `PUT +/snapshots/:modified/release`. This preserves snapshot identity and content, +while enforcing the adjacent semantic-version bounds. + +For a version-only correction, the STIX 2.1 SHA-256 changes because the +collection object contains `x_mitre_version`. The STIX 2.0 SHA-256 stays the +same: that format omits the collection object. The bundle ID is unchanged. +Hashes are generated before the version is changed and stored together with +the new version. If later history or catalogue updates fail, retry the same +PUT with the same version to finish them; a same-version request repairs +derived state rather than being a no-op. + +Virtual materialization holds the component release locks until its snapshot +is persisted. Concurrent release, rollback, retag, or materialization on a +shared component may return `409`; retry after the other operation finishes. +Once the virtual snapshot exists, rollback is blocked by its dependency. ### Tagging Endpoints @@ -193,18 +211,16 @@ POST /api/release-tracks/release--123/snapshots/latest/release POST /api/release-tracks/:id/snapshots/:modified/release ``` -Tags a specific snapshot as a tagged release. Can tag retroactively, (i.e., a non-latest snapshot can be tagged), granted no [versioning rules](#versioning-rules) are violated. +Publishes a specific draft. For a standard track, the server preserves that +draft and creates a tagged clone at the current time. **Use Cases:** -- You want to tag snapshot 3, then later also tag snapshot 5 -- You forgot to tag a snapshot and want to mark it retroactively -- You want to create multiple tagged releases from different development branches +- You want to release the content of an earlier retained draft +- You want to pin the operation to a snapshot rather than use `latest` -**Constraint:** The version must be greater than the nearest earlier tagged -snapshot and less than the nearest later tagged snapshot. Both bounds are -exclusive. This allows a forgotten historical draft to be tagged without -breaking the version order of the timeline. +**Constraint:** A standard release created from an earlier draft is not +backdated. Its version must be greater than the current latest release. ## Versioning Rules @@ -219,22 +235,21 @@ Collections use a **two-part versioning scheme** (MAJOR.MINOR), inspired by sema ### Version Constraints -1. **Chronologically increasing** - Tagged versions increase with snapshot - `modified` time. A retroactive tag is exclusively lower- and upper-bounded - by its adjacent tagged snapshots. -2. **Immutable once set** - Once a snapshot has `version` assigned, it cannot be changed -3. **Cannot re-tag** - A snapshot can only be tagged once (throws `AlreadyReleasedError` if attempted) +1. **Chronologically increasing** - Tagged versions increase with release + snapshot `modified` time. +2. **Immutable content** - Release contents and identity cannot be changed. + Administrators may correct the version within its adjacent bounds. +3. **Cannot release twice** - A draft already linked to a tagged standard + release cannot be released again. 4. **Valid version format** - Must match `/^\d+\.\d+$/` (MAJOR.MINOR only, no patch component) 5. **Unique within the track** - Exactly one snapshot may hold a given tagged version. If concurrent release requests race for the same version, one succeeds and the other receives `409 Conflict` with the conflicting `track_id` and `version`. -Relative `minor` and `major` increments are calculated from the nearest -earlier tagged snapshot, not from the numerically highest tag elsewhere in the -track. For example, a draft after explicit v19.1 previews as v19.2 for `minor` -and v20.0 for `major`. A historical draft between v1.0 and v3.0 previews as -v1.1 or v2.0 and may use any explicit version strictly inside that interval. +Relative `minor` and `major` increments are calculated from the latest tagged +release. For example, a track after explicit v19.1 previews as v19.2 for +`minor` and v20.0 for `major`, even when the selected source draft is older. ### First Tagged Release diff --git a/docs/user/release-tracks/workflow-examples.md b/docs/user/release-tracks/workflow-examples.md index d1d796f4..0d5d209f 100644 --- a/docs/user/release-tracks/workflow-examples.md +++ b/docs/user/release-tracks/workflow-examples.md @@ -26,19 +26,19 @@ POST /api/release-tracks/release--123/meta # 5. Ready for first release - staged objects become members POST /api/release-tracks/release--123/snapshots/latest/release { "increment": "major" } -# Updates: snapshot 4, version: "1.0" (in place) +# Preserves snapshot 4 and creates snapshot 5, version: "1.0" # 6. Continue development through the same candidate workflow POST /api/release-tracks/release--123/candidates { "object_refs": [{ "id": "malware--...", "modified": "latest" }] } POST /api/release-tracks/release--123/candidates/promote { "object_refs": ["malware--..."] } -# Creates snapshots 5 and 6 +# Creates snapshots 6 and 7 # 7. Minor release POST /api/release-tracks/release--123/snapshots/latest/release { "increment": "minor" } -# Updates: snapshot 6, version: "1.1" (in place) +# Preserves snapshot 7 and creates snapshot 8, version: "1.1" ``` **Resulting Timeline:** @@ -46,9 +46,11 @@ POST /api/release-tracks/release--123/snapshots/latest/release snapshot 1: initial empty draft snapshot 2: candidate added snapshot 3: candidate staged -snapshot 4: version "1.0" ← RELEASE -snapshot 5: next candidate added -snapshot 6: version "1.1" ← RELEASE +snapshot 4: preserved pre-1.0 draft +snapshot 5: version "1.0" ← RELEASE +snapshot 6: next candidate added +snapshot 7: preserved pre-1.1 draft +snapshot 8: version "1.1" ← RELEASE ``` ### Example 2: Selective Release Tagging @@ -60,7 +62,7 @@ POST /api/release-tracks/release--456/meta # draft 3 POST /api/release-tracks/release--456/meta # draft 4 POST /api/release-tracks/release--456/meta # draft 5 -# Tag draft 2 retroactively and then tag the latest draft +# Release draft 2 now and then release the latest remaining draft POST /api/release-tracks/release--456/snapshots//release { "version": "1.0" } @@ -71,13 +73,16 @@ POST /api/release-tracks/release--456/snapshots/latest/release **Resulting Timeline:** ``` snapshot 1: version: null (skipped) -snapshot 2: version: "1.0" ← RELEASE +snapshot 2: preserved pre-1.0 draft snapshot 3: version: null (skipped) snapshot 4: version: null (skipped) -snapshot 5: version: "1.1" ← RELEASE +snapshot 5: preserved pre-1.1 draft +snapshot 6: version: "1.0" ← RELEASE (created now from snapshot 2) +snapshot 7: version: "1.1" ← RELEASE (created now from snapshot 5) ``` -This mirrors Git's ability to tag any commit, not just the latest. +Selecting a historical draft does not backdate a release: its tagged clone is +created at the current time and must follow the current version lineage. ### Example 3: Handling Already-Released Snapshots From ac3dd03bda2a7ed2e1a78b10b534a1acaf972704 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:29:02 -0400 Subject: [PATCH 08/14] feat(release-tracks): support virtual schedule updates Persist validated schedule replacements without cloning snapshots and expose active schedules to configuration clients. --- AGENTS.md | 3 + .../definitions/components/release-tracks.yml | 7 ++ app/api/definitions/openapi.yml | 3 + .../paths/release-tracks-paths.yml | 42 ++++++++++++ app/controllers/release-tracks-controller.js | 23 +++++++ .../release-tracks/release-track-schemas.js | 1 + .../release-track-registry-model.js | 3 +- .../release-track-registry.repository.js | 19 ++++++ app/routes/release-tracks-routes.js | 8 +++ .../release-tracks/release-tracks-service.js | 19 +++++- .../release-tracks/snapshot-service.js | 7 +- .../release-tracks/virtual-track-service.js | 28 ++++++++ ...rtual-snapshot-schedule-validation.spec.js | 68 +++++++++++++++++++ docs/admin/virtual-track-schedules.md | 5 ++ docs/developer/TODO.md | 48 +++++++++++++ docs/developer/task-scheduler.md | 18 +++-- docs/user/release-tracks/api-reference.md | 18 +++++ docs/user/release-tracks/virtual-tracks.md | 5 ++ 18 files changed, 315 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ba9f8cdc..5f75a714 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -165,6 +165,9 @@ parameter semantics in the `docs { }` block. deletable "graph cache" to release-track exports; drafts inherit their predecessor's manifest and only member-changing writes seal a new one. The `x-mitre-collection` object is a projection, not a stored object. +- A virtual track's `snapshot_schedule` is live registry configuration, not + historical snapshot state. Schedule changes must update the registry without + cloning a draft; Workbench snapshot responses project the current schedule. - Historic full-suite flake (fixed 2026-07-10): per-spec-file mongod restarts hit "Port already in use", failing a random file's `before` hook (visible as `loginAnonymous` 404s). `database-in-memory.js` now reuses one diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index ceb2bc22..c1d364b6 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -106,6 +106,13 @@ components: one component, while conflicts include only IDs with genuinely different revisions. Each surviving member is attributed to exactly one component in objects_contributed. + snapshot_schedule: + readOnly: true + description: | + Current registry-backed materialization schedule for virtual tracks + in Workbench-format responses. It is not historical snapshot data. + allOf: + - $ref: '#/components/schemas/snapshot-schedule' scheduled_materialization: nullable: true description: | diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index 1fd73ccb..bb88629f 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -385,6 +385,9 @@ paths: /api/release-tracks/{id}/virtual/composition: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1composition' + /api/release-tracks/{id}/virtual/schedule: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1schedule' + /api/release-tracks/{id}/virtual/snapshots/create: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1virtual~1snapshots~1create' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index 6c127661..df5335d9 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -861,6 +861,48 @@ paths: '400': description: 'Track is not virtual or composition is invalid' + /api/release-tracks/{id}/virtual/schedule: + put: + summary: 'Update a virtual track snapshot schedule' + operationId: 'release-tracks-schedule-update' + description: | + Replace the registry-backed materialization schedule for a virtual + track without creating or mutating a content snapshot. Request bodies + are strictly validated via Zod: manual accepts only mode, cron requires + one five-field UTC expression, and dates requires at least one ISO UTC + timestamp. Scheduler reconciliation observes the replacement on its + next configured pass. + tags: + - 'Release Tracks' + parameters: + - name: id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + '200': + description: 'Snapshot schedule updated successfully' + content: + application/json: + schema: + type: object + required: + - snapshot_schedule + properties: + snapshot_schedule: + $ref: '../components/release-tracks.yml#/components/schemas/snapshot-schedule' + '400': + description: 'Track is not virtual or schedule is invalid' + '404': + description: 'Release track not found' + /api/release-tracks/{id}/virtual/snapshots/create: post: summary: 'Create a virtual track snapshot' diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 62842693..1ec25853 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -50,6 +50,7 @@ const { updateCandidateVersionBodySchema, updateConfigBodySchema, updateCompositionBodySchema, + updateScheduleBodySchema, createVirtualSnapshotBodySchema, promoteQuarantinedObjectBodySchema, reconstructSnapshotGraphBodySchema, @@ -1014,6 +1015,28 @@ exports.updateComposition = async function updateComposition(req, res, next) { } }; +/** PUT /api/release-tracks/:id/virtual/schedule */ +exports.updateSchedule = async function updateSchedule(req, res, next) { + try { + const bodyResult = updateScheduleBodySchema.safeParse(req.body); + if (!bodyResult.success) { + return next( + new BadRequestError({ + message: 'Invalid snapshot schedule update', + details: bodyResult.error.errors, + }), + ); + } + + const result = await releaseTracksService.updateSchedule(req.params.id, bodyResult.data); + logger.debug(`Success: Updated snapshot schedule for track ${req.params.id}`); + return res.status(200).send(result); + } catch (err) { + logger.error('Failed to update snapshot schedule: ' + err); + return next(err); + } +}; + /** POST /api/release-tracks/:id/virtual/snapshots/create */ exports.createVirtualSnapshot = async function createVirtualSnapshot(req, res, next) { try { diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index ef897365..559e3a68 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -685,6 +685,7 @@ module.exports = { updateCandidateVersionBodySchema, updateConfigBodySchema, updateCompositionBodySchema, + updateScheduleBodySchema: snapshotScheduleSchema, createVirtualSnapshotBodySchema, promoteQuarantinedObjectBodySchema, reconstructSnapshotGraphBodySchema, diff --git a/app/models/release-tracks/release-track-registry-model.js b/app/models/release-tracks/release-track-registry-model.js index fd74b71d..219b95ea 100644 --- a/app/models/release-tracks/release-track-registry-model.js +++ b/app/models/release-tracks/release-track-registry-model.js @@ -87,9 +87,10 @@ const releaseTrackRegistryDefinition = { default: undefined, validate: { validator: function validateRegistrySnapshotSchedule(value) { + const trackType = typeof this.getQuery === 'function' ? this.getQuery().type : this.type; return ( value === undefined || - (this.type === 'virtual' && validateSnapshotSchedule.validator(value)) + (trackType === 'virtual' && validateSnapshotSchedule.validator(value)) ); }, message: diff --git a/app/repository/release-tracks/release-track-registry.repository.js b/app/repository/release-tracks/release-track-registry.repository.js index 80e2bc72..0d3f4135 100644 --- a/app/repository/release-tracks/release-track-registry.repository.js +++ b/app/repository/release-tracks/release-track-registry.repository.js @@ -162,6 +162,25 @@ class ReleaseTrackRegistryRepository { } } + async setSnapshotSchedule(trackId, snapshotSchedule) { + try { + return await this.model + .findOneAndUpdate( + { track_id: trackId, type: 'virtual' }, + { + $set: { + snapshot_schedule: snapshotSchedule, + updated_at: new Date(), + }, + }, + { new: true, runValidators: true, lean: true }, + ) + .exec(); + } catch (err) { + throw new DatabaseError(err); + } + } + async replaceTaggedReleases(trackId, taggedReleases, latestTaggedVersion) { try { return await this.model diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 832bcd41..3febe1d3 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -309,6 +309,14 @@ router releaseTracksController.updateComposition, ); +router + .route('/release-tracks/:id/virtual/schedule') + .put( + authn.authenticate, + authz.requireRole(authz.editorOrHigher), + releaseTracksController.updateSchedule, + ); + // ============================================================================= // Delete release track (must be last -- :id is a catch-all param) // ============================================================================= diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 6b8e531d..85f12210 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -212,8 +212,12 @@ async function formatWorkbenchSnapshot(snapshot, options) { selectedTiers.flatMap((tierName) => snapshot[tierName] || []), ); const enriched = await addObjectInfoToSnapshot(snapshot); - // Registry-derived, read-only: lets clients build alias URLs for the track. - enriched.alias = await snapshotService.getTrackAlias(snapshot.id); + // Registry-derived, read-only metadata used alongside snapshot content. + const metadata = await snapshotService.getTrackMetadata(snapshot.id); + enriched.alias = metadata.alias; + if (snapshot.type === 'virtual') { + enriched.snapshot_schedule = metadata.snapshot_schedule || { mode: 'manual' }; + } return filterSnapshotTiers(enriched, options?.include); } @@ -539,6 +543,17 @@ exports.updateComposition = function updateComposition(trackId, composition, use }); }; +exports.updateSchedule = function updateSchedule(trackId, schedule) { + const scheduleResult = snapshotScheduleSchema.safeParse(schedule); + if (!scheduleResult.success) { + throw new BadRequestError({ + message: 'Invalid snapshot schedule', + details: scheduleResult.error.errors, + }); + } + return virtualTrackService.updateSchedule(trackId, scheduleResult.data); +}; + exports.createVirtualSnapshot = function createVirtualSnapshot(trackId, options) { let validatedOptions = options; if (options?.scheduledMaterialization !== undefined) { diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 102abdda..c617d83f 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -247,9 +247,12 @@ exports.resolveTrackAlias = async function resolveTrackAlias(alias) { /** * The alias registered for a track, or null. */ -exports.getTrackAlias = async function getTrackAlias(trackId) { +exports.getTrackMetadata = async function getTrackMetadata(trackId) { const entry = await registryRepo.findByTrackId(trackId); - return entry?.alias ?? null; + return { + alias: entry?.alias ?? null, + snapshot_schedule: entry?.snapshot_schedule, + }; }; // ============================================================================= diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index d7487215..a7da8d05 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -458,6 +458,34 @@ exports.updateComposition = async function updateComposition( return snapshot; }; +/** + * Replace the persisted materialization schedule for a virtual track. + * The registry is authoritative so schedule changes do not create or mutate a + * content snapshot. The scheduler reconciliation task observes the new value. + * + * @param {string} trackId + * @param {Object} schedule + * @returns {Promise<{snapshot_schedule: Object}>} + */ +exports.updateSchedule = async function updateSchedule(trackId, schedule) { + const registry = await registryRepo.findByTrackId(trackId); + if (!registry) { + throw new TrackNotFoundError(trackId); + } + if (registry.type !== 'virtual') { + throw new BadRequestError({ + message: 'This operation is only available for virtual release tracks', + details: `Track ${trackId} is a ${registry.type} track`, + }); + } + + const updated = await registryRepo.setSnapshotSchedule(trackId, schedule); + logger.verbose( + `VirtualTrackService: Updated snapshot schedule for track "${trackId}" to ${schedule.mode}`, + ); + return { snapshot_schedule: updated.snapshot_schedule }; +}; + /** * Create a new virtual snapshot by resolving the composition rules. * diff --git a/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js b/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js index 4f6b532e..9fdc8324 100644 --- a/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js +++ b/app/tests/api/release-tracks/virtual-snapshot-schedule-validation.spec.js @@ -52,6 +52,15 @@ describe('Virtual release-track snapshot schedule validation API', function () { return response.body.data[0]; } + async function updateSchedule(trackId, snapshotSchedule, status = 200) { + return request(app) + .put(`/api/release-tracks/${trackId}/virtual/schedule`) + .send(snapshotSchedule) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(status); + } + it('accepts and persists the fields defined by each schedule mode', async function () { const schedules = [ { mode: 'manual' }, @@ -69,6 +78,65 @@ describe('Virtual release-track snapshot schedule validation API', function () { } }); + it('updates a virtual track schedule without creating a snapshot', async function () { + const created = await createTrack({ mode: 'manual' }); + const trackId = created.body.id; + const schedule = { mode: 'cron', cron: '15 9 * * 1,3' }; + + const response = await updateSchedule(trackId, schedule); + + expect(response.body.snapshot_schedule).toEqual(schedule); + const registryTrack = await getRegistryTrack(created.name); + expect(registryTrack.snapshot_schedule).toEqual(schedule); + expect(registryTrack.snapshot_count).toBe(1); + }); + + it('returns the current registry schedule with workbench snapshots', async function () { + const created = await createTrack({ mode: 'manual' }); + const schedule = { + mode: 'dates', + dates: ['2027-01-15T09:30:00.000Z', '2027-07-15T09:30:00.000Z'], + }; + await updateSchedule(created.body.id, schedule); + + const response = await request(app) + .get(`/api/release-tracks/${created.body.id}/snapshots/latest`) + .set('Accept', 'application/json') + .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) + .expect(200); + + expect(response.body.snapshot_schedule).toEqual(schedule); + }); + + it('replaces schedule mode fields instead of retaining stale selectors', async function () { + const created = await createTrack({ mode: 'cron', cron: '0 0 * * *' }); + + const response = await updateSchedule(created.body.id, { mode: 'manual' }); + + expect(response.body.snapshot_schedule).toEqual({ mode: 'manual' }); + expect(await getRegistryTrack(created.name)).toEqual( + expect.objectContaining({ snapshot_schedule: { mode: 'manual' } }), + ); + }); + + it('rejects invalid updates and schedule updates on standard tracks', async function () { + const virtual = await createTrack({ mode: 'manual' }); + const standard = await createTrack(undefined, 201, 'standard'); + + await updateSchedule(virtual.body.id, { mode: 'cron' }, 400); + await updateSchedule(virtual.body.id, { mode: 'manual', cron: '0 0 * * *' }, 400); + await updateSchedule(standard.body.id, { mode: 'manual' }, 400); + await updateSchedule( + 'release-track--11111111-1111-4111-8111-111111111111', + { + mode: 'manual', + }, + 404, + ); + + expect(() => releaseTracksService.updateSchedule(virtual.body.id, { mode: 'cron' })).toThrow(); + }); + it('rejects fields that do not apply to manual schedules', async function () { const invalidSchedules = [ { mode: 'manual', cron: '0 0 1 1,7 *' }, diff --git a/docs/admin/virtual-track-schedules.md b/docs/admin/virtual-track-schedules.md index 7aad65cd..a184c35a 100644 --- a/docs/admin/virtual-track-schedules.md +++ b/docs/admin/virtual-track-schedules.md @@ -21,6 +21,11 @@ processed after startup. `manual` schedules register no executable work. Operators must call `POST /api/release-tracks/:id/virtual/snapshots/create`. +Editors can replace the active schedule through +`PUT /api/release-tracks/:id/virtual/schedule`. The change is visible +immediately in track and Workbench-format snapshot responses; executable jobs +are refreshed on the next `VIRTUAL_TRACK_SCHEDULES_CRON` reconciliation pass. + ## Idempotency and multiple instances The `virtualTrackScheduleOccurrences` collection stores one durable occurrence diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index e6ff635f..d0f86186 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,53 @@ # Release Track TODOs +## Virtual release-track schedule configuration + +- [x] Fix schedule saves submitting unsupported deduplication fields: remove + preferred tier/status controls and compare composition to its initial + editable state, including normalized priorities and defaults. +- [x] Verify server-shaped schedule-only save and strategy-change regressions, + full frontend tests, and production build. + Focused: 94 passing; full: 167 files / 410 tests passing; production build + and changed-file lint pass. Backend schema rejects precisely + `tier_resolution`/`status_resolution` and accepts the corrected payload. + Proposed commit: `fix(release-tracks): avoid invalid composition updates`. + Body: Remove unsupported deduplication controls and compare the edited + composition to its initial form state so schedule-only saves skip cloning. + +- [x] Add controlled natural-language schedule autocomplete with hourly and + 15/30-minute presets and guided customization. +- [x] Verify autocomplete regressions, full frontend suite, and production build. + Focused tests: 94 passing; full frontend: 167 files / 410 tests passing; + production build, changed-file lint, formatting, and diff checks pass. + Proposed commit: `feat(release-tracks): autocomplete schedule presets`. + Body: Map selected schedule phrases to deterministic UTC cron expressions + and support hourly and 15/30-minute guided customization. + +- [x] Review persisted schedule validation, storage, scheduler execution, and + existing regression coverage. +- [x] Add an authenticated virtual-track schedule update endpoint with strict + validation and persistence. +- [x] Add backend regression tests, OpenAPI documentation, user/developer + documentation, and Bruno coverage. +- [x] Add a controlled frontend schedule editor for manual, recurring cron, + and explicit-date schedules, without free-text cron entry. +- [x] Add frontend connector/component regressions and usage documentation. +- [x] Run focused checks, then the complete backend and frontend suites. +- [x] Propose conventional commit messages without committing. + +Verification (2026-09-09): + +- Backend focused schedule/API and scheduler specs: 19 passing; OpenAPI: 2 + passing; changed-file ESLint clean. +- Backend complete `npm test`: OpenAPI 2, config 22, API 1024, middleware 29, + and scheduler 10 passing. +- Frontend focused component/connector specs: 92 passing; complete suite: 167 + files and 408 tests passing; application TypeScript and production build + pass; changed-file ESLint has no errors. +- Proposed commits: `feat(release-tracks): add virtual schedule updates`, + `feat(release-tracks): add guided snapshot scheduling`, and + `docs(release-tracks): add virtual schedule request`. + ## Sealed snapshot content manifests (Problem 1) Design: [release-tracks/sealed-content-manifests.md](release-tracks/sealed-content-manifests.md). diff --git a/docs/developer/task-scheduler.md b/docs/developer/task-scheduler.md index 70824e73..f6877ea1 100644 --- a/docs/developer/task-scheduler.md +++ b/docs/developer/task-scheduler.md @@ -7,6 +7,7 @@ - All the scheduler does is load the task module. It is up to the module defining the task to (1) implement the task, (2) load the task with the `node-schedule` library, and (3) execute the loader in the global scope Example: + ```javascript /** * Initialize and schedule this task @@ -27,14 +28,16 @@ function initializeTask() { logger.info(`[here-is-my-task-name] Task scheduled successfully`); } -if (config.scheduler.enableScheduler) { // <-- make sure to condition the task to only load if globally enabled! +if (config.scheduler.enableScheduler) { + // <-- make sure to condition the task to only load if globally enabled! initializeTask(); } ``` + - The old task scheduler (formerly known as the "collection manager") is now defined in `app/scheduler/sync-collection-indexes-task.js` - Adds a new global runtime configuration setting for toggling on/off all scheduled tasks. The environment variable is `ENABLE_SCHEDULER` and it maps to `config.scheduler.enableScheduler`. - Adds a new CRON pattern for configuring when tasks are scheduled. - - The `SYNC_COLLECTION_INDEXES_CRON` environment variable is read at runtime to determine the periodicity that the scheduler should use for the former collection manager (now the `sync-collection-indexes-tasks`). It maps to `config.scheduler.syncCollectionIndexesCron`. + - The `SYNC_COLLECTION_INDEXES_CRON` environment variable is read at runtime to determine the periodicity that the scheduler should use for the former collection manager (now the `sync-collection-indexes-tasks`). It maps to `config.scheduler.syncCollectionIndexesCron`. - Future tasks must follow a similar pattern: - Add the task file @@ -64,11 +67,16 @@ after the scheduled snapshot was already created. Do not put release-track composition logic in the scheduler task. It delegates to `virtual-track-service`, which is also used by the explicit HTTP operation. +Schedule changes use `PUT /api/release-tracks/:id/virtual/schedule`. The write +replaces `releaseTrackRegistry.snapshot_schedule` atomically and does not clone +the latest snapshot. The next reconciliation pass refreshes or cancels the +track-local cron job and registers any due explicit dates. + ## TODO - [ ] Add robust documentation to `USAGE.md` explaining how task scheduling works and how to create new tasks - [ ] In the future we should add the ability to dynamically load tasks without having to clone the repository and modify the `app/` source code. This new design pattern makes it possible to define them elsewhere and mount them via Docker volume. - [ ] There is another task called `check-wip-attack-ids-task.js` that should probably be deleted - - It was created with the goal of restricting ATT&CK IDs to only exist on non-WIP objects - - That conversation is sort of out of scope - - I think we're going to move away from this approach and that the task will probably be moot + - It was created with the goal of restricting ATT&CK IDs to only exist on non-WIP objects + - That conversation is sort of out of scope + - I think we're going to move away from this approach and that the task will probably be moot diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index bc4ad84e..b6078442 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -105,6 +105,7 @@ GET /api/release-tracks/:id/objects/:objectRef/versions ``` PUT /api/release-tracks/:id/virtual/composition +PUT /api/release-tracks/:id/virtual/schedule POST /api/release-tracks/:id/virtual/snapshots/create POST /api/release-tracks/:id/virtual/quarantine/promote ``` @@ -1409,6 +1410,23 @@ each referenced track must already exist and must be a standard track. Virtual tracks cannot reference other virtual tracks, and unsupported top-level properties such as `native_members` return `400 Bad Request`. +### Update Virtual Track Schedule + +``` +PUT /api/release-tracks/:id/virtual/schedule +``` + +Replaces a virtual track's persisted `snapshot_schedule` without creating or +modifying a content snapshot. The body is one of the same strict `manual`, +`cron`, or `dates` shapes accepted during track creation. The response contains +the normalized value under `snapshot_schedule`. Standard tracks return +`400 Bad Request`; unknown tracks return `404 Not Found`. + +Workbench-format snapshot responses expose the current registry-backed +`snapshot_schedule` so configuration clients do not mistake a historical +snapshot for the active schedule. Scheduler reconciliation applies a saved +change on its next configured pass. + ### Update Virtual Track Composition ``` diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index 34127c42..b0c49a57 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -12,6 +12,11 @@ Virtual release tracks are computed aggregations of standard release tracks. The - Create snapshots **manually or on schedule** (never event-driven) - All snapshots start as **drafts** and must be explicitly tagged +The active schedule is registry metadata rather than historical snapshot +content. Replace it with `PUT /api/release-tracks/:id/virtual/schedule`; this +does not create a draft. Workbench-format snapshot responses project the +current schedule for configuration interfaces. + ## Use Cases ### Scenario 1: Different Cadences for Different Object Types From f186a28be2e25edb763fad8948de11672d3c3359 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:57:22 -0400 Subject: [PATCH 09/14] feat(release-tracks): expose virtual snapshot provenance Include recorded component snapshot metadata in virtual snapshot history summaries. Document exact source versions, creation timestamps, filters, and contribution counts in OpenAPI and user documentation, with regression coverage. --- .../definitions/components/release-tracks.yml | 128 ++++++++++++++++-- .../release-track-dynamic.repository.js | 1 + .../release-tracks/snapshot-service.js | 13 +- .../release-tracks/snapshot-history.spec.js | 48 +++++++ docs/developer/TODO.md | 39 ++++++ docs/user/release-tracks/api-reference.md | 7 +- docs/user/release-tracks/virtual-tracks.md | 12 ++ 7 files changed, 235 insertions(+), 13 deletions(-) diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index c1d364b6..95ad42c2 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -96,16 +96,7 @@ components: description: 'Component track references (virtual tracks only)' $ref: '#/components/schemas/composition' composition_resolution: - type: object - nullable: true - description: | - Immutable component-resolution provenance for a materialized virtual - draft. Null or absent means the virtual composition is configured - but has not been materialized and therefore cannot be previewed or - released. Duplicate counts include STIX IDs contributed by more than - one component, while conflicts include only IDs with genuinely - different revisions. Each surviving member is attributed to exactly - one component in objects_contributed. + $ref: '#/components/schemas/composition-resolution' snapshot_schedule: readOnly: true description: | @@ -276,10 +267,127 @@ components: scheduled_materialization: nullable: true $ref: '#/components/schemas/scheduled-materialization' + composition_resolution: + $ref: '#/components/schemas/composition-resolution' quarantine_count: type: integer minimum: 0 + composition-resolution: + type: object + nullable: true + readOnly: true + description: | + Immutable provenance for a materialized virtual snapshot. Each + component_snapshots entry names the exact tagged standard-track + snapshot used, including its version and creation timestamp. Null or + absent means the composition was configured but not materialized. + Duplicate counts include STIX IDs contributed by more than one + component, while conflicts include only genuinely different revisions. + Snapshot-history summaries return resolved_at and component_snapshots; + full snapshot responses additionally return deduplication and summary. + required: + - resolved_at + - component_snapshots + properties: + resolved_at: + type: string + format: date-time + description: 'When the virtual snapshot composition was resolved' + component_snapshots: + type: array + description: 'Exact standard-track snapshots constituting this virtual snapshot' + items: + type: object + required: + - track_id + - track_name + - track_type + - resolved_snapshot_id + - resolved_version + - strategy_used + - total_objects_in_source + - objects_after_filter + - objects_contributed + properties: + track_id: + type: string + track_name: + type: string + track_type: + type: string + enum: + - standard + resolved_snapshot_id: + type: string + format: date-time + description: 'Exact component snapshot timestamp; also identifies when that snapshot was created' + resolved_version: + type: string + description: 'Tagged version of the exact component snapshot' + strategy_used: + type: string + enum: + - latest_tagged + - specific_version + - specific_snapshot + filters_applied: + type: object + additionalProperties: false + properties: + object_types: + type: array + items: + type: string + domains: + type: array + items: + type: string + total_objects_in_source: + type: integer + minimum: 0 + objects_after_filter: + type: integer + minimum: 0 + objects_contributed: + type: integer + minimum: 0 + description: 'Surviving members attributed to this component after deduplication' + deduplication: + type: object + required: + - total_objects_before + - total_objects_after + - duplicates_found + - conflicts_resolved + properties: + total_objects_before: + type: integer + minimum: 0 + total_objects_after: + type: integer + minimum: 0 + duplicates_found: + type: integer + minimum: 0 + conflicts_resolved: + type: array + items: + type: object + additionalProperties: true + summary: + type: object + required: + - total_objects + - quarantined_objects + properties: + total_objects: + type: integer + minimum: 0 + quarantined_objects: + type: integer + minimum: 0 + tier-entry-base: type: object description: 'Shared fields for a release-track object reference' diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index 7d750d19..27187c7a 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -295,6 +295,7 @@ class ReleaseTrackDynamicRepository { name: 1, description: 1, scheduled_materialization: 1, + composition_resolution: 1, members_count: { $size: { $ifNull: ['$members', []] } }, staged_count: { $size: { $ifNull: ['$staged', []] } }, candidates_count: { $size: { $ifNull: ['$candidates', []] } }, diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index c617d83f..1e16c5ab 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -263,8 +263,9 @@ exports.getTrackMetadata = async function getTrackMetadata(trackId) { * List lightweight summaries of a track's snapshots. * * Standard summaries expose members/staged/candidates counts. Virtual - * summaries expose members/quarantine counts. Every summary exposes its - * content manifest ID and counts by manifest entry role. + * summaries expose members/quarantine counts plus their immutable composition + * resolution. Every summary exposes its content manifest ID and counts by + * manifest entry role. * * @param {string} trackId * @param {Object} options - { tagged?, limit, offset } @@ -302,9 +303,17 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { }; if (snapshot.type === 'virtual') { + const compositionResolution = snapshot.composition_resolution; return { ...common, scheduled_materialization: snapshot.scheduled_materialization, + composition_resolution: + compositionResolution == null + ? compositionResolution + : { + resolved_at: compositionResolution.resolved_at, + component_snapshots: compositionResolution.component_snapshots, + }, quarantine_count: snapshot.quarantine_count, }; } diff --git a/app/tests/api/release-tracks/snapshot-history.spec.js b/app/tests/api/release-tracks/snapshot-history.spec.js index f1e55800..3102145f 100644 --- a/app/tests/api/release-tracks/snapshot-history.spec.js +++ b/app/tests/api/release-tracks/snapshot-history.spec.js @@ -52,6 +52,7 @@ describe('GET /api/release-tracks/:id/snapshots', function () { let virtualTrack; let standardTaggedModified; let standardLatestModified; + let virtualResolvedAt; before(async function () { await database.initializeConnection(); @@ -132,11 +133,39 @@ describe('GET /api/release-tracks/:id/snapshots', function () { const virtualCreated = new Date(virtualTrack.modified); const virtualTaggedModified = new Date(virtualCreated.getTime() + 1000); + virtualResolvedAt = new Date(virtualCreated.getTime() + 500); await dynamicRepo.saveSnapshot(virtualTrack.id, { ...snapshotBase(virtualTrack), modified: virtualTaggedModified, version: '1.0', members: [memberEntry(0), memberEntry(1)], + composition_resolution: { + resolved_at: virtualResolvedAt, + component_snapshots: [ + { + track_id: standardTrack.id, + track_name: standardTrack.name, + track_type: 'standard', + resolved_snapshot_id: standardTaggedModified, + resolved_version: '1.0', + strategy_used: 'latest_tagged', + filters_applied: { domains: ['enterprise'] }, + total_objects_in_source: 2, + objects_after_filter: 2, + objects_contributed: 2, + }, + ], + deduplication: { + total_objects_before: 2, + total_objects_after: 2, + duplicates_found: 0, + conflicts_resolved: [], + }, + summary: { + total_objects: 2, + quarantined_objects: 1, + }, + }, quarantine: [ { ...memberEntry(2), @@ -263,7 +292,26 @@ describe('GET /api/release-tracks/:id/snapshots', function () { version: '1.0', members_count: 2, quarantine_count: 1, + composition_resolution: { + resolved_at: virtualResolvedAt.toISOString(), + component_snapshots: [ + { + track_id: standardTrack.id, + track_name: standardTrack.name, + track_type: 'standard', + resolved_snapshot_id: standardTaggedModified.toISOString(), + resolved_version: '1.0', + strategy_used: 'latest_tagged', + filters_applied: { domains: ['enterprise'] }, + total_objects_in_source: 2, + objects_after_filter: 2, + objects_contributed: 2, + }, + ], + }, }); + expect(response.body.data[0].composition_resolution).not.toHaveProperty('deduplication'); + expect(response.body.data[0].composition_resolution).not.toHaveProperty('summary'); expect(response.body.data[0]).not.toHaveProperty('staged_count'); expect(response.body.data[0]).not.toHaveProperty('candidates_count'); }); diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index d0f86186..b6fba4ab 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -48,6 +48,45 @@ Verification (2026-09-09): `feat(release-tracks): add guided snapshot scheduling`, and `docs(release-tracks): add virtual schedule request`. +## Snapshot-scoped virtual composition provenance + +- [x] Return each virtual snapshot's immutable `composition_resolution` from + snapshot history and declare it in OpenAPI. +- [x] Add backend regression coverage and update user/API documentation plus + the Bruno snapshot-history request. +- [x] Move the frontend Composition Resolution view from HEAD into each + virtual snapshot's Releases card, including exact component version, + snapshot timestamp, strategy, filters, and contribution counts. +- [x] Add frontend regression coverage and update frontend documentation. +- [x] Replace the dense seven-column provenance table with a responsive + component list whose identifiers wrap within their own regions and whose + counts use independently wrapping metric labels. +- [x] Run focused backend/frontend specs, then each repository's complete + required verification suite; propose conventional commit messages. + +Verification (2026-09-09): + +- REST API: snapshot-history spec 7 passing; OpenAPI 2 passing; lint clean; + full `npm test` under Node 24 clean (2 OpenAPI, 22 config, 1020 API, + 29 middleware, 10 scheduler). Node 22 full-suite attempts reproduced the + documented roaming HTTP-response flake; every affected spec passed alone. +- Frontend: release-track page spec 71 passing; full `npm test` 404 passing; + application TypeScript compilation clean; changed-file ESLint clean; + production build clean with existing bundle/style budget warnings. The + repository-wide lint command still reports pre-existing errors outside the + changed files. +- Proposed commits: REST API `feat(release-tracks): expose virtual snapshot + provenance`; frontend `feat(release-tracks): scope composition provenance to + snapshots`; Bruno `docs(release-tracks): document snapshot composition + provenance`. + +Post-merge review (2026-09-09): no blocking findings; live registry scheduling +and historical composition provenance remain separate. Reverified with Node 24: +7 focused backend tests, 77 focused frontend tests, full backend suite +(2 OpenAPI, 22 config, 1024 API, 29 middleware, 10 scheduler), and all 411 +frontend tests passed. Backend lint, changed-file frontend lint, and the +frontend production build passed (bundle/style budget warnings remain). + ## Sealed snapshot content manifests (Problem 1) Design: [release-tracks/sealed-content-manifests.md](release-tracks/sealed-content-manifests.md). diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index b6078442..48ae8afd 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -459,7 +459,12 @@ The UI groups supporting and LinkById targets together as **Dependencies**. Snapshot tier count keys continue to reflect the track type: - `type: "standard"` adds `staged_count` and `candidates_count`. -- `type: "virtual"` adds `quarantine_count`. +- `type: "virtual"` adds `quarantine_count` and the snapshot's immutable + `composition_resolution`. Its `component_snapshots` entries identify the + exact standard-track snapshot used by track ID, tagged version, and + `resolved_snapshot_id` creation timestamp, together with the resolution + strategy, filters, source count, filtered count, and final contributed + count. An unmaterialized virtual draft returns the field as `null`. Inapplicable count keys are omitted rather than returned as zero. diff --git a/docs/user/release-tracks/virtual-tracks.md b/docs/user/release-tracks/virtual-tracks.md index b0c49a57..7c8d8f1b 100644 --- a/docs/user/release-tracks/virtual-tracks.md +++ b/docs/user/release-tracks/virtual-tracks.md @@ -696,6 +696,18 @@ materialized, the virtual release still records the version that actually produced its frozen contents. Standard release history entries omit this virtual-only property. +The snapshot-history endpoint (`GET /api/release-tracks/:id/snapshots`) also +returns the provenance portion of each virtual snapshot's +`composition_resolution`: `resolved_at` and `component_snapshots`. This lets +clients present provenance beside the draft or release it describes rather +than presenting only the virtual track's current HEAD resolution. In each +component entry, `resolved_snapshot_id` is the exact component snapshot's +creation timestamp and stable retrieval key; `resolved_version` names its +tagged version. The stored source, filtered, and contributed counts belong to +that materialization and are not recomputed from the component track's current +state. Full snapshot retrieval additionally returns the deduplication report +and resolution summary. + **Business Logic:** 1. Validate snapshot exists and is a draft (version === null) From e9fd8562611ba3c687e1307bad4e497fb66a3e6e Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:20:22 -0400 Subject: [PATCH 10/14] feat(release-tracks): separate draft conversion from snapshot deletion Add confirmed release-to-draft conversion and restore guarded draft deletion. Preserve source drafts, composition provenance, dependency checks, and release serialization. --- app/api/definitions/openapi.yml | 3 + .../paths/release-tracks-paths.yml | 82 +++++-- app/controllers/release-tracks-controller.js | 29 ++- app/exceptions/index.js | 5 +- .../release-tracks/release-track-schemas.js | 2 + .../release-track-audit-event-model.js | 2 +- app/routes/release-tracks-routes.js | 8 + .../release-tracks/release-tracks-service.js | 41 ++-- .../release-tracks/snapshot-service.js | 65 ++++-- .../destructive-authorization.spec.js | 202 ++++++++++++++++-- .../release-tracks/releases-by-object.spec.js | 4 +- .../snapshot-immutability.spec.js | 8 +- docs/admin/release-track-audit.md | 7 +- docs/developer/TODO.md | 30 +++ .../developer/release-tracks/authorization.md | 20 +- docs/developer/release-tracks/entities.md | 4 +- .../release-tracks/error-handling.md | 4 +- .../sealed-content-manifests.md | 11 + docs/user/release-tracks/api-reference.md | 48 +++-- docs/user/release-tracks/summary.md | 3 +- docs/user/release-tracks/versioning.md | 22 +- 21 files changed, 483 insertions(+), 117 deletions(-) diff --git a/app/api/definitions/openapi.yml b/app/api/definitions/openapi.yml index bb88629f..6f70ed80 100644 --- a/app/api/definitions/openapi.yml +++ b/app/api/definitions/openapi.yml @@ -421,6 +421,9 @@ paths: /api/release-tracks/{id}/snapshots/{modified}/release: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1release' + /api/release-tracks/{id}/snapshots/{modified}/draft: + $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1draft' + /api/release-tracks/{id}/snapshots/{modified}/release/preview: $ref: 'paths/release-tracks-paths.yml#/paths/~1api~1release-tracks~1{id}~1snapshots~1{modified}~1release~1preview' diff --git a/app/api/definitions/paths/release-tracks-paths.yml b/app/api/definitions/paths/release-tracks-paths.yml index feafb8b5..4f2d6a18 100644 --- a/app/api/definitions/paths/release-tracks-paths.yml +++ b/app/api/definitions/paths/release-tracks-paths.yml @@ -1224,23 +1224,16 @@ paths: description: 'Requested format is not yet implemented' delete: - summary: 'Delete a specific snapshot' + summary: 'Delete the latest draft snapshot' operationId: 'release-tracks-snapshot-delete' description: | Delete the latest untagged draft snapshot by its modified timestamp (editor or higher); the track reverts to its immediately preceding - snapshot. Historical drafts have already been pruned. - - An administrator may also roll back the track's most recent standard - release by supplying `confirm_version` equal to that snapshot's - version. The tagged clone is deleted and its exact preserved source - draft becomes available again. The ledger and registry catalogue are - reconciled and a `delete_release` audit event is recorded. Rollback is - blocked if any persisted virtual snapshot resolved the exact release, - if it is followed by a later release, or if it predates preserved - source drafts. Virtual releases retain the existing irreversible - newest-release deletion behavior because virtual tagging remains - in-place. + snapshot. Tagged releases cannot be deleted, even by administrators: + first use POST /snapshots/{modified}/draft to convert to a draft. + Deletion is blocked for historical drafts, the only snapshot, preserved + source drafts of tagged releases, and snapshots resolved by downstream + virtual snapshots. Checks and deletion share the release lock. tags: - 'Release Tracks' parameters: @@ -1257,21 +1250,72 @@ paths: - name: confirm_version in: query description: | - Required to delete a release: must equal the release version of - the selected snapshot. + Deprecated compatibility parameter. It does not authorize tagged + deletion; tagged snapshots always return 409. Use the draft + conversion endpoint instead. + deprecated: true schema: type: string responses: '204': description: 'Snapshot deleted successfully' + '409': + description: 'Snapshot is tagged, protected, the only snapshot, or not the latest draft' + '404': + description: 'Snapshot not found' + + /api/release-tracks/{id}/snapshots/{modified}/draft: + post: + summary: 'Convert the latest tagged release back to a draft' + operationId: 'release-tracks-release-to-draft' + description: | + Administrator-only conversion with exact version confirmation. Standard + tracks restore the preserved source draft (whose modified timestamp is + returned); virtual releases become drafts in place, preserving their + sealed content and composition/creation provenance. Later drafts survive. + Only the latest tagged release may be converted. Downstream virtual + dependencies block conversion. Release history, catalogue, counters, and + backrefs are reconciled and convert_release_to_draft is audited. Draft + deletion is a separate operation with its own eligibility checks. + tags: ['Release Tracks'] + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: modified + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: false + required: [confirm_version] + properties: + confirm_version: + type: string + pattern: '^\d+\.\d+$' + responses: + '200': + description: 'Restored draft snapshot' + content: + application/json: + schema: + $ref: '../components/release-tracks.yml#/components/schemas/release-track-snapshot' '400': - description: 'Release confirmation missing or incorrect' + description: 'Version confirmation is missing, invalid, or incorrect' '403': - description: 'Deleting a release requires an administrator' - '409': - description: 'The release cannot be rolled back, a virtual snapshot depends on it, or the draft is not latest' + description: 'Administrator role required' '404': description: 'Snapshot not found' + '409': + description: 'Not the latest tagged release, source draft unavailable, downstream dependency, or concurrent operation' /api/release-tracks/{id}/snapshots/{modified}/clone: post: diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index b351bc79..6bb1cb5a 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -42,6 +42,7 @@ const { updateSnapshotDescriptionBodySchema, releaseBodySchema, retagReleaseBodySchema, + convertReleaseToDraftBodySchema, releaseVersionSelectionSchema, cloneBodySchema, addCandidatesBodySchema, @@ -713,13 +714,33 @@ exports.reconstructSnapshotManifest = async function reconstructSnapshotManifest } }; +/** POST /api/release-tracks/:id/snapshots/:modified/draft */ +exports.convertReleaseToDraft = async function convertReleaseToDraft(req, res, next) { + try { + const result = convertReleaseToDraftBodySchema.safeParse(req.body || {}); + if (!result.success) { + throw new BadRequestError({ + message: 'A valid confirm_version is required to convert a release to a draft', + }); + } + const draft = await releaseTracksService.convertReleaseToDraft( + req.params.id, + req.params.modified, + { + actor: destructiveActor(req), + confirmation: result.data.confirm_version, + }, + ); + return res.status(200).send(draft); + } catch (err) { + return next(err); + } +}; + /** DELETE /api/release-tracks/:id/snapshots/:modified */ exports.deleteSnapshotByModified = async function deleteSnapshotByModified(req, res, next) { try { - await releaseTracksService.deleteSnapshot(req.params.id, req.params.modified, { - actor: destructiveActor(req), - confirmation: req.query.confirm_version, - }); + await releaseTracksService.deleteSnapshot(req.params.id, req.params.modified); logger.debug(`Success: Deleted snapshot ${req.params.modified} from track ${req.params.id}`); return res.status(204).end(); } catch (err) { diff --git a/app/exceptions/index.js b/app/exceptions/index.js index 3360d26b..2f672542 100644 --- a/app/exceptions/index.js +++ b/app/exceptions/index.js @@ -347,7 +347,10 @@ class ReleaseTrackAuditError extends CustomError { class TaggedSnapshotDeletionError extends CustomError { constructor(version, options) { - super(`Tagged snapshot version ${version} cannot be deleted`, options); + super( + `Tagged snapshot version ${version} cannot be deleted; convert it to a draft first`, + options, + ); } } diff --git a/app/lib/release-tracks/release-track-schemas.js b/app/lib/release-tracks/release-track-schemas.js index 0903ad39..59858552 100644 --- a/app/lib/release-tracks/release-track-schemas.js +++ b/app/lib/release-tracks/release-track-schemas.js @@ -497,6 +497,7 @@ const releaseBodySchema = z /** PUT /release-tracks/:id/snapshots/:modified/release */ const retagReleaseBodySchema = z.object({ version: xMitreVersionSchema }).strict(); +const convertReleaseToDraftBodySchema = z.object({ confirm_version: xMitreVersionSchema }).strict(); /** POST /release-tracks/:id/clone */ const cloneBodySchema = z @@ -680,6 +681,7 @@ module.exports = { updateSnapshotDescriptionBodySchema, releaseBodySchema, retagReleaseBodySchema, + convertReleaseToDraftBodySchema, publicationConfigSchema, cloneBodySchema, addCandidatesBodySchema, diff --git a/app/models/release-tracks/release-track-audit-event-model.js b/app/models/release-tracks/release-track-audit-event-model.js index 48853cea..dad63313 100644 --- a/app/models/release-tracks/release-track-audit-event-model.js +++ b/app/models/release-tracks/release-track-audit-event-model.js @@ -9,7 +9,7 @@ const releaseTrackAuditEventSchema = new mongoose.Schema( action: { type: String, required: true, - enum: ['delete_track', 'delete_release', 'retag_release'], + enum: ['delete_track', 'delete_release', 'retag_release', 'convert_release_to_draft'], }, track_id: { type: String, required: true, validate: validateTrackId }, status: { diff --git a/app/routes/release-tracks-routes.js b/app/routes/release-tracks-routes.js index 01fef16f..2872c192 100644 --- a/app/routes/release-tracks-routes.js +++ b/app/routes/release-tracks-routes.js @@ -273,6 +273,14 @@ router releaseTracksController.retagRelease, ); +router + .route('/release-tracks/:id/snapshots/:modified/draft') + .post( + authn.authenticate, + authz.requireRole(authz.editorOrHigher), + releaseTracksController.convertReleaseToDraft, + ); + router .route('/release-tracks/:id/snapshots/:modified/clone') .post( diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index a9e5305b..34acb2c3 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -14,7 +14,12 @@ // Phase 6: Export, ephemeral, bundle import → export-service, ephemeral-service, bundle-import-service // ============================================================================= -const { BadRequestError, InsufficientRoleError, NotImplementedError } = require('../../exceptions'); +const { + BadRequestError, + InsufficientRoleError, + NotImplementedError, + ReleaseConflictError, +} = require('../../exceptions'); const authz = require('../../lib/authz-middleware'); const { compositionSchema, @@ -371,28 +376,38 @@ exports.deleteTrack = function deleteTrack(trackId, actor, confirmation) { }; /** - * Delete a snapshot. Drafts follow the ordinary editor rules. A release may - * only be deleted by an administrator who confirms its version, and the - * deletion is recorded as a `delete_release` audit event. + * Delete drafts only. Share the release lock with tagging and materialization + * so a draft cannot become released between validation and deletion. */ -exports.deleteSnapshot = async function deleteSnapshot(trackId, modified, options = {}) { - const snapshot = await snapshotService.getSnapshotByModified(trackId, modified); - if (snapshot.version == null) { - return snapshotService.deleteSnapshot(trackId, modified); - } +exports.deleteSnapshot = function deleteSnapshot(trackId, modified) { + return versioningService.withReleaseLock(trackId, () => + snapshotService.deleteSnapshot(trackId, modified), + ); +}; +/** Convert the latest tagged release back to a draft, with admin confirmation. */ +exports.convertReleaseToDraft = async function convertReleaseToDraft( + trackId, + modified, + options = {}, +) { return versioningService.withReleaseLock(trackId, async () => { const snapshot = await snapshotService.getSnapshotByModified(trackId, modified); if (options.actor?.role !== authz.userRoles.admin) { throw new InsufficientRoleError('administrator', { - details: 'Deleting a release requires an administrator.', + details: 'Converting a release to a draft requires an administrator.', track_id: trackId, version: snapshot.version, }); } + if (snapshot.version == null) { + throw new ReleaseConflictError('The selected snapshot is already a draft', { + track_id: trackId, + }); + } if (options.confirmation !== snapshot.version) { throw new BadRequestError({ - message: 'Destructive release confirmation is required', + message: 'Release conversion confirmation is required', details: `Set confirm_version to the exact release version '${snapshot.version}'.`, parameter_name: 'confirm_version', expected_version: snapshot.version, @@ -401,12 +416,12 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified, option return destructiveAuditService.execute( { - action: 'delete_release', + action: 'convert_release_to_draft', trackId, ...destructiveIdentity(trackId, options.actor, options.confirmation), request: { snapshot_modified: new Date(snapshot.modified).toISOString() }, }, - () => snapshotService.deleteRelease(trackId, modified), + () => snapshotService.convertReleaseToDraft(trackId, modified), ); }); }; diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 6a2a3a93..07f33adc 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -852,9 +852,9 @@ exports.deleteTrack = async function deleteTrack(trackId) { }; /** - * Delete the track's most recent release. + * Convert the track's most recent release back to a draft. * - * Only the newest tagged snapshot may be deleted, so the version order of the + * Only the newest tagged snapshot may be converted, so the version order of the * remaining releases and the provenance of any later release are never * disturbed. The release's ledger entry is retracted from every remaining * snapshot (the ledger is copied forward into clones), its manifest is @@ -863,9 +863,9 @@ exports.deleteTrack = async function deleteTrack(trackId) { * * @param {string} trackId * @param {string|Date} modified - * @returns {Promise} The deleted snapshot + * @returns {Promise} The restored draft snapshot */ -exports.deleteRelease = async function deleteRelease(trackId, modified) { +exports.convertReleaseToDraft = async function convertReleaseToDraft(trackId, modified) { const snapshot = await exports.getSnapshotByModified(trackId, modified); if (snapshot.version == null) { throw new ReleaseConflictError('The selected snapshot is not a release', { @@ -879,7 +879,7 @@ exports.deleteRelease = async function deleteRelease(trackId, modified) { new Date(latestTagged.modified).getTime() !== new Date(snapshot.modified).getTime() ) { throw new ReleaseConflictError( - 'Only the most recent release of a track can be deleted; delete later releases first.', + 'Only the most recent release of a track can be converted to a draft; convert later releases first.', { track_id: trackId, snapshot_modified: new Date(snapshot.modified).toISOString(), @@ -889,6 +889,7 @@ exports.deleteRelease = async function deleteRelease(trackId, modified) { ); } + let draft; if (snapshot.type === 'standard') { if (!snapshot.release_source_modified) { throw new ReleaseConflictError( @@ -900,21 +901,28 @@ exports.deleteRelease = async function deleteRelease(trackId, modified) { }, ); } - const dependents = await findVirtualSnapshotDependents(trackId, snapshot.modified); - if (dependents.length > 0) { - throw new ReleaseConflictError( - 'This release cannot be deleted because virtual track snapshots depend on it.', - { - track_id: trackId, - snapshot_modified: new Date(snapshot.modified).toISOString(), - version: snapshot.version, - dependent_snapshots: dependents, - }, - ); + draft = await dynamicRepo.getSnapshotByModified(trackId, snapshot.release_source_modified); + if (!draft || draft.version != null) { + throw new ReleaseConflictError('The preserved source draft is missing or no longer a draft', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }); } } + await assertNoVirtualDependents(trackId, snapshot.modified); - await dynamicRepo.deleteSnapshot(trackId, snapshot.modified); + if (snapshot.type === 'standard') { + // The tagged clone is retired; the exact source draft remains untouched. + await dynamicRepo.deleteSnapshot(trackId, snapshot.modified); + } else { + // Virtual tagging was in-place, so conversion keeps identity, manifest, + // composition resolution, notes, and immutable creation provenance. + await dynamicRepo.updateSnapshot(trackId, snapshot.modified, { + $set: { version: null }, + $unset: { publication: '', bundle_id: '', bundle_hashes: '' }, + }); + draft = snapshot; + } await dynamicRepo.pullVersionHistory(trackId, snapshot.version); await contentManifestService.discardUnreferenced(trackId, [snapshot.content_manifest_id]); const releaseHistoryService = require('./release-history-service'); @@ -925,11 +933,22 @@ exports.deleteRelease = async function deleteRelease(trackId, modified) { await emitContentsChanged(trackId, latest); logger.verbose( - `SnapshotService: Deleted release v${snapshot.version} (${modified}) from track "${trackId}"`, + `SnapshotService: Converted release v${snapshot.version} (${modified}) to draft in track "${trackId}"`, ); - return snapshot; + return dynamicRepo.getSnapshotByModified(trackId, draft.modified); }; +async function assertNoVirtualDependents(trackId, modified) { + const dependents = await findVirtualSnapshotDependents(trackId, modified); + if (dependents.length) { + throw new ReleaseConflictError('Virtual track snapshots depend on this snapshot', { + track_id: trackId, + snapshot_modified: new Date(modified).toISOString(), + dependent_snapshots: dependents, + }); + } +} + /** * Delete a specific snapshot from a track. * @@ -952,6 +971,14 @@ exports.deleteSnapshot = async function deleteSnapshot(trackId, modified) { throw new TaggedSnapshotDeletionError(snapshot.version); } + if (await dynamicRepo.getReleaseBySourceModified(trackId, snapshot.modified)) { + throw new ReleaseConflictError('This draft is the preserved source of a tagged release', { + track_id: trackId, + snapshot_modified: new Date(snapshot.modified).toISOString(), + }); + } + await assertNoVirtualDependents(trackId, snapshot.modified); + const latest = await dynamicRepo.getLatestSnapshot(trackId); if (!latest || new Date(latest.modified).getTime() !== new Date(snapshot.modified).getTime()) { throw new HistoricalSnapshotDeletionError(snapshot.modified, latest?.modified); diff --git a/app/tests/api/release-tracks/destructive-authorization.spec.js b/app/tests/api/release-tracks/destructive-authorization.spec.js index 8590d920..27fa1658 100644 --- a/app/tests/api/release-tracks/destructive-authorization.spec.js +++ b/app/tests/api/release-tracks/destructive-authorization.spec.js @@ -69,6 +69,10 @@ describe('Release-track destructive authorization and audit', function () { return (await api('post', path, body, status, query)).body; } + function convert(path, version, status = 200) { + return api('post', `${path}/draft`, { confirm_version: version }, status); + } + it('requires admin role, exact confirmation, and a durable outcome record', async function () { await setRole('admin'); const track = await post( @@ -171,20 +175,25 @@ describe('Release-track destructive authorization and audit', function () { // Editors cannot delete a release even with the right confirmation. await setRole('editor'); - await api('delete', secondPath, undefined, 403, { confirm_version: '1.1' }); + await convert(secondPath, '1.1', 403); await setRole('admin'); - await api('delete', secondPath, undefined, 400); - await api('delete', secondPath, undefined, 400, { confirm_version: '9.9' }); - expect(await ReleaseTrackAuditEvent.countDocuments({ action: 'delete_release' })).toBe(0); + await api('post', `${secondPath}/draft`, {}, 400); + await convert(secondPath, '9.9', 400); + expect( + await ReleaseTrackAuditEvent.countDocuments({ action: 'convert_release_to_draft' }), + ).toBe(0); // Only the most recent release can be deleted; the rejected attempt is // audited as failed, like any confirmed destructive request. - await api('delete', firstPath, undefined, 409, { confirm_version: '1.0' }); + await convert(firstPath, '1.0', 409); expect( - await ReleaseTrackAuditEvent.countDocuments({ action: 'delete_release', status: 'failed' }), + await ReleaseTrackAuditEvent.countDocuments({ + action: 'convert_release_to_draft', + status: 'failed', + }), ).toBe(1); - await api('delete', secondPath, undefined, 204, { confirm_version: '1.1' }); + await convert(secondPath, '1.1', 200); await api('get', secondPath, undefined, 404); const remaining = await api( @@ -208,7 +217,7 @@ describe('Release-track destructive authorization and audit', function () { expect(entry.latest_tagged_version).toBe('1.0'); const event = await ReleaseTrackAuditEvent.findOne({ - action: 'delete_release', + action: 'convert_release_to_draft', status: 'completed', }) .lean() @@ -218,7 +227,7 @@ describe('Release-track destructive authorization and audit', function () { confirmation: '1.1', status: 'completed', request: { snapshot_modified: new Date(second.modified).toISOString() }, - result: { snapshot_modified: expect.any(Date), version: '1.1', members_count: 1 }, + result: { snapshot_modified: expect.any(Date), version: null, members_count: 1 }, }); // The version is free again and the track keeps working. @@ -266,12 +275,10 @@ describe('Release-track destructive authorization and audit', function () { }); } - const response = await api( - 'delete', + const response = await convert( `/api/release-tracks/${component.id}/snapshots/${encodeURIComponent(released.modified)}`, - undefined, + '1.0', 409, - { confirm_version: '1.0' }, ); expect(response.body.dependent_snapshots).toHaveLength(2); expect(response.body.dependent_snapshots.map((item) => item.track_name).sort()).toEqual([ @@ -453,13 +460,13 @@ describe('Release-track destructive authorization and audit', function () { const path = `${base}/snapshots/${encodeURIComponent(released.modified)}`; const clone = snapshotService.cloneSnapshot; const stub = sinon.stub(snapshotService, 'cloneSnapshot').callsFake(async (...args) => { - await api('delete', path, undefined, 409, { confirm_version: '1.0' }); + await convert(path, '1.0', 409); return clone(...args); }); await post(`/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 201); expect(stub.calledOnce).toBe(true); stub.restore(); - const blocked = await api('delete', path, undefined, 409, { confirm_version: '1.0' }); + const blocked = await convert(path, '1.0', 409); expect(blocked.body.dependent_snapshots).toHaveLength(1); expect((await registryRepo.findByTrackId(component.id)).release_lock).toBeUndefined(); }); @@ -555,13 +562,7 @@ describe('Release-track destructive authorization and audit', function () { await api('post', `/api/release-tracks/${virtual.id}/virtual/snapshots/create`, {}, 409); return find.apply(dynamicRepo, args); }); - await api( - 'delete', - `${base}/snapshots/${encodeURIComponent(released.modified)}`, - undefined, - 204, - { confirm_version: '1.0' }, - ); + await convert(`${base}/snapshots/${encodeURIComponent(released.modified)}`, '1.0', 200); expect((await registryRepo.findByTrackId(component.id)).release_lock).toBeUndefined(); }); @@ -581,7 +582,7 @@ describe('Release-track destructive authorization and audit', function () { await api('put', `${path}/release`, { version: '1.1' }, 200); return acquire.apply(registryRepo, args); }); - const rejected = await api('delete', path, undefined, 400, { confirm_version: '1.0' }); + const rejected = await convert(path, '1.0', 400); expect(rejected.body.expected_version).toBe('1.1'); expect((await dynamicRepo.getSnapshotByModified(track.id, released.modified)).version).toBe( '1.1', @@ -630,6 +631,161 @@ describe('Release-track destructive authorization and audit', function () { } }); + it('rejects tagged DELETE for every role, then permits deleting an eligible restored standard draft', async function () { + await setRole('admin'); + const track = await post( + '/api/release-tracks/new', + { name: 'Three operations', type: 'standard' }, + 201, + ); + const base = `/api/release-tracks/${track.id}`; + const first = await post(`${base}/snapshots/latest/release`, { version: '1.0' }); + const draft = await post(`${base}/meta`, { description: 'Next cycle' }); + const released = await post(`${base}/snapshots/latest/release`, { version: '1.1' }); + const path = `${base}/snapshots/${encodeURIComponent(released.modified)}`; + for (const role of ['editor', 'admin']) { + await setRole(role); + await api('delete', path, undefined, 409); + await api('delete', path, undefined, 409, { confirm_version: '1.1' }); + } + await api('delete', `${base}/snapshots/${encodeURIComponent(draft.modified)}`, undefined, 409); + const restored = await convert(path, '1.1'); + expect(restored.body).toMatchObject({ + modified: draft.modified, + version: null, + }); + expect(restored.body.creation_cause).toEqual(draft.creation_cause); + expect(restored.body.creation_actor).toEqual(draft.creation_actor); + await setRole('editor'); + await api( + 'delete', + `${base}/snapshots/${encodeURIComponent(restored.body.modified)}`, + undefined, + 204, + ); + const latest = await api('get', `${base}/snapshots/latest`, undefined, 200); + expect(latest.body.modified).toBe(first.modified); + }); + + it('converts virtual releases in place, retaining provenance, before allowing draft deletion', async function () { + await setRole('admin'); + const component = await post( + '/api/release-tracks/new', + { name: 'Virtual source', type: 'standard' }, + 201, + ); + await post(`/api/release-tracks/${component.id}/snapshots/latest/release`, { version: '1.0' }); + const virtual = await post( + '/api/release-tracks/new', + { + name: 'Virtual conversion', + type: 'virtual', + composition: { + component_tracks: [ + { track_id: component.id, priority: 1, resolution_strategy: 'latest_tagged' }, + ], + }, + }, + 201, + ); + const base = `/api/release-tracks/${virtual.id}`; + const draft = await post( + `${base}/virtual/snapshots/create`, + { description: 'Preserve me' }, + 201, + ); + const release = await post(`${base}/snapshots/latest/release`, { version: '1.0' }); + const path = `${base}/snapshots/${encodeURIComponent(release.modified)}`; + await api('delete', path, undefined, 409); + const restored = await convert(path, '1.0'); + expect(restored.body).toMatchObject({ + modified: draft.modified, + version: null, + content_manifest_id: draft.content_manifest_id, + composition_resolution: draft.composition_resolution, + snapshot_description: draft.snapshot_description, + version_history: [], + }); + expect(restored.body.creation_cause).toEqual(draft.creation_cause); + expect(restored.body.creation_actor).toEqual(draft.creation_actor); + for (const field of ['publication', 'bundle_id', 'bundle_hashes']) + expect(restored.body).not.toHaveProperty(field); + expect( + await ReleaseTrackContentManifest.countDocuments({ manifest_id: draft.content_manifest_id }), + ).toBe(1); + await convert(path, '1.0', 409); + // The same materialized draft can be tagged again, then explicitly converted. + await post(`${path}/release`, { version: '1.0' }); + await convert(path, '1.0'); + await setRole('editor'); + await api('delete', path, undefined, 204); + expect((await api('get', `${base}/snapshots/latest`, undefined, 200)).body.modified).toBe( + virtual.modified, + ); + }); + + it('protects draft dependencies, historical drafts, and the only snapshot', async function () { + await setRole('admin'); + const track = await post( + '/api/release-tracks/new', + { name: 'Protected draft', type: 'standard' }, + 201, + ); + const path = `/api/release-tracks/${track.id}/snapshots/${encodeURIComponent(track.modified)}`; + await api('delete', path, undefined, 409); + const virtual = await post( + '/api/release-tracks/new', + { name: 'Legacy dependent', type: 'virtual' }, + 201, + ); + // Current composition resolution requires tagged sources. Model a retained + // historical dependency on a draft so deletion cannot assume none exist. + await dynamicRepo.updateSnapshot(virtual.id, virtual.modified, { + $set: { + composition_resolution: { + resolved_at: new Date(), + component_snapshots: [ + { + track_id: track.id, + track_name: track.name, + track_type: 'standard', + resolved_snapshot_id: track.modified, + resolved_version: '1.0', + strategy_used: 'specific_snapshot', + total_objects_in_source: 0, + objects_after_filter: 0, + objects_contributed: 0, + }, + ], + }, + }, + }); + const rejected = await api('delete', path, undefined, 409); + expect(rejected.body.dependent_snapshots).toHaveLength(1); + await post(`/api/release-tracks/${virtual.id}/meta`, { description: 'new draft' }); + await api( + 'delete', + `/api/release-tracks/${virtual.id}/snapshots/${encodeURIComponent(virtual.modified)}`, + undefined, + 409, + ); + }); + + it('does not retire a standard release whose preserved source is missing', async function () { + await setRole('admin'); + const track = await post( + '/api/release-tracks/new', + { name: 'Missing source', type: 'standard' }, + 201, + ); + const base = `/api/release-tracks/${track.id}`; + const released = await post(`${base}/snapshots/latest/release`, { version: '1.0' }); + await dynamicRepo.deleteSnapshot(track.id, released.release_source_modified); + const path = `${base}/snapshots/${encodeURIComponent(released.modified)}`; + await convert(path, '1.0', 409); + expect((await api('get', path, undefined, 200)).body.version).toBe('1.0'); + }); + it('reports an audit-finalization failure without hiding the persisted mutation', async function () { await setRole('admin'); const track = await post( diff --git a/app/tests/api/release-tracks/releases-by-object.spec.js b/app/tests/api/release-tracks/releases-by-object.spec.js index c82bfefd..54a53262 100644 --- a/app/tests/api/release-tracks/releases-by-object.spec.js +++ b/app/tests/api/release-tracks/releases-by-object.spec.js @@ -257,11 +257,11 @@ describe('GET /api/release-tracks/objects/:objectRef/releases', function () { await get(`/api/release-tracks/objects/${objectRevisionA.stix.id}/releases?limit=0`, 400); }); - it('requires a typed version confirmation before a release can be deleted', async function () { + it('rejects direct deletion of a tagged release', async function () { await request(app) .delete(`/api/release-tracks/${trackA}/snapshots/${trackATaggedSnapshot.modified}`) .set('Cookie', `${passportCookie.name}=${passportCookie.value}`) - .expect(400); + .expect(409); }); it('backfills missing registry refs from authoritative tagged snapshots', async function () { diff --git a/app/tests/api/release-tracks/snapshot-immutability.spec.js b/app/tests/api/release-tracks/snapshot-immutability.spec.js index b57b8bc5..2d4dae73 100644 --- a/app/tests/api/release-tracks/snapshot-immutability.spec.js +++ b/app/tests/api/release-tracks/snapshot-immutability.spec.js @@ -106,14 +106,14 @@ describe('Release-track snapshot immutability contract', function () { ); expect(reverted.body.modified).toBe(tagged.modified); - // A release is never deleted by the ordinary draft path: it requires an - // administrator's typed version confirmation. + // A tagged release must first be converted through the separate draft + // operation; DELETE never converts it, even for administrators. const taggedDelete = await api( 'delete', `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(tagged.modified)}`, undefined, - 400, + 409, ); - expect(taggedDelete.text).toContain('Destructive release confirmation is required'); + expect(taggedDelete.text).toContain('convert it to a draft first'); }); }); diff --git a/docs/admin/release-track-audit.md b/docs/admin/release-track-audit.md index e6427d8f..6e754a8f 100644 --- a/docs/admin/release-track-audit.md +++ b/docs/admin/release-track-audit.md @@ -2,10 +2,15 @@ Workbench stores administrator-initiated destructive attempts in `releaseTrackAuditEvents`: full-track deletion (`delete_track`), rollback of a -track's most recent release (`delete_release`), and release-version correction +track's most recent release (`convert_release_to_draft`), and release-version correction (`retag_release`). The collection is empty until an administrator performs one of those actions. +Older `delete_release` events retain their historical meaning. New snapshot +DELETE requests reject tagged releases; conversion and draft deletion are +separate operations. Conversion results identify the restored draft timestamp +and have `version: null`. + Each record contains: - `event_id`, `action`, and `track_id` diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index aa236eed..c3d6346e 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,35 @@ # Release Track TODOs +## Separate tagging, conversion to draft, and draft deletion + +- [x] Preserve creation provenance while separating conversion and deletion in the API. +- [x] Keep admin confirmation, latest/sole-snapshot, dependency, and release-lock guards. +- [x] Add distinct frontend Convert to draft and Delete draft controls. +- [x] Update regressions, OpenAPI, user/developer docs, and Bruno requests. +- [x] Run focused tests, full backend/frontend suites, lint, and build. + +Verification (Node 24): backend focused group 76 passing; full `npm test` +passes (OpenAPI 2, config 22, API 1048, middleware 29, scheduler 10). +The first full run identified two obsolete DELETE-confirmation expectations, +which now assert draft-only rejection. Unrelated HTTP/authentication failures +passed in isolation and in the complete rerun. Backend lint passes. +Frontend focused tests 108 passing; all 168 files / 425 tests pass, changed-file +lint passes, and production build succeeds with existing size-budget warnings. +Logic-specialist review: ROBUST for the scoped state transitions, dependency +checks, preserved-source protection, publication cleanup, and release-lock use. +Both main beta checkouts retain the other agent's uncommitted provenance work. +No commits were created; Bruno conversion/deletion requests are updated. + +Accepted commit: `feat(release-tracks): separate draft conversion from snapshot deletion` + +Body: Add an explicitly confirmed release-to-draft endpoint and matching UI +control. Restore guarded draft deletion while preserving source drafts, +composition provenance, dependency checks, and release serialization. + +Snapshot DELETE is draft-only. Convert tagged releases with +POST /release-tracks/:id/snapshots/:modified/draft and a confirm_version body +before attempting a separate eligible-draft DELETE. + ## Merge scheduling and rollback branches into local beta (2026-09-09) - [x] Inspect all worktrees and confirm scheduling branches are already merged. diff --git a/docs/developer/release-tracks/authorization.md b/docs/developer/release-tracks/authorization.md index 3254aaef..abef107b 100644 --- a/docs/developer/release-tracks/authorization.md +++ b/docs/developer/release-tracks/authorization.md @@ -14,31 +14,33 @@ history requires an administrator. | Create tracks and drafts; manage candidates/staged/config/composition | No | Yes | Yes | | Tag a standard or virtual snapshot | No | Yes | Yes | | Delete the latest untagged draft snapshot | No | Yes | Yes | -| Delete the track's most recent release | No | No | Yes | +| Convert the track's most recent release to draft | No | No | Yes | | Change a tagged release's semantic version | No | No | Yes | | Delete an entire track and all snapshot history | No | No | Yes | Full-track deletion also requires `confirm_track_id` to equal the `:id` path -parameter, and release deletion requires `confirm_version` to equal the -release version. Track deletion is authorized by route middleware; release -deletion shares the snapshot deletion route, so the service checks the -administrator role itself and answers `403` otherwise. Confirmation runs -before persistence in both cases. +parameter. `POST /snapshots/:modified/draft` requires a JSON `confirm_version` +equal to the release version and an administrator (service-checked, `403` +otherwise). `DELETE /snapshots/:modified` is draft-only, editor-or-higher; +tagged snapshots always return `409`, including for administrators. Draft +deletion keeps latest/sole-snapshot, preserved-source, and dependency guards. Release-version correction uses `PUT /snapshots/:modified/release`, is also checked in the service, and does not require destructive confirmation because it preserves the snapshot. It is serialized with release and rollback and is recorded as `retag_release`. -Release deletion re-reads the snapshot and checks `confirm_version` under the -release lock. Both deletion and retag capture audit identity under that same +Release conversion re-reads the snapshot and checks `confirm_version` under the +release lock. Both conversion and retag capture audit identity under that same lock, so a competing version correction cannot invalidate confirmation or change the version between audit capture and mutation. ## Audited destructive actions -The `delete_track`, `delete_release`, and `retag_release` actions create a +The `delete_track`, `convert_release_to_draft`, and `retag_release` actions create a `releaseTrackAuditEvents` record before the business operation begins. +The legacy `delete_release` value remains readable for historical audit events; +new requests never use it. Each event records the authenticated actor, confirmation value, target track, request summary, timestamps, and a `pending`, `completed`, or `failed` status. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 4da5ed91..474ed0f2 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -6,12 +6,12 @@ This document tracks new database schemas, interfaces, etc.; as well as changes | Collection | Purpose | Written by | Growth and retention | | ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | -| `releaseTrackRegistry` | One document per track: name, type, denormalized counters, the tagged-release catalogue (`tagged_releases`), the release lock, and virtual schedules. The index that maps a track to its own snapshot collection. | Track create/delete, every snapshot write (counters), release commit and release deletion (catalogue). | One document per track. | +| `releaseTrackRegistry` | One document per track: name, type, denormalized counters, the tagged-release catalogue (`tagged_releases`), the release lock, and virtual schedules. The index that maps a track to its own snapshot collection. | Track create/delete, every snapshot write (counters), release commit and conversion to draft (catalogue). | One document per track. | | `release-track--` | The track's snapshots: one active rolling draft, a preserved source draft per tagged standard release, and every tagged release; every materialized draft plus releases for a virtual track. | Snapshot service and release commit. | Standard tracks grow by two snapshots per release plus one active draft; virtual tracks by materializations. | | `releaseTrackContentManifests` | The sealed bill of materials each snapshot references (`content_manifest_id`). Several snapshots share one manifest when their member sets are identical. | Sealed whenever members are written; discarded when no snapshot references it. | Bounded by member-changing writes, not by snapshot count. | | `releaseTrackContentManifestEntries` | One exact-revision pointer per object a manifest emits or depends on. The `(object_ref, object_modified)` index is what protects referenced revisions from deletion. | With its manifest. | Roughly members + relationships + a few supporting objects per manifest. | | `releaseTrackReconciliations` | Outstanding backref reconciliation work only: a record is created before the `workspace.release_tracks` listeners run and deleted when they succeed, so anything present is pending or failed and needs repair. | Every snapshot write. | Normally empty. | -| `releaseTrackAuditEvents` | Audit trail for administrator-only track deletion, release rollback, and release retagging (`delete_track`, `delete_release`, `retag_release`). | Those operations. | Empty until an administrator performs one of those operations. | +| `releaseTrackAuditEvents` | Audit trail for administrator-only track deletion, release conversion to draft, and release retagging (`delete_track`, `convert_release_to_draft` (legacy: `delete_release`), `retag_release`). | Those operations. | Empty until an administrator performs one of those operations. | | `virtualTrackScheduleOccurrences` | Durable claims for scheduled virtual materialization (cron or dated schedules) so restarts and duplicate delivery execute each occurrence once. | The scheduler. | One record per scheduled occurrence; empty when no virtual track has a schedule. | Removed by the sealed-manifest work: the former `releaseTrackGraphManifests` diff --git a/docs/developer/release-tracks/error-handling.md b/docs/developer/release-tracks/error-handling.md index 435b07a3..d9d28f00 100644 --- a/docs/developer/release-tracks/error-handling.md +++ b/docs/developer/release-tracks/error-handling.md @@ -49,8 +49,8 @@ operation, then release the new snapshot. **HTTP Status:** 409 Conflict -Tagged snapshots are immutable release records. Create or modify a draft -snapshot instead; deleting an entire release track remains a separate +Convert the latest release with `POST /snapshots/:modified/draft` first, then +delete the returned draft if eligible; deleting an entire release track remains a separate track-level operation. ### HistoricalSnapshotDeletionError diff --git a/docs/developer/release-tracks/sealed-content-manifests.md b/docs/developer/release-tracks/sealed-content-manifests.md index dc59e079..eec15044 100644 --- a/docs/developer/release-tracks/sealed-content-manifests.md +++ b/docs/developer/release-tracks/sealed-content-manifests.md @@ -116,6 +116,17 @@ endpoint. ### Rollback and retag concurrency / recovery +Release conversion is now explicit: `POST /snapshots/:modified/draft` replaces +the old tagged-snapshot DELETE path. Standard conversion retires the release +clone and restores its preserved source; virtual conversion clears the tag and +release-only export fields in place, keeping the manifest and provenance. +`DELETE /snapshots/:modified` only removes eligible drafts. Both operations +check downstream virtual dependencies under the release lock. A preserved +standard source must still exist and be untagged before its release is retired. +Creation cause and creator describe the original snapshot creation and are +never rewritten by conversion. Legacy `delete_release` audit entries remain +valid; new conversions use `convert_release_to_draft`. + Virtual materialization acquires the existing database-backed release locks for all component tracks in sorted order, before resolving any release, and holds them through snapshot persistence. Partial acquisition and failed diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 2ba0a94c..05967872 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -59,6 +59,8 @@ GET /api/release-tracks/:id/snapshots GET /api/release-tracks/:id/snapshots/latest GET /api/release-tracks/:id/snapshots/:modified POST /api/release-tracks/:id/snapshots/:modified/release +PUT /api/release-tracks/:id/snapshots/:modified/release +POST /api/release-tracks/:id/snapshots/:modified/draft POST /api/release-tracks/:id/snapshots/:modified/clone PUT /api/release-tracks/:id/snapshots/:modified/description DELETE /api/release-tracks/:id/snapshots/:modified @@ -826,24 +828,46 @@ rule configured under `config.publication` (see configuration. Release freezes the resolved values onto the tagged snapshot as `publication`, so later changes never alter a published release. -### Delete Specific Snapshot +### Convert a Tagged Release Back to Draft + +```http +POST /api/release-tracks/:id/snapshots/:modified/draft +Content-Type: application/json + +{ "confirm_version": "1.1" } +``` + +Administrator-only; returns `200` with the restored draft. Standard releases +restore their exact preserved source draft, including its original timestamp, +tiers, manifest, notes, creation cause, and creator. Virtual releases become +drafts in place: identity, materialized contents, composition provenance, +notes, and creation attribution remain; release-only publication/bundle/hash +fields are cleared. Both retract the release ledger entry and reconcile the +catalogue, counters, and backrefs. The action is audited as +`convert_release_to_draft`. Later drafts survive and remain current if newer. + +Only the most recent tagged release may be converted. Downstream virtual +dependencies block conversion (`409`), as does a missing preserved standard +source. Missing/incorrect confirmation returns `400`, and non-administrators +receive `403`. The confirmation is checked under the release lock. + +### Delete a Draft Snapshot ``` DELETE /api/release-tracks/:id/snapshots/:modified -DELETE /api/release-tracks/:id/snapshots/:modified?confirm_version=1.1 ``` Editors may delete the latest untagged draft; the track reverts to the -preceding snapshot. Administrators may roll back the most recent standard -release by confirming its version. The tagged clone is removed, revealing its -exact preserved source draft; the release ledger and catalogue are reconciled -and a `delete_release` audit event is recorded. Rollback returns `409 Conflict` -if any persisted virtual snapshot resolved the exact release (whether through -`latest_tagged` or an explicit rule), or if the release predates preserved -source drafts. Deleting an older release also returns `409`; a missing or wrong -confirmation returns `400`; a non-administrator receives `403`. -The newest virtual release retains the existing irreversible deletion -behavior because virtual materializations are still tagged in place. +preceding snapshot and returns `204`. Tagged snapshots always return `409`, +even for administrators and even with the deprecated `confirm_version` query +parameter. Convert the release to a draft first, then delete the returned +draft timestamp in a separate request. + +Deletion also returns `409` for historical drafts, the only remaining +snapshot, preserved sources of tagged releases, or any draft resolved by a +downstream virtual snapshot. Thus conversion alone does not guarantee deletion +eligibility. Deletion and dependency checks share the same release lock as +tagging and component materialization. A missing snapshot returns `404`. --- diff --git a/docs/user/release-tracks/summary.md b/docs/user/release-tracks/summary.md index 04517060..4efc2b91 100644 --- a/docs/user/release-tracks/summary.md +++ b/docs/user/release-tracks/summary.md @@ -84,8 +84,9 @@ POST /api/release-tracks/:id/staged/demote # Snapshot-specific operations GET /api/release-tracks/:id/snapshots/:modified POST /api/release-tracks/:id/snapshots/:modified/clone -DELETE /api/release-tracks/:id/snapshots/:modified +DELETE /api/release-tracks/:id/snapshots/:modified # eligible drafts only POST /api/release-tracks/:id/snapshots/:modified/release +POST /api/release-tracks/:id/snapshots/:modified/draft # admin conversion with confirm_version POST /api/release-tracks/:id/snapshots/:modified/graph POST /api/release-tracks/:id/snapshots/:modified/graph/reconstruct # admin recovery DELETE /api/release-tracks/:id/snapshots/:modified/graph diff --git a/docs/user/release-tracks/versioning.md b/docs/user/release-tracks/versioning.md index 750adbf9..e1dc9d04 100644 --- a/docs/user/release-tracks/versioning.md +++ b/docs/user/release-tracks/versioning.md @@ -74,10 +74,24 @@ second draft is durably stored, the first draft is no longer retrievable. ### What Does Releasing Do? -The `release` operation **tags an existing snapshot as a release** by assigning -it a semantic version number (without the patch number). It does **not** create -a new snapshot. `release` is the command; `tagged` describes the resulting -snapshot state. +The `release` operation assigns a semantic version number (without the patch +number). Standard tracks create a separate tagged snapshot and preserve the +exact source draft; virtual tracks tag their materialized draft in place. +`release` is the command; `tagged` describes the resulting snapshot state. + +The snapshot lifecycle has three separate operations: + +- `POST /release-tracks/:id/snapshots/:modified/release` tags a draft. +- `POST /release-tracks/:id/snapshots/:modified/draft` converts the newest + tagged release back to a draft, with administrator authorization and an exact + `confirm_version` in the body. Standard tracks restore the preserved source; + virtual tracks retain the snapshot, its content, and composition provenance + but clear its tag and publication metadata. Downstream resolved dependencies + block conversion. +- `DELETE /release-tracks/:id/snapshots/:modified` deletes only the newest + draft. The sole snapshot, preserved release sources, and resolved components + of downstream virtual snapshots cannot be deleted. Tagged releases must + first be converted to drafts; administrators cannot bypass this requirement. This is analogous to Git's tagging system: From af6d6c19bf6987182e718618b9651554ae2856ac Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:21:40 -0400 Subject: [PATCH 11/14] docs(release-tracks): record snapshot header refinement verification Record the approved responsive header layout, timestamp metadata treatment, and regression verification while preserving unrelated provenance work. --- docs/developer/TODO.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index c3d6346e..6d014046 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,32 @@ # Release Track TODOs +## Snapshot card header hierarchy + +- [x] Give snapshot identity and status labels their own full-width header area. +- [x] Move controls below metadata; remove redundant draft copy and style tagged timestamps as metadata pills. +- [x] Verify template regressions, responsive layouts, frontend suite, lint, and build. + +Accepted lifecycle work committed without a breaking marker: backend e9fd8562, +frontend 880c49e0, Bruno a9b8a97. Other provenance changes remain uncommitted. + +UI verification: component spec 87 passing; full frontend suite 168 files / +428 tests passing. Changed-file lint, formatting, and production build pass +(existing bundle/style budget warnings remain). Isolated Angular-rendered +headers with compiled styles were visually checked in headless Chrome at +1400, 900, 390, and 320px: Latest stays in the title row, controls remain below +metadata, and neither draft nor tagged header overflows. Timestamp pills use +existing MITRE theme tokens with readable light/dark foregrounds. + +Backend lifecycle assertion verification: 20 passing; full suite passes +(OpenAPI 2, config 22, API 1048, middleware 29, scheduler 10), and lint passes. +A transient content-manifest HTTP 404 passed in isolation (10 tests) and on +the complete rerun. No unrelated test-harness changes were made. + +Header refinements approved for commit on 2026-09-10: +`fix(release-tracks): clarify snapshot card headers` +Body: Separate snapshot identity and status from controls, remove redundant +draft subtitles, and present snapshot/tagging timestamps as metadata pills. + ## Separate tagging, conversion to draft, and draft deletion - [x] Preserve creation provenance while separating conversion and deletion in the API. From a795b04bfbac203ac5f63f4f4c9ad3c5b9ec2cb0 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:28:44 -0400 Subject: [PATCH 12/14] feat(release-tracks): record snapshot creation provenance Persist creation causes and invoking users, distinguish standard release creation, and expose safe creator metadata through snapshot GETs. --- .../definitions/components/release-tracks.yml | 56 +++ app/controllers/release-tracks-controller.js | 13 +- .../release-tracks/snapshot-creation-actor.js | 7 + .../snapshot-creation-causes.js | 25 ++ .../release-track-snapshot-schema.js | 23 ++ .../release-track-dynamic.repository.js | 2 + .../release-tracks/bundle-import-service.js | 35 +- .../release-tracks/member-sync-service.js | 27 +- .../release-tracks/release-tracks-service.js | 51 ++- .../release-tracks/snapshot-service.js | 38 +- .../release-tracks/standard-track-service.js | 100 +++-- .../release-tracks/versioning-service.js | 8 +- .../release-tracks/virtual-track-service.js | 52 ++- .../release-tracks/workflow-service.js | 21 +- .../snapshot-creation-causes.spec.js | 351 ++++++++++++++++++ .../release-tracks/virtual-quarantine.spec.js | 2 + docs/README.md | 1 + docs/developer/TODO.md | 34 ++ docs/developer/release-tracks/entities.md | 10 + .../snapshot-creation-causes.md | 103 +++++ docs/user/release-tracks/api-reference.md | 16 + 21 files changed, 880 insertions(+), 95 deletions(-) create mode 100644 app/lib/release-tracks/snapshot-creation-actor.js create mode 100644 app/lib/release-tracks/snapshot-creation-causes.js create mode 100644 app/tests/api/release-tracks/snapshot-creation-causes.spec.js create mode 100644 docs/developer/release-tracks/snapshot-creation-causes.md diff --git a/app/api/definitions/components/release-tracks.yml b/app/api/definitions/components/release-tracks.yml index 25747d00..841d80ef 100644 --- a/app/api/definitions/components/release-tracks.yml +++ b/app/api/definitions/components/release-tracks.yml @@ -1,5 +1,53 @@ components: schemas: + snapshot-creation-actor: + type: object + readOnly: true + required: [kind] + description: Snapshot invoker, separate from the track creator. Missing historical attribution is unknown. User display data is resolved on GET and omitted if the account no longer exists. + properties: + kind: + type: string + enum: [user, system, unknown] + user_account_id: + type: string + description: Persisted invoking user account ID, present for kind=user. + user: + type: object + readOnly: true + properties: + id: + type: string + username: + type: string + displayName: + type: string + name: + type: string + snapshot-creation-cause: + type: string + readOnly: true + description: Server-recorded operation that created this snapshot. Standard releases use release_tagged; virtual tagging preserves the cause. Historical snapshots without provenance return unknown. + enum: + - unknown + - track_created + - release_tagged + - track_cloned + - bundle_imported + - metadata_updated + - configuration_updated + - candidates_added + - candidate_removed + - candidates_reviewed + - candidates_promoted + - candidate_version_updated + - staged_demoted + - candidates_auto_promoted + - member_synced + - composition_updated + - manual_snapshot + - scheduled_snapshot + - quarantine_promoted release-track-snapshot: type: object description: 'A snapshot document for a release track, containing versioned member objects and workflow tiers' @@ -8,6 +56,10 @@ components: type: string nullable: true description: 'The track alias from the registry (workbench responses), or null' + creation_cause: + $ref: '#/components/schemas/snapshot-creation-cause' + creation_actor: + $ref: '#/components/schemas/snapshot-creation-actor' id: type: string description: 'The release track ID (STIX identifier format)' @@ -229,6 +281,10 @@ components: content_statistics: $ref: '#/components/schemas/content-statistics' description: 'Counts of sealed manifest entries by role' + creation_cause: + $ref: '#/components/schemas/snapshot-creation-cause' + creation_actor: + $ref: '#/components/schemas/snapshot-creation-actor' snapshot_description: type: string maxLength: 4000 diff --git a/app/controllers/release-tracks-controller.js b/app/controllers/release-tracks-controller.js index 6bb1cb5a..ee9b1b41 100644 --- a/app/controllers/release-tracks-controller.js +++ b/app/controllers/release-tracks-controller.js @@ -395,7 +395,10 @@ exports.createReleaseTrackFromBundle = async function createReleaseTrackFromBund ); } - const result = await releaseTracksService.createTrackFromBundle(bodyResult.data); + const result = await releaseTracksService.createTrackFromBundle( + bodyResult.data, + req.user?.userAccountId, + ); logger.debug('Success: Created release track from bundle'); return res.status(201).send(result); } catch (err) { @@ -800,7 +803,11 @@ exports.listCandidates = async function listCandidates(req, res, next) { /** DELETE /api/release-tracks/:id/candidates/:objectRef */ exports.removeCandidate = async function removeCandidate(req, res, next) { try { - await releaseTracksService.removeCandidate(req.params.id, req.params.objectRef); + await releaseTracksService.removeCandidate( + req.params.id, + req.params.objectRef, + req.user?.userAccountId, + ); logger.debug(`Success: Removed candidate ${req.params.objectRef} from track ${req.params.id}`); return res.status(204).end(); } catch (err) { @@ -878,6 +885,7 @@ exports.updateCandidateVersion = async function updateCandidateVersion(req, res, req.params.id, req.params.objectRef, bodyResult.data, + req.user?.userAccountId, ); logger.debug(`Success: Updated version for candidate ${req.params.objectRef}`); return res.status(200).send(result); @@ -1130,6 +1138,7 @@ exports.promoteQuarantinedObject = async function promoteQuarantinedObject(req, const result = await releaseTracksService.promoteQuarantinedObject( req.params.id, bodyResult.data, + req.user?.userAccountId, ); logger.debug(`Success: Promoted quarantined object for track ${req.params.id}`); return res.status(200).send(result); diff --git a/app/lib/release-tracks/snapshot-creation-actor.js b/app/lib/release-tracks/snapshot-creation-actor.js new file mode 100644 index 00000000..71bfddc7 --- /dev/null +++ b/app/lib/release-tracks/snapshot-creation-actor.js @@ -0,0 +1,7 @@ +'use strict'; + +// Use trusted invocation context, never the source snapshot or track creator. +module.exports = function creationActor(userAccountId) { + if (userAccountId === 'system') return { kind: 'system' }; + return userAccountId ? { kind: 'user', user_account_id: userAccountId } : { kind: 'unknown' }; +}; diff --git a/app/lib/release-tracks/snapshot-creation-causes.js b/app/lib/release-tracks/snapshot-creation-causes.js new file mode 100644 index 00000000..81241779 --- /dev/null +++ b/app/lib/release-tracks/snapshot-creation-causes.js @@ -0,0 +1,25 @@ +'use strict'; + +// The operation that persisted this snapshot, not the operation that tagged it. +// Unknown is reserved for historical documents and low-level callers without provenance. +module.exports = Object.freeze({ + Unknown: 'unknown', + TrackCreated: 'track_created', + ReleaseTagged: 'release_tagged', + TrackCloned: 'track_cloned', + BundleImported: 'bundle_imported', + MetadataUpdated: 'metadata_updated', + ConfigurationUpdated: 'configuration_updated', + CandidatesAdded: 'candidates_added', + CandidateRemoved: 'candidate_removed', + CandidatesReviewed: 'candidates_reviewed', + CandidatesPromoted: 'candidates_promoted', + CandidateVersionUpdated: 'candidate_version_updated', + StagedDemoted: 'staged_demoted', + CandidatesAutoPromoted: 'candidates_auto_promoted', + MemberSynced: 'member_synced', + CompositionUpdated: 'composition_updated', + ManualSnapshot: 'manual_snapshot', + ScheduledSnapshot: 'scheduled_snapshot', + QuarantinePromoted: 'quarantine_promoted', +}); diff --git a/app/models/release-tracks/release-track-snapshot-schema.js b/app/models/release-tracks/release-track-snapshot-schema.js index 1fa7f312..09bf7ecf 100644 --- a/app/models/release-tracks/release-track-snapshot-schema.js +++ b/app/models/release-tracks/release-track-snapshot-schema.js @@ -1,6 +1,7 @@ 'use strict'; const mongoose = require('mongoose'); +const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes'); const revisionReference = require('../../lib/release-tracks/revision-reference'); const { validateTrackId, @@ -427,6 +428,28 @@ const releaseTrackSnapshotDefinition = { publication: { type: frozenPublicationSchema }, bundle_id: { type: String }, bundle_hashes: { type: bundleHashesSchema }, + creation_actor: { + type: new mongoose.Schema( + { + kind: { type: String, enum: ['user', 'system', 'unknown'], required: true }, + user_account_id: { + type: String, + required: function () { + return this.kind === 'user'; + }, + }, + }, + { _id: false }, + ), + default: () => ({ kind: 'unknown' }), + immutable: true, + }, + creation_cause: { + type: String, + enum: Object.values(CreationCause), + default: CreationCause.Unknown, + immutable: true, + }, snapshot_description: { type: String, maxlength: [4000, 'Snapshot description cannot exceed 4000 characters'], diff --git a/app/repository/release-tracks/release-track-dynamic.repository.js b/app/repository/release-tracks/release-track-dynamic.repository.js index b4663702..0358ad00 100644 --- a/app/repository/release-tracks/release-track-dynamic.repository.js +++ b/app/repository/release-tracks/release-track-dynamic.repository.js @@ -308,6 +308,8 @@ class ReleaseTrackDynamicRepository { bundle_hashes: 1, release_source_modified: 1, snapshot_description: 1, + creation_cause: { $ifNull: ['$creation_cause', 'unknown'] }, + creation_actor: { $ifNull: ['$creation_actor', { kind: 'unknown' }] }, name: 1, description: 1, scheduled_materialization: 1, diff --git a/app/services/release-tracks/bundle-import-service.js b/app/services/release-tracks/bundle-import-service.js index 6740c064..c39ca971 100644 --- a/app/services/release-tracks/bundle-import-service.js +++ b/app/services/release-tracks/bundle-import-service.js @@ -1,5 +1,7 @@ 'use strict'; +const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes'); + // ============================================================================= // Bundle Import Service // @@ -132,7 +134,7 @@ function sortByDependencyOrder(objects) { * @param {Object} serviceMap - Type → service mapping * @returns {Promise<{imported: boolean, ref: {object_ref: string, object_modified: string}}>} */ -async function importObject(stixObj, serviceMap) { +async function importObject(stixObj, serviceMap, userId) { const service = serviceMap[stixObj.type]; if (!service) { throw new BadRequestError({ @@ -183,7 +185,7 @@ async function importObject(stixObj, serviceMap) { workspace: {}, }; - await service.create(data, { import: true }); + await service.create(data, { import: true, userAccountId: userId }); logger.verbose(`BundleImportService: Imported "${stixObj.type}" "${stixObj.id}"`); return { imported: true, ref }; @@ -229,7 +231,7 @@ async function importObject(stixObj, serviceMap) { * @param {Object} bundleData - Validated bundle: { type: 'bundle', id, objects } * @returns {Promise} The created track's initial snapshot */ -exports.createTrackFromBundle = async function createTrackFromBundle(bundleData) { +exports.createTrackFromBundle = async function createTrackFromBundle(bundleData, userId) { if (!bundleData || !Array.isArray(bundleData.objects) || bundleData.objects.length === 0) { throw new BadRequestError({ message: 'Invalid bundle: must contain at least one object', @@ -261,7 +263,7 @@ exports.createTrackFromBundle = async function createTrackFromBundle(bundleData) let skippedCount = 0; for (const stixObj of sorted) { - const { imported, ref } = await importObject(stixObj, serviceMap); + const { imported, ref } = await importObject(stixObj, serviceMap, userId); if (ref) { importedRefs.push(ref); } @@ -306,17 +308,26 @@ exports.createTrackFromBundle = async function createTrackFromBundle(bundleData) // Step 4: Create the release track // ------------------------------------------------------------------ - const snapshot = await snapshotService.createTrack({ - name: trackName, - description: trackDescription, - type: 'standard', - }); + const snapshot = await snapshotService.createTrack( + { + name: trackName, + description: trackDescription, + type: 'standard', + userAccountId: userId, + }, + { creationCause: CreationCause.BundleImported, userAccountId: userId }, + ); // Add members by cloning the initial (empty) snapshot with the member entries if (memberEntries.length > 0) { - const finalSnapshot = await snapshotService.cloneSnapshot(snapshot.id, snapshot, { - members: memberEntries, - }); + const finalSnapshot = await snapshotService.cloneSnapshot( + snapshot.id, + snapshot, + { + members: memberEntries, + }, + { creationCause: CreationCause.BundleImported, userAccountId: userId }, + ); logger.verbose( `BundleImportService: Created track "${trackName}" (${snapshot.id}) ` + diff --git a/app/services/release-tracks/member-sync-service.js b/app/services/release-tracks/member-sync-service.js index a08d5d8d..2a7c4f34 100644 --- a/app/services/release-tracks/member-sync-service.js +++ b/app/services/release-tracks/member-sync-service.js @@ -1,5 +1,7 @@ 'use strict'; +const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes'); + // ============================================================================= // Member Sync Service // @@ -378,10 +380,15 @@ async function processMemberSync(trackId, snapshot, event) { } // Clone snapshot with updated tiers - const newSnapshot = await snapshotService.cloneSnapshot(trackId, snapshot, { - candidates: newCandidates, - staged: newStaged, - }); + const newSnapshot = await snapshotService.cloneSnapshot( + trackId, + snapshot, + { + candidates: newCandidates, + staged: newStaged, + }, + { creationCause: CreationCause.MemberSynced, userAccountId: modifiedBy || 'system' }, + ); logger.info( `[member-sync] Track ${trackId}: ${trigger} (${mode}) ${objectRef} → ` + @@ -478,9 +485,8 @@ async function handleStixObjectEvent(payload) { newModified: document.stix?.modified, oldModified: previousDocument?.stix?.modified, trigger: previousDocument ? 'in-place-update' : 'new-revision', - // Try to get user from options (create) or from document workflow metadata - modifiedBy: - options?.userAccountId || document.workspace?.workflow?.created_by_user_account || 'system', + // An object's original creator is not necessarily the user editing it. + modifiedBy: options?.userAccountId || 'system', }; try { @@ -530,10 +536,7 @@ async function handleStixObjectRevokedEvent(payload) { objectRef: stixId, newModified: revokedDocument?.stix?.modified, trigger: 'revocation', - modifiedBy: - options?.userAccountId || - revokedDocument?.workspace?.workflow?.created_by_user_account || - 'system', + modifiedBy: options?.userAccountId || 'system', }; try { @@ -578,7 +581,7 @@ async function handleStixObjectConvertedEvent(payload) { objectRef: stixId, newModified: document.stix.modified, trigger: 'new-revision', - modifiedBy: userAccountId || document.workspace?.workflow?.created_by_user_account || 'system', + modifiedBy: userAccountId || 'system', }; try { diff --git a/app/services/release-tracks/release-tracks-service.js b/app/services/release-tracks/release-tracks-service.js index 34acb2c3..1e935e0c 100644 --- a/app/services/release-tracks/release-tracks-service.js +++ b/app/services/release-tracks/release-tracks-service.js @@ -121,6 +121,32 @@ async function getUsersById(userIds) { return usersById; } +// Resolve each distinct creator once per response, including paginated history. +// Keep a missing/deleted user's ID without exposing account security metadata. +async function addCreationActors(snapshots) { + const ids = [ + ...new Set( + snapshots + .filter((snapshot) => snapshot.creation_actor?.kind === 'user') + .map((snapshot) => snapshot.creation_actor.user_account_id) + .filter(Boolean), + ), + ]; + const users = await getUsersById(ids); + return snapshots.map((snapshot) => { + const actor = snapshot.creation_actor || { kind: 'unknown' }; + snapshot.creation_actor = + actor.kind === 'user' + ? { + kind: actor.kind, + user_account_id: actor.user_account_id, + user: formatUser(users.get(actor.user_account_id)), + } + : { kind: actor.kind }; + return snapshot; + }); +} + function selectorKey(entry) { return `${entry.object_ref}:${revisionReference.modifiedKey(entry.object_modified)}`; } @@ -217,13 +243,15 @@ async function formatWorkbenchSnapshot(snapshot, options) { selectedTiers.flatMap((tierName) => snapshot[tierName] || []), ); const enriched = await addObjectInfoToSnapshot(snapshot); + const [attributed] = await addCreationActors([enriched]); // Registry-derived, read-only metadata used alongside snapshot content. const metadata = await snapshotService.getTrackMetadata(snapshot.id); enriched.alias = metadata.alias; + enriched.creation_cause = snapshot.creation_cause || 'unknown'; if (snapshot.type === 'virtual') { enriched.snapshot_schedule = metadata.snapshot_schedule || { mode: 'manual' }; } - return filterSnapshotTiers(enriched, options?.include); + return filterSnapshotTiers(attributed, options?.include); } exports.resolveTrackAlias = function resolveTrackAlias(alias) { @@ -303,12 +331,13 @@ exports.createTrack = async function createTrack(data) { }; // Phase 6 → bundle-import-service -exports.createTrackFromBundle = function createTrackFromBundle(bundleData) { - return bundleImportService.createTrackFromBundle(bundleData); +exports.createTrackFromBundle = function createTrackFromBundle(bundleData, userId) { + return bundleImportService.createTrackFromBundle(bundleData, userId); }; -exports.listSnapshots = function listSnapshots(trackId, options) { - return snapshotService.listSnapshots(trackId, options); +exports.listSnapshots = async function listSnapshots(trackId, options) { + const result = await snapshotService.listSnapshots(trackId, options); + return { ...result, data: await addCreationActors(result.data) }; }; // eslint-disable-next-line no-unused-vars @@ -455,8 +484,8 @@ exports.listCandidates = function listCandidates(trackId, options) { return standardTrackService.listCandidates(trackId, options); }; -exports.removeCandidate = function removeCandidate(trackId, objectRef) { - return standardTrackService.removeCandidate(trackId, objectRef); +exports.removeCandidate = function removeCandidate(trackId, objectRef, userId) { + return standardTrackService.removeCandidate(trackId, objectRef, userId); }; exports.reviewCandidates = function reviewCandidates(trackId, reviewData, userId) { @@ -467,8 +496,8 @@ exports.promoteCandidates = function promoteCandidates(trackId, objectRefs, user return standardTrackService.promoteCandidates(trackId, objectRefs, userId); }; -exports.updateCandidateVersion = function updateCandidateVersion(trackId, objectRef, data) { - return standardTrackService.updateCandidateVersion(trackId, objectRef, data); +exports.updateCandidateVersion = function updateCandidateVersion(trackId, objectRef, data, userId) { + return standardTrackService.updateCandidateVersion(trackId, objectRef, data, userId); }; // ----------------------------------------------------------------------------- @@ -610,8 +639,8 @@ exports.createVirtualSnapshot = function createVirtualSnapshot(trackId, options) return virtualTrackService.createVirtualSnapshot(trackId, validatedOptions); }; -exports.promoteQuarantinedObject = function promoteQuarantinedObject(trackId, selection) { - return virtualTrackService.promoteQuarantinedObject(trackId, selection); +exports.promoteQuarantinedObject = function promoteQuarantinedObject(trackId, selection, userId) { + return virtualTrackService.promoteQuarantinedObject(trackId, selection, userId); }; // ----------------------------------------------------------------------------- diff --git a/app/services/release-tracks/snapshot-service.js b/app/services/release-tracks/snapshot-service.js index 07f33adc..ca66f216 100644 --- a/app/services/release-tracks/snapshot-service.js +++ b/app/services/release-tracks/snapshot-service.js @@ -1,5 +1,8 @@ 'use strict'; +const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes'); +const creationActor = require('../../lib/release-tracks/snapshot-creation-actor'); + // ============================================================================= // Snapshot Service // @@ -207,7 +210,7 @@ exports.listTracks = async function listTracks(options) { * @param {Object} data - { name, description?, snapshot_description?, type, userAccountId?, composition?, snapshot_schedule?, scheduled_materialization?, config? } * @returns {Promise} The initial snapshot document */ -exports.createTrack = async function createTrack(data) { +exports.createTrack = async function createTrack(data, options = {}) { const trackId = `release-track--${uuidv4()}`; const now = new Date(); const trackType = data.type || 'standard'; @@ -215,6 +218,8 @@ exports.createTrack = async function createTrack(data) { const initialSnapshot = { id: trackId, + creation_cause: options.creationCause || CreationCause.TrackCreated, + creation_actor: creationActor(data.userAccountId), type: trackType, modified: now, version: null, @@ -328,6 +333,8 @@ exports.listSnapshots = async function listSnapshots(trackId, options) { bundle_hashes: snapshot.bundle_hashes, release_source_modified: snapshot.release_source_modified, snapshot_description: snapshot.snapshot_description, + creation_cause: snapshot.creation_cause || CreationCause.Unknown, + creation_actor: snapshot.creation_actor || { kind: 'unknown' }, content_statistics: snapshot.content_manifest_id ? statisticsByManifestId.get(snapshot.content_manifest_id) : undefined, @@ -456,6 +463,9 @@ exports.cloneSnapshot = async function cloneSnapshot( } } + // Never inherit provenance or accept it from snapshot overrides. + clone.creation_cause = options.creationCause || CreationCause.Unknown; + clone.creation_actor = creationActor(options.userAccountId); const normalized = tierRevisionInvariant.normalizeSnapshot(clone); let saved; if (rewritesMembers || !normalized.snapshot.content_manifest_id) { @@ -532,6 +542,8 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { delete clone.bundle_id; delete clone.bundle_hashes; clone.id = newTrackId; + clone.creation_cause = CreationCause.TrackCloned; + clone.creation_actor = creationActor(options.userAccountId); clone.modified = now; clone.version = null; clone.name = options.name || `${sourceSnapshot.name} (copy)`; @@ -594,11 +606,11 @@ async function _cloneToNewTrack(sourceSnapshot, options = {}) { * * @param {string} trackId * @param {Object} updates - { name?, description?, alias? } (alias null clears) - * @param {string} [_userId] + * @param {string} [userId] - Invoking snapshot creator * @returns {Promise} The new (or, for alias-only updates, latest) snapshot */ -// eslint-disable-next-line no-unused-vars -exports.updateMetadata = async function updateMetadata(trackId, updates, _userId) { + +exports.updateMetadata = async function updateMetadata(trackId, updates, userId) { const source = await exports.getLatestSnapshot(trackId); const overrides = {}; if (updates.name !== undefined) overrides.name = updates.name; @@ -619,7 +631,10 @@ exports.updateMetadata = async function updateMetadata(trackId, updates, _userId } if (Object.keys(overrides).length === 0) return source; - return exports.cloneSnapshot(trackId, source, overrides); + return exports.cloneSnapshot(trackId, source, overrides, { + creationCause: CreationCause.MetadataUpdated, + userAccountId: userId, + }); }; /** @@ -703,11 +718,11 @@ exports.getConfig = async function getConfig(trackId) { * * @param {string} trackId * @param {Object} config - Partial config to merge - * @param {string} [_userId] + * @param {string} [userId] - Invoking snapshot creator * @returns {Promise} The new snapshot */ -// eslint-disable-next-line no-unused-vars -exports.updateConfig = async function updateConfig(trackId, config, _userId) { + +exports.updateConfig = async function updateConfig(trackId, config, userId) { const source = await exports.getLatestSnapshot(trackId); const existing = source.config || {}; @@ -745,7 +760,12 @@ exports.updateConfig = async function updateConfig(trackId, config, _userId) { ); } - return exports.cloneSnapshot(trackId, source, { config: mergedConfig }); + return exports.cloneSnapshot( + trackId, + source, + { config: mergedConfig }, + { creationCause: CreationCause.ConfigurationUpdated, userAccountId: userId }, + ); }; // ============================================================================= diff --git a/app/services/release-tracks/standard-track-service.js b/app/services/release-tracks/standard-track-service.js index 60a94f0d..765a39c6 100644 --- a/app/services/release-tracks/standard-track-service.js +++ b/app/services/release-tracks/standard-track-service.js @@ -1,5 +1,7 @@ 'use strict'; +const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes'); + // ============================================================================= // Standard Track Service // @@ -133,7 +135,10 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId if (newEntries.length === 0) { return normalizedSource.removed.length > 0 - ? snapshotService.cloneSnapshot(trackId, source) + ? snapshotService.cloneSnapshot(trackId, source, undefined, { + creationCause: CreationCause.CandidatesAdded, + userAccountId: userId, + }) : source; } @@ -154,16 +159,25 @@ exports.addCandidates = async function addCandidates(trackId, objectRefs, userId ); } - let snapshot = await snapshotService.cloneSnapshot(trackId, source, { - candidates: mergedCandidates, - }); + let snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + candidates: mergedCandidates, + }, + { creationCause: CreationCause.CandidatesAdded, userAccountId: userId }, + ); logger.verbose( `StandardTrackService: Added ${newEntries.length} candidate(s) to track "${trackId}"`, ); // Evaluate auto-promotion for newly added candidates (Phase 3) - const autoPromotedSnapshot = await getWorkflowService().evaluateAutoPromotion(trackId, snapshot); + const autoPromotedSnapshot = await getWorkflowService().evaluateAutoPromotion( + trackId, + snapshot, + userId, + ); if (autoPromotedSnapshot) { snapshot = autoPromotedSnapshot; } @@ -199,7 +213,7 @@ exports.listCandidates = async function listCandidates(trackId, options = {}) { * @returns {Promise} The new snapshot * @throws {NotFoundError} If no candidate with that object_ref exists */ -exports.removeCandidate = async function removeCandidate(trackId, objectRef) { +exports.removeCandidate = async function removeCandidate(trackId, objectRef, userId) { const source = await snapshotService.getLatestSnapshot(trackId); assertStandardTrack(source); @@ -212,9 +226,14 @@ exports.removeCandidate = async function removeCandidate(trackId, objectRef) { }); } - const snapshot = await snapshotService.cloneSnapshot(trackId, source, { - candidates: remaining, - }); + const snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + candidates: remaining, + }, + { creationCause: CreationCause.CandidateRemoved, userAccountId: userId }, + ); logger.verbose(`StandardTrackService: Removed candidate "${objectRef}" from track "${trackId}"`); return snapshot; @@ -234,7 +253,7 @@ exports.removeCandidate = async function removeCandidate(trackId, objectRef) { * @param {string} [userId] * @returns {Promise} The new snapshot */ -// eslint-disable-next-line no-unused-vars + exports.reviewCandidates = async function reviewCandidates(trackId, reviewData, userId) { const { from, to, object_refs: filterRefs } = reviewData; @@ -269,16 +288,25 @@ exports.reviewCandidates = async function reviewCandidates(trackId, reviewData, }; }); - let snapshot = await snapshotService.cloneSnapshot(trackId, source, { - candidates: updatedCandidates, - }); + let snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + candidates: updatedCandidates, + }, + { creationCause: CreationCause.CandidatesReviewed, userAccountId: userId }, + ); logger.verbose( `StandardTrackService: Reviewed candidates "${from}" → "${to}" in track "${trackId}"`, ); // Evaluate auto-promotion after status transition (Phase 3) - const autoPromotedSnapshot = await getWorkflowService().evaluateAutoPromotion(trackId, snapshot); + const autoPromotedSnapshot = await getWorkflowService().evaluateAutoPromotion( + trackId, + snapshot, + userId, + ); if (autoPromotedSnapshot) { snapshot = autoPromotedSnapshot; } @@ -353,10 +381,15 @@ exports.promoteCandidates = async function promoteCandidates(trackId, objectRefs ...toPromote.filter((c) => rejectedRefs.has(c.object_ref)), ]; - const snapshot = await snapshotService.cloneSnapshot(trackId, source, { - candidates: finalCandidates, - staged: mergedStaged, - }); + const snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + candidates: finalCandidates, + staged: mergedStaged, + }, + { creationCause: CreationCause.CandidatesPromoted, userAccountId: userId }, + ); logger.verbose( `StandardTrackService: Promoted ${toPromote.length - rejected.length} candidate(s), ` + @@ -374,7 +407,12 @@ exports.promoteCandidates = async function promoteCandidates(trackId, objectRefs * @returns {Promise} The new snapshot * @throws {NotFoundError} If no matching candidate is found */ -exports.updateCandidateVersion = async function updateCandidateVersion(trackId, objectRef, data) { +exports.updateCandidateVersion = async function updateCandidateVersion( + trackId, + objectRef, + data, + userId, +) { const source = await snapshotService.getLatestSnapshot(trackId); assertStandardTrack(source); @@ -407,9 +445,14 @@ exports.updateCandidateVersion = async function updateCandidateVersion(trackId, await primaryRevisionService.assertRequestEntries([updatedEntry]); - const snapshot = await snapshotService.cloneSnapshot(trackId, source, { - candidates: updatedCandidates, - }); + const snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + candidates: updatedCandidates, + }, + { creationCause: CreationCause.CandidateVersionUpdated, userAccountId: userId }, + ); logger.verbose( `StandardTrackService: Updated version pin for "${objectRef}" in track "${trackId}"`, @@ -498,10 +541,15 @@ exports.demoteStaged = async function demoteStaged(trackId, objectRefs, userId) ); } - const snapshot = await snapshotService.cloneSnapshot(trackId, source, { - staged: remainingStaged, - candidates: mergedCandidates, - }); + const snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + staged: remainingStaged, + candidates: mergedCandidates, + }, + { creationCause: CreationCause.StagedDemoted, userAccountId: userId }, + ); logger.verbose( `StandardTrackService: Demoted ${demotedEntries.length} staged entry/entries in track "${trackId}"`, diff --git a/app/services/release-tracks/versioning-service.js b/app/services/release-tracks/versioning-service.js index 94ecd22f..1d67ba69 100644 --- a/app/services/release-tracks/versioning-service.js +++ b/app/services/release-tracks/versioning-service.js @@ -1,4 +1,6 @@ 'use strict'; +const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes'); +const creationActor = require('../../lib/release-tracks/snapshot-creation-actor'); // Plans and commits immutable releases from release-track snapshots. Planning // is side-effect free; persistence, reconciliation, and events occur only in @@ -204,7 +206,11 @@ function planRelease( modified: releaseModified, version, ...(sourceSnapshot.type === 'standard' - ? { release_source_modified: sourceSnapshot.modified } + ? { + release_source_modified: sourceSnapshot.modified, + creation_cause: CreationCause.ReleaseTagged, + creation_actor: creationActor(options.userAccountId || 'system'), + } : {}), members: mergedMembers, ...(updatesSnapshotDescription && options.description diff --git a/app/services/release-tracks/virtual-track-service.js b/app/services/release-tracks/virtual-track-service.js index 2091ad88..838d9957 100644 --- a/app/services/release-tracks/virtual-track-service.js +++ b/app/services/release-tracks/virtual-track-service.js @@ -1,5 +1,7 @@ 'use strict'; +const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes'); + // ============================================================================= // Virtual Track Service // @@ -426,7 +428,7 @@ async function resolveComposition(snapshot, registryMap) { * * @param {string} trackId * @param {Object} composition - The new composition configuration - * @param {string} [_userId] + * @param {string} [userId] - Invoking snapshot creator * @param {Object} [options] * @param {Object} [options.scheduledMaterialization] * @returns {Promise} The new snapshot @@ -434,7 +436,7 @@ async function resolveComposition(snapshot, registryMap) { exports.updateComposition = async function updateComposition( trackId, composition, - _userId, + userId, options = {}, ) { const source = await snapshotService.getLatestSnapshot(trackId); @@ -443,13 +445,18 @@ exports.updateComposition = async function updateComposition( // Validate all component tracks await validateComponentTracks(composition.component_tracks); - const snapshot = await snapshotService.cloneSnapshot(trackId, source, { - composition, - members: [], - quarantine: [], - composition_resolution: null, - scheduled_materialization: options.scheduledMaterialization, - }); + const snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + composition, + members: [], + quarantine: [], + composition_resolution: null, + scheduled_materialization: options.scheduledMaterialization, + }, + { creationCause: CreationCause.CompositionUpdated, userAccountId: userId }, + ); logger.verbose( `VirtualTrackService: Updated composition for track "${trackId}" ` + @@ -551,7 +558,13 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, op let snapshot; try { - snapshot = await snapshotService.cloneSnapshot(trackId, source, overrides); + snapshot = await snapshotService.cloneSnapshot(trackId, source, overrides, { + creationCause: options.scheduledMaterialization + ? CreationCause.ScheduledSnapshot + : CreationCause.ManualSnapshot, + userAccountId: + options.userAccountId || (options.scheduledMaterialization ? 'system' : undefined), + }); } catch (err) { if (!scheduledFor || !(err instanceof DuplicateIdError)) throw err; @@ -583,7 +596,11 @@ exports.createVirtualSnapshot = async function createVirtualSnapshot(trackId, op * @param {Object} selection - { object_ref, object_modified } * @returns {Promise} The new draft snapshot */ -exports.promoteQuarantinedObject = async function promoteQuarantinedObject(trackId, selection) { +exports.promoteQuarantinedObject = async function promoteQuarantinedObject( + trackId, + selection, + userId, +) { const source = await snapshotService.getLatestSnapshot(trackId); assertVirtualTrack(source); @@ -613,10 +630,15 @@ exports.promoteQuarantinedObject = async function promoteQuarantinedObject(track ); await primaryRevisionService.assertStoredEntries([...members, ...quarantine]); - const snapshot = await snapshotService.cloneSnapshot(trackId, source, { - members, - quarantine, - }); + const snapshot = await snapshotService.cloneSnapshot( + trackId, + source, + { + members, + quarantine, + }, + { creationCause: CreationCause.QuarantinePromoted, userAccountId: userId }, + ); logger.verbose( `VirtualTrackService: Promoted quarantined revision "${selected.object_ref}" ` + diff --git a/app/services/release-tracks/workflow-service.js b/app/services/release-tracks/workflow-service.js index dfe8085c..7c07b1f5 100644 --- a/app/services/release-tracks/workflow-service.js +++ b/app/services/release-tracks/workflow-service.js @@ -1,5 +1,7 @@ 'use strict'; +const CreationCause = require('../../lib/release-tracks/snapshot-creation-causes'); + // ============================================================================= // Workflow Service // @@ -56,7 +58,7 @@ exports.meetsThreshold = function meetsThreshold(candidateStatus, threshold) { * @param {Object} snapshot - The current snapshot (with updated candidates) * @returns {Promise} The new snapshot if promotion occurred, null otherwise */ -exports.evaluateAutoPromotion = async function evaluateAutoPromotion(trackId, snapshot) { +exports.evaluateAutoPromotion = async function evaluateAutoPromotion(trackId, snapshot, userId) { // Auto-promotion only applies to standard tracks if (snapshot.type !== 'standard') { return null; @@ -84,7 +86,7 @@ exports.evaluateAutoPromotion = async function evaluateAutoPromotion(trackId, sn ); // Promote qualifying candidates to staged - return _promoteToStaged(trackId, snapshot, qualifying); + return _promoteToStaged(trackId, snapshot, qualifying, userId); }; // ============================================================================= @@ -105,7 +107,7 @@ exports.evaluateAutoPromotion = async function evaluateAutoPromotion(trackId, sn * @param {Array} qualifyingCandidates - Candidates to promote * @returns {Promise} The new snapshot */ -async function _promoteToStaged(trackId, snapshot, qualifyingCandidates) { +async function _promoteToStaged(trackId, snapshot, qualifyingCandidates, userId) { const now = new Date(); const existingCandidates = snapshot.candidates || []; const existingStaged = snapshot.staged || []; @@ -155,10 +157,15 @@ async function _promoteToStaged(trackId, snapshot, qualifyingCandidates) { ]; // Clone snapshot with updated tiers - const newSnapshot = await snapshotService.cloneSnapshot(trackId, snapshot, { - candidates: finalCandidates, - staged: mergedStaged, - }); + const newSnapshot = await snapshotService.cloneSnapshot( + trackId, + snapshot, + { + candidates: finalCandidates, + staged: mergedStaged, + }, + { creationCause: CreationCause.CandidatesAutoPromoted, userAccountId: userId || 'system' }, + ); logger.verbose( `WorkflowService: Auto-promoted ${toPromote.length - rejected.length} candidate(s), ` + diff --git a/app/tests/api/release-tracks/snapshot-creation-causes.spec.js b/app/tests/api/release-tracks/snapshot-creation-causes.spec.js new file mode 100644 index 00000000..508de922 --- /dev/null +++ b/app/tests/api/release-tracks/snapshot-creation-causes.spec.js @@ -0,0 +1,351 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const snapshotService = require('../../../services/release-tracks/snapshot-service'); +const standard = require('../../../services/release-tracks/standard-track-service'); +const virtual = require('../../../services/release-tracks/virtual-track-service'); +const modelFactory = require('../../../models/release-tracks/model-factory'); +const Cause = require('../../../lib/release-tracks/snapshot-creation-causes'); + +describe('Snapshot creation causes', function () { + let app; + let cookie; + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + cookie = await login.loginAnonymous(app); + }); + after(async function () { + await database.closeConnection(); + }); + + async function api(method, path, body, status = 200) { + const call = request(app)[method](path).set('Cookie', `${cookie.name}=${cookie.value}`); + if (body !== undefined) call.send(body); + return (await call.expect(status)).body; + } + async function create(name, type = 'standard', extra = {}) { + return api('post', '/api/release-tracks/new', { name, type, ...extra }, 201); + } + async function assertPersisted(snapshot, cause, userId) { + expect(snapshot.creation_cause).toBe(cause); + const stored = await modelFactory + .getModel(snapshot.id) + .findOne({ modified: snapshot.modified }) + .lean(); + expect(stored.creation_cause).toBe(cause); + if (userId) { + expect(stored.creation_actor).toEqual({ kind: 'user', user_account_id: userId }); + } + return snapshot; + } + + it('attributes each new snapshot to its invoker, enriches GETs, and tolerates deleted users', async function () { + const User = require('../../../models/user-account-model'); + const id = 'identity--12345678-1234-4234-8234-123456789012'; + await User.create({ + id, + username: 'second.editor', + displayName: 'Second Editor', + email: 'private@example.test', + role: 'editor', + status: 'active', + created: new Date(), + modified: new Date(), + }); + await api( + 'post', + '/api/release-tracks/new', + { + name: 'Spoofed Actor', + type: 'virtual', + creation_actor: { kind: 'user', user_account_id: id }, + creation_cause: Cause.ScheduledSnapshot, + }, + 400, + ); + const first = await create('Creation Actor Track', 'virtual'); + expect(first.creation_cause).toBe(Cause.TrackCreated); + expect(first.creation_actor.kind).toBe('user'); + expect(first.creation_actor.user_account_id).toBe(first.created_by_ref); + expect(first.creation_actor.user_account_id).not.toBe(id); + const edited = await snapshotService.updateMetadata(first.id, { description: 'Changed' }, id); + expect(edited.created_by_ref).toBe(first.created_by_ref); + expect(edited.creation_actor).toEqual({ kind: 'user', user_account_id: id }); + const stored = await modelFactory + .getModel(first.id) + .findOne({ modified: edited.modified }) + .lean(); + expect(stored.creation_actor).toEqual(edited.creation_actor); + for (const suffix of ['latest', encodeURIComponent(new Date(edited.modified).toISOString())]) { + const response = await api('get', `/api/release-tracks/${first.id}/snapshots/${suffix}`); + expect(response.creation_actor.user).toEqual({ + id, + username: 'second.editor', + displayName: 'Second Editor', + name: 'Second Editor', + }); + } + const history = await api('get', `/api/release-tracks/${first.id}/snapshots`); + expect(history.data[0].creation_actor.user.displayName).toBe('Second Editor'); + expect(history.data[1].creation_actor.user_account_id).toBe(first.created_by_ref); + const copy = await snapshotService.cloneTrack(first.id, { + name: 'Creation Actor Copy', + userAccountId: first.created_by_ref, + }); + expect(copy.creation_actor.user_account_id).toBe(first.created_by_ref); + await User.deleteOne({ id }); + const deleted = await api('get', `/api/release-tracks/${first.id}/snapshots/latest`); + expect(deleted.creation_actor).toEqual({ kind: 'user', user_account_id: id }); + }); + + it('persists config causes and exposes them in latest, timestamp, and history GETs', async function () { + const initial = await create('Creation Cause Virtual', 'virtual'); + await assertPersisted(initial, Cause.TrackCreated); + await api('put', `/api/release-tracks/${initial.id}/virtual/schedule`, { + mode: 'cron', + cron: '*/15 * * * *', + }); + expect(await modelFactory.getModel(initial.id).countDocuments()).toBe(1); + const configured = await api('put', `/api/release-tracks/${initial.id}/config`, { + publication: { created_by_ref: { inherit: true } }, + }); + // Config endpoints return a config envelope; inspect the new snapshot itself. + expect(configured).toBeDefined(); + const latest = await api('get', `/api/release-tracks/${initial.id}/snapshots/latest`); + await assertPersisted(latest, Cause.ConfigurationUpdated); + expect(latest.creation_actor.user_account_id).toBe(initial.created_by_ref); + const selected = await api( + 'get', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(latest.modified)}`, + ); + expect(selected.creation_cause).toBe(Cause.ConfigurationUpdated); + const history = await api('get', `/api/release-tracks/${initial.id}/snapshots`); + expect(history.data.map((row) => row.creation_cause)).toEqual([ + Cause.ConfigurationUpdated, + Cause.TrackCreated, + ]); + }); + + it('retains creation cause when tagging, assigns new causes on metadata changes and track copies', async function () { + const initial = await create('Creation Cause Lifecycle'); + const metadata = await snapshotService.updateMetadata(initial.id, { + name: 'Creation Cause Renamed', + }); + await assertPersisted(metadata, Cause.MetadataUpdated); + const released = await api( + 'post', + `/api/release-tracks/${initial.id}/snapshots/latest/release`, + { version: '1.0' }, + ); + await assertPersisted(released, Cause.ReleaseTagged); + expect(released.creation_actor.user_account_id).toBe(initial.created_by_ref); + const source = await api( + 'get', + `/api/release-tracks/${initial.id}/snapshots/${encodeURIComponent(new Date(metadata.modified).toISOString())}`, + ); + expect(source.creation_cause).toBe(Cause.MetadataUpdated); + expect(source.creation_actor).toEqual({ kind: 'unknown' }); + const copy = await snapshotService.cloneTrack(initial.id, { name: 'Creation Cause Copy' }); + await assertPersisted(copy, Cause.TrackCloned); + const configured = await snapshotService.updateConfig(initial.id, { auto_promote: false }); + await assertPersisted(configured, Cause.ConfigurationUpdated); + }); + + it('distinguishes manual, scheduled, and composition-created virtual drafts', async function () { + const component = await create('Creation Cause Component'); + await api('post', `/api/release-tracks/${component.id}/snapshots/latest/release`, { + version: '1.0', + }); + const composition = { + component_tracks: [ + { track_id: component.id, resolution_strategy: 'latest_tagged', priority: 0 }, + ], + }; + const track = await create('Creation Cause Materialization', 'virtual', { composition }); + const manual = await virtual.createVirtualSnapshot(track.id, { + userAccountId: track.created_by_ref, + }); + await assertPersisted(manual, Cause.ManualSnapshot); + expect(manual.creation_actor.user_account_id).toBe(track.created_by_ref); + const tagged = await api('post', `/api/release-tracks/${track.id}/snapshots/latest/release`, { + version: '1.0', + }); + expect(tagged.creation_cause).toBe(Cause.ManualSnapshot); + expect(tagged.creation_actor.user_account_id).toBe(track.created_by_ref); + const scheduled = await virtual.createVirtualSnapshot(track.id, { + scheduledMaterialization: { + schedule_mode: 'cron', + scheduled_for: new Date('2027-01-01T00:00:00Z'), + }, + }); + await assertPersisted(scheduled, Cause.ScheduledSnapshot); + expect(scheduled.creation_actor).toEqual({ kind: 'system' }); + await assertPersisted( + await virtual.updateComposition(track.id, composition), + Cause.CompositionUpdated, + ); + }); + + it('records standard workflow operations and automatic promotion', async function () { + const object = await api( + 'post', + '/api/techniques', + { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { type: 'attack-pattern', spec_version: '2.1', name: 'Creation Cause Technique' }, + }, + 201, + ); + const track = await create('Creation Cause Workflow', 'standard', { + config: { auto_promote: false, member_sync: { strategy: 'manual' } }, + }); + const ref = { id: object.stix.id, modified: object.stix.modified }; + await assertPersisted( + await standard.addCandidates(track.id, [ref], 'cause-test'), + Cause.CandidatesAdded, + 'cause-test', + ); + await assertPersisted( + await standard.updateCandidateVersion( + track.id, + ref.id, + { + old_modified: ref.modified, + new_modified: 'latest', + }, + 'version-editor', + ), + Cause.CandidateVersionUpdated, + 'version-editor', + ); + await assertPersisted( + await standard.reviewCandidates( + track.id, + { from: 'work-in-progress', to: 'reviewed' }, + 'reviewer', + ), + Cause.CandidatesReviewed, + 'reviewer', + ); + await assertPersisted( + await standard.promoteCandidates(track.id, [ref.id], 'cause-test'), + Cause.CandidatesPromoted, + 'cause-test', + ); + await assertPersisted( + await standard.demoteStaged(track.id, [{ id: ref.id, modified: 'latest' }], 'cause-test'), + Cause.StagedDemoted, + 'cause-test', + ); + await api('delete', `/api/release-tracks/${track.id}/candidates/${ref.id}`, undefined, 204); + await assertPersisted( + await snapshotService.getLatestSnapshot(track.id), + Cause.CandidateRemoved, + track.created_by_ref, + ); + await snapshotService.updateConfig(track.id, { + auto_promote: true, + candidacy_threshold: 'work-in-progress', + }); + await assertPersisted( + await standard.addCandidates(track.id, [ref], 'cause-test'), + Cause.CandidatesAutoPromoted, + 'cause-test', + ); + }); + + it('records bundle imports and automatic synchronization of object edits', async function () { + const importer = require('../../../services/release-tracks/bundle-import-service'); + const track = await importer.createTrackFromBundle( + { + type: 'bundle', + id: 'bundle--11111111-1111-4111-8111-111111111111', + objects: [ + { + type: 'attack-pattern', + spec_version: '2.1', + id: 'attack-pattern--11111111-1111-4111-8111-111111111111', + name: 'Creation Cause Import', + description: 'Technique imported to verify snapshot provenance.', + x_mitre_domains: ['enterprise-attack'], + x_mitre_platforms: ['Windows'], + x_mitre_version: '1.0', + x_mitre_attack_spec_version: '3.3.0', + x_mitre_is_subtechnique: false, + kill_chain_phases: [{ kill_chain_name: 'mitre-attack', phase_name: 'execution' }], + external_references: [{ source_name: 'mitre-attack', external_id: 'T9998' }], + created: '2026-01-01T00:00:00.000Z', + modified: '2026-01-01T00:00:00.000Z', + }, + ], + }, + 'identity--12345678-1234-4234-8234-123456789012', + ); + await assertPersisted( + track, + Cause.BundleImported, + 'identity--12345678-1234-4234-8234-123456789012', + ); + const object = await api( + 'post', + '/api/techniques', + { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { type: 'attack-pattern', spec_version: '2.1', name: 'Creation Cause Sync' }, + }, + 201, + ); + const syncedTrack = await create('Creation Cause Sync Track'); + await standard.addCandidates( + syncedTrack.id, + [{ id: object.stix.id, modified: object.stix.modified }], + 'cause-test', + ); + const sync = require('../../../services/release-tracks/member-sync-service'); + const results = await sync.handleObjectModified({ + objectRef: object.stix.id, + newModified: object.stix.modified, + trigger: 'in-place-update', + modifiedBy: 'object-editor', + }); + expect(results).toHaveLength(1); + await assertPersisted(results[0], Cause.MemberSynced, 'object-editor'); + }); + + it('does not invent history for legacy snapshots or inherit a source cause on an unclassified clone', async function () { + const track = await create('Creation Cause Legacy', 'virtual'); + const Model = modelFactory.getModel(track.id); + await Model.collection.updateOne( + { id: track.id }, + { $unset: { creation_cause: '', creation_actor: '' } }, + ); + const latest = await api('get', `/api/release-tracks/${track.id}/snapshots/latest`); + expect(latest.creation_cause).toBe(Cause.Unknown); + expect(latest.creation_actor).toEqual({ kind: 'unknown' }); + const history = await api('get', `/api/release-tracks/${track.id}/snapshots`); + expect(history.data[0].creation_cause).toBe(Cause.Unknown); + expect(history.data[0].creation_actor).toEqual({ kind: 'unknown' }); + const clone = await snapshotService.cloneSnapshot( + track.id, + { ...latest, creation_cause: Cause.ScheduledSnapshot }, + { + creation_cause: Cause.TrackCreated, + creation_actor: { kind: 'user', user_account_id: 'spoofed' }, + }, + ); + await assertPersisted(clone, Cause.Unknown); + expect(clone.creation_actor).toEqual({ kind: 'unknown' }); + await expect( + new Model({ ...clone, _id: undefined, creation_cause: 'made_up' }).validate(), + ).rejects.toThrow(); + }); +}); diff --git a/app/tests/api/release-tracks/virtual-quarantine.spec.js b/app/tests/api/release-tracks/virtual-quarantine.spec.js index f63e6c0c..fcb18ee7 100644 --- a/app/tests/api/release-tracks/virtual-quarantine.spec.js +++ b/app/tests/api/release-tracks/virtual-quarantine.spec.js @@ -169,6 +169,8 @@ describe('Virtual release-track quarantine API', function () { }); expect(promoted.modified).not.toBe(materialized.modified); + expect(promoted.creation_cause).toBe('quarantine_promoted'); + expect(materialized.creation_cause).toBe('manual_snapshot'); expect(promoted.version).toBeNull(); expect(promoted.members).toEqual([ { diff --git a/docs/README.md b/docs/README.md index ae622cec..6b7eb575 100644 --- a/docs/README.md +++ b/docs/README.md @@ -48,6 +48,7 @@ Architecture, patterns, and implementation details for contributors. - [Frontend Handoff](developer/FRONTEND_TODO.md): Backend contract changes requiring downstream Angular updates - [Entities](developer/release-tracks/entities.md): Database schemas and data models +- [Snapshot Creation Causes](developer/release-tracks/snapshot-creation-causes.md): Persisted creation-cause enum, standard/virtual operation mapping, and historical fallback - [Sealed Content Manifests](developer/release-tracks/sealed-content-manifests.md): Why every snapshot seals its bill of materials, how the collection object is projected, and publication inheritance - [Backref Reconciliation](developer/release-tracks/backref-reconciliation.md): How `workspace.release_tracks` backrefs stay in sync with snapshots - [Member Sync Strategies](developer/release-tracks/member-sync-strategies.md): Automatic tracking of member object revisions diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 6d014046..fe26ccba 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -57,6 +57,40 @@ Snapshot DELETE is draft-only. Convert tagged releases with POST /release-tracks/:id/snapshots/:modified/draft and a confirm_version body before attempting a separate eligible-draft DELETE. +## Snapshot creation provenance and user attribution (2026-09-09) + +- [x] Inspect beta and carry forward the prior uncommitted creation-cause work. +- [x] Persist immutable creation cause and invoking user, including standard release creation. +- [x] Return safe user display metadata with snapshot GETs and history; show initials avatars. +- [x] Cover user changes, automation, legacy records, tagging, and spoofing with regressions. +- [x] Update OpenAPI, user/developer documentation, and Bruno requests. +- [x] Run focused tests, full suites, lint, and frontend build. + +Verification: backend provenance/release/quarantine group 32 passing; final +provenance spec 7 passing. Complete backend suite under Node 24 passes: +OpenAPI 2, config 22, API 1044, middleware 29, scheduler 10. Backend lint +passes. First full-run transient HTTP failures passed in isolation (100) +and on the complete rerun. Frontend focused tests 97 passing; complete suite +168 files / 420 tests passing, changed-file lint and production build pass. +The existing save-dialog timing failure passed in isolation and on full rerun; +existing bundle/style budget warnings remain. Production build required +execution outside the sandbox. No test-harness changes were made. + +Work is in both main beta checkouts, preserving the newer rollback/composition +provenance work and unrelated local files. Commit preparation (2026-09-10) +excludes unrelated working-tree changes, including shared-document formatting. + +Commit messages: + +- Backend: `feat(release-tracks): record snapshot creation provenance` + Body: Persist creation causes and invoking users, distinguish standard + release creation, and expose safe creator metadata through snapshot GETs. +- Frontend: `feat(release-tracks): show snapshot causes and creator avatars` + Body: Display snapshot-local creator names and initials, with explicit + automation and historical-attribution fallbacks on Releases cards. +- Bruno: `docs(release-tracks): document snapshot creation attribution` + Body: Describe creation cause and actor fields on snapshot GET requests. + ## Merge scheduling and rollback branches into local beta (2026-09-09) - [x] Inspect all worktrees and confirm scheduling branches are already merged. diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 474ed0f2..252c1a5c 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -21,6 +21,16 @@ and `releaseTrackGraphManifestEntries` collections (renamed in place by the ### Release Track +Every new snapshot persists an immutable `creation_cause` enum describing the +operation, together with immutable `creation_actor` (`kind`: `user`, `system`, +or `unknown`; `user_account_id` for user invocations). This is distinct from +the original track's `created_by_ref`. Workbench GETs resolve safe user display +metadata for the frontend's initials avatar. The cause identifies the +operation that created it. Latest/timestamp Workbench responses and snapshot +history expose it; historical missing values read as `unknown`. See +[snapshot creation causes](snapshot-creation-causes.md) for the complete +standard/virtual operation mapping and non-creating operations. + `ReleaseTrack` instances will be tracked as independent MongoDB Collections. The reason for this is because the volume of snapshot permutations is expected to be very high given the frequency of changes that typically occur between releases. #### Naming Conventions diff --git a/docs/developer/release-tracks/snapshot-creation-causes.md b/docs/developer/release-tracks/snapshot-creation-causes.md new file mode 100644 index 00000000..32eba523 --- /dev/null +++ b/docs/developer/release-tracks/snapshot-creation-causes.md @@ -0,0 +1,103 @@ +# Snapshot creation causes + +`creation_cause` is immutable, server-controlled metadata on each snapshot. +It identifies the operation that persisted that snapshot, not the operation +that later tagged it. The authoritative enum is +`app/lib/release-tracks/snapshot-creation-causes.js`; the frontend mirrors its +wire values and maps them to readable labels on Releases cards. + +## Causes shared by standard and virtual tracks + +| Value | Operation | +| ----------------------- | --------------------------------------------------------- | +| `track_created` | Create a track and its initial empty draft | +| `track_cloned` | Copy a track from its latest or a selected snapshot | +| `metadata_updated` | Write track name or description | +| `configuration_updated` | Write track configuration, including publication settings | + +## Standard-track causes + +| Value | Operation | +| --------------------------- | ------------------------------------------------------------------------- | +| `bundle_imported` | Create a track from a STIX bundle, including a metadata-only import | +| `release_tagged` | Create a standard release snapshot, preserving its source draft | +| `candidates_added` | Add candidate pins; includes normalization-only writes in that operation | +| `candidate_removed` | Remove a candidate | +| `candidates_reviewed` | Change candidate workflow status | +| `candidates_promoted` | Explicitly promote candidates to staged | +| `candidate_version_updated` | Change a candidate's revision selector | +| `staged_demoted` | Move staged entries back to candidates | +| `candidates_auto_promoted` | Automatically promote qualifying candidates after add/review | +| `member_synced` | Synchronize an object change into a track according to member-sync policy | + +Member sync includes newly created revisions, in-place changes to pinned +objects, revocations, and technique conversions. It can enroll, replace, or +queue pins. These are one snapshot-producing operation category. Events that +produce no write do not receive a new creation cause. + +## Virtual-track causes + +| Value | Operation | +| --------------------- | -------------------------------------------------------------- | +| `composition_updated` | Replace composition and reset materialized content | +| `manual_snapshot` | Explicitly materialize the composition | +| `scheduled_snapshot` | Materialize with scheduled occurrence metadata (cron or dates) | +| `quarantine_promoted` | Resolve a quarantined revision into members | + +The existing API also accepts scheduled occurrence metadata on explicit +materialization requests. Such requests receive `scheduled_snapshot`; this +field describes the materialization mode, not proof of scheduler identity. +Track creation and composition updates retain their own operation causes even +when their requests carry scheduled occurrence metadata. + +## Persistence and response behavior + +Creation and track-copy entry points set their own causes. Every production +caller of `cloneSnapshot` supplies a cause through its internal options; the +helper replaces inherited provenance after applying overrides. Client request +fields cannot select the cause. The Mongoose enum validates persisted values +and makes the field immutable. Snapshot latest/timestamp Workbench GETs and +paginated snapshot history expose the value. STIX bundle exports omit it. + +Historical documents without provenance return `unknown`, displayed as +"Creation cause unavailable". No migration guesses old causes from current +content. New documents persist a value; low-level callers without explicit +provenance use `unknown` rather than inheriting a misleading source cause. + +Virtual tagging, release rollback/retagging, editing draft notes, alias-only updates, manifest reconstruction, +deleting snapshots, and replacing registry-backed schedules do not themselves +create new snapshots. A subsequent operation may do so. In particular, the +frontend's Save Config flow saves the virtual schedule and then writes +publication configuration. That config write creates a draft labelled +"Configuration updated", even if the supplied config equals its prior value. +An actual scheduled materialization is labelled "Scheduled snapshot". + +Standard tracks keep only their latest rolling draft, so this is provenance +for each surviving snapshot, not a complete event log. If auto-promotion +immediately replaces a candidate-add/review draft, the surviving snapshot is +labelled "Candidates automatically promoted". Tagged snapshots retain their +creation cause throughout their lifetime. Standard release creation persists a +separate snapshot labelled "Release tagged"; its retained source draft keeps +its earlier provenance. Rollback restores an existing source, so does not +invent a new creation cause or actor. + +## Invoking user + +`creation_actor` is an immutable snapshot-local object. Trusted invocation +context supplies `{ kind: 'user', user_account_id }`, system jobs use +`{ kind: 'system' }`, and unclassified/internal or historical writes use +`{ kind: 'unknown' }`. Creation, copying, and cloning replace any inherited +actor after overrides, independently of the original track's `created_by_ref`. +Request bodies cannot choose attribution. Member-sync events use the invoking +user in event options, never an object's historical creator. Auto-promotion +passes through the user who initiated add/review; the scheduler supplies no +user and is attributed to the system. An explicit HTTP materialization with +scheduled metadata still records the authenticated human. + +Workbench GETs and history resolve distinct user IDs once per response through +the facade's existing user-enrichment path. Only ID, username, and display +name fields are exposed, not email, roles, or authentication metadata. Names +reflect the current account; missing accounts retain their persisted IDs. +The frontend uses the existing initials avatar and full display name for +users, "Automated" for system jobs, and "Creator unavailable" for unknown +attribution. No historical backfill guesses users from inherited track data. diff --git a/docs/user/release-tracks/api-reference.md b/docs/user/release-tracks/api-reference.md index 05967872..424c93cf 100644 --- a/docs/user/release-tracks/api-reference.md +++ b/docs/user/release-tracks/api-reference.md @@ -4,6 +4,22 @@ This document provides the complete API reference for Release Tracks V2 (formerly "Collections V2"). +Snapshot latest/timestamp GETs in Workbench format and the snapshot-history +list return `creation_cause`, a read-only enum identifying the operation that +created each snapshot. Standard release creation uses `release_tagged`; virtual +tagging retains the original cause. Historical missing +values return `unknown`. The [creation-cause reference](../../developer/release-tracks/snapshot-creation-causes.md) +lists all standard and virtual causes. + +`creation_actor` records the invoker independently of the track's original +`created_by_ref`. Its `kind` is `user`, `system`, or `unknown`. User actors +persist `user_account_id`; GETs additionally resolve a minimal `user` object +(`id`, `username`, `displayName`, `name`) for initials/name display. Missing +or deleted accounts retain their ID but omit `user`. Historical snapshots +return `{ "kind": "unknown" }`. Clients cannot set either provenance field. +Scheduled jobs are system actors; human-triggered automatic promotion retains +the initiating user. Neither field is included in STIX bundle exports. + **Related Documentation:** - [summary.md](./summary.md) - High-level design summary and problem statement From 47542361ff3e97486d6d7879e06b8bb9c373ce13 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:34:11 -0400 Subject: [PATCH 13/14] style(release-tracks): align entity documentation table Normalize Markdown table padding and separator widths without changing content. --- docs/developer/release-tracks/entities.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/developer/release-tracks/entities.md b/docs/developer/release-tracks/entities.md index 252c1a5c..44ad7218 100644 --- a/docs/developer/release-tracks/entities.md +++ b/docs/developer/release-tracks/entities.md @@ -4,15 +4,15 @@ This document tracks new database schemas, interfaces, etc.; as well as changes ### Collections at a glance -| Collection | Purpose | Written by | Growth and retention | -| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | -| `releaseTrackRegistry` | One document per track: name, type, denormalized counters, the tagged-release catalogue (`tagged_releases`), the release lock, and virtual schedules. The index that maps a track to its own snapshot collection. | Track create/delete, every snapshot write (counters), release commit and conversion to draft (catalogue). | One document per track. | -| `release-track--` | The track's snapshots: one active rolling draft, a preserved source draft per tagged standard release, and every tagged release; every materialized draft plus releases for a virtual track. | Snapshot service and release commit. | Standard tracks grow by two snapshots per release plus one active draft; virtual tracks by materializations. | -| `releaseTrackContentManifests` | The sealed bill of materials each snapshot references (`content_manifest_id`). Several snapshots share one manifest when their member sets are identical. | Sealed whenever members are written; discarded when no snapshot references it. | Bounded by member-changing writes, not by snapshot count. | -| `releaseTrackContentManifestEntries` | One exact-revision pointer per object a manifest emits or depends on. The `(object_ref, object_modified)` index is what protects referenced revisions from deletion. | With its manifest. | Roughly members + relationships + a few supporting objects per manifest. | -| `releaseTrackReconciliations` | Outstanding backref reconciliation work only: a record is created before the `workspace.release_tracks` listeners run and deleted when they succeed, so anything present is pending or failed and needs repair. | Every snapshot write. | Normally empty. | -| `releaseTrackAuditEvents` | Audit trail for administrator-only track deletion, release conversion to draft, and release retagging (`delete_track`, `convert_release_to_draft` (legacy: `delete_release`), `retag_release`). | Those operations. | Empty until an administrator performs one of those operations. | -| `virtualTrackScheduleOccurrences` | Durable claims for scheduled virtual materialization (cron or dated schedules) so restarts and duplicate delivery execute each occurrence once. | The scheduler. | One record per scheduled occurrence; empty when no virtual track has a schedule. | +| Collection | Purpose | Written by | Growth and retention | +| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `releaseTrackRegistry` | One document per track: name, type, denormalized counters, the tagged-release catalogue (`tagged_releases`), the release lock, and virtual schedules. The index that maps a track to its own snapshot collection. | Track create/delete, every snapshot write (counters), release commit and conversion to draft (catalogue). | One document per track. | +| `release-track--` | The track's snapshots: one active rolling draft, a preserved source draft per tagged standard release, and every tagged release; every materialized draft plus releases for a virtual track. | Snapshot service and release commit. | Standard tracks grow by two snapshots per release plus one active draft; virtual tracks by materializations. | +| `releaseTrackContentManifests` | The sealed bill of materials each snapshot references (`content_manifest_id`). Several snapshots share one manifest when their member sets are identical. | Sealed whenever members are written; discarded when no snapshot references it. | Bounded by member-changing writes, not by snapshot count. | +| `releaseTrackContentManifestEntries` | One exact-revision pointer per object a manifest emits or depends on. The `(object_ref, object_modified)` index is what protects referenced revisions from deletion. | With its manifest. | Roughly members + relationships + a few supporting objects per manifest. | +| `releaseTrackReconciliations` | Outstanding backref reconciliation work only: a record is created before the `workspace.release_tracks` listeners run and deleted when they succeed, so anything present is pending or failed and needs repair. | Every snapshot write. | Normally empty. | +| `releaseTrackAuditEvents` | Audit trail for administrator-only track deletion, release conversion to draft, and release retagging (`delete_track`, `convert_release_to_draft` (legacy: `delete_release`), `retag_release`). | Those operations. | Empty until an administrator performs one of those operations. | +| `virtualTrackScheduleOccurrences` | Durable claims for scheduled virtual materialization (cron or dated schedules) so restarts and duplicate delivery execute each occurrence once. | The scheduler. | One record per scheduled occurrence; empty when no virtual track has a schedule. | Removed by the sealed-manifest work: the former `releaseTrackGraphManifests` and `releaseTrackGraphManifestEntries` collections (renamed in place by the From 65ebb4e3740f510bd9421a0f766ce09a3998e2b7 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:42:46 -0400 Subject: [PATCH 14/14] fix(reports): reduce duplicate relationship report memory use Filter duplicate groups in MongoDB before fetching full relationships, limit endpoint lookups to their latest revisions, and consume results through a batched cursor. Add history and lifecycle regressions and document remaining response-size limits. --- app/repository/relationships-repository.js | 91 ++++++-- .../reports/parallel-relationships.spec.js | 200 ++++++++++++++++++ docs/README.md | 1 + docs/developer/TODO.md | 26 +++ docs/developer/data-quality-reports.md | 74 +++++++ docs/user/data-quality-reports.md | 13 +- 6 files changed, 381 insertions(+), 24 deletions(-) create mode 100644 app/tests/api/reports/parallel-relationships.spec.js create mode 100644 docs/developer/data-quality-reports.md diff --git a/app/repository/relationships-repository.js b/app/repository/relationships-repository.js index 0bec975c..76f25445 100644 --- a/app/repository/relationships-repository.js +++ b/app/repository/relationships-repository.js @@ -268,30 +268,79 @@ class RelationshipsRepository extends BaseRepository { } async retrieveParallelRelationships() { - const all_relationships = await this.retrieveAll({ - versions: 'latest', - lookupRefs: true, - }); + // Keep only compact selection fields through both grouping stages. In + // particular, never join endpoint histories for the entire relationship set. + const aggregation = [ + { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, + { + $project: { + 'stix.id': 1, + 'stix.source_ref': 1, + 'stix.target_ref': 1, + 'stix.relationship_type': 1, + 'stix.revoked': 1, + 'stix.x_mitre_deprecated': 1, + }, + }, + { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, + { $replaceRoot: { newRoot: '$document' } }, + // Filter after selecting latest revisions so old active revisions cannot reappear. + { + $match: { + 'stix.revoked': { $in: [null, false] }, + 'stix.x_mitre_deprecated': { $in: [null, false] }, + }, + }, + { + $group: { + _id: { + source: '$stix.source_ref', + type: '$stix.relationship_type', + target: '$stix.target_ref', + }, + ids: { $push: '$_id' }, + count: { $sum: 1 }, + }, + }, + { $match: { count: { $gt: 1 } } }, + { $unwind: '$ids' }, + { + $lookup: { + from: this.model.collection.name, + localField: 'ids', + foreignField: '_id', + as: 'relationship', + }, + }, + { $unwind: '$relationship' }, + { $replaceRoot: { newRoot: '$relationship' } }, + { $sort: { 'stix.id': 1 } }, + ]; + for (const endpoint of ['source', 'target']) { + aggregation.push({ + $lookup: { + from: 'attackObjects', + localField: `stix.${endpoint}_ref`, + foreignField: 'stix.id', + pipeline: [{ $sort: { 'stix.modified': -1 } }, { $limit: 1 }], + as: `${endpoint}_objects`, + }, + }); + } - // Create a mapping of rel_key (source_ref--relationship_type--target_ref) - // to an array of relationships that share it. - let rel_map = new Map(); - for (const rel of all_relationships) { - const rel_key = - rel.stix.source_ref + '--' + rel.stix.relationship_type + '--' + rel.stix.target_ref; - if (!rel_map.has(rel_key)) { - rel_map.set(rel_key, []); + const cursor = this.model.aggregate(aggregation).allowDiskUse(true).cursor({ batchSize: 100 }); + const relationshipMap = new Map(); + try { + for await (const relationship of cursor) { + const { source_ref, relationship_type, target_ref } = relationship.stix; + const key = `${source_ref}--${relationship_type}--${target_ref}`; + if (!relationshipMap.has(key)) relationshipMap.set(key, []); + relationshipMap.get(key).push(relationship); } - const entry = rel_map.get(rel_key); - entry.push(rel); + } finally { + await cursor.close(); } - - // Return only the rel_keys that have more than one item in the array. - const parallel_relationships = new Map( - [...rel_map.entries()].filter(([, value]) => value.length > 1), - ); - - return parallel_relationships; + return relationshipMap; } } diff --git a/app/tests/api/reports/parallel-relationships.spec.js b/app/tests/api/reports/parallel-relationships.spec.js new file mode 100644 index 00000000..c0c8bacf --- /dev/null +++ b/app/tests/api/reports/parallel-relationships.spec.js @@ -0,0 +1,200 @@ +'use strict'; + +const request = require('supertest'); +const { expect } = require('expect'); + +const config = require('../../../config/config'); +const database = require('../../../lib/database-in-memory'); +const databaseConfiguration = require('../../../lib/database-configuration'); +const login = require('../../shared/login'); +const Technique = require('../../../models/technique-model'); + +const markingDefinitionId = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9'; + +function technique(name, domains) { + const timestamp = new Date().toISOString(); + const killChains = { + 'enterprise-attack': 'mitre-attack', + 'mobile-attack': 'mitre-mobile-attack', + 'ics-attack': 'mitre-ics-attack', + }; + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + name, + description: `${name} description`, + spec_version: '2.1', + type: 'attack-pattern', + object_marking_refs: [markingDefinitionId], + kill_chain_phases: domains.map((domain) => ({ + kill_chain_name: killChains[domain], + phase_name: 'persistence', + })), + x_mitre_is_subtechnique: false, + x_mitre_platforms: ['Windows'], + x_mitre_domains: domains, + x_mitre_version: '1.0', + }, + }; +} + +function relationship(source, target, extra = {}) { + const timestamp = new Date().toISOString(); + return { + workspace: { workflow: { state: 'work-in-progress' } }, + stix: { + created: timestamp, + modified: timestamp, + spec_version: '2.1', + type: 'relationship', + relationship_type: 'subtechnique-of', + source_ref: source.stix.id, + target_ref: target.stix.id, + object_marking_refs: [markingDefinitionId], + ...extra, + }, + }; +} + +const Relationship = require('../../../models/relationship-model'); +const repository = require('../../../repository/relationships-repository'); +const { randomUUID } = require('node:crypto'); + +describe('GET /api/reports/parallel-relationships', function () { + let app; + let passportCookie; + let source; + let target; + + before(async function () { + await database.initializeConnection(); + await databaseConfiguration.checkSystemConfiguration(); + config.validateRequests.withAttackDataModel = true; + config.validateRequests.withOpenApi = true; + app = await require('../../../index').initializeApp(); + passportCookie = await login.loginAnonymous(app); + }); + + after(async function () { + await database.closeConnection(); + }); + + function authenticated(builder) { + return builder.set('Cookie', `${passportCookie.name}=${passportCookie.value}`); + } + + async function post(path, body) { + return (await authenticated(request(app).post(path).send(body)).expect(201)).body; + } + + async function report() { + return ( + await authenticated(request(app).get('/api/reports/parallel-relationships')).expect(200) + ).body; + } + + beforeEach(async function () { + await Relationship.deleteMany({}); + await Technique.deleteMany({}); + source = await post('/api/techniques', technique('Source', ['enterprise-attack'])); + target = await post('/api/techniques', technique('Target', ['enterprise-attack'])); + }); + + async function revision(model, id, changes) { + const document = await model.findOne({ 'stix.id': id }).sort({ 'stix.modified': -1 }).lean(); + delete document._id; + Object.assign(document.stix, changes, { + modified: new Date(new Date(document.stix.modified).getTime() + 1000), + }); + return model.create(document); + } + + it('filters singletons before returning data and bounds endpoint history to one revision', async function () { + const first = await post('/api/relationships', relationship(source, target)); + const second = await post('/api/relationships', relationship(source, target)); + await revision(Relationship, first.stix.id, { description: 'Newest relationship' }); + + // Simulate imported historical content using schema-backed copies of API-valid fixtures. + const template = await Technique.findOne({ 'stix.id': source.stix.id }).lean(); + delete template._id; + const history = Array.from({ length: 40 }, (_, index) => ({ + ...template, + stix: { + ...template.stix, + modified: new Date(new Date(template.stix.modified).getTime() + (index + 1) * 1000), + description: 'x'.repeat(8192), + name: `Source revision ${index + 1}`, + }, + })); + await Technique.insertMany(history); + const singleton = await Relationship.findOne({ 'stix.id': second.stix.id }).lean(); + delete singleton._id; + await Relationship.insertMany( + Array.from({ length: 100 }, () => ({ + ...singleton, + stix: { + ...singleton.stix, + id: `relationship--${randomUUID()}`, + target_ref: `attack-pattern--${randomUUID()}`, + }, + })), + ); + + const oldResults = await repository.retrieveAll({ versions: 'latest', lookupRefs: true }); + const groups = await repository.retrieveParallelRelationships(); + const results = [...groups.values()].flat(); + expect(results).toHaveLength(2); + for (const result of results) { + expect(result.source_objects).toHaveLength(1); + expect(result.source_objects[0].stix.name).toBe('Source revision 40'); + expect(result.target_objects).toHaveLength(1); + } + const oldBytes = Buffer.byteLength(JSON.stringify(oldResults)); + const newBytes = Buffer.byteLength(JSON.stringify(results)); + expect(newBytes).toBeLessThan(oldBytes / 100); + console.log( + `Parallel report fixture: old query ${oldBytes} bytes; new query ${newBytes} bytes`, + ); + + const response = await report(); + const key = `${source.stix.id}--subtechnique-of--${target.stix.id}`; + expect(Object.keys(response)).toEqual([key]); + expect(response[key].map((item) => item.stix.id).sort()).toEqual( + [first.stix.id, second.stix.id].sort(), + ); + expect(response[key].find((item) => item.stix.id === first.stix.id).stix.description).toBe( + 'Newest relationship', + ); + for (const result of response[key]) { + expect(result.source_object.stix.name).toBe('Source revision 40'); + expect(result.target_object.stix.id).toBe(target.stix.id); + expect(result.source_objects).toBeUndefined(); + expect(result.target_objects).toBeUndefined(); + } + }); + + it('does not resurrect old active revisions or count history as duplicates', async function () { + const first = await post('/api/relationships', relationship(source, target)); + const revoked = await post('/api/relationships', relationship(source, target)); + const deprecated = await post('/api/relationships', relationship(source, target)); + await revision(Relationship, first.stix.id, { description: 'Latest' }); + await revision(Relationship, revoked.stix.id, { revoked: true }); + await revision(Relationship, deprecated.stix.id, { x_mitre_deprecated: true }); + expect(await report()).toEqual({}); + }); + + it('keeps duplicate findings when endpoints are missing', async function () { + await post('/api/relationships', relationship(source, target)); + await post('/api/relationships', relationship(source, target)); + await Technique.deleteMany({}); + const response = await report(); + const results = Object.values(response).flat(); + expect(results).toHaveLength(2); + for (const result of results) { + expect(result.source_object).toBeUndefined(); + expect(result.target_object).toBeUndefined(); + } + }); +}); diff --git a/docs/README.md b/docs/README.md index 6b7eb575..7953c3fb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,6 +31,7 @@ Guides for consumers of the REST API — endpoints, workflows, and terminology. Architecture, patterns, and implementation details for contributors. - [Build Information](developer/build-information.md): Build metadata provenance, runtime configuration, and frontend integration +- [Data Quality Query Design](developer/data-quality-reports.md): Duplicate-report memory amplification, query optimization, and remaining response-size limits - [Data Model](developer/data-model.md): Database schema and STIX object structure - [Event Bus Architecture](developer/event-bus-architecture.md): Event-driven architecture for cross-document dependencies - [Lifecycle Hooks Guide](developer/lifecycle-hooks-guide.md): Service lifecycle hooks pattern diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index fe26ccba..2be02d79 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -1,5 +1,31 @@ # Release Track TODOs +## Duplicate relationship report memory (2026-09-10) + +- [x] Trace dashboard request and identify historical endpoint fan-out. +- [x] Filter duplicates in MongoDB before hydrating relationships and latest endpoints. +- [x] Add regressions for history, lifecycle filters, missing endpoints, and bounded query output. +- [x] Run focused specs, full npm test, and lint; document findings and limits. + +Verification: focused report + relationship specs 38 passing; full `npm test` +passes (OpenAPI 2, config 22, API 1051, middleware 29, scheduler 10); lint +and `git diff --check` pass. Synthetic query output falls from 38,162,701 to +23,131 serialized bytes; production peak heap has not been measured. See +[data-quality report design](data-quality-reports.md) for the analysis. +No API contract change or frontend/Bruno update is needed. + +Proposed commit: `fix(reports): reduce duplicate relationship report memory use` + +Body: Filter duplicate groups in MongoDB before fetching full relationships, +limit endpoint lookups to their latest revisions, and consume results through +a batched cursor. Add history and lifecycle regressions and document remaining +response-size limits. + +Proposed AGENTS.md lesson: Analytical reports should filter findings before +joining full documents, and latest-endpoint lookups should limit revisions in +MongoDB rather than discarding history after materialization. + + ## Snapshot card header hierarchy - [x] Give snapshot identity and status labels their own full-width header area. diff --git a/docs/developer/data-quality-reports.md b/docs/developer/data-quality-reports.md new file mode 100644 index 00000000..433fd0e7 --- /dev/null +++ b/docs/developer/data-quality-reports.md @@ -0,0 +1,74 @@ +# Data quality report query design + +## Duplicate relationships: September 2026 memory failure + +The Data Quality dashboard's Duplicate Relationships function calls +`GET /api/reports/parallel-relationships`. Previously, +`RelationshipsRepository.retrieveParallelRelationships()` called +`retrieveAll({ versions: 'latest', lookupRefs: true })`. That aggregation joined +**every revision** of each source and target object to **every active latest +relationship**. Awaiting `.exec()` materialized the entire expanded result in +Node. Only then did JavaScript group relationships and discard singleton groups; +the reports service subsequently discarded historical endpoint revisions. + +This means a small final report can require a very large intermediate heap. +The endpoint history is repeated for each incident relationship, including +relationships that will never appear in the report. Large descriptions and +workspace backrefs further increase each repeated document's size. Concurrent +dashboard reports can add to the process's overall memory demand. + +The supplied container log shows V8 exhausting its roughly 4 GB heap. It does +not identify the allocating JavaScript frame, and no production heap snapshot +or database reproduction was available. The code path and synthetic regression +establish a concrete amplification mechanism consistent with that failure, +not a measurement of the exact production allocation. + +## Query implementation + +1. Sort by STIX ID and descending modified time, project compact selection + fields, and choose one latest relationship revision per ID. +2. Exclude revoked/deprecated latest revisions. Filtering earlier would + incorrectly resurrect an older active revision. +3. Group by source, relationship type, and target, retaining MongoDB IDs and + counts. Discard groups with fewer than two distinct relationship IDs. +4. Unwind matching IDs and retrieve only those full relationship documents. +5. Join each endpoint with a descending modified sort and limit of one. + Existing `(stix.id, stix.modified descending)` indexes support latest-revision + access; the relationship hydration lookup uses the existing `_id` index. +6. Consume the aggregation with a 100-document cursor and explicit cursor + cleanup. Allow disk use for eligible MongoDB aggregation stages. + +The repository preserves the existing arrays of endpoint matches (now at most +one entry each), so identity enrichment and the public JSON map stay compatible. +Missing endpoints still preserve findings. Results retain ascending relationship +STIX-ID ordering. No OpenAPI, Bruno, or frontend contract change is required. + +MongoDB documents the pipeline form of +[`$lookup`](https://www.mongodb.com/docs/manual/reference/operator/aggregation/lookup/) +and the top-N optimization for +[`$sort` followed by `$limit`](https://www.mongodb.com/docs/manual/reference/operator/aggregation/sort/). + +## Evidence and limits + +`app/tests/api/reports/parallel-relationships.spec.js` exercises real MongoDB +aggregation and the HTTP endpoint with ADM validation enabled. Its synthetic +history fixture contains 102 latest active relationships, only two duplicates, +and 41 revisions of their shared source, with 8 KiB descriptions on 40 revisions. +The previous query returned 38,162,701 serialized bytes; the replacement returned +23,131 bytes (about 1,650 times smaller). This measures JSON query-result volume, +not peak heap or elapsed time. The test also covers newest relationship selection, +revoked/deprecated latest revisions, singleton exclusion, and missing endpoints. + +Database cursor batching is **not HTTP streaming**. The service still retains +all actual duplicate findings, and Express serializes the complete JSON map. +Memory therefore still scales with the real report size. A very large individual +duplicate group also grows MongoDB's grouped ID array; allowing disk use does not +remove all aggregation or BSON size limits. + +If real duplicate output itself is excessive, introduce bounded pagination of +group summaries with relationship details loaded on expansion, coordinated with +the frontend and OpenAPI/Bruno contracts. Incremental NDJSON is another option, +but requires response backpressure, disconnect cancellation, error semantics, +and a frontend streaming parser; the existing Angular JSON request waits for a +complete response. Merely streaming the current query or increasing Node's heap +would leave its unnecessary history fan-out intact. diff --git a/docs/user/data-quality-reports.md b/docs/user/data-quality-reports.md index 3505e842..a80c259c 100644 --- a/docs/user/data-quality-reports.md +++ b/docs/user/data-quality-reports.md @@ -21,10 +21,17 @@ result to one STIX type (`relationship` for relationships only). GET /api/reports/parallel-relationships ``` -Latest relationship revisions grouped by `source_ref--relationship_type--target_ref` -where more than one relationship shares the key — likely duplicates. The +Latest active (not revoked or deprecated) relationship revisions grouped by +`source_ref--relationship_type--target_ref` where more than one relationship +shares the key — likely duplicates. The response is a map from that key to the array of relationships, each carrying -its latest `source_object` and `target_object`. +its latest `source_object` and `target_object`. Missing endpoints do not remove +a duplicate finding; the corresponding endpoint property is omitted. Historical +revisions of one relationship do not count as separate duplicates. + +The report filters duplicates in the database before retrieving endpoint +details, and loads only the latest endpoint revisions. The response remains +a single JSON map, with no pagination or incremental delivery. ## Domain consistency