From 8dcf59da43d6d35b2ccf8b78975ae8bc1caf1c76 Mon Sep 17 00:00:00 2001 From: brontolosone <177225737+brontolosone@users.noreply.github.com> Date: Fri, 28 Nov 2025 17:34:48 +0000 Subject: [PATCH 1/2] Row event stamping for submissions table --- ...7-01-submission-event-stamping-01.down.sql | 27 ++++ ...127-01-submission-event-stamping-01.up.sql | 36 +++++ ...7-01-submission-event-stamping-02.down.sql | 11 ++ ...127-01-submission-event-stamping-02.up.sql | 126 ++++++++++++++++++ .../20251127-01-submission-event-stamping.js | 10 ++ 5 files changed, 210 insertions(+) create mode 100644 lib/model/migrations/20251127-01-submission-event-stamping-01.down.sql create mode 100644 lib/model/migrations/20251127-01-submission-event-stamping-01.up.sql create mode 100644 lib/model/migrations/20251127-01-submission-event-stamping-02.down.sql create mode 100644 lib/model/migrations/20251127-01-submission-event-stamping-02.up.sql create mode 100644 lib/model/migrations/20251127-01-submission-event-stamping.js diff --git a/lib/model/migrations/20251127-01-submission-event-stamping-01.down.sql b/lib/model/migrations/20251127-01-submission-event-stamping-01.down.sql new file mode 100644 index 000000000..138d47a5c --- /dev/null +++ b/lib/model/migrations/20251127-01-submission-event-stamping-01.down.sql @@ -0,0 +1,27 @@ +-- Copyright 2025 ODK Central Developers +-- See the NOTICE file at the top-level directory of this distribution and at +-- https://github.com/getodk/central-backend/blob/master/NOTICE. +-- This file is part of ODK Central. It is subject to the license terms in +-- the LICENSE file found in the top-level directory of this distribution and at +-- https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +-- including this file, may be copied, modified, propagated, or distributed +-- except according to the terms contained in the LICENSE file. + + +--- drop: get_event(submission_id integer) --- +DROP FUNCTION IF EXISTS "public"."get_event"(submission_id integer) CASCADE; + +--- drop: public.submissions.set_eventstamp_submissions_at_commit --- +DROP TRIGGER IF EXISTS set_eventstamp_submissions_at_commit ON "public"."submissions" CASCADE; + +--- drop: public.submissions.blank_submissions_event_on_insert --- +DROP TRIGGER IF EXISTS blank_submissions_event_on_insert ON "public"."submissions" CASCADE; + +--- drop: eventstamp_submissions_triggerfunction --- +DROP FUNCTION IF EXISTS "public"."eventstamp_submissions_triggerfunction"() CASCADE; + +--- drop: blank_submissions_event_triggerfunction --- +DROP FUNCTION IF EXISTS "public"."blank_submissions_event_triggerfunction"() CASCADE; + +--- drop: public.submissions.blank_submissions_event_on_update --- +DROP TRIGGER IF EXISTS blank_submissions_event_on_update ON "public"."submissions" CASCADE; diff --git a/lib/model/migrations/20251127-01-submission-event-stamping-01.up.sql b/lib/model/migrations/20251127-01-submission-event-stamping-01.up.sql new file mode 100644 index 000000000..d670af8be --- /dev/null +++ b/lib/model/migrations/20251127-01-submission-event-stamping-01.up.sql @@ -0,0 +1,36 @@ +-- Copyright 2025 ODK Central Developers +-- See the NOTICE file at the top-level directory of this distribution and at +-- https://github.com/getodk/central-backend/blob/master/NOTICE. +-- This file is part of ODK Central. It is subject to the license terms in +-- the LICENSE file found in the top-level directory of this distribution and at +-- https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +-- including this file, may be copied, modified, propagated, or distributed +-- except according to the terms contained in the LICENSE file. + +ALTER TABLE submissions + ADD COLUMN event bigint +; + +CREATE INDEX event_idx ON submissions (event) +; + +UPDATE + submissions s1 +SET + event = s2.rowno +FROM ( + SELECT + s_inner.id, + row_number() OVER (ORDER BY COALESCE(s_inner."updatedAt", s_inner."createdAt"), s_inner.id) AS rowno + FROM + submissions s_inner) AS s2 +WHERE + s1.id = s2.id +; + +CREATE TABLE current_event ( + event bigint NOT NULL +) +; + +INSERT INTO current_event (event) (SELECT coalesce(MAX(event), 0) FROM submissions); \ No newline at end of file diff --git a/lib/model/migrations/20251127-01-submission-event-stamping-02.down.sql b/lib/model/migrations/20251127-01-submission-event-stamping-02.down.sql new file mode 100644 index 000000000..8a0403744 --- /dev/null +++ b/lib/model/migrations/20251127-01-submission-event-stamping-02.down.sql @@ -0,0 +1,11 @@ +-- Copyright 2025 ODK Central Developers +-- See the NOTICE file at the top-level directory of this distribution and at +-- https://github.com/getodk/central-backend/blob/master/NOTICE. +-- This file is part of ODK Central. It is subject to the license terms in +-- the LICENSE file found in the top-level directory of this distribution and at +-- https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +-- including this file, may be copied, modified, propagated, or distributed +-- except according to the terms contained in the LICENSE file. + +DROP TABLE current_event; +ALTER TABLE submissions DROP COLUMN event; \ No newline at end of file diff --git a/lib/model/migrations/20251127-01-submission-event-stamping-02.up.sql b/lib/model/migrations/20251127-01-submission-event-stamping-02.up.sql new file mode 100644 index 000000000..22bdf886f --- /dev/null +++ b/lib/model/migrations/20251127-01-submission-event-stamping-02.up.sql @@ -0,0 +1,126 @@ +-- Copyright 2025 ODK Central Developers +-- See the NOTICE file at the top-level directory of this distribution and at +-- https://github.com/getodk/central-backend/blob/master/NOTICE. +-- This file is part of ODK Central. It is subject to the license terms in +-- the LICENSE file found in the top-level directory of this distribution and at +-- https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +-- including this file, may be copied, modified, propagated, or distributed +-- except according to the terms contained in the LICENSE file. + + +--- create: blank_submissions_event_triggerfunction --- +CREATE FUNCTION "public"."blank_submissions_event_triggerfunction"() +RETURNS trigger +AS + $BODY$ + BEGIN + NEW.event = NULL; + RETURN NEW; + END; + $BODY$ +LANGUAGE plpgsql +; + +--- sign: blank_submissions_event_triggerfunction --- +COMMENT ON FUNCTION "public"."blank_submissions_event_triggerfunction"() IS '{"dbsamizdat": {"version": 1, "definition_hash": "0982b1119302eb5c8afe455903c5ec8a"}}'; + +--- create: get_event --- +CREATE FUNCTION "public"."get_event"() +RETURNS bigint +AS + $BODY$ + -- Locks out invocations in other transactions until this transaction commits. + -- Use at the end of a transaction (the idea is to use this in a deferred constraint trigger). + + WITH current_event_locked AS ( + SELECT event FROM current_event + FOR UPDATE + ), + + -- figure out whether we've already acquired the advisory lock. + -- The lock indicates whether, within this transaction, we've already bumped the eventcounter, and thus shouldn't do it again. + -- (because this function is called in a constraint trigger which automatically runs at commit time, for every new/modified row) + test_lock AS ( + SELECT + EXISTS ( + SELECT + 1 + FROM + -- the `pg_locks` catalog is cluster-wide, so we need to select for the current database + pg_locks l INNER JOIN pg_database d ON (d.datname = current_database() AND d.oid = l.database) + WHERE + -- See PostgreSQL documentation (for version 14), section 52.74: + -- https://www.postgresql.org/docs/14/view-pg-locks.html#:~:text=A%20bigint%20key%20is%20displayed%20with%20its%20high%2Dorder%20half%20in%20the%20classid%20column%2C%20its%20low%2Dorder%20half%20in%20the%20objid%20column%2C%20and%20objsubid%20equal%20to%201%2E + l.locktype = 'advisory' + AND l.objsubid = 1 + AND ((l.classid::bigint << 32) | l.objid::bigint) = hash_text_to_bigint(pg_current_xact_id()::text, 'submissions-eventstamp-lock') + ) as lockfound, + pg_advisory_xact_lock(hash_text_to_bigint(pg_current_xact_id()::text, 'submissions-eventstamp-lock')) AS newlock + ), + + maybe_new_event AS ( + UPDATE current_event + SET event = event + 1 + FROM test_lock + WHERE test_lock.lockfound = false -- the lock would mean that we're not processing the first row (of potentially many) in this transaction, and the eventcounter is already incremented + RETURNING event + ) + + SELECT COALESCE( + (SELECT event FROM maybe_new_event), + (SELECT event FROM current_event_locked) + ) as event + + $BODY$ +LANGUAGE sql +; + +--- sign: get_event --- +COMMENT ON FUNCTION "public"."get_event"() IS '{"dbsamizdat": {"version": 1, "definition_hash": "dd6ab7bb6eef6087a892360fd85f4151"}}'; + +--- create: eventstamp_submissions_triggerfunction --- +CREATE FUNCTION "public"."eventstamp_submissions_triggerfunction"() +RETURNS trigger +AS + $BODY$ + BEGIN + -- as this is called from a constraint trigger, we can't modify the data through modification of NEW. + UPDATE submissions SET event = get_event() WHERE id = NEW.id; + RETURN NEW; + END; + $BODY$ +LANGUAGE plpgsql +; + +--- sign: eventstamp_submissions_triggerfunction --- +COMMENT ON FUNCTION "public"."eventstamp_submissions_triggerfunction"() IS '{"dbsamizdat": {"version": 1, "definition_hash": "9af0a3a329872908cdf8ad0e7fc34efd"}}'; + +--- create: public.submissions.blank_submissions_event_on_update --- +CREATE TRIGGER "blank_submissions_event_on_update" BEFORE UPDATE ON "public"."submissions" +-- Application transparency: +-- New rows are already created with a NULL event stamp by default. +-- Updates to rows need to have their event stamp reset to NULL as well. +FOR EACH ROW +WHEN ( + (NEW.event IS NOT NULL) + AND + (OLD.event IS NOT NULL) +) +EXECUTE PROCEDURE blank_submissions_event_triggerfunction() +; + +--- sign: public.submissions.blank_submissions_event_on_update --- +COMMENT ON TRIGGER "blank_submissions_event_on_update" ON "public"."submissions" IS '{"dbsamizdat": {"version": 1, "definition_hash": "c6d9c9e5191fa009a63a4155620f8eb2"}}'; + +--- create: public.submissions.set_eventstamp_submissions_at_commit --- +CREATE CONSTRAINT TRIGGER "set_eventstamp_submissions_at_commit" AFTER INSERT OR UPDATE ON "public"."submissions" +-- Runs at the end of a transaction. +-- The `NEW.event IS NULL` filter prevents recursion. +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW +WHEN (NEW.event IS NULL) +EXECUTE PROCEDURE eventstamp_submissions_triggerfunction(); +; + +--- sign: public.submissions.set_eventstamp_submissions_at_commit --- +COMMENT ON TRIGGER "set_eventstamp_submissions_at_commit" ON "public"."submissions" IS '{"dbsamizdat": {"version": 1, "definition_hash": "9bc8ef789e6e8cd458dccc98b88b0db9"}}'; diff --git a/lib/model/migrations/20251127-01-submission-event-stamping.js b/lib/model/migrations/20251127-01-submission-event-stamping.js new file mode 100644 index 000000000..1a47cf703 --- /dev/null +++ b/lib/model/migrations/20251127-01-submission-event-stamping.js @@ -0,0 +1,10 @@ +// Copyright 2025 ODK Central Developers +// See the NOTICE file at the top-level directory of this distribution and at +// https://github.com/getodk/central-backend/blob/master/NOTICE. +// This file is part of ODK Central. It is subject to the license terms in +// the LICENSE file found in the top-level directory of this distribution and at +// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +// including this file, may be copied, modified, propagated, or distributed +// except according to the terms contained in the LICENSE file. + +module.exports = require('../pure-sql-migration')(__filename); From 0186bf10c720261235ab9265fb8fa5b14a44c911 Mon Sep 17 00:00:00 2001 From: brontolosone <177225737+brontolosone@users.noreply.github.com> Date: Thu, 18 Dec 2025 14:55:54 +0000 Subject: [PATCH 2/2] Basic test for row event stamping --- test/integration/other/db-eventstamping.js | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 test/integration/other/db-eventstamping.js diff --git a/test/integration/other/db-eventstamping.js b/test/integration/other/db-eventstamping.js new file mode 100644 index 000000000..cf9b0b9d3 --- /dev/null +++ b/test/integration/other/db-eventstamping.js @@ -0,0 +1,59 @@ +const { testService } = require('../setup'); +const { sql } = require('slonik'); +const { instances } = require('../../data/xml'); + + +describe('DB: Event stamping', () => { + + it('Event stamps are applied upon submission insertion and modification', testService(async (service, { db }) => { + // As we are testing for the effect of a commit, we have to... commit. And then clean up. + try { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .set('Content-Type', 'application/xml') + .send(instances.simple.one) + .expect(200); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .set('Content-Type', 'application/xml') + .send(instances.simple.two) + .expect(200); + + await db.query(sql`commit;`); + // .one and .two were created in the same transaction, so they should receive the same event stamp + (await db.oneFirst(sql`select distinct(event) from submissions`)).should.equal(1); + + await asAlice.post('/v1/projects/1/forms/simple/submissions') + .set('Content-Type', 'application/xml') + .send(instances.simple.three) + .expect(200); + + await db.query(sql`commit;`); + (await db.oneFirst(sql`select event from submissions where "instanceId" = 'three'`)).should.equal(2); + + await asAlice.put('/v1/projects/1/forms/simple/submissions/one') + .set('Content-Type', 'application/xml') + .send('oneAonebla') + .expect(200); + + await db.query(sql`commit;`); + (await db.oneFirst(sql`select event from submissions where "instanceId" = 'one'`)).should.equal(3); + + await asAlice.patch('/v1/projects/1/forms/simple/submissions/one') + .set('Content-Type', 'application/json') + .send({ reviewState: 'approved' }) + .expect(200); + + await db.query(sql`commit;`); + (await db.oneFirst(sql`select event from submissions where "instanceId" = 'one'`)).should.equal(4); + + } finally { + await db.query(sql`truncate table submissions cascade`); + await db.query(sql`truncate table audits cascade`); + await db.query(sql`commit;`); + } + + })); + +});