Dataset access filters

lib/model/frames/dataset.js

@@ -29,7 +29,7 @@ class Dataset extends Frame.define(
     // Include the numeric dataset id (used for downloading entities of deleted dataset)
     const idField = this.deletedAt ? { id: this.id } : null;
 
-    return Object.assign(Frame.prototype.forApi.call(this), idField);
+    return Object.assign(Frame.prototype.forApi.call(this), idField, { accessFilter: null });
   }
 }
 
@@ -77,4 +77,10 @@ Dataset.LinkedForm = class extends Frame.define(
   }
 };
 
+Dataset.AccessFilter = Frame.define(
+  table('dataset_access_filters'),
+  'datasetProperty', readable,
+  'actorProperty', readable
+);
+
 module.exports = { Dataset };

lib/model/migrations/20260602-01-add-dataset-filters-01.down.sql

@@ -0,0 +1,3 @@
+DROP VIEW IF EXISTS actor_dataset_filter;
+DROP TABLE dataset_access_filters CASCADE;
+DROP INDEX IF EXISTS "dataset_access_filters__unique_for_composite_fk_referent";

lib/model/migrations/20260602-01-add-dataset-filters-01.up.sql

@@ -0,0 +1,41 @@
+-- Maps a dataset property to an actor property, defining a filter rule:
+-- entities are visible to an actor only if their value for "datasetPropertyId" matches the actor's value for "actorPropertyId".
+CREATE TABLE dataset_access_filters (
+    "datasetId" integer NOT NULL,  -- Redundant with "datasetPropertyId" but needed for the composite FK below to prevent insertion anomalies.
+    "datasetPropertyId" integer NOT NULL PRIMARY KEY,
+    "actorPropertyId" integer NOT NULL REFERENCES actor_properties (id) ON DELETE CASCADE
+);
+-- The unique index on ds_properties enables the composite FK, which ensures "datasetId" is always
+-- consistent with "datasetPropertyId" (preventing insertion anomalies).
+CREATE UNIQUE INDEX "dataset_access_filters__unique_for_composite_fk_referent" ON ds_properties USING btree ("datasetId", id);
+ALTER TABLE dataset_access_filters
+    ADD CONSTRAINT "dataset_access_filters__composite_fk" FOREIGN KEY ("datasetId", "datasetPropertyId") REFERENCES ds_properties ("datasetId", id) ON DELETE CASCADE;
+-- Ensures each actor property is used at most once per dataset.
+CREATE UNIQUE INDEX "dataset_access_filters__spend_actorprop_once_per_dsprop" ON dataset_access_filters ("datasetPropertyId", "actorPropertyId");
+-- Index to support FK lookup on actorPropertyId (CASCADE deletes from actor_properties).
+CREATE INDEX ON dataset_access_filters ("actorPropertyId");
+-- Index to support FK lookup on (datasetId, datasetPropertyId) (CASCADE deletes from ds_properties).
+CREATE INDEX ON dataset_access_filters ("datasetId", "datasetPropertyId");
+
+CREATE VIEW actor_dataset_filter AS (
+    WITH filter_as_json AS (
+        SELECT
+            pfilter."datasetId",
+            aprop."actorId",
+            jsonb_object_agg (dsprops."name", aprop."value") AS thefilter
+        FROM
+            dataset_access_filters pfilter
+            INNER JOIN ds_properties dsprops ON (pfilter."datasetPropertyId" = dsprops.id)
+            INNER JOIN actor_property_values aprop USING ("actorPropertyId")
+        GROUP BY
+            (pfilter."datasetId", aprop."actorId")
+    )
+    SELECT
+        *,
+        md5(thefilter::text) AS filterhash  -- Used to make the filter part of an HTTP resource for incremental entity list download.
+    FROM
+        filter_as_json
+);
+
+-- Commented out: add back w/ info about taking a long time
+-- CREATE INDEX "idx_entity_defs_data" on entity_defs using gin (data);

lib/model/migrations/20260602-01-add-dataset-filters.js

@@ -0,0 +1 @@
+module.exports = require('../pure-sql-migration')(__filename);

lib/model/query/datasets.js

@@ -508,9 +508,26 @@ const getMetadata = (dataset) => async ({ all, Datasets }) => {
     }
   }
 
+  const accessFilterRules = await Datasets.getAccessFilter(dataset.id);
+
+  // Compute the unified accessFilter field:
+  //   - { type: 'ownerOnly' }                        if ownerOnly flag is set
+  //   - { type: 'property', rules: [...] }           if property-based rules exist
+  //   - null                                         otherwise
+  let accessFilter = null;
+  if (dataset.ownerOnly) {
+    accessFilter = { type: 'ownerOnly' };
+  } else if (accessFilterRules.length > 0) {
+    accessFilter = {
+      type: 'property',
+      rules: accessFilterRules.map(r => r.forApi())
+    };
+  }
+
   // return all dataset metadata
   return {
     ...dataset.forApi(),
+    accessFilter,
     lastUpdate,
     linkedForms: linkedForms.filter(f => !f.draft).map(f => f.forApi()),
     sourceForms: sourceForms.filter(f => !f.draft).map(f => f.forApi()),
@@ -901,6 +918,56 @@ const getListWithProperties = (projectId, ids) => async ({ all }) => {
   return Array.from(rows.reduce(_dsPropertiesReducer, new Map()).values());
 };
 
+////////////////////////////////////////////////////////////////////////////
+// DATASET ACCESS FILTERS
+
+// Adds or updates a single access filter rule for a dataset.
+// datasetPropertyName and userPropertyName are names (strings).
+const setAccessFilter = (datasetId, datasetPropertyName, userPropertyName) => ({ maybeOne }) =>
+  maybeOne(sql`
+WITH ds_prop AS (
+  SELECT id FROM ds_properties
+  WHERE "datasetId" = ${datasetId} AND "name" = ${datasetPropertyName} AND "publishedAt" IS NOT NULL
+), actor_prop AS (
+  SELECT ap.id FROM actor_properties ap
+  JOIN datasets d ON d."projectId" = ap."projectId"
+  WHERE d.id = ${datasetId} AND ap."name" = ${userPropertyName}
+), inserted AS (
+  INSERT INTO dataset_access_filters ("datasetId", "datasetPropertyId", "actorPropertyId")
+  SELECT ${datasetId}, ds_prop.id, actor_prop.id FROM ds_prop, actor_prop
+  ON CONFLICT ("datasetPropertyId") DO UPDATE SET "actorPropertyId" = EXCLUDED."actorPropertyId"
+  RETURNING *
+)
+SELECT * FROM inserted`)
+    .then((rows) => rows.map((row) => new Dataset.AccessFilter(row)));
+
+// Returns all access filter rules for a dataset as DatasetAccessFilter instances.
+const getAccessFilter = (datasetId) => ({ all }) =>
+  all(sql`
+SELECT dsp."name" AS "datasetProperty", ap."name" AS "actorProperty"
+FROM dataset_access_filters daf
+JOIN ds_properties dsp ON dsp.id = daf."datasetPropertyId"
+JOIN actor_properties ap ON ap.id = daf."actorPropertyId"
+WHERE daf."datasetId" = ${datasetId}
+ORDER BY dsp."name"`)
+    .then((rows) => rows.map((row) => new Dataset.AccessFilter(row)));
+
+
+// Deletes all access filter rules for a dataset.
+const deleteAllAccessFilters = (datasetId) => ({ run }) =>
+  run(sql`DELETE FROM dataset_access_filters WHERE "datasetId" = ${datasetId}`);
+
+// Atomically replaces all access filter rules for a dataset.
+// rules is an array of { datasetProperty, actorProperty } objects.
+// Passing an empty array deletes all rules.
+const setAccessFilters = (datasetId, rules) => async ({ Datasets }) => {
+  await Datasets.deleteAllAccessFilters(datasetId);
+  for (const { datasetProperty, actorProperty } of rules) {
+    // eslint-disable-next-line no-await-in-loop
+    await Datasets.setAccessFilter(datasetId, datasetProperty, actorProperty);
+  }
+};
+
 module.exports = {
   createPublishedDataset, createPublishedProperty,
   createOrMerge, publishIfExists,
@@ -913,5 +980,6 @@ module.exports = {
   getLastUpdateTimestamp, canReadForOpenRosa,
   del, purge, deleteProperty,
   getTargetDatasetsAndProperties, getLinkedDatasets, getRelatedDatasetsOfForm,
-  getListWithProperties
+  getListWithProperties,
+  setAccessFilter, getAccessFilter, deleteAllAccessFilters, setAccessFilters
 };

lib/model/query/entities.js

@@ -909,7 +909,7 @@ const searchClause = (expr) => {
   return sql`(jsonb_to_tsvector('simple', entity_defs.data, '["string"]') || to_tsvector('simple', entity_defs.label)) @@ websearch_to_tsquery('simple', ${expr})`;
 };
 
-const streamForExport = (datasetId, options = QueryOptions.none) => ({ stream }) =>
+const streamForExport = (datasetId, options = QueryOptions.none, actorId) => ({ stream }) =>
   stream(sql`
 SELECT ${_exportUnjoiner.fields} FROM entity_defs
 INNER JOIN entities ON entities.id = entity_defs."entityId"
@@ -919,13 +919,31 @@ ${options.skiptoken ? sql`
   ( SELECT id, "createdAt" FROM entities WHERE "uuid" = ${options.skiptoken.uuid}::uuid) AS cursor
   ON entities."createdAt" <= cursor."createdAt" AND entities.id < cursor.id
   `: sql``} 
+${actorId != null ? sql`
+LEFT OUTER JOIN actor_dataset_filter propfilter
+  ON propfilter."datasetId" = entities."datasetId" AND propfilter."actorId" = ${actorId}
+LEFT JOIN (SELECT DISTINCT "datasetId" FROM dataset_access_filters) hasfilter
+  ON hasfilter."datasetId" = entities."datasetId"
+` : sql``}
 WHERE
   entities."datasetId" = ${datasetId}
   AND ${sqlEquals(options.condition)}
   AND ${odataExcludeDeleted(options.filter, odataToColumnMap)}
   AND entity_defs.current=true
   AND ${odataFilter(options.filter, odataToColumnMap)}
   AND ${searchClause(options.search)}
+${actorId != null ? sql`
+  AND (
+    NOT EXISTS (SELECT 1 FROM dataset_access_filters WHERE "datasetId" = entities."datasetId")
+    OR EXISTS (
+      SELECT 1 FROM dataset_access_filters daf
+      JOIN ds_properties dp ON dp.id = daf."datasetPropertyId"
+      JOIN actor_property_values apv ON apv."actorPropertyId" = daf."actorPropertyId"
+      WHERE daf."datasetId" = entities."datasetId"
+        AND apv."actorId" = ${actorId}
+        AND entity_defs.data ->> dp."name" = apv."value"
+    )
+  )` : sql``}
 ${options.orderby ? sql`
   ${odataOrderBy(options.orderby, odataToColumnMap, 'entities.id')}
   `: sql`ORDER BY entities."createdAt" DESC, entities.id DESC`}

lib/resources/datasets.js

@@ -51,7 +51,49 @@ module.exports = (service, endpoint) => {
     const autoConvert = newDataset.approvalRequired === false
       ? query.convert === 'true'
       : undefined;
-    const updatedDataset = await Datasets.update(dataset, newDataset, autoConvert);
+
+    // Collect all column updates into one object to call Datasets.update once.
+    // accessFilter wins over the legacy ownerOnly field if both are present.
+    const updates = {};
+    if (newDataset.approvalRequired !== undefined) updates.approvalRequired = newDataset.approvalRequired;
+    if (newDataset.ownerOnly !== undefined) {
+      updates.ownerOnly = newDataset.ownerOnly;
+      // Clear property rules when enabling ownerOnly so the two filter types don't coexist.
+      if (newDataset.ownerOnly) await Datasets.deleteAllAccessFilters(dataset.id);
+    }
+
+    // Handle accessFilter transitions if present in body (takes precedence over ownerOnly above).
+    if (Object.hasOwn(body, 'accessFilter')) {
+      const { accessFilter } = body;
+
+      if (accessFilter === null) {
+        // Clear any existing filter (both ownerOnly and property rules)
+        if (dataset.ownerOnly) updates.ownerOnly = false;
+        await Datasets.deleteAllAccessFilters(dataset.id);
+      } else if (accessFilter.type === 'ownerOnly') {
+        // Switch to ownerOnly: clear property rules first
+        await Datasets.deleteAllAccessFilters(dataset.id);
+        if (!dataset.ownerOnly) updates.ownerOnly = true;
+      } else if (accessFilter.type === 'property') {
+        const { rules } = accessFilter;
+        if (!Array.isArray(rules) || rules.length === 0)
+          throw Problem.user.unexpectedValue({ field: 'accessFilter.rules', value: rules, reason: 'rules must be a non-empty array.' });
+        for (const rule of rules) {
+          if (!rule.datasetProperty || !rule.actorProperty)
+            throw Problem.user.unexpectedValue({ field: 'accessFilter.rules', value: rule, reason: 'Each rule must have datasetProperty and actorProperty.' });
+        }
+        // Clear ownerOnly if set, then replace all property rules atomically
+        if (dataset.ownerOnly) updates.ownerOnly = false;
+        await Datasets.setAccessFilters(dataset.id, rules);
+      } else {
+        throw Problem.user.unexpectedValue({ field: 'accessFilter.type', value: accessFilter.type, reason: 'type must be ownerOnly or property.' });
+      }
+    }
+
+    const updatedDataset = Object.keys(updates).length > 0
+      ? await Datasets.update(dataset, updates, autoConvert)
+      : dataset;
+
     return Datasets.getMetadata(updatedDataset);
   }));
 
@@ -189,4 +231,6 @@ module.exports = (service, endpoint) => {
 
     return entityList({ entities });
   }));
+
+
 };

lib/resources/forms.js

@@ -54,7 +54,7 @@ const getOwnerOnlyOptions = async (auth, project) => ((await auth.can('entity.li
   ? QueryOptions.none
   : QueryOptions.none.withCondition({ 'entities.creatorId': auth.actor.get().id }));
 
-const streamAttachment = async (container, attachment, ownerOnlyOptions, response) => {
+const streamAttachment = async (container, attachment, ownerOnlyOptions, response, auth) => {
   const { s3, Blobs, Datasets, Entities } = container;
 
   if (attachment.blobId == null && attachment.datasetId == null) {
@@ -66,7 +66,8 @@ const streamAttachment = async (container, attachment, ownerOnlyOptions, respons
     const properties = await Datasets.getProperties(attachment.datasetId);
     return withEtag(attachment.openRosaHash, async () => {
       const options = attachment.ownerOnly ? ownerOnlyOptions : QueryOptions.none;
-      const entities = await Entities.streamForExport(attachment.datasetId, options);
+      const actorId = auth != null ? auth.actor.get().id : null;
+      const entities = await Entities.streamForExport(attachment.datasetId, options, actorId);
       response.set('Content-Disposition', contentDisposition(`${attachment.name}`));
       response.set('Content-Type', 'text/csv');
       return streamEntityCsvAttachment(entities, properties);
@@ -341,7 +342,7 @@ module.exports = (service, endpoint, anonymousEndpoint) => {
       const options = await getOwnerOnlyOptions(auth, project);
       const attachment = await FormAttachments.getByFormDefIdAndNameForOpenRosa(form.def.id, params.name, options)
         .then(getOrNotFound);
-      return streamAttachment(container, attachment, options, response);
+      return streamAttachment(container, attachment, options, response, auth);
     }));
   };
 

test/assertions.js

@@ -373,7 +373,7 @@ should.Assertion.add('Dataset', function assertDataset(extraKeys = []) {
   this.params = { operator: 'to be a Dataset' };
 
   Object.keys(this.obj).should.be.a.subsetOf([
-    'projectId', 'name', 'approvalRequired', 'ownerOnly', 'createdAt', 'deletedAt',
+    'projectId', 'name', 'approvalRequired', 'ownerOnly', 'accessFilter', 'createdAt', 'deletedAt',
     // Optional metadata
     'properties', 'linkedForms', 'sourceForms', 'lastUpdate', 'draftLinkedForms', 'draftSourceForms',
     ...extraKeys