feat: add traceability, identities router, install plugin from an archive

.github/workflows/reuse-quality.yml

@@ -12,7 +12,7 @@ jobs:
 
     - uses: actions/setup-node@v3
       with:
-        node-version: 22
+        node-version: 24
         cache: 'npm'
 
     - name: Install dependencies

.nvmrc

@@ -1 +1 @@
-22
+24

Dockerfile

@@ -1,7 +1,7 @@
 # =============================
 # Base Node image
 # =============================
-FROM node:22.14.0-alpine AS base
+FROM node:24-alpine AS base
 
 WORKDIR /app
 ENV NODE_ENV=production
@@ -68,6 +68,7 @@ RUN npm i --no-save @rollup/rollup-linux-x64-musl
 COPY --from=types /app/api/config api/config
 COPY --from=types /app/api/types api/types
 ADD /api/src/config.ts api/src/config.ts
+COPY shared shared
 ADD /ui ui
 RUN npm -w ui run build
 
@@ -106,7 +107,7 @@ COPY package.json README.md LICENSE BUILD.json* ./
 EXPOSE 9090
 # USER node # This would be great to use, but not possible as the volumes are mounted as root
 WORKDIR /app/worker
-CMD ["node", "--experimental-strip-types", "index.ts"]
+CMD ["node", "--disable-warning=ExperimentalWarning", "index.ts"]
 
 # =============================
 # Install production dependencies for API
@@ -141,4 +142,4 @@ EXPOSE 8080
 EXPOSE 9090
 # USER node # This would be great to use, but not possible as the volumes are mounted as root
 WORKDIR /app/api
-CMD ["node", "--max-http-header-size", "65536", "--experimental-strip-types", "index.ts"]
+CMD ["node", "--max-http-header-size", "65536", "--disable-warning=ExperimentalWarning", "index.ts"]

api/config/custom-environment-variables.cjs → api/config/custom-environment-variables.mjs

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
   cipherPassword: 'CIPHER_PASSWORD',
   dataDir: 'DATA_DIR',
   tmpDir: 'TMP_DIR',
@@ -12,8 +12,11 @@ module.exports = {
   },
   port: 'PORT',
   privateDirectoryUrl: 'PRIVATE_DIRECTORY_URL',
+  privateEventsUrl: 'PRIVATE_EVENTS_URL',
   secretKeys: {
-    limits: 'SECRET_LIMITS'
+    limits: 'SECRET_LIMITS',
+    events: 'SECRET_EVENTS',
+    identities: 'SECRET_IDENTITIES'
   },
   observer: {
     active: 'OBSERVER_ACTIVE',

api/config/default.cjs → api/config/default.mjs

@@ -1,4 +1,4 @@
-module.exports = {
+export default {
   cipherPassword: undefined,
   dataDir: '/app/data',
   tmpDir: null, // will be dataDir + '/tmp' if null
@@ -9,8 +9,11 @@ module.exports = {
   },
   pluginCategories: ['Essentiels', 'Mes plugins', 'Données de références', 'Tests'],
   privateDirectoryUrl: 'http://simple-directory:8080',
+  privateEventsUrl: undefined,
   secretKeys: {
-    limits: null
+    limits: null,
+    events: undefined,
+    identities: undefined
   },
   mongoUrl: 'mongodb://localhost:27017/data-fair-processings',
   port: 8080,

api/config/development.cjs → api/config/development.mjs

@@ -1,11 +1,16 @@
-module.exports = {
+export default {
   cipherPassword: 'dev',
   dataDir: '../data/development',
-  port: 8082,
-  privateDirectoryUrl: 'http://localhost:8080',
   mongoUrl: 'mongodb://localhost:27017/data-fair-processings-development',
   observer: {
     port: 9092
   },
+  port: 8082,
+  privateDirectoryUrl: 'http://localhost:8080',
+  privateEventsUrl: 'http://localhost:8084',
+  secretKeys: {
+    identities: 'secret-identities',
+    events: 'secret-events'
+  },
   serveUi: false
 }

api/config/production.mjs

@@ -0,0 +1 @@
+export default {}

api/config/test.cjs → api/config/test.mjs

@@ -1,11 +1,16 @@
-module.exports = {
+export default {
   cipherPassword: 'test',
   dataDir: './data/test',
-  port: 8082,
-  privateDirectoryUrl: 'http://localhost:8080',
   mongoUrl: 'mongodb://localhost:27017/data-fair-processings-test',
   observer: {
     port: 9092
   },
+  port: 8082,
+  privateDirectoryUrl: 'http://localhost:8080',
+  privateEventsUrl: 'http://localhost:8084',
+  secretKeys: {
+    identities: 'secret-identities',
+    events: 'secret-events'
+  },
   serveUi: false
 }

api/config/type/schema.json

@@ -54,11 +54,23 @@
       "type": "string",
       "pattern": "^https?://"
     },
+    "privateEventsUrl": {
+      "type": "string"
+    },
     "secretKeys": {
       "type": "object",
       "properties": {
         "limits": {
-          "type": ["string", "null"]
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "events": {
+          "type": "string"
+        },
+        "identities": {
+          "type": "string"
         }
       }
     },

api/package.json

@@ -3,7 +3,7 @@
   "main": "index.ts",
   "type": "module",
   "scripts": {
-    "dev": "NODE_ENV=development DEBUG=upgrade node --watch --experimental-strip-types index.ts"
+    "dev": "NODE_ENV=development DEBUG=upgrade node --watch index.ts"
   },
   "imports": {
     "#config": "./src/config.ts",
@@ -16,24 +16,26 @@
   "devDependencies": {
     "@types/fs-extra": "^11.0.4",
     "@types/memoizee": "^0.4.11",
+    "@types/multer": "^1.4.13",
     "@types/resolve-path": "^1.4.3",
     "@types/semver": "^7.5.8",
     "@types/ws": "^8.5.12"
   },
   "dependencies": {
+    "@data-fair/lib-express": "^1.20.4",
+    "@data-fair/lib-node": "^2.8.1",
+    "@data-fair/lib-utils": "^1.6.1",
     "@data-fair/processings-shared": "*",
-    "@data-fair/lib-express": "^1.12.6",
-    "@data-fair/lib-node": "^2.2.3",
-    "@data-fair/lib-utils": "^1.3.0",
     "ajv": "^8.17.1",
     "ajv-formats": "^3.0.1",
-    "config": "^3.3.12",
+    "config": "^4.0.0",
     "crypto-random-string": "^5.0.0",
     "express": "^5.0.1",
     "fs-extra": "^11.2.0",
     "http-terminator": "^3.2.0",
     "memoizee": "^0.4.17",
     "mongodb": "^6.10.0",
+    "multer": "^2.0.1",
     "nanoid": "^5.0.8",
     "resolve-path": "^1.4.0",
     "semver": "^7.6.3",

api/src/app.ts

@@ -1,6 +1,7 @@
 import { resolve } from 'node:path'
 import express from 'express'
 import { session, errorHandler, createSiteMiddleware, createSpaMiddleware } from '@data-fair/lib-express/index.js'
+import identitiesRouter from './routers/identities.ts'
 import limitsRouter from './routers/limits.ts'
 import pluginsRegistryRouter from './routers/plugins-registry.ts'
 import pluginsRouter from './routers/plugins.ts'
@@ -22,6 +23,7 @@ app.set('json spaces', 2)
 app.use(express.json())
 app.use(express.urlencoded({ extended: true }))
 
+app.use('/api/identities', identitiesRouter)
 app.use('/api/v1/plugins-registry', pluginsRegistryRouter)
 app.use('/api/v1/plugins', pluginsRouter)
 app.use('/api/v1/processings', processingsRouter)

api/src/config.ts

@@ -1,5 +1,4 @@
-import type { ApiConfig } from '../config/type/index.ts'
-import { assertValid } from '../config/type/index.ts'
+import { assertValid, type ApiConfig } from '../config/type/index.ts'
 import config from 'config'
 
 // we reload the config instead of using the singleton from the config module for testing purposes

api/src/routers/identities.ts

@@ -0,0 +1,65 @@
+// Webhooks for Simple Directory
+
+import Debug from 'debug'
+import { createIdentitiesRouter } from '@data-fair/lib-express/identities/index.js'
+import config from '#config'
+import mongo from '#mongo'
+
+const debug = Debug('webhooks-simple-directory')
+
+/** Helper function to update all collections */
+const updateAllCollections = async (filter: any, update: any) => {
+  await Promise.all([
+    mongo.processings.updateMany(filter, update),
+    mongo.runs.updateMany(filter, update)
+  ])
+}
+
+export default createIdentitiesRouter(
+  config.secretKeys.identities,
+  // onUpdate
+  async (identity) => {
+    debug('Incoming sd webhook for update', identity)
+
+    // Update owner name
+    await updateAllCollections(
+      { 'owner.type': identity.type, 'owner.id': identity.id },
+      { $set: { 'owner.name': identity.name } }
+    )
+
+    if (identity.type === 'user') {
+      // Update created/updated name
+      await Promise.all([
+        updateAllCollections(
+          { 'created.id': identity.id },
+          { $set: { 'created.name': identity.name } }
+        ),
+        updateAllCollections(
+          { 'updated.id': identity.id },
+          { $set: { 'updated.name': identity.name } }
+        )
+      ])
+    }
+
+    // If the identity has departments, update the department names in processings and runs
+    if (identity.departments) {
+      const departmentUpdates = identity.departments.map(department =>
+        updateAllCollections(
+          { 'owner.type': identity.type, 'owner.id': identity.id, 'owner.department': department.id },
+          { $set: { 'owner.departmentName': department.name } }
+        )
+      )
+      await Promise.all(departmentUpdates)
+    }
+  },
+
+  // onDelete
+  async (identity) => {
+    debug('Incoming sd webhook for delete', identity)
+    // Delete all processings and runs for this identity
+    const filter = { 'owner.type': identity.type, 'owner.id': identity.id }
+    await mongo.processings.deleteMany(filter)
+    await mongo.runs.deleteMany(filter)
+    // When departments are deleted, do nothing
+  }
+)