diff --git a/.gitignore b/.gitignore index 53607c0a5..9e95cf440 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,7 @@ dataEval.json # Ignore generated credentials from google-github-actions/auth gha-creds-*.json + +# Matching Service (models and dbs) +./src/competence-matcher/src/db/dbs/ +./src/competence-matcher/src/models/ diff --git a/src/competence-matcher/.env b/src/competence-matcher/.env new file mode 100644 index 000000000..e69de29bb diff --git a/src/competence-matcher/openAPI.json b/src/competence-matcher/openAPI.json new file mode 100644 index 000000000..dc04d3fad --- /dev/null +++ b/src/competence-matcher/openAPI.json @@ -0,0 +1,720 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Matching Server API", + "version": "0.1.0", + "description": "API for managing resource competence lists and matching tasks to resources." + }, + "servers": [ + { + "url": "http://localhost:8501" + } + ], + "paths": { + "/": { + "get": { + "tags": [ + "Default" + ], + "summary": "Welcome endpoint", + "responses": { + "200": { + "description": "Hello World!", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/resource-competence-list": { + "get": { + "tags": [ + "Resources" + ], + "summary": "Get all resource lists (only if multipleDBs=true)", + "responses": { + "200": { + "description": "Array of resource list IDs", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/resource-competence-list/jobs": { + "post": { + "tags": [ + "Resources" + ], + "summary": "Create a resource competence list job", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ResourceInput" + } + } + } + } + }, + "responses": { + "202": { + "description": "Job accepted", + "headers": { + "Location": { + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + } + } + } + }, + "/resource-competence-list/jobs/{jobId}": { + "get": { + "tags": [ + "Resources" + ], + "summary": "Get status of a resource creation job", + "parameters": [ + { + "name": "jobId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "202": { + "description": "Job pending/preprocessing/running", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + }, + "201": { + "description": "Completed with competenceListId", + "headers": { + "Location": { + "schema": { + "type": "string" + }, + "description": "Complete Path to created Competence-List" + } + }, + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/JobResponse" + }, + { + "type": "object", + "properties": { + "competenceListId": { + "type": "string" + } + }, + "required": [ + "competenceListId" + ] + } + ] + } + } + } + }, + "500": { + "description": "Job failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + } + } + } + }, + "/resource-competence-list/{competenceListId}": { + "get": { + "tags": [ + "Resources" + ], + "summary": "Get a specific resource competence list", + "parameters": [ + { + "name": "competenceListId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Resource list details", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResourceList" + } + } + } + }, + "400": { + "description": "Missing resourceListId" + }, + "404": { + "description": "Not found" + } + } + } + }, + "/matching-task-to-resource/jobs": { + "post": { + "tags": [ + "Matching" + ], + "summary": "Create a matching job", + "description": "Start a matching job by providing either an existing competenceListId or an inline competenceList.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/MatchByListIdRequest" + }, + { + "$ref": "#/components/schemas/MatchByListRequest" + } + ] + }, + "examples": { + "Match existing list": { + "summary": "Use an existing list ID", + "value": { + "competenceListId": "123e4567-e89b-12d3-a456-426614174000", + "tasks": [ + { + "taskId": "task1", + "name": "Task 1", + "description": "This is what we currently use for matching", + "executionInstructions": "Some execution instructions", + "requiredCompetencies": [ + { + "competenceId": "comp1", + "name": "Competence 1", + "description": "Description of competence 1", + "externalQualificationNeeded": false, + "renewTime": 30, + "proficiencyLevel": "advanced", + "qualificationDates": [ + "2025-07-01" + ], + "lastUsages": [ + "2025-07-10T14:30:00Z" + ] + } + ] + } + ] + } + }, + "Match inline list": { + "summary": "Pass a full competenceList inline", + "value": { + "competenceList": [ + { + "resourceId": "string", + "competencies": [ + { + "competenceId": "string", + "name": "string", + "description": "string", + "externalQualificationNeeded": true, + "renewTime": 0, + "proficiencyLevel": "string", + "qualificationDates": [ + "2025-07-15" + ], + "lastUsages": [ + "2025-07-15T10:37:09.695Z" + ] + } + ] + } + ], + "tasks": [ + { + "taskId": "task1", + "name": "Task 1", + "description": "This is what we currently use for matching", + "executionInstructions": "Some execution instructions", + "requiredCompetencies": [ + { + "competenceId": "comp1", + "name": "Competence 1", + "description": "Description of competence 1", + "externalQualificationNeeded": false, + "renewTime": 30, + "proficiencyLevel": "advanced", + "qualificationDates": [ + "2025-07-01" + ], + "lastUsages": [ + "2025-07-10T14:30:00Z" + ] + } + ] + } + ] + } + } + } + } + } + }, + "responses": { + "202": { + "description": "Match job accepted", + "headers": { + "Location": { + "schema": { + "type": "string" + }, + "description": "Complete Path to created Competence-List" + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + } + } + } + }, + "/matching-task-to-resource/jobs/{jobId}": { + "get": { + "tags": [ + "Matching" + ], + "summary": "Get match job results", + "parameters": [ + { + "name": "jobId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "rankBy", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "avgFit", + "bestFit" + ] + }, + "description": "Optional ranking method: 'avgFit' or 'bestFit'" + } + ], + "responses": { + "202": { + "description": "Job pending/running", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + }, + "200": { + "description": "Match results", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GroupedMatchResults" + } + } + } + }, + "404": { + "description": "Job not found" + }, + "500": { + "description": "Job failed" + } + } + } + } + }, + "components": { + "schemas": { + "ResourceInput": { + "type": "object", + "required": [ + "resourceId", + "competencies" + ], + "properties": { + "resourceId": { + "type": "string" + }, + "competencies": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CompetenceInput" + } + } + } + }, + "CompetenceInput": { + "type": "object", + "required": [ + "competenceId" + ], + "properties": { + "competenceId": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "externalQualificationNeeded": { + "type": "boolean" + }, + "renewTime": { + "type": "number" + }, + "proficiencyLevel": { + "type": "string" + }, + "qualificationDates": { + "type": "array", + "items": { + "type": "string", + "format": "date" + } + }, + "lastUsages": { + "type": "array", + "items": { + "type": "string", + "format": "date-time" + } + } + } + }, + "ResourceList": { + "type": "object", + "required": [ + "competenceListId", + "resources" + ], + "properties": { + "competenceListId": { + "type": "string" + }, + "resources": { + "type": "array", + "items": { + "type": "object", + "required": [ + "resourceId", + "competencies" + ], + "properties": { + "resourceId": { + "type": "string" + }, + "competencies": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CompetenceInput" + } + } + } + } + } + } + }, + "JobResponse": { + "type": "object", + "required": [ + "jobId", + "status" + ], + "properties": { + "jobId": { + "type": "string" + }, + "status": { + "type": "string" + } + } + }, + "MatchByListIdRequest": { + "type": "object", + "required": [ + "competenceListId", + "tasks" + ], + "properties": { + "competenceListId": { + "type": "string" + }, + "tasks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MatchingTask" + } + } + } + }, + "MatchByListRequest": { + "type": "object", + "required": [ + "competenceList", + "tasks" + ], + "properties": { + "competenceList": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ResourceInput" + } + }, + "tasks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MatchingTask" + } + } + } + }, + "MatchingTask": { + "type": "object", + "required": [ + "taskId" + ], + "properties": { + "taskId": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "executionInstructions": { + "type": "string" + }, + "requiredCompetencies": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/components/schemas/CompetenceInput" + } + ] + } + } + } + }, + "GroupedMatchResults": { + "type": "object", + "required": [ + "taskOverview", + "resourceRanking" + ], + "properties": { + "tasks": { + "$ref": "#/components/schemas/TaskOverview" + }, + "resourceRanking": { + "$ref": "#/components/schemas/ResourceRanking" + } + } + }, + "TaskOverview": { + "type": "array", + "items": { + "type": "object", + "required": [ + "taskId", + "taskText" + ], + "properties": { + "taskId": { + "type": "string" + }, + "taskText": { + "type": "string" + } + } + } + }, + "ResourceRanking": { + "type": "array", + "items": { + "type": "object", + "required": [ + "resourceId", + "taskMatchings", + "avgTaskMatchProbability" + ], + "properties": { + "resourceId": { + "type": "string" + }, + "taskMatchings": { + "type": "array", + "items": { + "type": "object", + "required": [ + "taskId", + "taskText", + "competenceMatchings", + "maxMatchProbability" + ], + "properties": { + "taskId": { + "type": "string" + }, + "competenceMatchings": { + "type": "array", + "items": { + "type": "object", + "required": [ + "competenceId", + "matchings", + "avgMatchProbability" + ], + "properties": { + "competenceId": { + "type": "string" + }, + "matchings": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MatchDetail" + } + }, + "avgMatchProbability": { + "type": "number" + }, + "avgBestFitTaskMatchProbability": { + "type": "number" + } + } + } + }, + "maxMatchProbability": { + "type": "number" + }, + "maxBestFitMatchProbability": { + "type": "number" + } + } + } + }, + "avgTaskMatchProbability": { + "type": "number" + }, + "avgBestFitTaskMatchProbability": { + "type": "number" + }, + "contradicting": { + "type": "boolean" + } + } + } + }, + "MatchDetail": { + "type": "object", + "required": [ + "text", + "type", + "matchProbability" + ], + "properties": { + "text": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "name", + "description", + "proficiencyLevel" + ] + }, + "matchProbability": { + "type": "number" + }, + "alignment": { + "type": "string", + "enum": [ + "aligning", + "neutral", + "contradicting" + ] + }, + "reason": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/src/competence-matcher/package.json b/src/competence-matcher/package.json new file mode 100644 index 000000000..032b82a8d --- /dev/null +++ b/src/competence-matcher/package.json @@ -0,0 +1,40 @@ +{ + "name": "competence-matcher", + "version": "0.1.0", + "description": "Matching microservice that allows to allows to define and match on data criteria", + "main": "dist/server.js", + "scripts": { + "dev": "ts-node-dev --respawn --transpile-only src/server.ts", + "build": "tsc", + "run-production": "node dist/server.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/PROCEED-Labs/proceed.git" + }, + "keywords": [ + "embedding", + "matching" + ], + "author": "PROCEED Project", + "license": "MIT", + "bugs": { + "url": "https://github.com/PROCEED-Labs/proceed/issues" + }, + "homepage": "https://github.com/PROCEED-Labs/proceed#readme", + "dependencies": { + "@huggingface/transformers": "^3.5.2", + "express": "^5.1.0", + "ollama": "^0.5.16", + "sqlite-vec": "^0.1.7-alpha.2" + }, + "devDependencies": { + "@types/express": "^5.0.2", + "@types/node": "^22.15.30", + "ts-node-dev": "^2.0.0", + "typescript": "^5.8.3" + }, + "engines": { + "node": ">=23.5.0" + } +} \ No newline at end of file diff --git a/src/competence-matcher/src/config.ts b/src/competence-matcher/src/config.ts new file mode 100644 index 000000000..02a8a7001 --- /dev/null +++ b/src/competence-matcher/src/config.ts @@ -0,0 +1,19 @@ +import * as os from 'node:os'; + +export const config = { + dbPath: process.env.DB_PATH || 'src/db/dbs/', + embeddingModel: process.env.EMBEDDING_MODEL || 'onnx-community/Qwen3-Embedding-0.6B-ONNX', + embeddingDim: parseInt(process.env.EMBEDDING_DIM || '1024', 10), + nliModel: process.env.NLI_MODEL || './src/models/roberta_mnli_onnx', + modelCache: process.env.MODEL_CACHE || 'src/models/', + useGPU: process.env.USE_GPU === 'true' || false, + port: parseInt(process.env.PORT || '8501', 10), + multipleDBs: process.env.MULTIPLE_DBS === 'true' || false, + ollamaPath: process.env.OLLAMA_PATH || 'http://localhost:11434', + ollamaBatchSize: parseInt(process.env.OLLAMA_BATCH_SIZE || '5', 10), + splittingModel: process.env.SPLITTING_MODEL || 'llama3.2', + reasonModel: process.env.REASON_MODEL || 'llama3.2', + splittingSymbol: process.env.SPLITTING_SYMBOL || 'SPLITTING_SYMBOL', + maxWorkerThreads: parseInt(process.env.NUMBER_OF_THREADS || String(os.cpus().length - 1), 10), // -1 for main thread + maxJobTime: parseInt(process.env.MAX_JOB_TIME || '600', 10) * 1_000, // converted from seconds to milliseconds +}; diff --git a/src/competence-matcher/src/db/db-manager.ts b/src/competence-matcher/src/db/db-manager.ts new file mode 100644 index 000000000..d5ab99d4f --- /dev/null +++ b/src/competence-matcher/src/db/db-manager.ts @@ -0,0 +1,162 @@ +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import VectorDataBase from './db'; +import { config } from '../config'; + +const { dbPath: rawDbPath, embeddingDim } = config; + +/** + * DBManager: Singleton that manages multiple VectorDataBase instances keyed by name. + * + */ +class DBManager { + private static managerInstance: DBManager; + private dbInstances = new Map(); + private static activeDB: VectorDataBase | null = null; + private dbPath: string; + private embeddingDim: number; + + private constructor() { + this.embeddingDim = embeddingDim; + + // Resolve absolute path for storage directory + this.dbPath = path.resolve(rawDbPath); + // Ensure directory exists + if (!fs.existsSync(this.dbPath)) { + fs.mkdirSync(this.dbPath, { recursive: true }); + } + // Load existing databases + this.loadSavedDBs(); + } + + /** + * Retrieve the singleton DBManager instance, initialising if necessary. + * @returns DBManager singleton + */ + public static getInstance(): DBManager { + if (!DBManager.managerInstance) { + DBManager.managerInstance = new DBManager(); + } + return DBManager.managerInstance; + } + + /** + * Initialise the DBManager and load any existing databases. + */ + private loadSavedDBs(): void { + // Load existing databases from the storage directory + const files = fs.readdirSync(this.dbPath); + files.forEach((file) => { + if (file.endsWith('.db')) { + const dbName = path.basename(file, '.db'); + this.addDBInstance(dbName); + } + }); + } + + /** + * Normalise a database name by stripping any extension and enforcing `.db`. + * @param dbName Name provided; may include extension. + * @returns Normalised filename ending with `.db`. + */ + private normaliseDBName(dbName: string): string { + const base = path.basename(dbName, path.extname(dbName)); + return `${base}.db`; + } + + /** + * Resolve full absolute path to the DB file under storage directory. + * @param dbName Name provided by user; normalised to `.db` and joined with storage dir. + * @returns Absolute file path for the database. + */ + private resolveDbPath(dbName: string): string { + const normalisedDBName = this.normaliseDBName(dbName); + return path.join(this.dbPath, normalisedDBName); + } + + /** + * Internal: create and cache a new VectorDataBase instance for the given name. + * Uses resolveDbPath to obtain the absolute file path. + * @param dbName Name provided by user; normalised internally. + * @returns Newly created VectorDataBase instance. + */ + private addDBInstance(dbName: string): VectorDataBase { + const normalisedDBName = this.normaliseDBName(dbName); + const filePath = this.resolveDbPath(normalisedDBName); + const db = new VectorDataBase({ filePath, embeddingDim: this.embeddingDim }); + this.dbInstances.set(normalisedDBName, db); + return db; + } + + /** + * Get the currently active VectorDataBase instance, if set via setActiveDB. + * @returns Active VectorDataBase or null if none is set. + */ + public static getActiveDB(): VectorDataBase | null { + return DBManager.activeDB; + } + + /** + * Set the active database by name. Creates the instance if it does not exist. + * @param dbName Name of the database (without extension or with any extension). + */ + public static setActiveDB(dbName: string): void { + const manager = DBManager.getInstance(); + const normalisedDBName = manager.normaliseDBName(dbName); + if (!manager.dbInstances.has(normalisedDBName)) { + manager.addDBInstance(normalisedDBName); + } + DBManager.activeDB = manager.dbInstances.get(normalisedDBName)!; + } + + /** + * Retrieve (or create) the VectorDataBase instance for given name. + * @param dbName Name of the database (without extension or with any extension). + * @returns VectorDataBase instance corresponding to the name. + */ + public getDB(dbName: string): VectorDataBase { + const normalisedDBName = this.normaliseDBName(dbName); + if (this.dbInstances.has(normalisedDBName)) { + return this.dbInstances.get(normalisedDBName)!; + } + return this.addDBInstance(normalisedDBName); + } + + /** + * Close and remove the VectorDataBase instance for given name. + * @param dbName Name of the database to close. + * @returns True if instance existed and was closed; false otherwise. + */ + public closeDB(dbName: string): boolean { + const normalisedDBName = this.normaliseDBName(dbName); + const db = this.dbInstances.get(normalisedDBName); + if (db) { + db.close(); + this.dbInstances.delete(normalisedDBName); + if (DBManager.activeDB === db) { + DBManager.activeDB = null; + } + return true; + } + return false; + } + + /** + * Close and remove all managed VectorDataBase instances. + */ + public closeAllDBs(): void { + this.dbInstances.forEach((db) => db.close()); + this.dbInstances.clear(); + DBManager.activeDB = null; + } + + /** + * List the names (normalised) of all managed databases. + * @returns Array of database filenames (e.g. ['tenant1.db', 'other.db']). + */ + public listDBs(): string[] { + return Array.from(this.dbInstances.keys()); + } +} + +export default DBManager; diff --git a/src/competence-matcher/src/db/db.ts b/src/competence-matcher/src/db/db.ts new file mode 100644 index 000000000..c92974a24 --- /dev/null +++ b/src/competence-matcher/src/db/db.ts @@ -0,0 +1,868 @@ +// db.ts +import { DatabaseSync } from 'node:sqlite'; +import * as path from 'node:path'; +import * as sqliteVec from 'sqlite-vec'; +import { v4 as uuid } from 'uuid'; +import { CompetenceDBOutput, VectorDBOptions } from '../utils/types'; + +class VectorDataBase { + private db: DatabaseSync; + private embeddingDim: number; + private transactionInProgress = false; + + /** + * Opens (or creates) the SQLite DB, enables FKs, loads sqlite-vec, and sets up schema. + */ + constructor(opts: VectorDBOptions) { + this.embeddingDim = opts.embeddingDim; + const dbPath = + !opts.filePath || opts.filePath === ':memory:' + ? ':memory:' + : path.isAbsolute(opts.filePath) + ? opts.filePath + : path.join(process.cwd(), opts.filePath); + + this.db = new DatabaseSync(dbPath, { allowExtension: true }); + this.db.exec(`PRAGMA foreign_keys = ON;`); + sqliteVec.load(this.db); + this.initSchema(); + } + + /** Close the database connection */ + public close(): void { + this.db.close(); + } + + /** Run a set of operations atomically (in a transaction) */ + public atomicStep(cb: () => void): void { + if (this.transactionInProgress) throw new Error('Transaction already in progress'); + this.transactionInProgress = true; + this.db.exec('BEGIN'); + try { + cb(); + this.db.exec('COMMIT'); + } catch (e) { + this.db.exec('ROLLBACK'); + throw e; + } finally { + this.transactionInProgress = false; + } + } + + /** Set up all tables, indexes and virtual tables */ + private initSchema() { + // jobs + this.db.exec(` + CREATE TABLE IF NOT EXISTS jobs ( + id TEXT PRIMARY KEY, + status TEXT NOT NULL DEFAULT 'pending', + reference_id TEXT + ); + `); + + // matches + this.db.exec(` + CREATE TABLE IF NOT EXISTS match_results ( + id TEXT PRIMARY KEY, -- UUID for this match record + job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, + task_id TEXT NOT NULL, -- task ID this match belongs to, + task_text TEXT NOT NULL, -- task text that was used for matching + competence_id TEXT NOT NULL, -- matched competence + resource_id TEXT NOT NULL, -- resource ID the competence belongs to + distance REAL NOT NULL, -- similarity score + text TEXT NOT NULL, -- the matched snippet + type TEXT NOT NULL, -- 'name' | 'description' | 'proficiencyLevel' + alignment TEXT NOT NULL, -- 'contradicting' | 'neutral' | 'aligning' + reason TEXT -- llm based reason for the match + ); + `); + this.db.exec(` + CREATE INDEX IF NOT EXISTS ix_match_results_job + ON match_results(job_id); + `); + + // resource_list + this.db.exec(` + CREATE TABLE IF NOT EXISTS resource_list ( + id TEXT PRIMARY KEY + ); + `); + + // resources (internal PK + user‐facing ID) + this.db.exec(` + CREATE TABLE IF NOT EXISTS resource ( + _rid INTEGER PRIMARY KEY AUTOINCREMENT, + resource_id TEXT NOT NULL, + list_id TEXT NOT NULL REFERENCES resource_list(id) ON DELETE CASCADE + ); + `); + this.db.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS ux_resource_list_resid + ON resource(list_id, resource_id); + `); + + // competences (internal PK + user‐facing ID) + this.db.exec(` + CREATE TABLE IF NOT EXISTS competence ( + _cid INTEGER PRIMARY KEY AUTOINCREMENT, + competence_id TEXT NOT NULL, + resource_rid INTEGER NOT NULL REFERENCES resource(_rid) ON DELETE CASCADE, + competence_name TEXT, + competence_description TEXT, + external_qualification_needed BOOLEAN DEFAULT FALSE, + renew_time INTEGER, + proficiency_level TEXT, + qualification_dates TEXT, + last_usages TEXT + ); + `); + this.db.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS ux_competence_rescid + ON competence(resource_rid, competence_id); + `); + + // embeddings (virtual vec0 table; explicit deletes required) + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS competence_embedding + USING vec0( + cid INTEGER NOT NULL REFERENCES competence(_cid) ON DELETE CASCADE, + text TEXT, + type TEXT, + embedding FLOAT32[${this.embeddingDim}] + ); + `); + } + + /*-------------------------------------------------------------------- + * Helper Lookups + *------------------------------------------------------------------*/ + + /** Get the internal `_rid` for a given user‐facing resourceId + listId */ + private getResourceRid(resourceId: string, listId: string): number { + const row = this.db + .prepare(`SELECT _rid FROM resource WHERE resource_id = ? AND list_id = ?`) + .get(resourceId, listId); + if (!row) throw new Error(`Resource '${resourceId}' not found in list '${listId}'`); + return row._rid as number; + } + + /** Get the internal `_cid` for a given user‐facing competenceId */ + private getCompetenceCidByCompetenceId( + listId: string, + resourceId: string, + competenceId: string, + ): number { + const _rid = this.getResourceRid(resourceId, listId); + const row = this.db + .prepare( + ` + SELECT _cid + FROM competence + WHERE competence_id = ? + AND resource_rid = ? + `, + ) + .get(competenceId, _rid); + if (!row) + throw new Error( + `Competence '${competenceId}' in List ${listId} in Resource ${resourceId} not found`, + ); + return row._cid as number; + } + + /*-------------------------------------------------------------------- + * Job Methods + *------------------------------------------------------------------*/ + + /** + * Create a new background‐job record. + * @param referenceId Optionally point back to a resource-list, resource, or competence ID. + * @returns the new job’s UUID. + */ + public createJob(referenceId?: string): string { + const jobId = uuid(); + this.db + .prepare(`INSERT INTO jobs(id, reference_id) VALUES (?, ?)`) + .run(jobId, referenceId ?? null); + return jobId; + } + + /** + * Change a job’s status. + * @throws if no such job exists. + */ + public updateJobStatus( + jobId: string, + status: 'pending' | 'preprocessing' | 'running' | 'completed' | 'failed', + ): void { + const result = this.db.prepare(`UPDATE jobs SET status = ? WHERE id = ?`).run(status, jobId); + if (result.changes === 0) throw new Error(`Job with id ${jobId} not found`); + } + + /** + * Look up a job’s current status and its referenceId. + * @throws if no such job exists. + */ + public getJob(jobId: string): { jobId: string; status: string; referenceId?: string } { + const row = this.db + .prepare(`SELECT id, status, reference_id FROM jobs WHERE id = ?`) + .get(jobId) as { id: string; status: string; reference_id: string } | undefined; + if (!row) throw new Error(`Job with id ${jobId} not found`); + return { jobId: row.id, status: row.status, referenceId: row.reference_id ?? undefined }; + } + + /*-------------------------------------------------------------------- + * Match Methods + *------------------------------------------------------------------*/ + + /** + * Add a match result for a job, task, and competence. + * + * @param opts Options for adding a match result. + * @throws if the jobId, taskId, or competenceId do not exist. + */ + public addMatchResult(opts: { + jobId: string; + taskId: string; + taskText: string; + competenceId: string; + resourceId: string; + distance: number; + text: string; + type: string; // 'name' | 'description' | 'proficiencyLevel' + alignment: string; // 'contradicting' | 'neutral' | 'aligning' + reason?: string; // optional reason for the match + }): void { + const id = uuid(); + this.db + .prepare( + ` + INSERT INTO match_results + (id, job_id, task_id, task_text, competence_id, resource_id, distance, text, type, alignment, reason) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ) + .run( + id, + opts.jobId, + opts.taskId, + opts.taskText, + opts.competenceId, + opts.resourceId, + opts.distance, + opts.text, + opts.type, + opts.alignment, + opts.reason ?? null, + ); + } + + /** + * Fetch all match results for a given jobId. + * @returns an array of match results, sorted by taskId and distance. + */ + public getMatchResults(jobId: string): Array<{ + taskId: string; + taskText: string; + competenceId: string; + resourceId: string; + distance: number; + text: string; + type: string; + alignment: string; // 'contradicting' | 'neutral' | 'aligning' + reason?: string; + }> { + return this.db + .prepare( + ` + SELECT task_id, task_text, competence_id, resource_id, distance, text, type, alignment, reason + FROM match_results + WHERE job_id = ? + ORDER BY task_id, distance + `, + ) + .all(jobId) + .map((r: any) => ({ + taskId: r.task_id, + taskText: r.task_text, + competenceId: r.competence_id, + resourceId: r.resource_id, + distance: r.distance, + text: r.text, + type: r.type, + alignment: r.alignment, + reason: r.reason ?? undefined, + })); + } + + /*-------------------------------------------------------------------- + * ResourceList Methods + *------------------------------------------------------------------*/ + + /** Create a fresh, empty resource‐list and return its UUID */ + public createResourceList(): string { + const listId = uuid(); + this.db.prepare(`INSERT INTO resource_list(id) VALUES (?)`).run(listId); + return listId; + } + + /** + * Delete an entire list—this cascades down to resources, competences, + * explicitly wipes embeddings. + */ + public deleteResourceList(listId: string): void { + this.atomicStep(() => { + this.db + .prepare( + ` + DELETE FROM competence_embedding + WHERE cid IN ( + SELECT c._cid + FROM competence c + JOIN resource r ON c.resource_rid = r._rid + WHERE r.list_id = ? + ) + `, + ) + .run(listId); + this.db.prepare(`DELETE FROM resource_list WHERE id = ?`).run(listId); + }); + } + + /** Enumerate all list IDs */ + public getAvailableResourceLists(): string[] { + return this.db + .prepare(`SELECT id FROM resource_list`) + .all() + .map((r) => (r as any).id); + } + + /** + * Fetch a list plus all its resources and each resource’s competences. + * @throws if listId doesn’t exist. + */ + public getResourceList(listId: string): { + competenceListId: string; + resources: Array<{ + resourceId: string; + competencies: Array<{ + competenceId: string; + name?: string; + description?: string; + externalQualificationNeeded: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates: string[]; + lastUsages: string[]; + }>; + }>; + } { + const exists = this.db.prepare(`SELECT 1 FROM resource_list WHERE id = ?`).get(listId); + if (!exists) throw new Error(`Resource list '${listId}' not found`); + + const resources = this.db + .prepare(`SELECT _rid, resource_id FROM resource WHERE list_id = ?`) + .all(listId) as Array<{ _rid: number; resource_id: string }>; + + return { + competenceListId: listId, + resources: resources.map(({ _rid, resource_id }) => { + const comps = this.db + .prepare( + ` + SELECT competence_id, competence_name, competence_description, + external_qualification_needed, renew_time, + proficiency_level, qualification_dates, last_usages + FROM competence + WHERE resource_rid = ? + `, + ) + .all(_rid) as CompetenceDBOutput[]; + return { + resourceId: resource_id, + competencies: comps.map((c) => ({ + competenceId: c.competence_id, + name: c.competence_name ?? undefined, + description: c.competence_description ?? undefined, + externalQualificationNeeded: Boolean(c.external_qualification_needed), + renewTime: c.renew_time ?? undefined, + proficiencyLevel: c.proficiency_level ?? undefined, + qualificationDates: c.qualification_dates ? JSON.parse(c.qualification_dates) : [], + lastUsages: c.last_usages ? JSON.parse(c.last_usages) : [], + })), + }; + }), + }; + } + + /*-------------------------------------------------------------------- + * Resource Methods + *------------------------------------------------------------------*/ + + /** + * Add a resource (user‐facing ID) into a list. + * Returns the user‐facing resourceId. + */ + public addResource(listId: string, resourceId?: string): string { + const rid = resourceId ?? uuid(); + this.db.prepare(`INSERT INTO resource(resource_id, list_id) VALUES (?, ?)`).run(rid, listId); + return rid; + } + + /** + * Move a resource from one list to another. + * @param oldListId current list + * @param resourceId user‐facing ID + * @param newListId target list + */ + public updateResource(oldListId: string, resourceId: string, newListId: string): void { + const _rid = this.getResourceRid(resourceId, oldListId); + this.db.prepare(`UPDATE resource SET list_id = ? WHERE _rid = ?`).run(newListId, _rid); + } + + /** + * Delete a single resource (and its subtree) by user‐facing ID + list. + */ + public deleteResource(listId: string, resourceId: string): void { + this.atomicStep(() => { + const _rid = this.getResourceRid(resourceId, listId); + this.db + .prepare( + ` + DELETE FROM competence_embedding + WHERE cid IN (SELECT _cid FROM competence WHERE resource_rid = ?) + `, + ) + .run(_rid); + this.db.prepare(`DELETE FROM resource WHERE _rid = ?`).run(_rid); + }); + } + + /** + * Fetch one resource + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing ID of the resource + * @returns an object with the resource’s metadata. + * @throws if not found. + */ + public getResource( + listId: string, + resourceId: string, + ): { + listId: string; + resourceId: string; + competencies: Array<{ + competenceId: string; + name?: string; + description?: string; + externalQualificationNeeded: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates: string[]; + lastUsages: string[]; + }>; + } { + const row = this.db + .prepare( + ` + SELECT _rid, resource_id, list_id + FROM resource + WHERE resource_id = ? + AND list_id = ? + `, + ) + .get(resourceId, listId) as + | { _rid: number; resource_id: string; list_id: string } + | undefined; + if (!row) throw new Error(`Resource '${resourceId}' in List '${listId}' not found`); + + const comps = this.db + .prepare( + ` + SELECT competence_id, competence_name, competence_description, + external_qualification_needed, renew_time, + proficiency_level, qualification_dates, last_usages + FROM competence + WHERE resource_rid = ? + `, + ) + .all(row._rid) as CompetenceDBOutput[]; + + return { + listId: row.list_id, + resourceId, + competencies: comps.map((c) => ({ + competenceId: c.competence_id, + name: c.competence_name ?? undefined, + description: c.competence_description ?? undefined, + externalQualificationNeeded: Boolean(c.external_qualification_needed), + renewTime: c.renew_time ?? undefined, + proficiencyLevel: c.proficiency_level ?? undefined, + qualificationDates: c.qualification_dates ? JSON.parse(c.qualification_dates) : [], + lastUsages: c.last_usages ? JSON.parse(c.last_usages) : [], + })), + }; + } + + /*-------------------------------------------------------------------- + * Competence Methods + *------------------------------------------------------------------*/ + + /** + * Add a competence under a given resource. + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing + * @param competence Input object (may supply its own competenceId) + * @returns the user‐facing competenceId + */ + public addCompetence( + listId: string, + resourceId: string, + competence: { + competenceId?: string; + name?: string; + description?: string; + externalQualificationNeeded?: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates?: string[]; + lastUsages?: string[]; + }, + ): string { + const cidUser = competence.competenceId ?? uuid(); + const _rid = this.getResourceRid(resourceId, listId); + + this.db + .prepare( + ` + INSERT INTO competence + (competence_id, resource_rid, + competence_name, competence_description, + external_qualification_needed, renew_time, + proficiency_level, qualification_dates, last_usages) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ) + .run( + cidUser, + _rid, + competence.name ?? null, + competence.description ?? null, + competence.externalQualificationNeeded ? 1 : 0, + competence.renewTime ?? null, + competence.proficiencyLevel ?? null, + competence.qualificationDates ? JSON.stringify(competence.qualificationDates) : null, + competence.lastUsages ? JSON.stringify(competence.lastUsages) : null, + ); + + return cidUser; + } + + /** + * Update a competence’s metadata. + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing ID of the resource + * @param competenceId user‐facing ID of the competence to update + * @param fields object with fields to update; only those provided will be changed + * @returns nothing, but throws if the competence does not exist. + * @throws if no such competence exists on the resource. + */ + public updateCompetence( + listId: string, + resourceId: string, + competenceId: string, + fields: { + name?: string; + description?: string; + externalQualificationNeeded?: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates?: string[]; + lastUsages?: string[]; + }, + ): void { + const _rid = this.getResourceRid(resourceId, listId); + const _cid = this.db + .prepare(`SELECT _cid FROM competence WHERE competence_id = ? AND resource_rid = ?`) + .get(competenceId, _rid)?.['_cid']; + if (!_cid) throw new Error(`Competence '${competenceId}' not on resource '${resourceId}'`); + + const sets: string[] = []; + const params: any[] = []; + if (fields.name !== undefined) { + sets.push(`competence_name = ?`); + params.push(fields.name); + } + if (fields.description !== undefined) { + sets.push(`competence_description = ?`); + params.push(fields.description); + } + if (fields.externalQualificationNeeded !== undefined) { + sets.push(`external_qualification_needed = ?`); + params.push(fields.externalQualificationNeeded ? 1 : 0); + } + if (fields.renewTime !== undefined) { + sets.push(`renew_time = ?`); + params.push(fields.renewTime); + } + if (fields.proficiencyLevel !== undefined) { + sets.push(`proficiency_level = ?`); + params.push(fields.proficiencyLevel); + } + if (fields.qualificationDates !== undefined) { + sets.push(`qualification_dates = ?`); + params.push(JSON.stringify(fields.qualificationDates)); + } + if (fields.lastUsages !== undefined) { + sets.push(`last_usages = ?`); + params.push(JSON.stringify(fields.lastUsages)); + } + + if (sets.length > 0) { + params.push(_cid); + this.db.prepare(`UPDATE competence SET ${sets.join(', ')} WHERE _cid = ?`).run(...params); + } + } + + /** + * Delete a competence from a resource. + * @param resourceId user‐facing ID + * @param listId user‐facing ID of the resource list + * @param competenceId user‐facing ID of the competence to delete + * @throws if no such competence exists on the resource. + */ + public deleteCompetence(listId: string, resourceId: string, competenceId: string): void { + this.atomicStep(() => { + const _rid = this.getResourceRid(resourceId, listId); + const _cid = this.db + .prepare(`SELECT _cid FROM competence WHERE competence_id = ? AND resource_rid = ?`) + .get(competenceId, _rid)?._cid; + if (!_cid) throw new Error(`Competence '${competenceId}' not on resource '${resourceId}'`); + + // explicitly delete embeddings + this.db.prepare(`DELETE FROM competence_embedding WHERE cid = ?`).run(_cid); + // then delete competence row + this.db.prepare(`DELETE FROM competence WHERE _cid = ?`).run(_cid); + }); + } + + /** + * Fetch one competence’s metadata (including listId + resourceId). + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing ID of the resource + * @param competenceId user‐facing ID of the competence to fetch + * @returns an object with the competence’s metadata. + * @throws if no such competence exists on the resource. + * @throws if the competenceId is not found on the resource. + */ + public getCompetence( + listId: string, + resourceId: string, + competenceId: string, + ): { + listId: string; + resourceId: string; + competenceId: string; + name?: string; + description?: string; + externalQualificationNeeded: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates: string[]; + lastUsages: string[]; + } { + const _rid = this.getResourceRid(resourceId, listId); + const row = this.db + .prepare( + ` + SELECT c.competence_id, c.competence_name, c.competence_description, + c.external_qualification_needed, c.renew_time, + c.proficiency_level, c.qualification_dates, c.last_usages + FROM competence c + WHERE c.competence_id = ? AND c.resource_rid = ? + `, + ) + .get(competenceId, _rid) as + | { + competence_id: string; + competence_name: string | null; + competence_description: string | null; + external_qualification_needed: number; + renew_time: number | null; + proficiency_level: string | null; + qualification_dates: string | null; + last_usages: string | null; + } + | undefined; + if (!row) throw new Error(`Competence '${competenceId}' not on resource '${resourceId}'`); + + return { + listId, + resourceId, + competenceId: row.competence_id, + name: row.competence_name ?? undefined, + description: row.competence_description ?? undefined, + externalQualificationNeeded: Boolean(row.external_qualification_needed), + renewTime: row.renew_time ?? undefined, + proficiencyLevel: row.proficiency_level ?? undefined, + qualificationDates: row.qualification_dates ? JSON.parse(row.qualification_dates) : [], + lastUsages: row.last_usages ? JSON.parse(row.last_usages) : [], + }; + } + + /*-------------------------------------------------------------------- + * Embedding Methods + *------------------------------------------------------------------*/ + + /** + * Insert or replace a text embedding for a competence. + * This will overwrite any existing embedding for the same text and type. + * @param embeddingInput object with competenceId, text, type, and embedding vector. + * @throws if the embedding vector does not match the configured dimension. + */ + public upsertEmbedding(embeddingInput: { + listId: string; + resourceId: string; + competenceId: string; + text: string; + type: 'name' | 'description' | 'proficiencyLevel'; + embedding: number[]; + }): void { + const { listId, resourceId, competenceId, text, type, embedding } = embeddingInput; + if (embedding.length !== this.embeddingDim) { + throw new Error(`Embedding must have length ${this.embeddingDim}`); + } + const cid = this.getCompetenceCidByCompetenceId(listId, resourceId, competenceId); + + const cidInt = `${Math.floor(cid)}`; // This + the cast is a workaround, sqlite-vec or sqlite read the cid as a float even though it is an integer. (Could be the lib or the fact that it is a virtual table, not sure) + + this.db + .prepare( + ` + INSERT OR REPLACE INTO competence_embedding + (cid, text, type, embedding) + VALUES (CAST(? AS INTEGER), ?, ?, vec_f32(?)) + `, + ) + .run(cidInt, text, type, new Float32Array(embedding)); + } + + /** Delete all embeddings for one competence + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing ID of the resource + * @param competenceId user‐facing ID of the competence + */ + public deleteEmbeddingsForCompetence( + listId: string, + resourceId: string, + competenceId: string, + ): void { + const cid = this.getCompetenceCidByCompetenceId(listId, resourceId, competenceId); + this.db.prepare(`DELETE FROM competence_embedding WHERE cid = ?`).run(cid); + } + + /** + * kNN‐search over embeddings, returning user‐facing competenceIds + distances. + * + * @param embedding the query vector to search for + * @param options optional parameters: + * - k: number of nearest neighbors to return (default: all) + * - filter: optional filter by resourceId and/or listId + * - similarityMetric: 'cosine', 'hamming', or 'euclidean' (default: 'cosine') + * @returns an array of objects with competenceId, text, type, and distance. + * @throws if the embedding length does not match the configured dimension. + * @throws if the metric is unsupported or k is not a positive integer. + */ + public searchEmbedding( + embedding: number[], + options?: { + k?: number; + filter?: { resourceId?: string; listId?: string }; + similarityMetric?: 'cosine' | 'hamming' | 'euclidean'; + }, + ): Array<{ + competenceId: string; + resourceId: string; + text: string; + type: string; + distance: number; + }> { + const { k, filter, similarityMetric } = options || {}; + const metrics = { + cosine: 'vec_distance_cosine', + hamming: 'vec_distance_hamming', + euclidean: 'vec_distance_L2', + }; + const metric = metrics[similarityMetric || 'cosine']; + if (!metric) throw new Error(`Unsupported metric: ${similarityMetric}`); + if (embedding.length !== this.embeddingDim) throw new Error(`Embedding length mismatch`); + if (k !== undefined && k <= 0) throw new Error('k must be > 0'); + + let sql = ` + SELECT c.competence_id, r.resource_id, ce.text, ce.type, + ${metric}(ce.embedding, vec_f32(?)) AS distance + FROM competence_embedding ce + JOIN competence c ON ce.cid = c._cid + JOIN resource r ON c.resource_rid = r._rid + `; + const params: any[] = [new Float32Array(embedding)]; + + const whereClauses: string[] = []; + if (filter?.resourceId) { + whereClauses.push(`r.resource_id = ?`); + params.push(filter.resourceId); + } + if (filter?.listId) { + whereClauses.push(`r.list_id = ?`); + params.push(filter.listId); + } + if (whereClauses.length > 0) { + sql += ` WHERE ` + whereClauses.join(' AND '); + } + // sql += ` GROUP BY c.competence_id, ce.type`; + sql += ` ORDER BY distance ASC`; + + if (k) { + sql += ` LIMIT ?`; + params.push(k); + } + + const rows = this.db.prepare(sql).all(...params) as Array; + + let result = rows.map((r) => ({ + competenceId: r.competence_id, + resourceId: r.resource_id, + distance: r.distance, + text: r.text, + type: r.type, + })); + + // Normalise distances to [0, 1], depending on the metric: + if (similarityMetric === 'cosine') { + // Cosine distance is in [0, 2] + result = result.map((row) => ({ + ...row, + distance: row.distance / 2, + })); + } else if (similarityMetric === 'hamming') { + // Hamming distance is in [0, 1], so we leave it as is + } else if (similarityMetric === 'euclidean') { + // Euclidean distance is in [0, sqrt(embeddingDim)] + const maxDistance = Math.sqrt(this.embeddingDim); + result = result.map((row) => ({ + ...row, + distance: row.distance / maxDistance, + })); + } + + // Since similariy is now normalised to [0, 1], + // we need to adapt it, as it should be somewhat interpretable as a probability + result = result.map((row) => ({ + ...row, + distance: 1 - row.distance, // Convert distance to similarity + })); + return result; + } +} + +export default VectorDataBase; diff --git a/src/competence-matcher/src/middleware/db-locator.ts b/src/competence-matcher/src/middleware/db-locator.ts new file mode 100644 index 000000000..3b65c0cd6 --- /dev/null +++ b/src/competence-matcher/src/middleware/db-locator.ts @@ -0,0 +1,22 @@ +import { Request, Response, NextFunction } from 'express'; +import { config } from '../config'; + +const { multipleDBs } = config; + +export function dbHeader(req: Request, res: Response, next: NextFunction): void { + // This middleware allows for the use f multiple databases instead of a single one. + // 'x-proceed-db-id' is a custom header that should be included in the request, which specifies the database name to use. + if (multipleDBs) { + const dbName = req.header('x-proceed-db-id'); + if (!dbName || typeof dbName !== 'string' || dbName.trim() === '') { + res.status(400).json({ error: 'Missing x-proceed-db-id header' }); + return; + } + req.dbName = dbName; + } else { + // For now, we use a single database, so we can just set a default name. + req.dbName = 'PROCEED-Matching.db'; + } + + next(); +} diff --git a/src/competence-matcher/src/middleware/logging.ts b/src/competence-matcher/src/middleware/logging.ts new file mode 100644 index 000000000..af93af303 --- /dev/null +++ b/src/competence-matcher/src/middleware/logging.ts @@ -0,0 +1,17 @@ +import { Request, Response, NextFunction } from 'express'; + +export function requestLogger(req: Request, res: Response, next: NextFunction): void { + const { method, query, body, headers, params } = req; + const logData = { + time: new Date().toISOString(), + method, + path: req.path, + query: JSON.stringify(query, null, 2), + body, + headers: JSON.stringify(headers, null, 2), + params: JSON.stringify(params, null, 2), + ip: req.ip, + }; + console.table([logData]); + next(); +} diff --git a/src/competence-matcher/src/middleware/match.ts b/src/competence-matcher/src/middleware/match.ts new file mode 100644 index 000000000..86c44bb56 --- /dev/null +++ b/src/competence-matcher/src/middleware/match.ts @@ -0,0 +1,406 @@ +import { Request, Response, NextFunction } from 'express'; +import { PATHS } from '../server'; +import { getDB } from '../utils/db'; +import workerManager from '../worker/worker-manager'; +import { + CompetenceInput, + GroupedMatchResults, + MatchingJob, + MatchingTask, + ResourceListInput, + ResourceRanking, + TaskOverview, +} from '../utils/types'; +import { handleCreateResourceList } from './resource'; + +export function matchCompetenceList(req: Request, res: Response, next: NextFunction): void { + try { + let listId: string; + let list: ResourceListInput; + let taskInput: MatchingTask[]; + const db = getDB(req.dbName!); + + /**-------------------------------------------- + * Checks + *---------------------------------------------*/ + if ('competenceList' in req.body) { + // Handle case where competenceList is provided + const { competenceList, tasks } = req.body as { + competenceList: ResourceListInput; + tasks: MatchingTask[]; + }; + list = competenceList; + taskInput = tasks; + } else if ('competenceListId' in req.body) { + // Handle case where competenceListId is provided + const { competenceListId, tasks } = req.body as { + competenceListId: string; + tasks: MatchingTask[]; + }; + listId = competenceListId; + taskInput = tasks; + } + + if (!listId! && !list!) { + res.status(400).json({ + error: 'Either competenceListId or competenceList must be provided.', + }); + return; + } + + if (!taskInput! || !Array.isArray(taskInput) || taskInput?.length === 0) { + res.status(400).json({ + error: 'An array of tasks must be provided for matching.', + }); + return; + } + + if (listId! && !(typeof listId === 'string')) { + res.status(400).json({ + error: 'competenceListId must be an UUIDStrings.', + }); + return; + } else if ( + list! && + (!Array.isArray(list) || + !list.every( + (entry) => typeof entry === 'object' && !Array.isArray(entry) && entry !== null, + )) + ) { + res.status(400).json({ + error: 'competenceList must be an array of ResourceInput objects.', + }); + return; + } + + /**-------------------------------------------- + * Case existing competenceListId was passed + *---------------------------------------------*/ + if (listId!) { + // Check if the competence list exists + const competenceLists = db.getAvailableResourceLists(); + if (!competenceLists.includes(listId)) { + res.status(404).json({ + error: `Competence list with ID ${listId} not found.`, + }); + return; + } + + const jobId = db.createJob(listId); + const job: MatchingJob = { + jobId, + dbName: req.dbName!, + listId, + resourceId: undefined, // For now, we don't support matching against a single resource + tasks: taskInput.map((task) => { + return { + taskId: task.taskId, + name: task.name, + description: task.description, + executionInstructions: task.executionInstructions, + requiredCompetencies: (task.requiredCompetencies ?? []).map((competence) => + typeof competence === 'string' + ? (competence as string) + : ({ + competenceId: competence.competenceId, + name: competence.name, + description: competence.description, + externalQualificationNeeded: competence.externalQualificationNeeded, + renewTime: competence.renewTime, + proficiencyLevel: competence.proficiencyLevel, + qualificationDates: competence.qualificationDates, + lastUsages: competence.lastUsages, + } as CompetenceInput), + ) as string[] | CompetenceInput[], + }; + }), + }; + + workerManager.enqueue(job, 'matcher'); + + // Respond with jobId in location header + res + .setHeader('Location', `${PATHS.match}/jobs/${jobId}`) + // Accepted response + .status(202) + .json({ jobId, status: 'pending' }); + return; + } + + /**-------------------------------------------- + * Case new Competence-List was passed + *---------------------------------------------*/ + // Create a new competence list + const matchingJobId = db.createJob(); + if (list!) { + db.updateJobStatus(matchingJobId, 'preprocessing'); + handleCreateResourceList(req.dbName!, list, (job, code, jobId) => { + try { + // Embedding fails -> no matching possible (i.e. fail the matching job) + if (code !== 0) { + db.updateJobStatus(matchingJobId, 'failed'); + return; + } + db.updateJobStatus(matchingJobId, 'pending'); + + // Retrieve the competence list ID + const { referenceId: listId } = db.getJob(jobId); + // Create the matching job + const matchingJob: MatchingJob = { + jobId: matchingJobId, + dbName: req.dbName!, + listId, + resourceId: undefined, // For now, we don't support matching against a single resource + tasks: taskInput.map((task) => { + return { + taskId: task.taskId, + name: task.name, + description: task.description, + executionInstructions: task.executionInstructions, + requiredCompetencies: (task.requiredCompetencies ?? []).map((competence) => + typeof competence === 'string' + ? (competence as string) + : ({ + competenceId: competence.competenceId, + name: competence.name, + description: competence.description, + externalQualificationNeeded: competence.externalQualificationNeeded, + renewTime: competence.renewTime, + proficiencyLevel: competence.proficiencyLevel, + qualificationDates: competence.qualificationDates, + lastUsages: competence.lastUsages, + } as CompetenceInput), + ) as string[] | CompetenceInput[], + }; + }), + }; + // Enqueue the matching job + workerManager.enqueue(matchingJob, 'matcher'); + } catch (error) { + db.updateJobStatus(matchingJobId, 'failed'); + console.error('Error creating (inline) matching job:', error); + } + }); + + res + .setHeader('Location', `${PATHS.match}/jobs/${matchingJobId}`) + .status(202) + .json({ jobId: matchingJobId, status: 'pending' }); + } + } catch (error) { + console.error('Error matching:', error); + res.status(500).json({ error: 'Internal Server Error' }); + } +} + +export function getMatchJobResults(req: Request, res: Response, next: NextFunction): void { + // Get jobId from path + const { jobId } = req.params; + // Get sorter from query params + const requestedSorter = req.query.rankBy as string | undefined; + const sorter = requestedSorter == 'bestFit' ? 'bestFit' : 'avgFit'; // Default to avgFit + const db = getDB(req.dbName!); + + // Check if job exists + const job = db.getJob(jobId); + if (!job) { + res.status(404).json({ error: `Job with ID ${jobId} not found.` }); + return; + } + + // Job can be pending, preprocessing, running, completed, or failed + switch (job.status) { + case 'pending': + case 'running': + case 'preprocessing': + res.status(202).json({ + jobId, + status: job.status, + }); + return; + case 'failed': + res.status(500).json({ + error: `Job with ID ${jobId} failed.`, + }); + return; + case 'completed': + // Proceed to return results below + break; + default: + console.error(`Unexpected job status: ${job.status} for jobId: ${jobId}`); + res.status(500).json({ + error: `Job with ID ${jobId} failed.`, + }); + return; + } + + // Return match results + const results = db.getMatchResults(jobId); + + const tasks: TaskOverview = results.reduce((acc, result) => { + const { taskId, taskText } = result; + // Check if task already exists in the overview + if (!acc.some((task) => task.taskId === taskId)) { + acc.push({ taskId, taskText }); + } + return acc; + }, [] as TaskOverview); + + // Get the structure of the results + let groupedResults: ResourceRanking = results.reduce((acc, result) => { + const { taskId, competenceId, resourceId, distance, text, type, alignment, reason } = result; + + // resourceId + let resourceGroup = acc.find((group) => group.resourceId === resourceId); + if (!resourceGroup) { + resourceGroup = { + resourceId, + taskMatchings: [], + avgTaskMatchProbability: 0, + avgBestFitTaskMatchProbability: 0, + contradicting: false, + }; + acc.push(resourceGroup); + } + // taskMatchings + let taskMatches = resourceGroup.taskMatchings.find((task) => task.taskId === taskId); + if (!taskMatches) { + taskMatches = { + taskId, + competenceMatchings: [], + maxMatchProbability: 0, + maxBestFitMatchProbability: 0, + }; + resourceGroup.taskMatchings.push(taskMatches); + } + + // competenceMatchings + let competenceMatches = taskMatches.competenceMatchings.find( + (competence) => competence.competenceId === competenceId, + ); + if (!competenceMatches) { + competenceMatches = { + competenceId, + matchings: [], + avgMatchProbability: 0, + avgBestFitMatchProbability: 0, + }; + taskMatches.competenceMatchings.push(competenceMatches); + } + + // Add the match to competenceMatches + competenceMatches.matchings.push({ + text, + type: type as 'name' | 'description' | 'proficiencyLevel', + matchProbability: distance, + alignment: alignment as 'contradicting' | 'neutral' | 'aligning', + reason: reason || undefined, + }); + + return acc; + }, [] as ResourceRanking); + + // Aggregate and sort + groupedResults = groupedResults + .map((resourceGroup) => { + const { resourceId, taskMatchings, avgTaskMatchProbability, avgBestFitTaskMatchProbability } = + resourceGroup; + + const newTaskMatchings = taskMatchings.map((taskGroup) => { + const { taskId, competenceMatchings, maxMatchProbability, maxBestFitMatchProbability } = + taskGroup; + + const newCompetenceMatchings = competenceMatchings.map((competenceGroup) => { + const { competenceId, matchings, avgMatchProbability, avgBestFitMatchProbability } = + competenceGroup; + + // Calculate average match probability for this competence (i.e. avg over all parts of this competence) + const totalMatchProbability = matchings.reduce( + (sum, match) => sum + match.matchProbability, + 0, + ); + + let numberOfBestFits = 0; + const totalBestFitMatchProbability = matchings.reduce((sum, match) => { + if (match.alignment === 'aligning') { + numberOfBestFits++; + return sum + match.matchProbability; + } + return sum; + }, 0); + + // Return sorted + return { + competenceId, + matchings: matchings.sort((a, b) => b.matchProbability - a.matchProbability), + avgMatchProbability: totalMatchProbability / matchings.length, + avgBestFitMatchProbability: + numberOfBestFits > 0 ? totalBestFitMatchProbability / numberOfBestFits : 0, // If no best fit, set to 0 + }; + }); + + // Return sorted + return { + taskId, + competenceMatchings: newCompetenceMatchings.sort((a, b) => { + const key = sorter === 'bestFit' ? 'avgBestFitMatchProbability' : 'avgMatchProbability'; + return b[key] - a[key]; + }), + maxMatchProbability: Math.max( + ...newCompetenceMatchings.map((c) => c.avgMatchProbability), + ), + maxBestFitMatchProbability: Math.max( + ...newCompetenceMatchings.map((c) => c.avgBestFitMatchProbability), + ), + }; + }); + + // Calculate average task match probability for this resource + const totalTaskMatchProbability = newTaskMatchings.reduce( + (sum, task) => sum + task.maxMatchProbability, + 0, + ); + const totalBestFitTaskMatchProbability = newTaskMatchings.reduce( + (sum, task) => sum + task.maxBestFitMatchProbability, + 0, + ); + + // Return sorted + return { + resourceId, + taskMatchings: newTaskMatchings.sort((a, b) => { + const key = sorter === 'bestFit' ? 'maxBestFitMatchProbability' : 'maxMatchProbability'; + return b[key] - a[key]; + }), + avgTaskMatchProbability: totalTaskMatchProbability / newTaskMatchings.length, + avgBestFitTaskMatchProbability: + totalBestFitTaskMatchProbability / newTaskMatchings.length || 0, // If no best fit, set to 0 + contradicting: newTaskMatchings.some((task) => + task.competenceMatchings.some((competence) => + competence.matchings.some((match) => match.alignment === 'contradicting'), + ), + ), + }; + }) + .sort((a, b) => { + const key = + sorter === 'bestFit' ? 'avgBestFitTaskMatchProbability' : 'avgTaskMatchProbability'; + + // Sort in two levels: Contradicting, key + // First not contradicting resources, then contradicting ones + // Case one is contradicting, the other is not + if (a.contradicting !== b.contradicting) { + return a.contradicting ? 1 : -1; // Non-contradicting first + } + // Both are contradicting or both are not + // Sort by the key + return b[key] - a[key]; + }); + + const load: GroupedMatchResults = { + tasks, + resourceRanking: groupedResults, + }; + + res.status(200).json(load); +} diff --git a/src/competence-matcher/src/middleware/resource.ts b/src/competence-matcher/src/middleware/resource.ts new file mode 100644 index 000000000..cf5d9848e --- /dev/null +++ b/src/competence-matcher/src/middleware/resource.ts @@ -0,0 +1,213 @@ +import { Request, Response, NextFunction } from 'express'; +import { PATHS } from '../server'; +import { getDB } from '../utils/db'; +import workerManager from '../worker/worker-manager'; +import { splitSemantically } from '../tasks/semantic-split'; +import { CompetenceInput, EmbeddingJob, EmbeddingTask, ResourceInput } from '../utils/types'; + +export function getResourceLists(req: Request, res: Response, next: NextFunction): void { + try { + const db = getDB(req.dbName!); + + const availableResourceLists = db.getAvailableResourceLists(); + + res.status(200).json(availableResourceLists); // string[] + } catch (error) { + console.error('Error retrieving resource lists:', error); + res.status(500).json({ error: 'Internal Server Error' }); + } +} + +export function getResourceList(req: Request, res: Response, next: NextFunction): void { + try { + const db = getDB(req.dbName!); + const resourceListId = req.params.resourceListId; + + if (!resourceListId) { + res.status(400).json({ error: 'Resource list ID is required' }); + return; + } + + let resourceList; + try { + resourceList = db.getResourceList(resourceListId); + } catch (error) { + res.status(404).json({ error: 'Resource list not found' }); + return; + } + + res.status(200).json(resourceList); + // type: + // resourceList: { + // competenceListId: string; + // resources: Array<{ + // resourceId: string; + // competencies: Array<{ + // competenceId: string; + // name?: string; + // description?: string; + // externalQualificationNeeded: boolean; + // renewTime?: number; + // proficiencyLevel?: string; + // qualificationDates: string[]; + // lastUsages: string[]; + // }>; + // }>; + // } + } catch (error) { + console.error('Error retrieving resource list:', error); + res.status(500).json({ error: 'Internal Server Error' }); + } +} + +// Helper function to handle the creation logic +export async function handleCreateResourceList( + dbName: string, + resources: ResourceInput[], + onWorkerExit?: (job: any, code: number, jobId: string) => void, +): Promise<{ jobId: string; status: string }> { + let resourceIds: string[] = []; + let competences: CompetenceInput[][] = []; + + // Validate and extract data + resources.forEach(({ resourceId, competencies }: ResourceInput) => { + if (!resourceId || typeof resourceId !== 'string') { + throw new Error('Invalid resourceId in request body'); + } + if (!Array.isArray(competencies)) { + throw new Error('Invalid competencies in request body'); + } + resourceIds.push(resourceId); + const checkedCompetences = competencies.map((c: CompetenceInput) => { + if (!c.competenceId || typeof c.competenceId !== 'string') { + throw new Error('Invalid competenceId in request body'); + } + return { + competenceId: c.competenceId, + name: c.name, + description: c.description, + externalQualificationNeeded: c.externalQualificationNeeded, + renewTime: c.renewTime, + proficiencyLevel: c.proficiencyLevel, + qualificationDates: c.qualificationDates, + lastUsages: c.lastUsages, + }; + }); + competences.push(checkedCompetences); + }); + + // Create a new resource list in the database + let listId: string; + let jobId: string; + const db = getDB(dbName); + db.atomicStep(() => { + listId = db.createResourceList(); + resourceIds.forEach((resourceId) => { + db.addResource(listId, resourceId); + }); + competences.forEach((competenceArray, resourceIndex) => { + competenceArray.forEach((competence) => { + db.addCompetence(listId, resourceIds[resourceIndex], competence); + }); + }); + jobId = db.createJob(listId); + }); + + // Prepare embedding tasks + const descriptionEmbeddingInput = competences + .map((competenceArray, resourceIndex) => { + return competenceArray.map((competence) => { + return { + listId: listId!, + resourceId: resourceIds[resourceIndex], + competenceId: competence.competenceId, + text: competence.description, + type: 'description', + }; + }); + }) + .flat() as EmbeddingTask[]; + + // Workaround for now + // Ideally, the worker should handle the splitting as well + db.updateJobStatus(jobId!, 'preprocessing'); + let job: EmbeddingJob | undefined; + + splitSemantically(descriptionEmbeddingInput) + .then((tasks) => { + job = { + jobId: jobId!, + dbName: dbName, + tasks, + }; + }) + .catch((err) => { + console.error('Error splitting semantically:', err); + job = { + jobId: jobId!, + dbName: dbName, + tasks: descriptionEmbeddingInput, + }; + }) + .finally(() => { + db.updateJobStatus(jobId!, 'pending'); + workerManager.enqueue(job!, 'embedder', { + onExit: (job, code) => onWorkerExit?.(job, code, jobId!), + }); + }); + + return { jobId: jobId!, status: 'pending' }; +} + +export function createResourceList(req: Request, res: Response, next: NextFunction): void { + if (!Array.isArray(req.body) || req.body.length === 0) { + res.status(400).json({ error: 'Invalid request body. Expected an array of resources.' }); + return; + } + try { + handleCreateResourceList(req.dbName!, req.body) + .then(({ jobId, status }) => { + res + .setHeader('Location', `${PATHS.resource}/jobs/${jobId}`) + .status(202) + .json({ jobId, status }); + }) + .catch((error) => { + console.error('Error adding resource list:', error); + res.status(400).json({ error: error.message || 'Invalid request body format' }); + }); + } catch (error) { + console.error('Error processing request body:', error); + res.status(400).json({ error: 'Invalid request body format' }); + } +} + +export function getJobStatus(req: Request, res: Response) { + try { + const db = getDB(req.dbName!); + const job = db.getJob(req.params.jobId); + + switch (job.status) { + case 'pending': + case 'preprocessing': + case 'running': + res.status(202).json({ jobId: job.jobId, status: job.status }); // both strings + return; + case 'completed': + res + .status(201) + .setHeader('Location', `${PATHS.resource}/${job.referenceId}`) + .json({ jobId: job.jobId, status: job.status, competenceListId: job.referenceId }); + return; + case 'failed': + res.status(500).json({ jobId: job.jobId, status: job.status }); //both strings + return; + default: + res.status(500).json({ error: 'Internal Server Error' }); + return; + } + } catch (err) { + // console.error(err); + res.status(404).json({ error: 'Job not found' }); + } +} diff --git a/src/competence-matcher/src/models/onnx-community/Qwen3-Embedding-0.6B-ONNX/config.json b/src/competence-matcher/src/models/onnx-community/Qwen3-Embedding-0.6B-ONNX/config.json new file mode 100644 index 000000000..1b0d51dce --- /dev/null +++ b/src/competence-matcher/src/models/onnx-community/Qwen3-Embedding-0.6B-ONNX/config.json @@ -0,0 +1,70 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.53.0.dev0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151669, + "transformers.js_config": { + "kv_cache_dtype": { + "q4f16": "float16", + "fp16": "float16" + }, + "use_external_data_format": { + "model.onnx": true, + "model_fp16.onnx": true + } + } +} \ No newline at end of file diff --git a/src/competence-matcher/src/routes/match.ts b/src/competence-matcher/src/routes/match.ts new file mode 100644 index 000000000..0cb894f73 --- /dev/null +++ b/src/competence-matcher/src/routes/match.ts @@ -0,0 +1,21 @@ +import express from 'express'; +import { getMatchJobResults, matchCompetenceList } from '../middleware/match'; + +const router = express.Router(); + +// .all() +// .get() +// .put() +// .post(); +// .patch(); +// .delete(); + +// '/:resource-list-id' +// '/:resource-list-id/:resource-id' +// '/:resource-list-id/:resource-id/:competence-id' + +router.route('/jobs/').post(matchCompetenceList); + +router.route('/jobs/:jobId').get(getMatchJobResults); + +export default router; diff --git a/src/competence-matcher/src/routes/resource.ts b/src/competence-matcher/src/routes/resource.ts new file mode 100644 index 000000000..2096119ec --- /dev/null +++ b/src/competence-matcher/src/routes/resource.ts @@ -0,0 +1,34 @@ +import express from 'express'; +import { + createResourceList, + getJobStatus, + getResourceList, + getResourceLists, +} from '../middleware/resource'; +import { config } from '../config'; + +const { multipleDBs } = config; + +const router = express.Router(); + +// .all() +// .get() +// .put() +// .post(); +// .patch(); +// .delete(); + +// '/:resource-list-id' +// '/:resource-list-id/:resource-id' +// '/:resource-list-id/:resource-id/:competence-id' + +// Getting an overview of all resource lists should be tennant-specific, +// so it should only be available if multipleDBs is true. +if (multipleDBs) router.route('/').get(getResourceLists); + +router.route('/jobs').post(createResourceList); +router.route('/jobs/:jobId').get(getJobStatus); + +router.route('/:resourceListId').get(getResourceList); + +export default router; diff --git a/src/competence-matcher/src/server.ts b/src/competence-matcher/src/server.ts new file mode 100644 index 000000000..158a5faac --- /dev/null +++ b/src/competence-matcher/src/server.ts @@ -0,0 +1,112 @@ +import express from 'express'; + +import ResourceRouter from './routes/resource'; +import MatchRouter from './routes/match'; +import { config } from './config'; +import { dbHeader } from './middleware/db-locator'; +import { requestLogger } from './middleware/logging'; +import Embedding from './tasks/embedding'; +import { ensureAllOllamaModelsAreAvailable } from './utils/ollama'; +import { splitSemantically } from './tasks/semantic-split'; +import { createWorker } from './utils/worker'; +import { ensureAllHuggingfaceModelsAreAvailable } from './utils/huggingface'; +import { EmbeddingTask } from './utils/types'; + +const { port: PORT } = config; + +export const PATHS = { + resource: '/resource-competence-list', + match: '/matching-task-to-resource', +}; + +// Extend Express Request interface +declare module 'express-serve-static-core' { + interface Request { + dbName?: string; + } +} + +async function main() { + const app = express(); + + // Ensure all required models are available + // Hugging Face models + await ensureAllHuggingfaceModelsAreAvailable(); + // Ollama models + await ensureAllOllamaModelsAreAvailable(); + + // const tasks = [ + // { + // listId: 'test-list', + // resourceId: 'test-resource', + // competenceId: 'test-competence', + // text: 'This competence covers the principles and best practices of designing scalable software systems. It includes high-level architecture, component interaction, and trade-off analysis. Practitioners will need to balance performance, reliability, and maintainability when making design decisions.', + // type: 'description', + // }, + // { + // listId: 'test-list', + // resourceId: 'test-resource', + // competenceId: 'test-competence', + // text: 'This competence focuses on building and maintaining RESTful and GraphQL APIs. It covers endpoint design, versioning strategies, and error handling. Learners will gain hands-on experience with request validation, authentication, and performance tuning.', + // type: 'description', + // }, + // { + // listId: 'test-list', + // resourceId: 'test-resource', + // competenceId: 'test-competence', + // text: 'This competence entails designing effective database schemas to represent business domains. It involves normalization, denormalization, and indexing strategies for optimal query performance. Real-world scenarios will illustrate when to choose relational versus NoSQL approaches.', + // type: 'description', + // }, + // { + // listId: 'test-list', + // resourceId: 'test-resource', + // competenceId: 'test-competence', + // text: 'This competence covers fundamental security principles for web applications. Topics include authentication, authorization, encryption, and secure configuration management. Practical exercises demonstrate common vulnerabilities and how to mitigate them effectively.', + // type: 'description', + // }, + // { + // listId: 'test-list', + // resourceId: 'test-resource', + // competenceId: 'test-competence', + // text: "This person can not swim at all. Please don't let them close water at all.", + // type: 'description', + // }, + // ] as EmbeddingTask[]; + + // const testworker = createWorker('test'); + // testworker.on('message', (message) => { + // console.log(message); + // }); + // testworker.postMessage(tasks); + + // const result = await splitSemantically(tasks); + // console.log(result); + + // Parse JSON + app.use(express.json()); + // Parse URL-encoded data + app.use(express.urlencoded({ extended: true })); + // Middleware to handle database header + app.use(dbHeader); + // Logging middleware + // app.use(requestLogger); + + // Hello World + app.get('/', (req, res, next) => { + console.log('Received a GET request on /'); + res.status(200).send('Welcome to the Matching Server'); + }); + + // Routes + app.use(PATHS.resource, ResourceRouter); + app.use(PATHS.match, MatchRouter); + + app.listen(PORT, () => { + console.log(`Matching-Server is running on http://localhost:${PORT}`); + }); +} + +main().catch((error) => { + console.error('Server shutdown due to error:', error); + process.exit(1); +}); diff --git a/src/competence-matcher/src/tasks/embedding.ts b/src/competence-matcher/src/tasks/embedding.ts new file mode 100644 index 000000000..0f320214c --- /dev/null +++ b/src/competence-matcher/src/tasks/embedding.ts @@ -0,0 +1,48 @@ +import { + PipelineType, + FeatureExtractionPipeline, + FeatureExtractionPipelineOptions, +} from '@huggingface/transformers'; +import { config } from '../config'; +import { TransformerPipeline } from '../utils/model'; +import { TransformerPipelineOptions } from '../utils/types'; + +export default class Embedding extends TransformerPipeline { + protected static override getPipelineOptions(): TransformerPipelineOptions { + return { + task: 'feature-extraction' as PipelineType, + model: config.embeddingModel, + options: { + // progress_callback: (progress) => { + // console.log(`Embedding progress: ${progress}`); + // }, + }, + }; + } + + /** + * Turn text (or array of text) into one or more vectors. + * @param texts string or array + * @param opts override default mean-pooling/normalize + */ + public static async embed( + texts: string | string[], + opts: FeatureExtractionPipelineOptions = { pooling: 'mean', normalize: true }, + ): Promise { + // Pipeline is loaded & cached by TransformerPipeline + const pipe = await this.getInstance(); + const input = Array.isArray(texts) ? texts : [texts]; + // call the pipeline + const raw = await pipe(input, opts); + const arrs = (Array.isArray(raw) ? raw : [raw]).map((tensor) => { + // each tensor.data is a Float32Array + const data = (tensor as any).data as Float32Array; + const vec = Array.from(data); + if (vec.length !== config.embeddingDim) { + throw new Error(`Expected embeddingDim=${config.embeddingDim}, got ${vec.length}`); + } + return vec; + }) as number[][]; + return arrs; + } +} diff --git a/src/competence-matcher/src/tasks/reason.ts b/src/competence-matcher/src/tasks/reason.ts new file mode 100644 index 000000000..bb171e083 --- /dev/null +++ b/src/competence-matcher/src/tasks/reason.ts @@ -0,0 +1,43 @@ +import { ollama } from '../utils/ollama'; +import { config } from '../config'; +import { MATCH_REASON as intructPrompt } from '../utils/prompts'; +import type { Message } from 'ollama'; +import { Match } from '../utils/types'; + +const { reasonModel } = config; + +export async function addReason(matches: T[], targetText: string): Promise { + if (matches.length === 0) { + return matches; // No matches to reason about + } + const reasonMatches: T[] = await Promise.all( + matches.map(async (match) => { + const messages: Message[] = [ + ...intructPrompt, + { + role: 'user', + content: `Task: ${targetText}\nCompetence: ${match.text}\nSimilarity Score: ${match.distance}`, + }, + ]; + try { + const response = await ollama.chat({ + model: reasonModel, + messages: messages, + }); + + // Extract the reason from the response + const reason = response.message.content.trim(); + return { + ...match, + reason, // Add the reason to the match + }; + } catch (error) { + console.error('Error during reasoning:', error); + // If there's an error, just keep the original match without a reason + return match; + } + }), + ); + + return reasonMatches; +} diff --git a/src/competence-matcher/src/tasks/semantic-split.ts b/src/competence-matcher/src/tasks/semantic-split.ts new file mode 100644 index 000000000..c07583a8b --- /dev/null +++ b/src/competence-matcher/src/tasks/semantic-split.ts @@ -0,0 +1,155 @@ +import { ollama } from '../utils/ollama'; +import { config } from '../config'; +import { SEMANTIC_SPLITTER as intructPrompt } from '../utils/prompts'; +import type { Message } from 'ollama'; +import { EmbeddingTask } from '../utils/types'; + +const { splittingModel, splittingSymbol, ollamaBatchSize } = config; + +const MIN_TEXT_LENGTH = 60; // Minimum length of text to consider for splitting (I noticed that text inputs that are too short often lead to errors in the splitting process - and since they are so small, they can be embedded directly without splitting) + +// async function ollamaChat(messages: Array<{ role: string; content: string }>) { +// const res = await fetch(`${ollamaPath}/api/chat`, { +// method: 'POST', +// headers: { 'Content-Type': 'application/json' }, +// body: JSON.stringify({ +// model: config.ollamaSplittingModel, +// messages, +// stream: false, +// }), +// }); +// if (!res.ok) { +// const t = await res.text(); +// throw new Error(`Ollama REST chat failed: ${res.status} ${t}`); +// } +// const data = (await res.json()) as { message: { content: string } }; +// return data.message.content as string; +// } + +// export async function splitSemantically(tasks: EmbeddingTask[]): Promise { +// const splittedTasks: EmbeddingTask[] = []; + +// // First, for each task, decide whether it needs splitting (filteredMessages) +// // or can be passed through as‐is. +// const toSplit: { task: EmbeddingTask; messages: Message[] }[] = []; +// for (const task of tasks) { +// const messages: Message[] = [...intructPrompt, { role: 'user', content: task.text }]; + +// // Filter out too‐short or empty +// const filtered = messages.filter(({ content }) => { +// const c = content.replace(/\s+/g, ' ').trim(); +// return c.length > MIN_TEXT_LENGTH; +// }); + +// if (filtered.length === 0) { +// // no splitting needed +// splittedTasks.push({ ...task, text: task.text }); +// } else { +// toSplit.push({ task, messages: filtered }); +// } +// } + +// // Now process in batches of size ollamaBatchSize +// for (let i = 0; i < toSplit.length; i += ollamaBatchSize) { +// const batch = toSplit.slice(i, i + ollamaBatchSize); + +// // Kick off all requests in this batch in parallel +// const promises = batch.map(async ({ task, messages }) => { +// try { +// const response = await ollamaChat(messages); +// const parts = response +// .replace(/\s+/g, ' ') +// .trim() +// .split(splittingSymbol) +// .map((p: string) => p.trim()) +// .filter((p: string) => p.length > 0); + +// if (parts.length === 0) { +// // fallback to original text if splitting yields nothing +// splittedTasks.push({ ...task, text: task.text }); +// } else { +// for (const part of parts) { +// splittedTasks.push({ ...task, text: part }); +// } +// } +// } catch (err) { +// console.error('Error during semantic splitting:', err); +// // in case of error, include the original +// splittedTasks.push({ ...task, text: task.text }); +// } +// }); + +// // Wait for this batch to finish before launching the next +// await Promise.all(promises); +// } + +// return splittedTasks; +// } + +// _______________________________________________ + +export async function splitSemantically(tasks: EmbeddingTask[]): Promise { + const splittedTasks: EmbeddingTask[] = []; + const toSplit: { task: EmbeddingTask; messages: Message[] }[] = []; + + for (const task of tasks) { + const messages: Message[] = [ + ...intructPrompt, + { + role: 'user', + content: task.text, + }, + ]; + + // Filter out empty, whitespace-only and too short messages + const filteredMessages = messages.filter((message) => { + const content = message.content + // Remove excessive whitespace + .replace(/\s+/g, ' ') + // Remove potential occurences of splittingSymbol in upper and lower case + .replace(new RegExp(splittingSymbol, 'gi'), ' ') + // Trim leading and trailing whitespace + .trim(); + return content !== '' && content.length > MIN_TEXT_LENGTH; + }); + + if (filteredMessages.length === 0) { + splittedTasks.push({ ...task, text: task.text }); + } else { + toSplit.push({ task, messages: filteredMessages }); + } + } + + // Process in batches + for (let i = 0; i < toSplit.length; i += ollamaBatchSize) { + const batch = toSplit.slice(i, i + ollamaBatchSize); + + const promises = batch.map(async ({ task, messages }) => { + try { + const response = await ollama.chat({ + model: splittingModel, + messages, + }); + const parts = response.message.content + .split(splittingSymbol) + .map((part: string) => part.trim()) + .filter((part: string) => part !== ''); + + if (parts.length === 0) { + splittedTasks.push({ ...task, text: task.text }); + } else { + for (const part of parts) { + splittedTasks.push({ ...task, text: part }); + } + } + } catch (error) { + console.error('Error during semantic splitting:', error); + splittedTasks.push({ ...task, text: task.text }); + } + }); + + await Promise.all(promises); + } + + return splittedTasks; +} diff --git a/src/competence-matcher/src/tasks/semantic-zeroshot.ts b/src/competence-matcher/src/tasks/semantic-zeroshot.ts new file mode 100644 index 000000000..23556ee84 --- /dev/null +++ b/src/competence-matcher/src/tasks/semantic-zeroshot.ts @@ -0,0 +1,43 @@ +import { + PipelineType, + TextClassificationPipeline, + ZeroShotClassificationPipeline, +} from '@huggingface/transformers'; +import { config } from '../config'; +import { TransformerPipeline } from '../utils/model'; +import { TransformerPipelineOptions } from '../utils/types'; + +export default class ZeroShot extends TransformerPipeline { + protected static override getPipelineOptions(): TransformerPipelineOptions { + return { + task: 'zero-shot-classification' as PipelineType, + model: config.nliModel, + options: { + // progress_callback: (progress) => { + // console.log(`Embedding progress: ${progress}`); + // }, + model_file_name: 'model.onnx', + use_external_data_format: true, + local_files_only: true, + }, + }; + } + + /** + * Classify text against a set of labels using zero-shot classification. + * @param text The text to classify. + * @param labels The labels to classify against. + * @param hypothesisTemplate Optional hypothesis template for classification - should include '{}' as placeholder for label. + */ + public static async classify(text: string, labels?: string[], hypothesisTemplate?: string) { + const _labels = labels || [ + 'contradicting', + 'aligning, thus a good match', + 'neither aligning nor contradicting', + ]; + const hypothesis_template = + hypothesisTemplate || 'Task description and Skill/Capability descriptions are {}.'; + const pipe = await this.getInstance(); + return pipe(text, _labels, { hypothesis_template }); + } +} diff --git a/src/competence-matcher/src/utils/db.ts b/src/competence-matcher/src/utils/db.ts new file mode 100644 index 000000000..5d02224b2 --- /dev/null +++ b/src/competence-matcher/src/utils/db.ts @@ -0,0 +1,7 @@ +import DBManager from '../db/db-manager'; + +export function getDB(name: string) { + const dbManager = DBManager.getInstance(); + DBManager.setActiveDB(name); + return dbManager.getDB(name); +} diff --git a/src/competence-matcher/src/utils/huggingface.ts b/src/competence-matcher/src/utils/huggingface.ts new file mode 100644 index 000000000..3d6d5128f --- /dev/null +++ b/src/competence-matcher/src/utils/huggingface.ts @@ -0,0 +1,13 @@ +import Embedding from '../tasks/embedding'; +import ZeroShotSemanticOpposites from '../tasks/semantic-zeroshot'; + +export async function ensureAllHuggingfaceModelsAreAvailable() { + try { + Embedding.getInstance(); + ZeroShotSemanticOpposites.getInstance(); + } catch (error) { + throw error; + } + + console.log('All required Hugging Face models are available.'); +} diff --git a/src/competence-matcher/src/utils/model.ts b/src/competence-matcher/src/utils/model.ts new file mode 100644 index 000000000..885c6fce4 --- /dev/null +++ b/src/competence-matcher/src/utils/model.ts @@ -0,0 +1,63 @@ +import { config } from '../config'; + +import { + pipeline, + env as huggingfaceEnv, + PipelineType, + PretrainedModelOptions, +} from '@huggingface/transformers'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { TransformerPipelineOptions } from './types'; + +import { isMainThread } from 'node:worker_threads'; + +/** + * Base class that handles: + * - singleton pipeline instance + * - model cache dir + * - on‑demand loading with optional progressCallback + */ +export abstract class TransformerPipeline { + protected static instance: any = null; + protected static loaded = false; + + protected static getPipelineOptions(): TransformerPipelineOptions { + throw new Error('getPipelineOptions must be implemented in subclasses'); + } + + private static async initEnv(cacheDir?: string) { + if (cacheDir) { + const abs = path.resolve(cacheDir); + if (!fs.existsSync(abs)) fs.mkdirSync(abs, { recursive: true }); + huggingfaceEnv.cacheDir = abs; + } + huggingfaceEnv.allowLocalModels = true; + } + + public static async getInstance(): Promise { + if (this.instance === null) { + const { task, model, options } = this.getPipelineOptions(); + await this.initEnv(options?.cache_dir || config.modelCache); + + const opts: PretrainedModelOptions = { + cache_dir: options?.cache_dir || config.modelCache, + use_external_data_format: options?.use_external_data_format ?? true, + device: options?.device || (config.useGPU ? 'cuda' : 'cpu'), + dtype: options?.dtype || 'fp32', + progress_callback: options?.progress_callback, + }; + + // actually load the pipeline + this.instance = await pipeline(task as PipelineType, model, opts); + + // mark it as loaded and log on first load + if (!this.loaded && isMainThread) { + console.log(`${model} (${task}) is ready`); + this.loaded = true; + } + } + + return this.instance; + } +} diff --git a/src/competence-matcher/src/utils/ollama.ts b/src/competence-matcher/src/utils/ollama.ts new file mode 100644 index 000000000..cb53af134 --- /dev/null +++ b/src/competence-matcher/src/utils/ollama.ts @@ -0,0 +1,42 @@ +import { Ollama } from 'ollama'; +import { config } from '../config'; + +const { ollamaPath, splittingModel, reasonModel } = config; + +export const ollama = new Ollama({ + host: ollamaPath, + headers: { + 'User-Agent': 'PROCEED Competence Matcher', + }, +}); + +/** + * Ensures that all required models are available by checking their existence + * in the Ollama server. If a model is not available, it will be downloaded. + * If the model cannot be downloaded or is not available, an error will be thrown. + * (Ensures all needed models are actually available) + */ +export async function ensureAllOllamaModelsAreAvailable() { + const models = [splittingModel, reasonModel]; + + const availableModels = (await ollama.list()).models.map((model) => model.model); + + for (const model of models) { + if (!availableModels.includes(model)) { + const modelpull = await ollama.pull({ + model, + insecure: false, + stream: false, + }); + + // Check if the model was successfully pulled + if (!modelpull || modelpull.status !== 'success') { + throw new Error( + `Model ${model} could not be pulled: ${modelpull?.status || 'Unknown error'}`, + ); + } + } + } + + console.log('All required models are available in ollama.'); +} diff --git a/src/competence-matcher/src/utils/prompts.ts b/src/competence-matcher/src/utils/prompts.ts new file mode 100644 index 000000000..023fe5697 --- /dev/null +++ b/src/competence-matcher/src/utils/prompts.ts @@ -0,0 +1,189 @@ +import type { Message } from 'ollama'; +import { config } from '../config'; + +const { splittingSymbol } = config; + +const SEMANTIC_SPLITTER_INTRUCT: Message = { + role: 'system', + content: ` + Your task is to segment the following text (i.e. user input such as plain prose, bullet points or listings) into semantically independent parts. + Do not add, remove, or modify any words - under no circumstances should you ever add any additional text, comments, or explanations. + Preserve the original ordering of words within each group — but groups themselves need not follow the original sequence. + Separate each group only by the delimiter + ${splittingSymbol} + (i.e. exactly as shown, on a line by themselves, no additional whitespaces, just '${splittingSymbol}'). + If the entire input is already one coherent semantic unit, return it verbatim without any delimiter. + Grouping need do not be adjacent - just semantically related (i.e. two related text parts might be separated by other text parts). + If the input is empty, return an empty string. + If the input is a single word, return it verbatim without any delimiter. + If the input is a single sentence, return it verbatim without any delimiter. + If you are unsure about the grouping, return the entire input as a single group without any delimiters. + Under no circumstances should you ever add any additional text, comments, or explanations. + `, +}; +const SEMANTIC_SPLITTER_EXAMPLES: Message[] = [ + { + role: 'user', + content: ` + The job requires welding experience. Tick welding would be preferable if the person is familiar with it. The cage is designed for small pets like rabbits. Therefore, it must not contain any sharp edges that could harm them. Experience with welding small wires could be beneficial. + `, + }, + { + role: 'assistant', + content: ` + The job requires welding experience. Tick welding would be preferable if the person is familiar with it. Experience with welding small wires could be beneficial. + ${splittingSymbol} + The cage is designed for small pets like rabbits. Therefore, it must not contain any sharp edges that could harm them. + `, + }, + { + role: 'user', + content: ` + - Assemble circuit boards according to schematic diagrams + - Test each board for continuity and signal integrity + - Package finished units in protective casing + - Ship completed orders to customers worldwide + `, + }, + { + role: 'assistant', + content: ` + - Assemble circuit boards according to schematic diagrams + - Test each board for continuity and signal integrity + ${splittingSymbol} + - Package finished units in protective casing + - Ship completed orders to customers worldwide + `, + }, + { + role: 'user', + content: ` + 1. Prepare raw materials for production + 2. Record daily output and machine performance. + 3. Clean workstations and restock supplies + 4. Order order new materials when needed. + 5. Calibrate and maintain measurement instruments. + `, + }, + { + role: 'assistant', + content: ` + 1. Prepare raw materials for production + 4. Order order new materials when needed. + ${splittingSymbol} + 2. Record daily output and machine performance. + ${splittingSymbol} + 3. Clean workstations and restock supplies + ${splittingSymbol} + 5. Calibrate and maintain measurement instruments. + `, + }, + { + role: 'user', + content: ` + Operate CNC milling machines to produce precision metal parts. Perform quality inspections using calipers, micrometers, and gauges. Monitor machine operation and adjust feed rates, speeds, and tooling as needed. Maintain a clean and safe workspace, following all OSHA safety guidelines. Collaborate with engineers to troubleshoot design issues and implement improvements. Document production logs, inspection reports, and maintenance records daily. Assist in training new operators on standard operating procedures and best practices. + `, + }, + { + role: 'assistant', + content: ` + Operate CNC milling machines to produce precision metal parts. + Monitor machine operation and adjust feed rates, speeds, and tooling as needed. + ${splittingSymbol} + Perform quality inspections using calipers, micrometers, and gauges. + Collaborate with engineers to troubleshoot design issues and implement improvements. + ${splittingSymbol} + Maintain a clean and safe workspace, following all OSHA safety guidelines. + Document production logs, inspection reports, and maintenance records daily. + Assist in training new operators on standard operating procedures and best practices. + `, + }, +]; + +export const SEMANTIC_SPLITTER: Message[] = [ + SEMANTIC_SPLITTER_INTRUCT, + ...SEMANTIC_SPLITTER_EXAMPLES, +]; + +/** + * ------------------------------------------------------------- + */ + +const MATCH_REASON_INTRUCT: Message = { + role: 'system', + content: ` + You are an expert in generating reasons for matching scores between tasks and competences. + Your task is to generate a reason for the matching score between a task and a competence. + The reason should be one to three short, concise sentence that explain why the task and competence match as well as they did or why they did not match that well. + Do not mention the similarity score in your response. + The reason should be based on the text of the task and the competence and their estimated normalized similarity score. + The similarity score is a number between 0 and 1, where 0 means no similarity and 1 means perfect similarity. + Do not mention the similarity score in your response. + `, +}; + +const MATCH_REASON_EXAMPLES: Message[] = [ + { + role: 'user', + content: ` + Task: Operate CNC milling machines to produce precision metal parts. + Competence: Experience with CNC milling machines and precision machining. + Similarity Score: 0.95 + `, + }, + { + role: 'assistant', + content: ` + The statements match very well because the task requires operating CNC milling machines, which is exactly what the competence is about. + `, + }, + { + role: 'user', + content: ` + Task: Assemble circuit boards according to schematic diagrams. + Competence: Basic knowledge of electronics and soldering skills. + Similarity Score: 0.65 + `, + }, + { + role: 'assistant', + content: ` + The the statements have a moderate match because while assembling circuit boards requires some knowledge of electronics, it does not specifically require advanced soldering skills. + `, + }, + { + role: 'user', + content: ` + Task: Prepare raw materials for production. + Competence: Experience with inventory management and supply chain logistics. + Similarity Score: 0.30 + `, + }, + { + role: 'assistant', + content: ` + The statements have a low match because preparing raw materials is a basic task that does not require advanced inventory management or supply chain logistics skills. + `, + }, +]; + +export const MATCH_REASON: Message[] = [MATCH_REASON_INTRUCT, ...MATCH_REASON_EXAMPLES]; + +/** + * ------------------------------------------------------------- + */ +// """""""""" +// The warehouse must maintain ambient temperatures between 15°C and 25°C to protect sensitive goods. Humidity levels should not exceed 60% to prevent corrosion and mold growth. Inventory audits are scheduled weekly to ensure accuracy and compliance with safety standards. +// """""""""" + +// """""""""" +// Our catering service provides vegetarian, vegan, and gluten-free menu options to accommodate diverse dietary needs. All dishes are prepared fresh daily using locally sourced ingredients whenever possible. Orders must be placed at least 48 hours in advance to guarantee availability. Delivery times range from 8 AM to 6 PM on weekdays. +// """""""""" + +// """""""""" +// Employees are required to complete the annual cybersecurity training module before accessing the new intranet portal. The module covers password hygiene, phishing identification, and secure remote-access procedures. Failure to complete training by the deadline will result in temporary revocation of network privileges. +// """""""""" + +// """""""""" +// The production line must operate continuously in three shifts—morning, afternoon, and night—to meet daily target outputs of 5,000 units. All machinery, including conveyor belts and hydraulic presses, requires a thorough safety inspection at the start of each shift to ensure proper lubrication and guard alignment. Operators are responsible for logging any unusual vibrations, noise anomalies, or temperature spikes immediately in the maintenance ledger. Raw material deliveries arrive twice weekly and must be verified against purchase orders for quantity, grade, and certificate of analysis before being released to the staging area. Finished goods undergo a final quality check where dimensional tolerances, surface finish, and functional performance are recorded. Any nonconforming parts are quarantined and tagged, then reported to quality assurance for root‐cause analysis. At the end of each month, team leads compile production metrics, downtime reasons, and scrap rates into a summarized report for review at the management safety and efficiency meeting. +// """""""""" diff --git a/src/competence-matcher/src/utils/types.ts b/src/competence-matcher/src/utils/types.ts new file mode 100644 index 000000000..fb51ca786 --- /dev/null +++ b/src/competence-matcher/src/utils/types.ts @@ -0,0 +1,146 @@ +import { PretrainedModelOptions } from '@huggingface/transformers'; + +export type Competence = { + listId: string; // UUIDString + resourceId: string; // UUIDString + competenceId: string; // UUIDString + name?: string; // optional + description?: string; // optional but recommended to have content + externalQualificationNeeded?: boolean; // optional + renewTime?: number; // DaysAsInteger, optional + proficiencyLevel?: string; // optional + qualificationDates?: string[]; // ISO date strings, optional + lastUsages?: string[]; // ISO date strings, optional +}; + +export type CompetenceInput = { + competenceId?: string; + name?: string; + description?: string; + externalQualificationNeeded?: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates?: string[]; + lastUsages?: string[]; +}; + +export type ResourceInput = { + resourceId?: string; + competencies: CompetenceInput[]; +}; + +export type ResourceListInput = ResourceInput[]; + +export type MatchingTask = { + taskId: string; // UUIDString + name?: string; // optional + description?: string; // optional but recommended to have content + executionInstructions?: string; // optional, e.g. HTML + requiredCompetencies?: string[] | CompetenceInput[]; // either array of competenceIds or array of CompetenceInput +}; + +export type Match = { + competenceId: string; + resourceId: string; + text: string; + type: string; + distance: number; + reason?: string; +}; + +export interface VectorDBOptions { + filePath?: string; // If undefined or ":memory:", use in-memory; else path to file - Note: memory will not work with workers!! + embeddingDim: number; +} + +export type CompetenceDBOutput = { + competence_id: string; + competence_name: string | null; + competence_description: string | null; + external_qualification_needed: number; // 0 or 1 + renew_time: number | null; + proficiency_level: string | null; + qualification_dates: string | null; // JSON string + last_usages: string | null; // JSON string +}; + +export type EmbeddingTask = { + listId: string; // UUIDString + resourceId: string; // UUIDString + competenceId: string; // UUIDString + text: string; // Text to embed + type: 'name' | 'description' | 'proficiencyLevel'; // Type of text +}; + +export interface Job { + jobId: string; + dbName: string; +} + +export interface EmbeddingJob extends Job { + tasks: EmbeddingTask[]; +} + +export interface MatchingJob extends Job { + listId?: string; // Which List to match against + resourceId?: string; // Optional: If matching against a single resource + tasks: MatchingTask[]; // Tasks to match +} + +export type ResourceRanking = { + resourceId: string; + taskMatchings: { + taskId: string; // Which of the tasks this matching is referring to + competenceMatchings: { + competenceId: string; + matchings: { + text: string; + type: 'name' | 'description' | 'proficiencyLevel'; + // Sorted DESC by + matchProbability: number; // Normalised inverted distance (, where distance refers to the cosine similarity) + alignment: 'contradicting' | 'neutral' | 'aligning'; // Semantic opposite classification + reason?: string; // Reason for the match + }[]; // Array: Competence-Parts matched to task + // Sorted DESC by either: + avgMatchProbability: number; // Average matchProbability of all parts of this competence + avgBestFitMatchProbability: number; // Average of the parts that align well with the task, 0 means there is none + }[]; // Array: Competences matched to task + // Sorted DESC by either: + maxMatchProbability: number; // Best avgMatchingProbability of all competences for this task + maxBestFitMatchProbability: number; // Best avgBestFitMatchProbability of all competences for this task, 0 means there is none + }[]; // Array: Matching of the resource to each task, respectively + // Sorted DESC first by [not contradicting , contradicting] then by either: + avgTaskMatchProbability: number; // Average maxMatchProbability of all tasks for this resource + avgBestFitTaskMatchProbability: number; // Average maxBestFitMatchProbability of all tasks for this resource, 0 means there is none + + contradicting: boolean; // Whether there is a part in a competence of this resource that contradicts the task +}[]; + +export type TaskOverview = { + taskId: string; // UUIDString + taskText: string; // Text of the task +}[]; + +export type GroupedMatchResults = { + tasks: TaskOverview; + resourceRanking: ResourceRanking; +}; + +export type workerTypes = 'embedder' | 'matcher'; + +export interface WorkerQueue { + job: any; + workerScript: workerTypes; + options?: { + onOnline?: (job: any) => void; + onExit?: (job: any, code: number) => void; + onError?: (job: any, error: Error) => void; + onMessage?: (job: any, message: any) => void; + }; +} + +export interface TransformerPipelineOptions { + task: string; + model: string; + options?: PretrainedModelOptions; +} diff --git a/src/competence-matcher/src/utils/worker.ts b/src/competence-matcher/src/utils/worker.ts new file mode 100644 index 000000000..d7b051825 --- /dev/null +++ b/src/competence-matcher/src/utils/worker.ts @@ -0,0 +1,79 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { Worker, parentPort } from 'worker_threads'; +import VectorDataBase from '../db/db'; +import { getDB } from './db'; +import { config } from '../config'; + +const { maxJobTime } = config; + +export function createWorker(filename: string): Worker { + const tsPath = path.resolve(__dirname, `../worker/${filename}.ts`); + const jsPath = path.resolve(__dirname, `../worker/${filename}.js`); + const isTs = fs.existsSync(tsPath); + + const workerFile = isTs ? tsPath : jsPath; + + const execArgv = isTs + ? [...process.execArgv, '-r', 'ts-node/register/transpile-only'] + : process.execArgv; + + const worker = new Worker(workerFile, { execArgv }); + + return worker; +} + +export async function withJobUpdates( + job: { jobId: string; dbName: string }, + cb: (db: VectorDataBase, payload: T) => Promise, + options?: { + onStart?: () => void; + onDone?: () => void; + onError?: (error: Error) => void; + }, +) { + const db = getDB(job.dbName); + let exitCode = 0; // success by default + let maxTimeCheck = setTimeout(() => { + // if not completed by then, timeout + process.exit(2); + }, maxJobTime); + try { + if (options && options.onStart) { + options.onStart(); + } else { + db.updateJobStatus(job.jobId, 'running'); + parentPort!.postMessage({ type: 'status', jobId: job.jobId, status: 'running' }); + } + + await cb(db, job as any as T); + + if (options && options.onDone) { + options.onDone(); + } else { + db.updateJobStatus(job.jobId, 'completed'); + parentPort!.postMessage({ type: 'status', jobId: job.jobId, status: 'completed' }); + } + } catch (err) { + if (options && options.onError) { + options.onError(err as Error); + } else { + exitCode = 1; // indicate failure + parentPort!.postMessage({ + type: 'error', + jobId: job.jobId, + error: err instanceof Error ? err.message : String(err), + }); + db.updateJobStatus(job.jobId, 'failed'); + } + } finally { + clearTimeout(maxTimeCheck); + db.close(); + parentPort!.close(); + process.exit(exitCode); + } +} + +export function log(...args: any[]) { + parentPort?.postMessage({ type: 'log', message: args.map(String).join(' ') }); +} diff --git a/src/competence-matcher/src/worker/embedder.ts b/src/competence-matcher/src/worker/embedder.ts new file mode 100644 index 000000000..e51f65949 --- /dev/null +++ b/src/competence-matcher/src/worker/embedder.ts @@ -0,0 +1,25 @@ +import { parentPort } from 'worker_threads'; +import Embedding from '../tasks/embedding'; +import { splitSemantically } from '../tasks/semantic-split'; +import { withJobUpdates } from '../utils/worker'; +import { config } from '../config'; +import { EmbeddingJob } from '../utils/types'; + +parentPort!.once('message', async (job: EmbeddingJob) => { + (global as any).CURRENT_JOB = job.jobId; + + await withJobUpdates(job, async (db, { tasks, jobId }) => { + let work = tasks; + // TODO: This appears to cause the worker to crash silently + // Split tasks semantically + // work = await splitSemantically(tasks); + + // For each task: embed & upsert + for (const { listId, resourceId, competenceId, text, type } of work) { + const [vector] = await Embedding.embed(text); + // console.log(`Embedded text for job ${jobId}:`, text, '->', vector); + + db.upsertEmbedding({ listId, resourceId, competenceId, text, type, embedding: vector }); + } + }); +}); diff --git a/src/competence-matcher/src/worker/matcher.ts b/src/competence-matcher/src/worker/matcher.ts new file mode 100644 index 000000000..7e2fe7a93 --- /dev/null +++ b/src/competence-matcher/src/worker/matcher.ts @@ -0,0 +1,130 @@ +import { parentPort } from 'worker_threads'; +import Embedding from '../tasks/embedding'; +import { withJobUpdates } from '../utils/worker'; +import { addReason } from '../tasks/reason'; +import { Match, MatchingJob } from '../utils/types'; +import ZeroShot from '../tasks/semantic-zeroshot'; + +parentPort!.once('message', async (job: MatchingJob) => { + // For workaround: + const matchResults: { [description: string]: any[] } = {}; + for (const task of job.tasks) { + const { taskId, name, description, executionInstructions, requiredCompetencies } = task; + if (!description) { + continue; // Skip tasks without description + } + // Add task description to match results + matchResults[description] = []; + } + + await withJobUpdates( + job, + async (db, { jobId, tasks, listId: listIdFilter, resourceId: resourceIdFilter }) => { + for (const task of tasks) { + const { taskId, name, description, executionInstructions, requiredCompetencies } = task; + if (!description) { + continue; // Skip tasks without description + } + // Embed the task description + const [vector] = await Embedding.embed(description); + + // Search for matches in the competence list (and resource if provided) + let matches: Match[] = db.searchEmbedding(vector, { + filter: { + listId: listIdFilter, + resourceId: resourceIdFilter, // Optional: If matching against a single resource + }, + }); + + // TODO: This appears to cause the worker to not start at all + // Invert potentially contrastive matches + // Add reasoning for matching score + // matches = await addReason(matches, description); + + for (const match of matches) { + // Check for semantic opposites + const zeroshotText = `Task description: ${description}\nSkill/Capability description: ${match.text}`; + // From unsuitable to suitable + const contraLabels = ['contradicting', 'aligning']; + const contraHypothesis = 'Task description and Skill/Capability descriptions are {}.'; + const scalingLabls = ['perfect', 'mediocre']; + const scalingHypothesis = + 'Task description and Skill/Capability descriptions are a {} match.'; + const labelScalar = [ + 0.25, // Contradicting matches should be penalised + 0.5, // Scale it down a bit to avoid too high scores for irrelevant matches + 1, // keep the best matches as is + ]; + const contraClassification = await ZeroShot.classify( + zeroshotText, + contraLabels, + contraHypothesis, + ); + let flag: 'contradicting' | 'neutral' | 'aligning' = 'neutral'; + // console.log(contraClassification); + + // @ts-ignore + if (contraClassification.labels[0] === contraLabels[0]) { + // Invert the match distance (since it's normalised to [0,1]: 1 - distance) + match.distance = (1 - match.distance) * labelScalar[0]; + flag = 'contradicting'; + } else { + const scalingClassification = await ZeroShot.classify( + zeroshotText, + scalingLabls, + scalingHypothesis, + ); + + // console.log(scalingClassification); + if ( + // @ts-ignore + scalingClassification.labels[0] === scalingLabls[0] && + // @ts-ignore + scalingClassification.scores[0] > 0.65 + ) { + // Keep the match as is + match.distance *= labelScalar[2]; + flag = 'aligning'; + } + // @ts-ignore + else if (scalingClassification.labels[0] === scalingLabls[1]) { + // Scale it down a bit + match.distance *= labelScalar[1]; + flag = 'neutral'; + } + } + + // db.addMatchResult({ + // jobId, + // taskId, + // competenceId: match.competenceId, + // text: match.text, + // type: match.type as 'name' | 'description' | 'proficiencyLevel', + // distance: match.distance, + // reason: match.reason, + // }); + // } + + // Workaround to avoid the worker crashing silently + matchResults[description].push({ + jobId, + taskId, + taskText: description, + competenceId: match.competenceId, + resourceId: match.resourceId, + text: match.text, + type: match.type as 'name' | 'description' | 'proficiencyLevel', + alignment: flag, + distance: match.distance, + reason: match.reason, + }); + } + } + }, + { + onDone: () => { + parentPort!.postMessage({ type: 'job', job: 'reason', workload: matchResults }); + }, + }, + ); +}); diff --git a/src/competence-matcher/src/worker/test.ts b/src/competence-matcher/src/worker/test.ts new file mode 100644 index 000000000..fde1d468a --- /dev/null +++ b/src/competence-matcher/src/worker/test.ts @@ -0,0 +1,9 @@ +import { parentPort } from 'worker_threads'; +import { splitSemantically } from '../tasks/semantic-split'; +import { EmbeddingJob } from '../utils/types'; + +parentPort!.once('message', async (job: EmbeddingJob) => { + const { tasks, jobId } = job; + parentPort!.postMessage({ type: 'status', jobId, status: 'running' }); + parentPort!.postMessage(await splitSemantically(tasks)); +}); diff --git a/src/competence-matcher/src/worker/worker-manager.ts b/src/competence-matcher/src/worker/worker-manager.ts new file mode 100644 index 000000000..0bb7c1a00 --- /dev/null +++ b/src/competence-matcher/src/worker/worker-manager.ts @@ -0,0 +1,141 @@ +import { Worker } from 'worker_threads'; +import { config } from '../config'; +import { createWorker } from '../utils/worker'; +import { splitSemantically } from '../tasks/semantic-split'; +import { Match, WorkerQueue, workerTypes } from '../utils/types'; +import { addReason } from '../tasks/reason'; +import { getDB } from '../utils/db'; + +class WorkerManager { + private concurrency: number; + private queue: WorkerQueue[] = []; + private active: Set = new Set(); + + constructor(concurrency: number) { + this.concurrency = concurrency; + } + + /** + * Enqueue a job for the named worker script + */ + public enqueue(job: any, workerScript: workerTypes, options: WorkerQueue['options'] = {}) { + this.queue.push({ job, workerScript, options }); + this.dispatch(); + } + + /** Try to start as many queued jobs as we have free threads */ + private dispatch() { + while (this.active.size < this.concurrency && this.queue.length > 0) { + const { job, workerScript, options } = this.queue.shift()!; + this.startWorker(job, workerScript, options); + } + } + + /** Spawn one worker, hook up its lifecycle, and send the job */ + private startWorker(job: any, workerScript: workerTypes, options: WorkerQueue['options']) { + const worker = createWorker(workerScript); + + this.active.add(worker); + + worker.once('online', () => { + // console.log(`[WorkerManager] Worker for ${workerScript} started`); + worker.postMessage(job); + + options?.onOnline?.(job); + }); + + // When the worker exits (success or failure), remove from active set & dispatch next + worker.once('exit', (code) => { + this.active.delete(worker); + if (code === 1) { + console.error(`[WorkerManager] ${workerScript} exited (failed) with code`, code); + } else if (code === 0) { + // console.log(`[WorkerManager] ${workerScript} exited successfully`); + } else if (code === 2) { + console.error(`[WorkerManager] ${workerScript} timed out`); + } + this.dispatch(); + + options?.onExit?.(job, code); + }); + + worker.once('error', (err) => { + console.error(`[WorkerManager] ${workerScript} error:`, err); + + options?.onError?.(job, err); + }); + + worker.on('message', async (message) => { + switch (message.type) { + case 'status': + console.log(`[WorkerManager] Worker for job ${message.jobId} status:`, message.status); + break; + case 'error': + console.error(`[WorkerManager] Worker for job ${message.jobId} error:`, message.error); + break; + case 'log': + console.log(`[WorkerManager] Worker for job ${message.jobId} log:`, message.message); + break; + + // Workaround for adding reasoning before saving in DB + case 'job': + switch (message.job) { + case 'reason': + await handleReasoning(job, message); + break; + } + break; + } + options?.onMessage?.(job, message); + }); + } +} + +async function handleReasoning(job: any, message: any) { + const finalMatches = []; + // Add reasoning before saving in DB + for (const [task, matches] of Object.entries(message.workload)) { + const taskMatches = await addReason< + Match & { + taskId: string; + taskText: string; + type: 'name' | 'description' | 'proficiencyLevel'; + alignment: 'contradicting' | 'neutral' | 'aligning'; + } + >( + matches as (Match & { + taskId: string; + taskText: string; + type: 'name' | 'description' | 'proficiencyLevel'; + alignment: 'contradicting' | 'neutral' | 'aligning'; + })[], + task, + ); + finalMatches.push(...taskMatches); + } + + // Save in DB + const db = getDB(job.dbName); + + for (const match of finalMatches) { + db.addMatchResult({ + jobId: job.jobId, + taskId: match.taskId, + taskText: match.taskText, + competenceId: match.competenceId, + resourceId: match.resourceId, + distance: match.distance, + text: match.text, + type: match.type, + reason: match.reason, + alignment: match.alignment, + }); + } + + // Update job status + db.updateJobStatus(job.jobId, 'completed'); +} + +// export a singleton instance +const manager = new WorkerManager(config.maxWorkerThreads); +export default manager; diff --git a/src/competence-matcher/tools/onnx-model-external-data.py b/src/competence-matcher/tools/onnx-model-external-data.py new file mode 100644 index 000000000..f2c74c216 --- /dev/null +++ b/src/competence-matcher/tools/onnx-model-external-data.py @@ -0,0 +1,32 @@ +import onnx +import argparse +import sys + +def main(): + parser = argparse.ArgumentParser(description="Convert ONNX model weights to external data format.") + parser.add_argument('--input', '-i', required=True, help='Path to the input ONNX model') + parser.add_argument('--output', '-o', required=False, help='Path to save the modified ONNX model (defaults to input path)', default=None) + + # If only -h/--help is present, show help and exit + if len(sys.argv) == 2 and sys.argv[1] in ('-h', '--help'): + parser.print_help() + sys.exit(0) + + args = parser.parse_args() + + if args.output is None: + args.output = args.input + + model = onnx.load(args.input) + + onnx.external_data_helper.convert_model_to_external_data( + model, + convert_attribute=True, + all_tensors_to_one_file=True, + location="model.onnx_data" + ) + + onnx.save(model, args.output) + +if __name__ == "__main__": + main() diff --git a/src/competence-matcher/tsconfig.json b/src/competence-matcher/tsconfig.json new file mode 100644 index 000000000..c612291b4 --- /dev/null +++ b/src/competence-matcher/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["esnext"], + "target": "ES2020", + "module": "commonjs", + "rootDir": "src", + "outDir": "dist", + "strict": true, + "esModuleInterop": true, + "moduleResolution": "node" + }, + "include": ["src"] +} diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/competences/page.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/competences/page.tsx new file mode 100644 index 000000000..f67c8b2c3 --- /dev/null +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/competences/page.tsx @@ -0,0 +1,34 @@ +import Content from '@/components/content'; +import { env } from 'process'; +import FeatureFlags from 'FeatureFlags'; +import CompentencesContainer from '@/components/competences/competences-container'; +import CompetencesTable from '@/components/competences/competences-table'; +import CompetencesViewer from '@/components/competences/competences-viewer'; +import { getAllCompetencesOfUser, getAllSpaceCompetences } from '@/lib/data/competences'; +import { getCurrentEnvironment, getCurrentUser } from '@/components/auth'; +import { CompetenceTypes } from '@/lib/data/competence-schema'; +import type { CompetenceType } from '@/lib/data/competence-schema'; + +const CompetencesPage = async ({ + params: { environmentId }, +}: { + params: { environmentId: string }; +}) => { + const { activeEnvironment, ability } = await getCurrentEnvironment(environmentId); + const { spaceId } = activeEnvironment; + const { userId } = await getCurrentUser(); + + const spaceCompetences = await getAllSpaceCompetences(environmentId); + const userCompetences = await getAllCompetencesOfUser(environmentId); + + return ( + + + + ); +}; + +export default CompetencesPage; diff --git a/src/management-system-v2/app/(dashboard)/[environmentId]/layout.tsx b/src/management-system-v2/app/(dashboard)/[environmentId]/layout.tsx index 63a4a82cd..d08d76d5c 100644 --- a/src/management-system-v2/app/(dashboard)/[environmentId]/layout.tsx +++ b/src/management-system-v2/app/(dashboard)/[environmentId]/layout.tsx @@ -21,6 +21,7 @@ import { SettingOutlined, SolutionOutlined, HomeOutlined, + OrderedListOutlined, } from '@ant-design/icons'; import Link from 'next/link'; @@ -122,6 +123,11 @@ const DashboardLayout = async ({ label: Machines, icon: , }, + automationSettings.automations?.active !== false && { + key: 'competences', + label: Competences, + icon: , + }, ].filter(truthyFilter); if (children.length) diff --git a/src/management-system-v2/components/competences/competence-creation-modal.tsx b/src/management-system-v2/components/competences/competence-creation-modal.tsx new file mode 100644 index 000000000..15bc46a5f --- /dev/null +++ b/src/management-system-v2/components/competences/competence-creation-modal.tsx @@ -0,0 +1,122 @@ +'use client'; + +import { Button, Input, message, Modal, Row, Space, Switch, Typography } from 'antd'; +import { CheckOutlined, CloseOutlined } from '@ant-design/icons'; +import { FC, useState } from 'react'; +import { set } from 'zod'; +import { wrapServerCall } from '@/lib/wrap-server-call'; +import { useRouter } from 'next/navigation'; +import { addSpaceCompetence } from '@/lib/data/competences'; +import { useEnvironment } from '../auth-can'; +import { useSession } from 'next-auth/react'; + +type CompetenceCreationModalProps = React.PropsWithChildren<{ + open: boolean; + onClose?: () => void; + environmentId: string; +}>; + +const CompetenceCreationModal: FC = ({ + children, + open, + onClose, + environmentId, +}) => { + const router = useRouter(); + const [loading, setLoading] = useState(false); + + const { spaceId } = useEnvironment(); + const session = useSession(); + const userId = session.data?.user.id; + + // User Input + const [name, setName] = useState(''); + const [description, setDescription] = useState(''); + const [externalQualificationNeeded, setExternalQualificationNeeded] = useState(false); + const [renewalTimeInterval, setRenewalTimeInterval] = useState(undefined); + + const createSpaceCompetence = async () => { + setLoading(true); + await wrapServerCall({ + fn: async () => { + if (!spaceId || !userId) { + message.error(' creating competence'); + setLoading(false); + return; + } + await addSpaceCompetence(environmentId, { + name, + description, + externalQualificationNeeded, + renewalTimeInterval, + }); + }, + onSuccess: () => { + setLoading(false); + + setName(''); + setDescription(''); + setExternalQualificationNeeded(false); + setRenewalTimeInterval(undefined); + + onClose?.(); + router.refresh(); + }, + onError: (err) => { + console.error('Error creating competence', err); + message.error('Error creating competence'); + setLoading(false); + }, + }); + }; + return ( + <> + + Create + , + ]} + > + + {/* Name */} + setName(e.target.value)} /> + {/* Description */} + setDescription(e.target.value)} + autoSize={{ minRows: 3, maxRows: 5 }} + /> + {/* Requires External Qualification */} +
+ Requires External Qualification: +
+ } + unCheckedChildren={} + checked={externalQualificationNeeded} + onChange={(checked) => setExternalQualificationNeeded(checked)} + /> +
+ {/* Renewal Time */} +
+ Renewal Time Interval: +
+
TIME
+
+ + + + ); +}; + +export default CompetenceCreationModal; diff --git a/src/management-system-v2/components/competences/competences-container.module.scss b/src/management-system-v2/components/competences/competences-container.module.scss new file mode 100644 index 000000000..0630a7823 --- /dev/null +++ b/src/management-system-v2/components/competences/competences-container.module.scss @@ -0,0 +1,8 @@ +.container { + width: 100%; + height: 100%; + display: flex; + gap: 1rem; + padding: 1rem; + // background-color: rgba(0, 255, 0, 0.219); +} diff --git a/src/management-system-v2/components/competences/competences-container.tsx b/src/management-system-v2/components/competences/competences-container.tsx new file mode 100644 index 000000000..ff923a4a4 --- /dev/null +++ b/src/management-system-v2/components/competences/competences-container.tsx @@ -0,0 +1,58 @@ +'use client'; +import style from './competences-container.module.scss'; +import { Space } from 'antd'; +import { FC, useLayoutEffect, useRef, useState } from 'react'; +import CompetencesTable from './competences-table'; +import CompetencesViewer from './competences-viewer'; +import { SpaceCompetence } from '@/lib/data/competence-schema'; + +type CompentencesContainerProps = React.PropsWithChildren<{ + competences: SpaceCompetence[]; + environmentId: string; +}>; + +const CompentencesContainer: FC = ({ + children, + competences, + environmentId, +}) => { + const [containerWidth, setContainerWidth] = useState(0); + const containerRef = useRef(null); + // console.log(containerWidth); + + useLayoutEffect(() => { + if (!window) return; + + const handleResize = () => { + if (containerRef.current) { + setContainerWidth(containerRef.current.clientWidth); + } + }; + + handleResize(); + window.addEventListener('resize', handleResize); + + return () => { + window.removeEventListener('resize', handleResize); + }; + }, [containerRef]); + + const [selectedCompetence, setSelectedCompetence] = useState(null); + + return ( + <> +
+ + +
+ + ); +}; + +export default CompentencesContainer; diff --git a/src/management-system-v2/components/competences/competences-table.module.scss b/src/management-system-v2/components/competences/competences-table.module.scss new file mode 100644 index 000000000..b603d8e51 --- /dev/null +++ b/src/management-system-v2/components/competences/competences-table.module.scss @@ -0,0 +1,21 @@ +.outerHandle { + width: 4px; + cursor: ew-resize; + display: flex; + justify-content: center; +} + +.innerHandle { + width: 2px; + background-color: #a6adb5; + height: 100%; + border-radius: 4px; +} +.card { + flex: auto; + height: 100%; +} + +.table { + height: 100%; +} diff --git a/src/management-system-v2/components/competences/competences-table.tsx b/src/management-system-v2/components/competences/competences-table.tsx new file mode 100644 index 000000000..48841c008 --- /dev/null +++ b/src/management-system-v2/components/competences/competences-table.tsx @@ -0,0 +1,271 @@ +'use client'; +import { useUserPreferences } from '@/lib/user-preferences'; +import styles from './competences-table.module.scss'; +import { Button, Card, message, Space, Table, TableProps, Tag } from 'antd'; +import { CheckOutlined, CloseOutlined, DeleteOutlined, PlusOutlined } from '@ant-design/icons'; +import { FC, useState, useRef, ReactNode, useEffect, useLayoutEffect } from 'react'; +import { ResizableBox } from 'react-resizable'; +import { SpaceCompetence, User } from '@/lib/data/competence-schema'; +import { ResizeableTitle, useResizeableColumnWidth } from '@/lib/useColumnWidth'; +import { set } from 'zod'; +import ConfirmationButton from '@/components/confirmation-button'; +import CompetenceCreationModal from './competence-creation-modal'; +import { wrapServerCall } from '@/lib/wrap-server-call'; +import { deleteSpaceCompetence } from '@/lib/data/competences'; +import { useRouter } from 'next/navigation'; + +type CompetencesTableProps = React.PropsWithChildren<{ + containerWidth: number; + competences: SpaceCompetence[]; + selectedCompetence: SpaceCompetence | null; + setSelectedSpaceCompetence: (competence: SpaceCompetence | null) => void; + environmentId: string; +}>; + +const CompetencesTable: FC = ({ + children, + containerWidth, + competences, + selectedCompetence, + setSelectedSpaceCompetence, + environmentId, +}) => { + const addPreferences = useUserPreferences.use.addPreferences(); + const { cardWidth: width } = useUserPreferences.use['competences-table'](); + const preferencesHydrated = useUserPreferences.use._hydrated(); + + const router = useRouter(); + + const [maxTableHeigth, setMaxTableHeigth] = useState(0); + const cardRef = useRef(null); + useLayoutEffect(() => { + if (!cardRef.current) return; + const handleResize = () => { + setMaxTableHeigth(cardRef.current?.clientHeight || 0); + }; + handleResize(); + window.addEventListener('resize', handleResize); + return () => { + window.removeEventListener('resize', handleResize); + }; + }, [cardRef]); + + const [creationModalOpen, setCreationModalOpen] = useState(false); + + const maxWidth = Math.round(containerWidth * 0.7), + minWidth = Math.round(containerWidth * 0.4), + intitalWidth = Math.round(containerWidth * 0.6); + + const setWidth = (newWidth: number) => { + if (!preferencesHydrated) return; + addPreferences({ + 'competences-table': { cardWidth: Math.max(Math.min(newWidth, maxWidth), minWidth) }, + }); + }; + const oldContainerWidth = useRef(containerWidth); + + /* Handle updates of container width */ + useEffect(() => { + /* Set new width */ + /* Edge-Case: Initial render (parent has not figuered out the actual width in px, yet) */ + if (oldContainerWidth.current === 0) { + oldContainerWidth.current = containerWidth; + /* Check whther Preferences yield any value */ + if (preferencesHydrated && Number.isNaN(width)) setWidth(intitalWidth); + return; + } + /* Other resize: */ + /* Get old width in % */ + const ratio = width / oldContainerWidth.current; + /* Set old width to new width */ + oldContainerWidth.current = containerWidth; + setWidth(Math.round(containerWidth * ratio)); + }, [containerWidth]); + + let columns: TableProps['columns'] = [ + { + title: 'Name', + dataIndex: 'name', + render: (value, record, index) => <>{value}, + }, + // { + // title: 'Description', + // dataIndex: 'description', + // render: (text) => <>TEST, + // }, + { + title: 'Creatd By', + dataIndex: 'creatorUserId', + /* TODO: Use ID to user mapping */ + render: (value, record, index) => <>{value}, + }, + { + title: 'Requires Qualification', + dataIndex: 'externalQualificationNeeded', + render: (value, record, index) => ( + <> +
+ {value ? ( + + ) : ( + + )} +
+ + ), + }, + { + title: 'Renewal Time', + dataIndex: 'renewalTimeInterval', + render: (value, record, index) => <>{value}, + }, + { + title: 'Claimed By', + dataIndex: ['claimedBy'], + render: (value, record, index) => ( + <> + {value.map((user: User) => ( + {user.userId} + ))} + + ), + }, + { + title: '', + key: 'delete-entry', + width: 40, + render: (value, record, index) => ( + <> + + {/*