diff --git a/.gitignore b/.gitignore index 81e501ecf..6ca80c0f4 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,80 @@ gha-creds-*.json # Ignore custom claude project files CLAUDE.md + +# Matching models & DB +src/competence-matcher/dist +src/competence-matcher/src/db/dbs +src/competence-matcher/src/models +src/competence-matcher/.env +src/competence-matcher/logs +node_modules/ +.yarn-cache/ +/*.zip +coverage/ + +# PROCEED files +src/engine/native/node/**/data_files +src/engine/native/node/native-config/src/config.json + +src/engine/native/web/server/public/* +src/engine/native/web/server/node_modules + +src/engine/e2e_tests/testEngine/config.json + +# OS files +.DS_Store +.directory + +# Build dirs +jsdoc/output_html/ +!build/index.html +build/* + +# local env files +.env.local +.env.*.local + +# Log files +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# yarn is used as package manager, so npm files +package-lock.json + +# Editor directories and files +.idea +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +# git modules +.gitmodules + +# env configs +./src/management-system/src/backend/server/environment-configurations/ +./src/backend/server/environment-configurations/ + +# Playwright +/test-results/ +/playwright-report/ +/playwright/.cache/ +dataEval.json +/blob-report/ + +# Ignore generated credentials from google-github-actions/auth +gha-creds-*.json + +# Ignore custom claude project files +CLAUDE.md + +# Matching models & DB +src/competence-matcher/dist +src/competence-matcher/src/db/dbs +src/competence-matcher/src/models +src/competence-matcher/.env +src/competence-matcher/logs +src/competence-matcher/server.pid diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 000000000..fdf2b9577 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,4 @@ +src/competence-matcher/dist/ +src/competence-matcher/src/db/dbs/ +src/competence-matcher/src/models/ +src/competence-matcher/.env \ No newline at end of file diff --git a/src/competence-matcher/.env b/src/competence-matcher/.env new file mode 100644 index 000000000..e69de29bb diff --git a/src/competence-matcher/openAPI.json b/src/competence-matcher/openAPI.json new file mode 100644 index 000000000..10a3af685 --- /dev/null +++ b/src/competence-matcher/openAPI.json @@ -0,0 +1,636 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Matching Server API", + "version": "0.1.0", + "description": "API for managing resource competence lists and matching tasks to resources." + }, + "servers": [ + { + "url": "http://localhost:8501" + } + ], + "paths": { + "/": { + "get": { + "tags": ["Default"], + "summary": "Welcome endpoint", + "responses": { + "200": { + "description": "Hello World!", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/resource-competence-list": { + "get": { + "tags": ["Resources"], + "summary": "Get all resource lists (only if multipleDBs=true)", + "responses": { + "200": { + "description": "Array of resource list IDs", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/resource-competence-list/jobs": { + "post": { + "tags": ["Resources"], + "summary": "Create a resource competence list job", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ResourceInput" + } + } + } + } + }, + "responses": { + "202": { + "description": "Job accepted", + "headers": { + "Location": { + "schema": { + "type": "string" + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + } + } + } + }, + "/resource-competence-list/jobs/{jobId}": { + "get": { + "tags": ["Resources"], + "summary": "Get status of a resource creation job", + "parameters": [ + { + "name": "jobId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "202": { + "description": "Job pending/preprocessing/running", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + }, + "201": { + "description": "Completed with competenceListId", + "headers": { + "Location": { + "schema": { + "type": "string" + }, + "description": "Complete Path to created Competence-List" + } + }, + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/JobResponse" + }, + { + "type": "object", + "properties": { + "competenceListId": { + "type": "string" + } + }, + "required": ["competenceListId"] + } + ] + } + } + } + }, + "500": { + "description": "Job failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + } + } + } + }, + "/resource-competence-list/{competenceListId}": { + "get": { + "tags": ["Resources"], + "summary": "Get a specific resource competence list", + "parameters": [ + { + "name": "competenceListId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Resource list details", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResourceList" + } + } + } + }, + "400": { + "description": "Missing resourceListId" + }, + "404": { + "description": "Not found" + } + } + } + }, + "/matching-task-to-resource/jobs": { + "post": { + "tags": ["Matching"], + "summary": "Create a matching job", + "description": "Start a matching job by providing either an existing competenceListId or an inline competenceList.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/MatchByListIdRequest" + }, + { + "$ref": "#/components/schemas/MatchByListRequest" + } + ] + }, + "examples": { + "Match existing list": { + "summary": "Use an existing list ID", + "value": { + "competenceListId": "123e4567-e89b-12d3-a456-426614174000", + "tasks": [ + { + "taskId": "task1", + "name": "Task 1", + "description": "This is what we currently use for matching", + "executionInstructions": "Some execution instructions", + "requiredCompetencies": [ + { + "competenceId": "comp1", + "name": "Competence 1", + "description": "Description of competence 1", + "externalQualificationNeeded": false, + "renewTime": 30, + "proficiencyLevel": "advanced", + "qualificationDates": ["2025-07-01"], + "lastUsages": ["2025-07-10T14:30:00Z"] + } + ] + } + ] + } + }, + "Match inline list": { + "summary": "Pass a full competenceList inline", + "value": { + "competenceList": [ + { + "resourceId": "string", + "competencies": [ + { + "competenceId": "string", + "name": "string", + "description": "string", + "externalQualificationNeeded": true, + "renewTime": 0, + "proficiencyLevel": "string", + "qualificationDates": ["2025-07-15"], + "lastUsages": ["2025-07-15T10:37:09.695Z"] + } + ] + } + ], + "tasks": [ + { + "taskId": "task1", + "name": "Task 1", + "description": "This is what we currently use for matching", + "executionInstructions": "Some execution instructions", + "requiredCompetencies": [ + { + "competenceId": "comp1", + "name": "Competence 1", + "description": "Description of competence 1", + "externalQualificationNeeded": false, + "renewTime": 30, + "proficiencyLevel": "advanced", + "qualificationDates": ["2025-07-01"], + "lastUsages": ["2025-07-10T14:30:00Z"] + } + ] + } + ] + } + } + } + } + } + }, + "responses": { + "202": { + "description": "Match job accepted", + "headers": { + "Location": { + "schema": { + "type": "string" + }, + "description": "Complete Path to created Competence-List" + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + } + } + } + }, + "/matching-task-to-resource/jobs/{jobId}": { + "get": { + "tags": ["Matching"], + "summary": "Get match job results", + "parameters": [ + { + "name": "jobId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "rankBy", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": ["avgFit", "bestFit"] + }, + "description": "Optional ranking method: 'avgFit' or 'bestFit'" + } + ], + "responses": { + "202": { + "description": "Job pending/running", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobResponse" + } + } + } + }, + "200": { + "description": "Match results", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GroupedMatchResults" + } + } + } + }, + "404": { + "description": "Job not found" + }, + "500": { + "description": "Job failed" + } + } + } + } + }, + "components": { + "schemas": { + "ResourceInput": { + "type": "object", + "required": ["resourceId", "competencies"], + "properties": { + "resourceId": { + "type": "string" + }, + "competencies": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CompetenceInput" + } + } + } + }, + "CompetenceInput": { + "type": "object", + "required": ["competenceId"], + "properties": { + "competenceId": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "externalQualificationNeeded": { + "type": "boolean" + }, + "renewTime": { + "type": "number" + }, + "proficiencyLevel": { + "type": "string" + }, + "qualificationDates": { + "type": "array", + "items": { + "type": "string", + "format": "date" + } + }, + "lastUsages": { + "type": "array", + "items": { + "type": "string", + "format": "date-time" + } + } + } + }, + "ResourceList": { + "type": "object", + "required": ["competenceListId", "resources"], + "properties": { + "competenceListId": { + "type": "string" + }, + "resources": { + "type": "array", + "items": { + "type": "object", + "required": ["resourceId", "competencies"], + "properties": { + "resourceId": { + "type": "string" + }, + "competencies": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CompetenceInput" + } + } + } + } + } + } + }, + "JobResponse": { + "type": "object", + "required": ["jobId", "status"], + "properties": { + "jobId": { + "type": "string" + }, + "status": { + "type": "string" + } + } + }, + "MatchByListIdRequest": { + "type": "object", + "required": ["competenceListId", "tasks"], + "properties": { + "competenceListId": { + "type": "string" + }, + "tasks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MatchingTask" + } + } + } + }, + "MatchByListRequest": { + "type": "object", + "required": ["competenceList", "tasks"], + "properties": { + "competenceList": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ResourceInput" + } + }, + "tasks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MatchingTask" + } + } + } + }, + "MatchingTask": { + "type": "object", + "required": ["taskId"], + "properties": { + "taskId": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "executionInstructions": { + "type": "string" + }, + "requiredCompetencies": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/components/schemas/CompetenceInput" + } + ] + } + } + } + }, + "GroupedMatchResults": { + "type": "object", + "required": ["taskOverview", "resourceRanking"], + "properties": { + "tasks": { + "$ref": "#/components/schemas/TaskOverview" + }, + "resourceRanking": { + "$ref": "#/components/schemas/ResourceRanking" + } + } + }, + "TaskOverview": { + "type": "array", + "items": { + "type": "object", + "required": ["taskId", "taskText"], + "properties": { + "taskId": { + "type": "string" + }, + "taskText": { + "type": "string" + } + } + } + }, + "ResourceRanking": { + "type": "array", + "items": { + "type": "object", + "required": ["resourceId", "taskMatchings", "avgTaskMatchProbability"], + "properties": { + "resourceId": { + "type": "string" + }, + "taskMatchings": { + "type": "array", + "items": { + "type": "object", + "required": ["taskId", "taskText", "competenceMatchings", "maxMatchProbability"], + "properties": { + "taskId": { + "type": "string" + }, + "competenceMatchings": { + "type": "array", + "items": { + "type": "object", + "required": ["competenceId", "matchings", "avgMatchProbability"], + "properties": { + "competenceId": { + "type": "string" + }, + "matchings": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MatchDetail" + } + }, + "avgMatchProbability": { + "type": "number" + }, + "avgBestFitTaskMatchProbability": { + "type": "number" + } + } + } + }, + "maxMatchProbability": { + "type": "number" + }, + "maxBestFitMatchProbability": { + "type": "number" + } + } + } + }, + "avgTaskMatchProbability": { + "type": "number" + }, + "avgBestFitTaskMatchProbability": { + "type": "number" + }, + "contradicting": { + "type": "boolean" + } + } + } + }, + "MatchDetail": { + "type": "object", + "required": ["text", "type", "matchProbability"], + "properties": { + "text": { + "type": "string" + }, + "type": { + "type": "string", + "enum": ["name", "description", "proficiencyLevel"] + }, + "matchProbability": { + "type": "number" + }, + "alignment": { + "type": "string", + "enum": ["aligning", "neutral", "contradicting"] + }, + "reason": { + "type": "string" + } + } + } + } + } +} diff --git a/src/competence-matcher/package.json b/src/competence-matcher/package.json new file mode 100644 index 000000000..78d8001ce --- /dev/null +++ b/src/competence-matcher/package.json @@ -0,0 +1,44 @@ +{ + "name": "competence-matcher", + "version": "0.1.0", + "description": "Matching microservice that allows to allows to define and match on data criteria", + "main": "dist/server.js", + "scripts": { + "dev": "ts-node-dev --respawn --transpile-only --watch .env src/server.ts", + "build": "tsc", + "production": "if [ -f ./server.pid ]; then PID=$(cat ./server.pid); if ps -p $PID > /dev/null 2>&1 && ps -p $PID -o args= | grep -q node; then echo \"Server already running (pid $PID)\"; exit 1; fi; fi; node dist/server.js", + "production:detached": "if [ -f ./server.pid ]; then PID=$(cat ./server.pid); if ps -p $PID > /dev/null 2>&1 && ps -p $PID -o args= | grep -q node; then echo \"Server already running (pid $PID)\"; exit 1; fi; fi; nohup node dist/server.js >/dev/null 2>&1 & echo $! > ./server.pid", + "production:stop": "if [ -f ./server.pid ]; then PID=$(cat ./server.pid); if ps -p $PID > /dev/null 2>&1; then kill $PID >/dev/null 2>&1 || true; sleep 1; fi; rm -f ./server.pid; else echo \"No server.pid file found\"; fi" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/PROCEED-Labs/proceed.git" + }, + "keywords": [ + "embedding", + "matching" + ], + "author": "PROCEED Project", + "license": "MIT", + "bugs": { + "url": "https://github.com/PROCEED-Labs/proceed/issues" + }, + "homepage": "https://github.com/PROCEED-Labs/proceed#readme", + "dependencies": { + "@huggingface/transformers": "^3.7.6", + "dotenv": "^17.2.1", + "express": "^5.1.0", + "ollama": "^0.5.16", + "sqlite-vec": "^0.1.7-alpha.2", + "uuid": "^11.1.0" + }, + "devDependencies": { + "@types/express": "^5.0.2", + "@types/node": "^22.15.30", + "ts-node-dev": "^2.0.0", + "typescript": "^5.8.3" + }, + "engines": { + "node": ">=23.5.0" + } +} diff --git a/src/competence-matcher/src/config.ts b/src/competence-matcher/src/config.ts new file mode 100644 index 000000000..58eb40272 --- /dev/null +++ b/src/competence-matcher/src/config.ts @@ -0,0 +1,46 @@ +import dotenv from 'dotenv'; +dotenv.config({ path: '.env' }); + +export const config = { + dbPath: process.env.DB_PATH || 'src/db/dbs/', + embeddingModel: process.env.EMBEDDING_MODEL || 'onnx-community/Qwen3-Embedding-0.6B-ONNX', + embeddingDim: parseInt(process.env.EMBEDDING_DIM || '1024', 10), + nliModel: process.env.NLI_MODEL || 'Maxi-Lein/roberta-large-mnli-onnx', + crossEncoderModel: process.env.CROSS_ENCODER_MODEL || 'local/ms-marco-MiniLM-L6-v2-onnx', + // crossEncoderModel: process.env.CROSS_ENCODER_MODEL || 'cross-encoder/ms-marco-MiniLM-L6-v2', + modelCache: process.env.MODEL_CACHE || 'src/models/', + useGPU: process.env.USE_GPU === 'true' || false, + port: parseInt(process.env.PORT || '8501', 10), + multipleDBs: process.env.MULTIPLE_DBS === 'true' || false, + ollamaPath: process.env.OLLAMA_PATH || 'http://localhost:11434', + ollamaBearerToken: process.env.OLLAMA_BEARER_TOKEN || '', + ollamaBatchSize: parseInt(process.env.OLLAMA_BATCH_SIZE || '20', 10), + splittingModel: process.env.SPLITTING_MODEL || 'llama3.2', + splittingLength: parseInt(process.env.SPLITTING_LENGTH || '1000', 0), // Set this to 0 to disable splitting + reasonModel: process.env.REASON_MODEL || 'llama3.2', + splittingSymbol: process.env.SPLITTING_SYMBOL || '', + workerHeartbeatInterval: parseInt(process.env.WORKER_HEARTBEAT_INTERVAL || '120', 10) * 1_000, + workerDeathTimeout: parseInt(process.env.WORKER_DEATH_TIMEOUT || '360', 10) * 1_000, + logLevel: process.env.LOG_LEVEL || 'INFO', // Levels: 'DEBUG', 'INFO', 'WARN', 'ERROR' + logTypes: process.env.LOG_TYPES || 'server,request,worker,database,model,system', + logToConsole: process.env.LOG_CONSOLE !== 'false', // Default to true unless explicitly set to false + logToFile: process.env.LOG_FILE === 'true' || false, // Default to false unless explicitly set to true + logPath: process.env.LOG_PATH || 'logs/', + jobMaxRetries: parseInt(process.env.JOB_MAX_RETRIES || '3', 10), // Maximum number of retries for a job before failing it + maxOllamaRetries: parseInt(process.env.MAX_OLLAMA_RETRIES || '5', 10), // Maximum model pull retry attempts + ollamaRetryDelay: parseInt(process.env.OLLAMA_RETRY_DELAY || '30', 10) * 1_000, // Base delay between retries in seconds (converted to ms) + ollamaRetryBackoff: parseFloat(process.env.OLLAMA_RETRY_BACKOFF || '1.5'), // Exponential backoff multiplier + // Matching algorithm scaling parameters + matchDistanceOffset: parseFloat(process.env.MATCH_DISTANCE_OFFSET || '0.45'), // Offset subtracted from match distance before scaling + matchDistanceMultiplier: parseFloat(process.env.MATCH_DISTANCE_MULTIPLIER || '2'), // Multiplier for distance after offset + contradictionThreshold: parseFloat(process.env.CONTRADICTION_THRESHOLD || '0.3'), // Threshold for contradiction detection + entailmentThreshold: parseFloat(process.env.ENTAILMENT_THRESHOLD || '0.55'), // Threshold for entailment detection + alignmentDistanceThreshold: parseFloat(process.env.ALIGNMENT_DISTANCE_THRESHOLD || '0.65'), // Minimum distance required for alignment boost + alignmentBoostMultiplier: parseFloat(process.env.ALIGNMENT_BOOST_MULTIPLIER || '1.2'), // Multiplier to boost distance for aligning matches + neutralReductionMultiplier: parseFloat(process.env.NEUTRAL_REDUCTION_MULTIPLIER || '0.65'), // Multiplier to reduce distance for neutral matches + + // Deprecated settings + /* For multiworker part, can be removed, if we actually settle for */ + embeddingWorkers: parseInt(process.env.EMBEDDING_WORKERS || '1', 10), // Number of embedding workers + matchingWorkers: parseInt(process.env.MATCHING_WORKERS || '1', 10), // Number of matching workers +}; diff --git a/src/competence-matcher/src/db/db-manager.ts b/src/competence-matcher/src/db/db-manager.ts new file mode 100644 index 000000000..d5ab99d4f --- /dev/null +++ b/src/competence-matcher/src/db/db-manager.ts @@ -0,0 +1,162 @@ +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import VectorDataBase from './db'; +import { config } from '../config'; + +const { dbPath: rawDbPath, embeddingDim } = config; + +/** + * DBManager: Singleton that manages multiple VectorDataBase instances keyed by name. + * + */ +class DBManager { + private static managerInstance: DBManager; + private dbInstances = new Map(); + private static activeDB: VectorDataBase | null = null; + private dbPath: string; + private embeddingDim: number; + + private constructor() { + this.embeddingDim = embeddingDim; + + // Resolve absolute path for storage directory + this.dbPath = path.resolve(rawDbPath); + // Ensure directory exists + if (!fs.existsSync(this.dbPath)) { + fs.mkdirSync(this.dbPath, { recursive: true }); + } + // Load existing databases + this.loadSavedDBs(); + } + + /** + * Retrieve the singleton DBManager instance, initialising if necessary. + * @returns DBManager singleton + */ + public static getInstance(): DBManager { + if (!DBManager.managerInstance) { + DBManager.managerInstance = new DBManager(); + } + return DBManager.managerInstance; + } + + /** + * Initialise the DBManager and load any existing databases. + */ + private loadSavedDBs(): void { + // Load existing databases from the storage directory + const files = fs.readdirSync(this.dbPath); + files.forEach((file) => { + if (file.endsWith('.db')) { + const dbName = path.basename(file, '.db'); + this.addDBInstance(dbName); + } + }); + } + + /** + * Normalise a database name by stripping any extension and enforcing `.db`. + * @param dbName Name provided; may include extension. + * @returns Normalised filename ending with `.db`. + */ + private normaliseDBName(dbName: string): string { + const base = path.basename(dbName, path.extname(dbName)); + return `${base}.db`; + } + + /** + * Resolve full absolute path to the DB file under storage directory. + * @param dbName Name provided by user; normalised to `.db` and joined with storage dir. + * @returns Absolute file path for the database. + */ + private resolveDbPath(dbName: string): string { + const normalisedDBName = this.normaliseDBName(dbName); + return path.join(this.dbPath, normalisedDBName); + } + + /** + * Internal: create and cache a new VectorDataBase instance for the given name. + * Uses resolveDbPath to obtain the absolute file path. + * @param dbName Name provided by user; normalised internally. + * @returns Newly created VectorDataBase instance. + */ + private addDBInstance(dbName: string): VectorDataBase { + const normalisedDBName = this.normaliseDBName(dbName); + const filePath = this.resolveDbPath(normalisedDBName); + const db = new VectorDataBase({ filePath, embeddingDim: this.embeddingDim }); + this.dbInstances.set(normalisedDBName, db); + return db; + } + + /** + * Get the currently active VectorDataBase instance, if set via setActiveDB. + * @returns Active VectorDataBase or null if none is set. + */ + public static getActiveDB(): VectorDataBase | null { + return DBManager.activeDB; + } + + /** + * Set the active database by name. Creates the instance if it does not exist. + * @param dbName Name of the database (without extension or with any extension). + */ + public static setActiveDB(dbName: string): void { + const manager = DBManager.getInstance(); + const normalisedDBName = manager.normaliseDBName(dbName); + if (!manager.dbInstances.has(normalisedDBName)) { + manager.addDBInstance(normalisedDBName); + } + DBManager.activeDB = manager.dbInstances.get(normalisedDBName)!; + } + + /** + * Retrieve (or create) the VectorDataBase instance for given name. + * @param dbName Name of the database (without extension or with any extension). + * @returns VectorDataBase instance corresponding to the name. + */ + public getDB(dbName: string): VectorDataBase { + const normalisedDBName = this.normaliseDBName(dbName); + if (this.dbInstances.has(normalisedDBName)) { + return this.dbInstances.get(normalisedDBName)!; + } + return this.addDBInstance(normalisedDBName); + } + + /** + * Close and remove the VectorDataBase instance for given name. + * @param dbName Name of the database to close. + * @returns True if instance existed and was closed; false otherwise. + */ + public closeDB(dbName: string): boolean { + const normalisedDBName = this.normaliseDBName(dbName); + const db = this.dbInstances.get(normalisedDBName); + if (db) { + db.close(); + this.dbInstances.delete(normalisedDBName); + if (DBManager.activeDB === db) { + DBManager.activeDB = null; + } + return true; + } + return false; + } + + /** + * Close and remove all managed VectorDataBase instances. + */ + public closeAllDBs(): void { + this.dbInstances.forEach((db) => db.close()); + this.dbInstances.clear(); + DBManager.activeDB = null; + } + + /** + * List the names (normalised) of all managed databases. + * @returns Array of database filenames (e.g. ['tenant1.db', 'other.db']). + */ + public listDBs(): string[] { + return Array.from(this.dbInstances.keys()); + } +} + +export default DBManager; diff --git a/src/competence-matcher/src/db/db.ts b/src/competence-matcher/src/db/db.ts new file mode 100644 index 000000000..af2a97fd1 --- /dev/null +++ b/src/competence-matcher/src/db/db.ts @@ -0,0 +1,959 @@ +// db.ts +import { DatabaseSync } from 'node:sqlite'; +import * as path from 'node:path'; +import * as sqliteVec from 'sqlite-vec'; +import { v4 as uuid } from 'uuid'; +import { CompetenceDBOutput, VectorDBOptions } from '../utils/types'; + +class VectorDataBase { + private db: DatabaseSync; + private embeddingDim: number; + private transactionInProgress = false; + + /** + * Opens (or creates) the SQLite DB, enables FKs, loads sqlite-vec, and sets up schema. + */ + constructor(opts: VectorDBOptions) { + this.embeddingDim = opts.embeddingDim; + const dbPath = + !opts.filePath || opts.filePath === ':memory:' + ? ':memory:' + : path.isAbsolute(opts.filePath) + ? opts.filePath + : path.join(process.cwd(), opts.filePath); + + this.db = new DatabaseSync(dbPath, { allowExtension: true }); + this.db.exec(`PRAGMA foreign_keys = ON;`); + sqliteVec.load(this.db); + this.initSchema(); + } + + /** Close the database connection */ + public close(): void { + this.db.close(); + } + + /** Run a set of operations atomically (in a transaction) */ + public atomicStep(cb: () => void): void { + if (this.transactionInProgress) throw new Error('Transaction already in progress'); + this.transactionInProgress = true; + this.db.exec('BEGIN'); + try { + cb(); + this.db.exec('COMMIT'); + } catch (e) { + this.db.exec('ROLLBACK'); + throw e; + } finally { + this.transactionInProgress = false; + } + } + + /** Set up all tables, indexes and virtual tables */ + private initSchema() { + // jobs + this.db.exec(` + CREATE TABLE IF NOT EXISTS jobs ( + id TEXT PRIMARY KEY, + status TEXT NOT NULL DEFAULT 'pending', + reference_id TEXT + ); + `); + + // matches + this.db.exec(` + CREATE TABLE IF NOT EXISTS match_results ( + id TEXT PRIMARY KEY, -- UUID for this match record + job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, + task_id TEXT NOT NULL, -- task ID this match belongs to, + task_text TEXT NOT NULL, -- task text that was used for matching + competence_id TEXT NOT NULL, -- matched competence + resource_id TEXT NOT NULL, -- resource ID the competence belongs to + distance REAL NOT NULL, -- similarity score + text TEXT NOT NULL, -- the matched snippet + type TEXT NOT NULL, -- 'name' | 'description' | 'proficiencyLevel' + alignment TEXT NOT NULL, -- 'contradicting' | 'neutral' | 'aligning' + reason TEXT -- llm based reason for the match + ); + `); + this.db.exec(` + CREATE INDEX IF NOT EXISTS ix_match_results_job + ON match_results(job_id); + `); + + // task embeddings per job (transient, cleared after matching) + this.db.exec(` + CREATE TABLE IF NOT EXISTS task_embedding ( + job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, + task_id TEXT NOT NULL, + embedding FLOAT32[${this.embeddingDim}], + PRIMARY KEY (job_id, task_id) + ); + `); + + // resource_list + this.db.exec(` + CREATE TABLE IF NOT EXISTS resource_list ( + id TEXT PRIMARY KEY + ); + `); + + // resources (internal PK + user‐facing ID) + this.db.exec(` + CREATE TABLE IF NOT EXISTS resource ( + _rid INTEGER PRIMARY KEY AUTOINCREMENT, + resource_id TEXT NOT NULL, + list_id TEXT NOT NULL REFERENCES resource_list(id) ON DELETE CASCADE + ); + `); + this.db.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS ux_resource_list_resid + ON resource(list_id, resource_id); + `); + + // competences (internal PK + user‐facing ID) + this.db.exec(` + CREATE TABLE IF NOT EXISTS competence ( + _cid INTEGER PRIMARY KEY AUTOINCREMENT, + competence_id TEXT NOT NULL, + resource_rid INTEGER NOT NULL REFERENCES resource(_rid) ON DELETE CASCADE, + competence_name TEXT, + competence_description TEXT, + external_qualification_needed BOOLEAN DEFAULT FALSE, + renew_time INTEGER, + proficiency_level TEXT, + qualification_dates TEXT, + last_usages TEXT + ); + `); + this.db.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS ux_competence_rescid + ON competence(resource_rid, competence_id); + `); + + // embeddings (virtual vec0 table; explicit deletes required) + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS competence_embedding + USING vec0( + cid INTEGER NOT NULL REFERENCES competence(_cid) ON DELETE CASCADE, + text TEXT, + type TEXT, + embedding FLOAT32[${this.embeddingDim}] + ); + `); + } + + /*-------------------------------------------------------------------- + * Helper Lookups + *------------------------------------------------------------------*/ + + /** Get the internal `_rid` for a given user‐facing resourceId + listId */ + private getResourceRid(resourceId: string, listId: string): number { + const row = this.db + .prepare(`SELECT _rid FROM resource WHERE resource_id = ? AND list_id = ?`) + .get(resourceId, listId); + if (!row) throw new Error(`Resource '${resourceId}' not found in list '${listId}'`); + return row._rid as number; + } + + /** Get the internal `_cid` for a given user‐facing competenceId */ + private getCompetenceCidByCompetenceId( + listId: string, + resourceId: string, + competenceId: string, + ): number { + const _rid = this.getResourceRid(resourceId, listId); + const row = this.db + .prepare( + ` + SELECT _cid + FROM competence + WHERE competence_id = ? + AND resource_rid = ? + `, + ) + .get(competenceId, _rid); + if (!row) + throw new Error( + `Competence '${competenceId}' in List ${listId} in Resource ${resourceId} not found`, + ); + return row._cid as number; + } + + /*-------------------------------------------------------------------- + * Job Methods + *------------------------------------------------------------------*/ + + /** + * Create a new background‐job record. + * @param referenceId Optionally point back to a resource-list, resource, or competence ID. + * @returns the new job’s UUID. + */ + public createJob(referenceId?: string): string { + const jobId = uuid(); + this.db + .prepare(`INSERT INTO jobs(id, reference_id) VALUES (?, ?)`) + .run(jobId, referenceId ?? null); + return jobId; + } + + /** + * Change a job’s status. + * @throws if no such job exists. + */ + public updateJobStatus( + jobId: string, + status: 'pending' | 'preprocessing' | 'running' | 'completed' | 'failed', + ): void { + const result = this.db.prepare(`UPDATE jobs SET status = ? WHERE id = ?`).run(status, jobId); + if (result.changes === 0) throw new Error(`Job with id ${jobId} not found`); + } + + /** + * Look up a job’s current status and its referenceId. + * @throws if no such job exists. + */ + public getJob(jobId: string): { jobId: string; status: string; referenceId?: string } { + const row = this.db + .prepare(`SELECT id, status, reference_id FROM jobs WHERE id = ?`) + .get(jobId) as { id: string; status: string; reference_id: string } | undefined; + if (!row) throw new Error(`Job with id ${jobId} not found`); + return { jobId: row.id, status: row.status, referenceId: row.reference_id ?? undefined }; + } + + /*-------------------------------------------------------------------- + * Match Methods + *------------------------------------------------------------------*/ + + /** + * Add a match result for a job, task, and competence. + * + * @param opts Options for adding a match result. + * @throws if the jobId, taskId, or competenceId do not exist. + */ + public addMatchResult(opts: { + jobId: string; + taskId: string; + taskText: string; + competenceId: string; + resourceId: string; + distance: number; + text: string; + type: string; // 'name' | 'description' | 'proficiencyLevel' + alignment: string; // 'contradicting' | 'neutral' | 'aligning' + reason?: string; // optional reason for the match + }): void { + const id = uuid(); + this.db + .prepare( + ` + INSERT INTO match_results + (id, job_id, task_id, task_text, competence_id, resource_id, distance, text, type, alignment, reason) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ) + .run( + id, + opts.jobId, + opts.taskId, + opts.taskText, + opts.competenceId, + opts.resourceId, + opts.distance, + opts.text, + opts.type, + opts.alignment, + opts.reason ?? null, + ); + } + + /** + * Fetch all match results for a given jobId. + * @returns an array of match results, sorted by taskId and distance. + */ + public getMatchResults(jobId: string): Array<{ + taskId: string; + taskText: string; + competenceId: string; + resourceId: string; + distance: number; + text: string; + type: string; + alignment: string; // 'contradicting' | 'neutral' | 'aligning' + reason?: string; + }> { + return this.db + .prepare( + ` + SELECT task_id, task_text, competence_id, resource_id, distance, text, type, alignment, reason + FROM match_results + WHERE job_id = ? + ORDER BY task_id, distance + `, + ) + .all(jobId) + .map((r: any) => ({ + taskId: r.task_id, + taskText: r.task_text, + competenceId: r.competence_id, + resourceId: r.resource_id, + distance: r.distance, + text: r.text, + type: r.type, + alignment: r.alignment, + reason: r.reason ?? undefined, + })); + } + + /*-------------------------------------------------------------------- + * ResourceList Methods + *------------------------------------------------------------------*/ + + /** Create a fresh, empty resource‐list and return its UUID */ + public createResourceList(): string { + const listId = uuid(); + this.db.prepare(`INSERT INTO resource_list(id) VALUES (?)`).run(listId); + return listId; + } + + /** + * Delete an entire list—this cascades down to resources, competences, + * explicitly wipes embeddings. + */ + public deleteResourceList(listId: string): void { + this.atomicStep(() => { + this.db + .prepare( + ` + DELETE FROM competence_embedding + WHERE cid IN ( + SELECT c._cid + FROM competence c + JOIN resource r ON c.resource_rid = r._rid + WHERE r.list_id = ? + ) + `, + ) + .run(listId); + this.db.prepare(`DELETE FROM resource_list WHERE id = ?`).run(listId); + }); + } + + /** Enumerate all list IDs */ + public getAvailableResourceLists(): string[] { + return this.db + .prepare(`SELECT id FROM resource_list`) + .all() + .map((r) => (r as any).id); + } + + /** + * Fetch a list plus all its resources and each resource’s competences. + * @throws if listId doesn’t exist. + */ + public getResourceList(listId: string): { + competenceListId: string; + resources: Array<{ + resourceId: string; + competencies: Array<{ + competenceId: string; + name?: string; + description?: string; + externalQualificationNeeded: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates: string[]; + lastUsages: string[]; + }>; + }>; + } { + const exists = this.db.prepare(`SELECT 1 FROM resource_list WHERE id = ?`).get(listId); + if (!exists) throw new Error(`Resource list '${listId}' not found`); + + const resources = this.db + .prepare(`SELECT _rid, resource_id FROM resource WHERE list_id = ?`) + .all(listId) as Array<{ _rid: number; resource_id: string }>; + + return { + competenceListId: listId, + resources: resources.map(({ _rid, resource_id }) => { + const comps = this.db + .prepare( + ` + SELECT competence_id, competence_name, competence_description, + external_qualification_needed, renew_time, + proficiency_level, qualification_dates, last_usages + FROM competence + WHERE resource_rid = ? + `, + ) + .all(_rid) as CompetenceDBOutput[]; + return { + resourceId: resource_id, + competencies: comps.map((c) => ({ + competenceId: c.competence_id, + name: c.competence_name ?? undefined, + description: c.competence_description ?? undefined, + externalQualificationNeeded: Boolean(c.external_qualification_needed), + renewTime: c.renew_time ?? undefined, + proficiencyLevel: c.proficiency_level ?? undefined, + qualificationDates: c.qualification_dates ? JSON.parse(c.qualification_dates) : [], + lastUsages: c.last_usages ? JSON.parse(c.last_usages) : [], + })), + }; + }), + }; + } + + /*-------------------------------------------------------------------- + * Resource Methods + *------------------------------------------------------------------*/ + + /** + * Add a resource (user‐facing ID) into a list. + * Returns the user‐facing resourceId. + */ + public addResource(listId: string, resourceId?: string): string { + const rid = resourceId ?? uuid(); + this.db.prepare(`INSERT INTO resource(resource_id, list_id) VALUES (?, ?)`).run(rid, listId); + return rid; + } + + /** + * Move a resource from one list to another. + * @param oldListId current list + * @param resourceId user‐facing ID + * @param newListId target list + */ + public updateResource(oldListId: string, resourceId: string, newListId: string): void { + const _rid = this.getResourceRid(resourceId, oldListId); + this.db.prepare(`UPDATE resource SET list_id = ? WHERE _rid = ?`).run(newListId, _rid); + } + + /** + * Delete a single resource (and its subtree) by user‐facing ID + list. + */ + public deleteResource(listId: string, resourceId: string): void { + this.atomicStep(() => { + const _rid = this.getResourceRid(resourceId, listId); + this.db + .prepare( + ` + DELETE FROM competence_embedding + WHERE cid IN (SELECT _cid FROM competence WHERE resource_rid = ?) + `, + ) + .run(_rid); + this.db.prepare(`DELETE FROM resource WHERE _rid = ?`).run(_rid); + }); + } + + /** + * Fetch one resource + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing ID of the resource + * @returns an object with the resource’s metadata. + * @throws if not found. + */ + public getResource( + listId: string, + resourceId: string, + ): { + listId: string; + resourceId: string; + competencies: Array<{ + competenceId: string; + name?: string; + description?: string; + externalQualificationNeeded: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates: string[]; + lastUsages: string[]; + }>; + } { + const row = this.db + .prepare( + ` + SELECT _rid, resource_id, list_id + FROM resource + WHERE resource_id = ? + AND list_id = ? + `, + ) + .get(resourceId, listId) as + | { _rid: number; resource_id: string; list_id: string } + | undefined; + if (!row) throw new Error(`Resource '${resourceId}' in List '${listId}' not found`); + + const comps = this.db + .prepare( + ` + SELECT competence_id, competence_name, competence_description, + external_qualification_needed, renew_time, + proficiency_level, qualification_dates, last_usages + FROM competence + WHERE resource_rid = ? + `, + ) + .all(row._rid) as CompetenceDBOutput[]; + + return { + listId: row.list_id, + resourceId, + competencies: comps.map((c) => ({ + competenceId: c.competence_id, + name: c.competence_name ?? undefined, + description: c.competence_description ?? undefined, + externalQualificationNeeded: Boolean(c.external_qualification_needed), + renewTime: c.renew_time ?? undefined, + proficiencyLevel: c.proficiency_level ?? undefined, + qualificationDates: c.qualification_dates ? JSON.parse(c.qualification_dates) : [], + lastUsages: c.last_usages ? JSON.parse(c.last_usages) : [], + })), + }; + } + + /*-------------------------------------------------------------------- + * Competence Methods + *------------------------------------------------------------------*/ + + /** + * Add a competence under a given resource. + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing + * @param competence Input object (may supply its own competenceId) + * @returns the user‐facing competenceId + */ + public addCompetence( + listId: string, + resourceId: string, + competence: { + competenceId?: string; + name?: string; + description?: string; + externalQualificationNeeded?: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates?: string[]; + lastUsages?: string[]; + }, + ): string { + const cidUser = competence.competenceId ?? uuid(); + const _rid = this.getResourceRid(resourceId, listId); + + this.db + .prepare( + ` + INSERT INTO competence + (competence_id, resource_rid, + competence_name, competence_description, + external_qualification_needed, renew_time, + proficiency_level, qualification_dates, last_usages) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ) + .run( + cidUser, + _rid, + competence.name ?? null, + competence.description ?? null, + competence.externalQualificationNeeded ? 1 : 0, + competence.renewTime ?? null, + competence.proficiencyLevel ?? null, + competence.qualificationDates ? JSON.stringify(competence.qualificationDates) : null, + competence.lastUsages ? JSON.stringify(competence.lastUsages) : null, + ); + + return cidUser; + } + + /** + * Update a competence’s metadata. + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing ID of the resource + * @param competenceId user‐facing ID of the competence to update + * @param fields object with fields to update; only those provided will be changed + * @returns nothing, but throws if the competence does not exist. + * @throws if no such competence exists on the resource. + */ + public updateCompetence( + listId: string, + resourceId: string, + competenceId: string, + fields: { + name?: string; + description?: string; + externalQualificationNeeded?: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates?: string[]; + lastUsages?: string[]; + }, + ): void { + const _rid = this.getResourceRid(resourceId, listId); + const _cid = this.db + .prepare(`SELECT _cid FROM competence WHERE competence_id = ? AND resource_rid = ?`) + .get(competenceId, _rid)?.['_cid']; + if (!_cid) throw new Error(`Competence '${competenceId}' not on resource '${resourceId}'`); + + const sets: string[] = []; + const params: any[] = []; + if (fields.name !== undefined) { + sets.push(`competence_name = ?`); + params.push(fields.name); + } + if (fields.description !== undefined) { + sets.push(`competence_description = ?`); + params.push(fields.description); + } + if (fields.externalQualificationNeeded !== undefined) { + sets.push(`external_qualification_needed = ?`); + params.push(fields.externalQualificationNeeded ? 1 : 0); + } + if (fields.renewTime !== undefined) { + sets.push(`renew_time = ?`); + params.push(fields.renewTime); + } + if (fields.proficiencyLevel !== undefined) { + sets.push(`proficiency_level = ?`); + params.push(fields.proficiencyLevel); + } + if (fields.qualificationDates !== undefined) { + sets.push(`qualification_dates = ?`); + params.push(JSON.stringify(fields.qualificationDates)); + } + if (fields.lastUsages !== undefined) { + sets.push(`last_usages = ?`); + params.push(JSON.stringify(fields.lastUsages)); + } + + if (sets.length > 0) { + params.push(_cid); + this.db.prepare(`UPDATE competence SET ${sets.join(', ')} WHERE _cid = ?`).run(...params); + } + } + + /** + * Delete a competence from a resource. + * @param resourceId user‐facing ID + * @param listId user‐facing ID of the resource list + * @param competenceId user‐facing ID of the competence to delete + * @throws if no such competence exists on the resource. + */ + public deleteCompetence(listId: string, resourceId: string, competenceId: string): void { + this.atomicStep(() => { + const _rid = this.getResourceRid(resourceId, listId); + const _cid = this.db + .prepare(`SELECT _cid FROM competence WHERE competence_id = ? AND resource_rid = ?`) + .get(competenceId, _rid)?._cid; + if (!_cid) throw new Error(`Competence '${competenceId}' not on resource '${resourceId}'`); + + // explicitly delete embeddings + this.db.prepare(`DELETE FROM competence_embedding WHERE cid = ?`).run(_cid); + // then delete competence row + this.db.prepare(`DELETE FROM competence WHERE _cid = ?`).run(_cid); + }); + } + + /** + * Fetch one competence’s metadata (including listId + resourceId). + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing ID of the resource + * @param competenceId user‐facing ID of the competence to fetch + * @returns an object with the competence’s metadata. + * @throws if no such competence exists on the resource. + * @throws if the competenceId is not found on the resource. + */ + public getCompetence( + listId: string, + resourceId: string, + competenceId: string, + ): { + listId: string; + resourceId: string; + competenceId: string; + name?: string; + description?: string; + externalQualificationNeeded: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates: string[]; + lastUsages: string[]; + } { + const _rid = this.getResourceRid(resourceId, listId); + const row = this.db + .prepare( + ` + SELECT c.competence_id, c.competence_name, c.competence_description, + c.external_qualification_needed, c.renew_time, + c.proficiency_level, c.qualification_dates, c.last_usages + FROM competence c + WHERE c.competence_id = ? AND c.resource_rid = ? + `, + ) + .get(competenceId, _rid) as + | { + competence_id: string; + competence_name: string | null; + competence_description: string | null; + external_qualification_needed: number; + renew_time: number | null; + proficiency_level: string | null; + qualification_dates: string | null; + last_usages: string | null; + } + | undefined; + if (!row) throw new Error(`Competence '${competenceId}' not on resource '${resourceId}'`); + + return { + listId, + resourceId, + competenceId: row.competence_id, + name: row.competence_name ?? undefined, + description: row.competence_description ?? undefined, + externalQualificationNeeded: Boolean(row.external_qualification_needed), + renewTime: row.renew_time ?? undefined, + proficiencyLevel: row.proficiency_level ?? undefined, + qualificationDates: row.qualification_dates ? JSON.parse(row.qualification_dates) : [], + lastUsages: row.last_usages ? JSON.parse(row.last_usages) : [], + }; + } + + /*-------------------------------------------------------------------- + * Embedding Methods + *------------------------------------------------------------------*/ + + /** + * Insert or replace a text embedding for a competence. + * This will overwrite any existing embedding for the same text and type. + * @param embeddingInput object with competenceId, text, type, and embedding vector. + * @throws if the embedding vector does not match the configured dimension. + */ + public upsertEmbedding(embeddingInput: { + listId: string; + resourceId: string; + competenceId: string; + text: string; + type: 'name' | 'description' | 'proficiencyLevel'; + embedding: number[]; + }): void { + const { listId, resourceId, competenceId, text, type, embedding } = embeddingInput; + if (embedding.length !== this.embeddingDim) { + throw new Error(`Embedding must have length ${this.embeddingDim}`); + } + const cid = this.getCompetenceCidByCompetenceId(listId, resourceId, competenceId); + + const cidInt = `${Math.floor(cid)}`; // This + the cast is a workaround, sqlite-vec or sqlite read the cid as a float even though it is an integer. (Could be the lib or the fact that it is a virtual table, not sure) + + this.db + .prepare( + ` + INSERT OR REPLACE INTO competence_embedding + (cid, text, type, embedding) + VALUES (CAST(? AS INTEGER), ?, ?, vec_f32(?)) + `, + ) + .run(cidInt, text, type, new Float32Array(embedding)); + } + + /** Delete all embeddings for one competence + * @param listId user‐facing ID of the resource list + * @param resourceId user‐facing ID of the resource + * @param competenceId user‐facing ID of the competence + */ + public deleteEmbeddingsForCompetence( + listId: string, + resourceId: string, + competenceId: string, + ): void { + const cid = this.getCompetenceCidByCompetenceId(listId, resourceId, competenceId); + this.db.prepare(`DELETE FROM competence_embedding WHERE cid = ?`).run(cid); + } + + /** + * kNN‐search over embeddings, returning user‐facing competenceIds + distances. + * + * @param embedding the query vector to search for + * @param options optional parameters: + * - k: number of nearest neighbors to return (default: all) + * - filter: optional filter by resourceId and/or listId + * - similarityMetric: 'cosine', 'hamming', or 'euclidean' (default: 'cosine') + * @returns an array of objects with competenceId, text, type, and distance. + * @throws if the embedding length does not match the configured dimension. + * @throws if the metric is unsupported or k is not a positive integer. + */ + public searchEmbedding( + embedding: number[], + options?: { + k?: number; + filter?: { resourceId?: string; listId?: string }; + similarityMetric?: 'cosine' | 'hamming' | 'euclidean'; + }, + ): Array<{ + competenceId: string; + resourceId: string; + text: string; + type: string; + distance: number; + }> { + const { k, filter, similarityMetric } = options || {}; + const metrics = { + cosine: 'vec_distance_cosine', + hamming: 'vec_distance_hamming', + euclidean: 'vec_distance_L2', + }; + const metric = metrics[similarityMetric || 'cosine']; + if (!metric) throw new Error(`Unsupported metric: ${similarityMetric}`); + if (embedding.length !== this.embeddingDim) throw new Error(`Embedding length mismatch`); + if (k !== undefined && k <= 0) throw new Error('k must be > 0'); + + let sql = ` + SELECT c.competence_id, r.resource_id, ce.text, ce.type, + ${metric}(ce.embedding, vec_f32(?)) AS distance + FROM competence_embedding ce + JOIN competence c ON ce.cid = c._cid + JOIN resource r ON c.resource_rid = r._rid + `; + const params: any[] = [new Float32Array(embedding)]; + + const whereClauses: string[] = []; + if (filter?.resourceId) { + whereClauses.push(`r.resource_id = ?`); + params.push(filter.resourceId); + } + if (filter?.listId) { + whereClauses.push(`r.list_id = ?`); + params.push(filter.listId); + } + if (whereClauses.length > 0) { + sql += ` WHERE ` + whereClauses.join(' AND '); + } + // sql += ` GROUP BY c.competence_id, ce.type`; + sql += ` ORDER BY distance ASC`; + + if (k) { + sql += ` LIMIT ?`; + params.push(k); + } + + const rows = this.db.prepare(sql).all(...params) as Array; + + let result = rows.map((r) => ({ + competenceId: r.competence_id, + resourceId: r.resource_id, + distance: r.distance, + text: r.text, + type: r.type, + })); + + // Normalise distances to [0, 1], depending on the metric: + if (similarityMetric === 'cosine') { + // Cosine distance is in [0, 2] + result = result.map((row) => ({ + ...row, + distance: row.distance / 2, + })); + } else if (similarityMetric === 'hamming') { + // Hamming distance is in [0, 1], so we leave it as is + } else if (similarityMetric === 'euclidean') { + // Euclidean distance is in [0, sqrt(embeddingDim)] + const maxDistance = Math.sqrt(this.embeddingDim); + result = result.map((row) => ({ + ...row, + distance: row.distance / maxDistance, + })); + } + + // Since similariy is now normalised to [0, 1], + // we need to adapt it, as it should be somewhat interpretable as a probability + result = result.map((row) => ({ + ...row, + distance: 1 - row.distance, // Convert distance to similarity + })); + return result; + } + + /** + * Store or update the embedding vector for a specific task within a job. + */ + public upsertTaskEmbedding(jobId: string, taskId: string, embedding: number[]): void { + if (embedding.length !== this.embeddingDim) { + throw new Error(`Task embedding must have length ${this.embeddingDim}`); + } + + this.db + .prepare( + ` + INSERT INTO task_embedding (job_id, task_id, embedding) + VALUES (?, ?, vec_f32(?)) + ON CONFLICT(job_id, task_id) + DO UPDATE SET embedding = excluded.embedding + `, + ) + .run(jobId, taskId, new Float32Array(embedding)); + } + + /** + * Retrieve the embedding for a given task within a job. + */ + public getTaskEmbedding(jobId: string, taskId: string): number[] | null { + const row = this.db + .prepare( + ` + SELECT embedding + FROM task_embedding + WHERE job_id = ? AND task_id = ? + `, + ) + .get(jobId, taskId) as { embedding: unknown } | undefined; + + if (!row || row.embedding == null) return null; + + let vector: number[] | null = null; + const raw = row.embedding as any; + + if (raw instanceof Float32Array) { + vector = Array.from(raw); + } else if (Buffer.isBuffer(raw)) { + // sqlite-vec returns the FLOAT32 column as a Buffer; reinterpret it as Float32 values + const floatView = new Float32Array( + raw.buffer, + raw.byteOffset, + raw.byteLength / Float32Array.BYTES_PER_ELEMENT, + ); + vector = Array.from(floatView); + } else if (ArrayBuffer.isView(raw)) { + const view = raw as ArrayBufferView; + const floatView = new Float32Array( + view.buffer, + view.byteOffset, + view.byteLength / Float32Array.BYTES_PER_ELEMENT, + ); + vector = Array.from(floatView); + } else if (raw instanceof ArrayBuffer) { + vector = Array.from(new Float32Array(raw)); + } + + if (!vector) { + throw new Error('Unsupported embedding format fetched from task_embedding table'); + } + + if (vector.length !== this.embeddingDim) { + throw new Error( + `Task embedding length mismatch: expected ${this.embeddingDim}, received ${vector.length}`, + ); + } + + return vector; + } + + /** + * Remove all task embeddings associated with a job. + */ + public deleteTaskEmbeddings(jobId: string): void { + this.db.prepare(`DELETE FROM task_embedding WHERE job_id = ?`).run(jobId); + } +} + +export default VectorDataBase; diff --git a/src/competence-matcher/src/middleware/db-locator.ts b/src/competence-matcher/src/middleware/db-locator.ts new file mode 100644 index 000000000..3b65c0cd6 --- /dev/null +++ b/src/competence-matcher/src/middleware/db-locator.ts @@ -0,0 +1,22 @@ +import { Request, Response, NextFunction } from 'express'; +import { config } from '../config'; + +const { multipleDBs } = config; + +export function dbHeader(req: Request, res: Response, next: NextFunction): void { + // This middleware allows for the use f multiple databases instead of a single one. + // 'x-proceed-db-id' is a custom header that should be included in the request, which specifies the database name to use. + if (multipleDBs) { + const dbName = req.header('x-proceed-db-id'); + if (!dbName || typeof dbName !== 'string' || dbName.trim() === '') { + res.status(400).json({ error: 'Missing x-proceed-db-id header' }); + return; + } + req.dbName = dbName; + } else { + // For now, we use a single database, so we can just set a default name. + req.dbName = 'PROCEED-Matching.db'; + } + + next(); +} diff --git a/src/competence-matcher/src/middleware/error-handler.ts b/src/competence-matcher/src/middleware/error-handler.ts new file mode 100644 index 000000000..5ded0a6d5 --- /dev/null +++ b/src/competence-matcher/src/middleware/error-handler.ts @@ -0,0 +1,67 @@ +import { Request, Response, NextFunction } from 'express'; +import { CompetenceMatcherError } from '../utils/errors'; +import { getLogger } from '../utils/logger'; + +const logger = getLogger(); + +/** + * Error handler middleware using the logging system + */ +export function errorHandler( + error: Error | CompetenceMatcherError, + req: Request, + res: Response, + next: NextFunction, +): void { + const requestId = req.requestId || 'unknown'; + + if (error instanceof CompetenceMatcherError) { + // Handle our custom errors + logger.error( + 'request', + `${error.context}: ${error.message}`, + error, + { + statusCode: error.statusCode, + details: error.details, + path: req.path, + method: req.method, + requestId, + }, + requestId, + ); + + res.status(error.statusCode).json({ + error: { + message: error.message, + context: error.context, + requestId: requestId, + ...(error.details && { details: error.details }), + }, + }); + } else { + // Handle unexpected errors + logger.error( + 'system', + 'Unhandled error occurred', + error, + { + path: req.path, + method: req.method, + body: req.body, + query: req.query, + params: req.params, + requestId, + }, + requestId, + ); + + res.status(500).json({ + error: { + message: 'An unexpected error occurred', + context: 'internal_server_error', + requestId: requestId, + }, + }); + } +} diff --git a/src/competence-matcher/src/middleware/match.ts b/src/competence-matcher/src/middleware/match.ts new file mode 100644 index 000000000..e85d36cb7 --- /dev/null +++ b/src/competence-matcher/src/middleware/match.ts @@ -0,0 +1,550 @@ +import { Request, Response, NextFunction } from 'express'; +import { PATHS } from '../server'; +import { getDB } from '../utils/db'; +import { + CompetenceInput, + GroupedMatchResults, + MatchingJob, + MatchingTask, + ResourceListInput, + ResourceRanking, + TaskOverview, +} from '../utils/types'; +import { handleCreateResourceList } from './resource'; +import { + ValidationError, + ResourceNotFoundError, + DatabaseError, + CompetenceMatcherError, +} from '../utils/errors'; +import { getLogger } from '../utils/logger'; +import { scheduleMatchingPipeline } from '../tasks/matching-pipeline'; + +const logger = getLogger(); + +export function matchCompetenceList(req: Request, res: Response, next: NextFunction): void { + const requestId = (req as any).requestId; + + try { + let listId: string | undefined; + let list: ResourceListInput | undefined; + let taskInput: MatchingTask[] | undefined; + const db = getDB(req.dbName!); + + /**-------------------------------------------- + * Input Validation + *---------------------------------------------*/ + if ('competenceList' in req.body) { + // Handle case where competenceList is provided + const { competenceList, tasks } = req.body as { + competenceList: ResourceListInput; + tasks: MatchingTask[]; + }; + list = competenceList; + taskInput = tasks; + } else if ('competenceListId' in req.body) { + // Handle case where competenceListId is provided + const { competenceListId, tasks } = req.body as { + competenceListId: string; + tasks: MatchingTask[]; + }; + listId = competenceListId; + taskInput = tasks; + } + + // Validate input presence + if (!listId && !list) { + throw new ValidationError( + 'Either competenceListId or competenceList must be provided', + 'competenceListId|competenceList', + { competenceListId: listId, competenceList: list }, + requestId, + ); + } + + // Validate tasks input + if (!taskInput || !Array.isArray(taskInput) || taskInput.length === 0) { + throw new ValidationError( + 'Must provide a non-empty array of tasks for matching', + 'tasks', + taskInput, + requestId, + ); + } + + // Validate competenceListId format + if (listId && typeof listId !== 'string') { + throw new ValidationError( + 'Must be a valid UUID string', + 'competenceListId', + listId, + requestId, + ); + } + + // Validate competenceList structure + if ( + list && + (!Array.isArray(list) || + !list.every( + (entry) => typeof entry === 'object' && !Array.isArray(entry) && entry !== null, + )) + ) { + throw new ValidationError( + 'Must be an array of ResourceInput objects', + 'competenceList', + list, + requestId, + ); + } + + /**-------------------------------------------- + * Case existing competenceListId was passed + *---------------------------------------------*/ + if (listId) { + let competenceLists: string[]; + + try { + // Check if the competence list exists + competenceLists = db.getAvailableResourceLists(); + } catch (error) { + throw new DatabaseError( + 'getAvailableResourceLists', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + if (!competenceLists.includes(listId)) { + throw new ResourceNotFoundError('Competence list', listId, requestId); + } + + let jobId: string; + try { + jobId = db.createJob(listId); + } catch (error) { + throw new DatabaseError( + 'createJob', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + try { + db.updateJobStatus(jobId, 'preprocessing'); + } catch (error) { + throw new DatabaseError( + 'updateJobStatus', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + const job: MatchingJob = { + jobId, + dbName: req.dbName!, + listId, + resourceId: undefined, // For now, we don't support matching against a single resource + tasks: taskInput.map((task) => { + return { + taskId: task.taskId, + name: task.name, + description: task.description, + executionInstructions: task.executionInstructions, + requiredCompetencies: (task.requiredCompetencies ?? []).map((competence) => + typeof competence === 'string' + ? (competence as string) + : ({ + competenceId: competence.competenceId, + name: competence.name, + description: competence.description, + externalQualificationNeeded: competence.externalQualificationNeeded, + renewTime: competence.renewTime, + proficiencyLevel: competence.proficiencyLevel, + qualificationDates: competence.qualificationDates, + lastUsages: competence.lastUsages, + } as CompetenceInput), + ) as string[] | CompetenceInput[], + }; + }), + }; + + try { + scheduleMatchingPipeline(job, job.tasks, requestId); + } catch (error) { + throw new CompetenceMatcherError( + `Failed to enqueue matching job: ${error instanceof Error ? error.message : String(error)}`, + 'job_enqueue', + 500, + requestId, + { jobId, listId }, + ); + } + + // Respond with jobId in location header + res + .setHeader('Location', `${PATHS.match}/jobs/${jobId}`) + // Accepted response + .status(202) + .json({ jobId, status: 'preprocessing' }); + return; + } + + /**-------------------------------------------- + * Case new Competence-List was passed + *---------------------------------------------*/ + let matchingJobId: string; + try { + // Create a new competence list + matchingJobId = db.createJob(); + } catch (error) { + throw new DatabaseError( + 'createJob', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + if (list) { + try { + db.updateJobStatus(matchingJobId, 'preprocessing'); + } catch (error) { + throw new DatabaseError( + 'updateJobStatus', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + // Start the embedding job and get the promise for completion + const embeddingResult = handleCreateResourceList(req.dbName!, list, requestId); + + // Send response immediately with the matching job ID + res + .setHeader('Location', `${PATHS.match}/jobs/${matchingJobId}`) + .status(202) + .json({ jobId: matchingJobId, status: 'preprocessing' }); // Match the actual database status + + // Chain the matching job to start after embedding completes + embeddingResult.promise + .then(() => { + const matchingJob: MatchingJob = { + jobId: matchingJobId, + dbName: req.dbName!, + listId: embeddingResult.listId, // Use the embedding result's listId + resourceId: undefined, + tasks: taskInput!.map((task) => { + return { + taskId: task.taskId, + name: task.name, + description: task.description, + executionInstructions: task.executionInstructions, + requiredCompetencies: (task.requiredCompetencies ?? []).map((competence) => + typeof competence === 'string' + ? (competence as string) + : ({ + competenceId: competence.competenceId, + name: competence.name, + description: competence.description, + externalQualificationNeeded: competence.externalQualificationNeeded, + renewTime: competence.renewTime, + proficiencyLevel: competence.proficiencyLevel, + qualificationDates: competence.qualificationDates, + lastUsages: competence.lastUsages, + } as CompetenceInput), + ) as string[] | CompetenceInput[], + }; + }), + }; + + scheduleMatchingPipeline(matchingJob, matchingJob.tasks, requestId); + }) + .catch((error) => { + // Embedding failed, mark matching job as failed too + try { + db.updateJobStatus(matchingJobId, 'failed'); + } catch (dbError) { + // Log but don't throw + logger.error( + 'system', + 'Failed to update matching job status to failed', + dbError instanceof Error ? dbError : new Error(String(dbError)), + {}, + requestId, + ); + } + }); + + return; + } + } catch (error) { + // Pass error to error handler middleware + next(error); + } +} + +export function getMatchJobResults(req: Request, res: Response, next: NextFunction): void { + const requestId = (req as any).requestId; + + try { + // Get jobId from path + const { jobId } = req.params; + // Get sorter from query params + const requestedSorter = req.query.rankBy as string | undefined; + const sorter = requestedSorter == 'bestFit' ? 'bestFit' : 'avgFit'; // Default to avgFit + + if (!jobId) { + throw new ValidationError( + 'Job ID is required in the request path', + 'jobId', + jobId, + requestId, + ); + } + + let db; + try { + db = getDB(req.dbName!); + } catch (error) { + throw new DatabaseError( + 'getDB', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + // Check if job exists + let job; + try { + job = db.getJob(jobId); + } catch (error) { + throw new DatabaseError( + 'getJob', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + if (!job) { + throw new ResourceNotFoundError('Job', jobId, requestId); + } + + // Job can be pending, preprocessing, running, completed, or failed + switch (job.status) { + case 'pending': + case 'running': + case 'preprocessing': + res.status(202).json({ + jobId, + status: job.status, + }); + return; + case 'failed': + throw new CompetenceMatcherError( + `Job with ID '${jobId}' has failed during processing`, + 'job_execution_failed', + 500, + requestId, + { jobId, jobStatus: job.status }, + ); + case 'completed': + // Proceed to return results below + break; + default: + throw new CompetenceMatcherError( + `Job with ID '${jobId}' has unexpected status: ${job.status}`, + 'unexpected_job_status', + 500, + requestId, + { jobId, jobStatus: job.status }, + ); + } + + // Return match results + let results; + try { + results = db.getMatchResults(jobId); + } catch (error) { + throw new DatabaseError( + 'getMatchResults', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + const tasks: TaskOverview = results.reduce((acc, result) => { + const { taskId, taskText } = result; + // Check if task already exists in the overview + if (!acc.some((task) => task.taskId === taskId)) { + acc.push({ taskId, taskText }); + } + return acc; + }, [] as TaskOverview); + + // Get the structure of the results + let groupedResults: ResourceRanking = results.reduce((acc, result) => { + const { taskId, competenceId, resourceId, distance, text, type, alignment, reason } = result; + + // resourceId + let resourceGroup = acc.find((group) => group.resourceId === resourceId); + if (!resourceGroup) { + resourceGroup = { + resourceId, + taskMatchings: [], + avgTaskMatchProbability: 0, + avgBestFitTaskMatchProbability: 0, + contradicting: false, + }; + acc.push(resourceGroup); + } + // taskMatchings + let taskMatches = resourceGroup.taskMatchings.find((task) => task.taskId === taskId); + if (!taskMatches) { + taskMatches = { + taskId, + competenceMatchings: [], + maxMatchProbability: 0, + maxBestFitMatchProbability: 0, + }; + resourceGroup.taskMatchings.push(taskMatches); + } + + // competenceMatchings + let competenceMatches = taskMatches.competenceMatchings.find( + (competence) => competence.competenceId === competenceId, + ); + if (!competenceMatches) { + competenceMatches = { + competenceId, + matchings: [], + avgMatchProbability: 0, + avgBestFitMatchProbability: 0, + }; + taskMatches.competenceMatchings.push(competenceMatches); + } + + // Add the match to competenceMatches + competenceMatches.matchings.push({ + text, + type: type as 'name' | 'description' | 'proficiencyLevel', + matchProbability: distance, + alignment: alignment as 'contradicting' | 'neutral' | 'aligning', + reason: reason || undefined, + }); + + return acc; + }, [] as ResourceRanking); + + // Aggregate and sort + groupedResults = groupedResults + .map((resourceGroup) => { + const { + resourceId, + taskMatchings, + avgTaskMatchProbability, + avgBestFitTaskMatchProbability, + } = resourceGroup; + + const newTaskMatchings = taskMatchings.map((taskGroup) => { + const { taskId, competenceMatchings, maxMatchProbability, maxBestFitMatchProbability } = + taskGroup; + + const newCompetenceMatchings = competenceMatchings.map((competenceGroup) => { + const { competenceId, matchings, avgMatchProbability, avgBestFitMatchProbability } = + competenceGroup; + + // Calculate average match probability for this competence (i.e. avg over all parts of this competence) + const totalMatchProbability = matchings.reduce( + (sum, match) => sum + match.matchProbability, + 0, + ); + + let numberOfBestFits = 0; + const totalBestFitMatchProbability = matchings.reduce((sum, match) => { + if (match.alignment === 'aligning') { + numberOfBestFits++; + return sum + match.matchProbability; + } + return sum; + }, 0); + + // Return sorted + return { + competenceId, + matchings: matchings.sort((a, b) => b.matchProbability - a.matchProbability), + avgMatchProbability: totalMatchProbability / matchings.length, + avgBestFitMatchProbability: + numberOfBestFits > 0 ? totalBestFitMatchProbability / numberOfBestFits : 0, // If no best fit, set to 0 + }; + }); + + // Return sorted + return { + taskId, + competenceMatchings: newCompetenceMatchings.sort((a, b) => { + const key = + sorter === 'bestFit' ? 'avgBestFitMatchProbability' : 'avgMatchProbability'; + return b[key] - a[key]; + }), + maxMatchProbability: Math.max( + ...newCompetenceMatchings.map((c) => c.avgMatchProbability), + ), + maxBestFitMatchProbability: Math.max( + ...newCompetenceMatchings.map((c) => c.avgBestFitMatchProbability), + ), + }; + }); + + // Calculate average task match probability for this resource + const totalTaskMatchProbability = newTaskMatchings.reduce( + (sum, task) => sum + task.maxMatchProbability, + 0, + ); + const totalBestFitTaskMatchProbability = newTaskMatchings.reduce( + (sum, task) => sum + task.maxBestFitMatchProbability, + 0, + ); + + // Return sorted + return { + resourceId, + taskMatchings: newTaskMatchings.sort((a, b) => { + const key = sorter === 'bestFit' ? 'maxBestFitMatchProbability' : 'maxMatchProbability'; + return b[key] - a[key]; + }), + avgTaskMatchProbability: totalTaskMatchProbability / newTaskMatchings.length, + avgBestFitTaskMatchProbability: + totalBestFitTaskMatchProbability / newTaskMatchings.length || 0, // If no best fit, set to 0 + contradicting: newTaskMatchings.some((task) => + task.competenceMatchings.some((competence) => + competence.matchings.some((match) => match.alignment === 'contradicting'), + ), + ), + }; + }) + .sort((a, b) => { + const key = + sorter === 'bestFit' ? 'avgBestFitTaskMatchProbability' : 'avgTaskMatchProbability'; + + // Sort in two levels: Contradicting, key + // First not contradicting resources, then contradicting ones + // Case one is contradicting, the other is not + if (a.contradicting !== b.contradicting) { + return a.contradicting ? 1 : -1; // Non-contradicting first + } + // Both are contradicting or both are not + // Sort by the key + return b[key] - a[key]; + }); + + const load: GroupedMatchResults = { + tasks, + resourceRanking: groupedResults, + }; + + res.status(200).json(load); + } catch (error) { + // Pass error to error handler middleware + next(error); + } +} diff --git a/src/competence-matcher/src/middleware/request-logger.ts b/src/competence-matcher/src/middleware/request-logger.ts new file mode 100644 index 000000000..31539d585 --- /dev/null +++ b/src/competence-matcher/src/middleware/request-logger.ts @@ -0,0 +1,103 @@ +import { Request, Response, NextFunction } from 'express'; +import { getLogger } from '../utils/logger'; + +const logger = getLogger(); + +// Extend Request interface to include requestId +declare global { + namespace Express { + interface Request { + requestId: string; + startTime: number; + } + } +} + +/** + * Extract real IP address from request, considering proxy headers + */ +function getRealIP(req: Request): string { + // Check for common proxy headers in order of preference + const forwardedFor = req.headers['x-forwarded-for']; + const realIP = req.headers['x-real-ip']; + const clientIP = req.headers['x-client-ip']; + + // x-forwarded-for can be a comma-separated list, take the first (original) IP + if (forwardedFor) { + const ips = Array.isArray(forwardedFor) ? forwardedFor[0] : forwardedFor; + return ips.split(',')[0].trim(); + } + + // Single IP headers + if (realIP) { + return Array.isArray(realIP) ? realIP[0] : realIP; + } + + if (clientIP) { + return Array.isArray(clientIP) ? clientIP[0] : clientIP; + } + + // Fallback to express's req.ip (which might be the proxy IP) + return req.ip || 'unknown'; +} + +/** + * Request logger middleware + */ +export function requestLogger(req: Request, res: Response, next: NextFunction): void { + const requestId = logger.generateRequestId(); + const startTime = Date.now(); + + // Add requestId and startTime to request for later use + req.requestId = requestId; + req.startTime = startTime; + + // Log incoming request + const realIP = getRealIP(req); + logger.debug( + 'request', + `Incoming ${req.method} ${req.path}`, + { + query: req.query, + params: req.params, + ip: realIP, + proxyIP: req.ip, // Keep original for debugging + userAgent: req.headers['user-agent'], + contentType: req.headers['content-type'], + // Include proxy headers for debugging if they exist + ...(req.headers['x-forwarded-for'] && { 'x-forwarded-for': req.headers['x-forwarded-for'] }), + ...(req.headers['x-real-ip'] && { 'x-real-ip': req.headers['x-real-ip'] }), + }, + requestId, + ); + + // Override res.send to capture response + // res.json seems to use .send internally, so we can override send + const originalSend = res.send; + res.send = function (body) { + const responseTime = Date.now() - startTime; + + // Log the request completion + logger.request(req.method, req.path, res.statusCode, responseTime, requestId, { + ip: getRealIP(req), + contentLength: res.get('content-length'), + responseSize: body ? Buffer.byteLength(body, 'utf8') : 0, + }); + + return originalSend.call(this, body); + }; + + next(); +} + +/** + * Middleware to extract requestId from existing requests for backward compatibility + */ +export function requestIdExtractor(req: Request, res: Response, next: NextFunction): void { + // If requestId is not already set, generate one + if (!req.requestId) { + req.requestId = logger.generateRequestId(); + req.startTime = Date.now(); + } + next(); +} diff --git a/src/competence-matcher/src/middleware/resource.ts b/src/competence-matcher/src/middleware/resource.ts new file mode 100644 index 000000000..9954901d6 --- /dev/null +++ b/src/competence-matcher/src/middleware/resource.ts @@ -0,0 +1,264 @@ +import { Request, Response, NextFunction } from 'express'; +import { PATHS } from '../server'; +import { getDB } from '../utils/db'; +import workerManager from '../worker/worker-manager'; +import { splitSemantically } from '../tasks/semantic-split'; +import { CompetenceInput, EmbeddingJob, EmbeddingTask, ResourceInput } from '../utils/types'; +import { + ValidationError, + ResourceNotFoundError, + DatabaseError, + CompetenceMatcherError, +} from '../utils/errors'; +import { getLogger } from '../utils/logger'; + +const logger = getLogger(); + +export function getResourceLists(req: Request, res: Response, next: NextFunction): void { + const requestId = (req as any).requestId; + + try { + let db; + try { + db = getDB(req.dbName!); + } catch (error) { + throw new DatabaseError( + 'getDB', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + let availableResourceLists; + try { + availableResourceLists = db.getAvailableResourceLists(); + } catch (error) { + throw new DatabaseError( + 'getAvailableResourceLists', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + res.status(200).json(availableResourceLists); + } catch (error) { + next(error); + } +} + +export function getResourceList(req: Request, res: Response, next: NextFunction): void { + const requestId = (req as any).requestId; + + try { + const resourceListId = req.params.resourceListId; + + if (!resourceListId) { + throw new ValidationError( + 'Resource list ID is required in the request parameters', + 'resourceListId', + resourceListId, + requestId, + ); + } + + let db; + try { + db = getDB(req.dbName!); + } catch (error) { + throw new DatabaseError( + 'getDB', + error instanceof Error ? error : new Error(String(error)), + requestId, + ); + } + + let resourceList; + try { + resourceList = db.getResourceList(resourceListId); + } catch (error) { + throw new ResourceNotFoundError('Resource list', resourceListId, requestId); + } + + res.status(200).json(resourceList); + } catch (error) { + next(error); + } +} + +// Helper function to handle the creation logic +export function handleCreateResourceList( + dbName: string, + resources: ResourceInput[], + requestId?: string, +): { jobId: string; listId: string; status: string; promise: Promise } { + let resourceIds: string[] = []; + let competences: CompetenceInput[][] = []; + + // Validate and extract data + resources.forEach(({ resourceId, competencies }: ResourceInput) => { + if (!resourceId || typeof resourceId !== 'string') { + throw new Error('Invalid resourceId in request body'); + } + if (!Array.isArray(competencies)) { + throw new Error('Invalid competencies in request body'); + } + resourceIds.push(resourceId); + const checkedCompetences = competencies.map((c: CompetenceInput) => { + if (!c.competenceId || typeof c.competenceId !== 'string') { + throw new Error('Invalid competenceId in request body'); + } + return { + competenceId: c.competenceId, + name: c.name, + description: c.description, + externalQualificationNeeded: c.externalQualificationNeeded, + renewTime: c.renewTime, + proficiencyLevel: c.proficiencyLevel, + qualificationDates: c.qualificationDates, + lastUsages: c.lastUsages, + }; + }); + competences.push(checkedCompetences); + }); + + // Create a new resource list in the database + let listId: string; + let jobId: string; + const db = getDB(dbName); + db.atomicStep(() => { + listId = db.createResourceList(); + resourceIds.forEach((resourceId) => { + db.addResource(listId, resourceId); + }); + competences.forEach((competenceArray, resourceIndex) => { + competenceArray.forEach((competence) => { + db.addCompetence(listId, resourceIds[resourceIndex], competence); + }); + }); + jobId = db.createJob(listId); + }); + + // Prepare embedding tasks + const descriptionEmbeddingInput = competences + .map((competenceArray, resourceIndex) => { + return competenceArray.map((competence) => { + return { + listId: listId!, + resourceId: resourceIds[resourceIndex], + competenceId: competence.competenceId, + text: competence.description, + type: 'description', + }; + }); + }) + .flat() as EmbeddingTask[]; + + // Workaround for now + // Ideally, the worker should handle the splitting as well + db.updateJobStatus(jobId!, 'preprocessing'); + + const promise = splitSemantically(descriptionEmbeddingInput) + .then((tasks) => { + const job: EmbeddingJob = { + jobId: jobId!, + dbName: dbName, + tasks, + }; + + db.updateJobStatus(jobId!, 'pending'); + return workerManager.enqueue(job, 'embedder'); + }) + .catch((err) => { + logger.warn( + 'system', + 'Error during semantic splitting, falling back to original tasks', + { + error: err instanceof Error ? err.message : String(err), + taskCount: descriptionEmbeddingInput.length, + }, + requestId, + ); + + const job: EmbeddingJob = { + jobId: jobId!, + dbName: dbName, + tasks: descriptionEmbeddingInput, + }; + + db.updateJobStatus(jobId!, 'pending'); + return workerManager.enqueue(job, 'embedder'); + }); + + return { jobId: jobId!, listId: listId!, status: 'pending', promise }; +} + +export function createResourceList(req: Request, res: Response, next: NextFunction): void { + const requestId = req.requestId; + + if (!Array.isArray(req.body) || req.body.length === 0) { + res.status(400).json({ error: 'Invalid request body. Expected an array of resources.' }); + return; + } + try { + const { jobId, status } = handleCreateResourceList(req.dbName!, req.body, requestId); + + logger.debug( + 'request', + 'Resource list creation job created', + { + jobId, + status, + resourceCount: req.body.length, + }, + requestId, + ); + + res + .setHeader('Location', `${PATHS.resource}/jobs/${jobId}`) + .status(202) + .json({ jobId, status }); + } catch (error) { + logger.error( + 'request', + 'Error adding resource list', + error instanceof Error ? error : new Error(String(error)), + { + resourceCount: req.body.length, + }, + requestId, + ); + res + .status(400) + .json({ error: error instanceof Error ? error.message : 'Invalid request body format' }); + } +} + +export function getJobStatus(req: Request, res: Response) { + try { + const db = getDB(req.dbName!); + const job = db.getJob(req.params.jobId); + + switch (job.status) { + case 'pending': + case 'preprocessing': + case 'running': + res.status(202).json({ jobId: job.jobId, status: job.status }); // both strings + return; + case 'completed': + res + .status(201) + .setHeader('Location', `${PATHS.resource}/${job.referenceId}`) + .json({ jobId: job.jobId, status: job.status, competenceListId: job.referenceId }); + return; + case 'failed': + res.status(500).json({ jobId: job.jobId, status: job.status }); //both strings + return; + default: + res.status(500).json({ error: 'Internal Server Error' }); + return; + } + } catch (err) { + // logger.error("system", err); + res.status(404).json({ error: 'Job not found' }); + } +} diff --git a/src/competence-matcher/src/routes/match.ts b/src/competence-matcher/src/routes/match.ts new file mode 100644 index 000000000..0cb894f73 --- /dev/null +++ b/src/competence-matcher/src/routes/match.ts @@ -0,0 +1,21 @@ +import express from 'express'; +import { getMatchJobResults, matchCompetenceList } from '../middleware/match'; + +const router = express.Router(); + +// .all() +// .get() +// .put() +// .post(); +// .patch(); +// .delete(); + +// '/:resource-list-id' +// '/:resource-list-id/:resource-id' +// '/:resource-list-id/:resource-id/:competence-id' + +router.route('/jobs/').post(matchCompetenceList); + +router.route('/jobs/:jobId').get(getMatchJobResults); + +export default router; diff --git a/src/competence-matcher/src/routes/resource.ts b/src/competence-matcher/src/routes/resource.ts new file mode 100644 index 000000000..2096119ec --- /dev/null +++ b/src/competence-matcher/src/routes/resource.ts @@ -0,0 +1,34 @@ +import express from 'express'; +import { + createResourceList, + getJobStatus, + getResourceList, + getResourceLists, +} from '../middleware/resource'; +import { config } from '../config'; + +const { multipleDBs } = config; + +const router = express.Router(); + +// .all() +// .get() +// .put() +// .post(); +// .patch(); +// .delete(); + +// '/:resource-list-id' +// '/:resource-list-id/:resource-id' +// '/:resource-list-id/:resource-id/:competence-id' + +// Getting an overview of all resource lists should be tennant-specific, +// so it should only be available if multipleDBs is true. +if (multipleDBs) router.route('/').get(getResourceLists); + +router.route('/jobs').post(createResourceList); +router.route('/jobs/:jobId').get(getJobStatus); + +router.route('/:resourceListId').get(getResourceList); + +export default router; diff --git a/src/competence-matcher/src/server.ts b/src/competence-matcher/src/server.ts new file mode 100644 index 000000000..f0a5bf3b0 --- /dev/null +++ b/src/competence-matcher/src/server.ts @@ -0,0 +1,107 @@ +import express from 'express'; + +import { config } from './config'; +import { Logger, createLoggerConfig } from './utils/logger'; + +// Initilise logger first, before any other imports that might use it +const loggerConfig = createLoggerConfig(); +const logger = Logger.getInstance(loggerConfig); + +import ResourceRouter from './routes/resource'; +import MatchRouter from './routes/match'; +import { dbHeader } from './middleware/db-locator'; +import { requestLogger } from './middleware/request-logger'; +import { errorHandler } from './middleware/error-handler'; +import { ensureAllOllamaModelsAreAvailable } from './utils/ollama'; +import { ensureAllHuggingfaceModelsAreAvailable } from './utils/huggingface'; +import { CompetenceMatcherError } from './utils/errors'; + +const { port: PORT } = config; + +export const PATHS = { + resource: '/resource-competence-list', + match: '/matching-task-to-resource', +}; + +// Extend Express Request interface +declare module 'express-serve-static-core' { + interface Request { + dbName?: string; + } +} + +async function main() { + const app = express(); + + try { + logger.info('server', 'Initialising competence matcher service...'); + + // Ensure all required models are available + + // Hugging Face models + logger.info('server', 'Checking HuggingFace models availability...'); + await ensureAllHuggingfaceModelsAreAvailable(); + + // Ollama models + logger.info('server', 'Checking Ollama models availability...'); + await ensureAllOllamaModelsAreAvailable(); + + logger.info('server', 'All required models are available'); + } catch (error) { + const initError = new CompetenceMatcherError( + `Failed to initialise service: ${error instanceof Error ? error.message : String(error)}`, + 'server_initialisation', + 503, + undefined, + { + stage: + error instanceof Error && error.message.includes('worker') + ? 'worker_initialisation' + : 'model_initialisation', + originalError: error instanceof Error ? error.message : String(error), + }, + ); + + logger.error('server', 'Failed to start due to initialisation error', initError); + throw initError; // Rethrow to be caught by outer catch + } + + // Parse JSON + app.use(express.json()); + // Parse URL-encoded data + app.use(express.urlencoded({ extended: true })); + // Middleware to handle database header + app.use(dbHeader); + // Logging middleware + app.use(requestLogger); + + // Hello World + app.get('/', (req, res, next) => { + res.status(200).send('Welcome to the Matching Server'); + }); + + // Routes + app.use(PATHS.resource, ResourceRouter); + app.use(PATHS.match, MatchRouter); + + // Error handler middleware (must be last, only invoked if error occurs) + app.use(errorHandler); + + app.listen(PORT, () => { + logger.info('server', `Matching-Server is running on http://localhost:${PORT}`); + }); +} + +main().catch((error) => { + const startupError = new CompetenceMatcherError( + `Server startup failed: ${error instanceof Error ? error.message : String(error)}`, + 'server_startup', + 500, + undefined, + { originalError: error instanceof Error ? error.message : String(error) }, + ); + + logger.error('server', 'Server startup failed', startupError); + logger.error('system', '[Server] Server shutdown due to startup error:', error); + process.exit(1); +}); diff --git a/src/competence-matcher/src/tasks/cross-encode.ts b/src/competence-matcher/src/tasks/cross-encode.ts new file mode 100644 index 000000000..c00439ade --- /dev/null +++ b/src/competence-matcher/src/tasks/cross-encode.ts @@ -0,0 +1,76 @@ +import { TransformerPipeline } from '../utils/model'; +import { TransformerPipelineOptions } from '../utils/types'; +import { PipelineType, TextClassificationPipeline } from '@huggingface/transformers'; +import { config } from '../config'; + +export default class CrossEncoder extends TransformerPipeline { + protected static override getPipelineOptions(): TransformerPipelineOptions { + return { + task: 'text-classification' as PipelineType, + model: config.crossEncoderModel, + options: { + model_file_name: 'model.onnx', + use_external_data_format: true, + local_files_only: true, + }, + }; + } + + /** + * Score an array of (task, candidateText) pairs. + * Returns an array of { index, raw, score } where `score` is normalised to [0,1]. + */ + public static async scorePairs(pairs: Array<{ task: string; comptence: string }>) { + if (!pairs.length) return []; + + const pipe = await this.getInstance(); + + // Convert objects to arrays-of-pairs + const inputs = pairs.map((p) => [p.task, p.comptence]); // -> string[][] + + // call the pipeline in a batch + // @ts-ignore + const out = await pipe(inputs); + + // out format varies by model/pipeline: often each item is { label, score } or {score} + // We'll defensively normalise results to [0,1]. + const sigmoid = (x: number) => 1 / (1 + Math.exp(-x)); + + console.log('CrossEncoder input:', inputs); + console.log('CrossEncoder output:', out); + + // return out.map((item: any, i: number) => { + // // Try common shapes: + // // - item = { label: 'LABEL_0', score: 0.72 } -> take score + // // - item = { scores: [..], labels: [..] } or a logits array -> handle below + // let raw: number; + + // if (typeof item === 'object' && 'score' in item && typeof item.score === 'number') { + // raw = item.score; + // } else if (Array.isArray(item) && item.length === 1 && typeof item[0].score === 'number') { + // raw = item[0].score; + // } else if (typeof item === 'object' && 'logits' in item) { + // // If logits are returned (raw model outputs), use sigmoid on the first logit + // const logits = item.logits; + // if (Array.isArray(logits)) { + // raw = sigmoid(logits[0]); + // } else { + // raw = 0; + // } + // } else if (typeof item === 'number') { + // // sometimes minimal wrappers return a numeric score + // raw = item; + // } else { + // // fallback: try to find a numeric value in the object + // const val = Object.values(item).find((v) => typeof v === 'number'); + // raw = typeof val === 'number' ? val : 0; + // } + + // // if raw is outside 0..1, apply sigmoid as fallback (common if raw is logit) + // let score = raw; + // if (score < 0 || score > 1) score = sigmoid(raw); + + // return { index: pairs[i].index, raw, score }; + // }); + } +} diff --git a/src/competence-matcher/src/tasks/embedding.ts b/src/competence-matcher/src/tasks/embedding.ts new file mode 100644 index 000000000..4148e6980 --- /dev/null +++ b/src/competence-matcher/src/tasks/embedding.ts @@ -0,0 +1,48 @@ +import { + PipelineType, + FeatureExtractionPipeline, + FeatureExtractionPipelineOptions, +} from '@huggingface/transformers'; +import { config } from '../config'; +import { TransformerPipeline } from '../utils/model'; +import { TransformerPipelineOptions } from '../utils/types'; + +export default class Embedding extends TransformerPipeline { + protected static override getPipelineOptions(): TransformerPipelineOptions { + return { + task: 'feature-extraction' as PipelineType, + model: config.embeddingModel, + options: { + // progress_callback: (progress) => { + // logger.debug("system", `Embedding progress: ${progress}`); + // }, + }, + }; + } + + /** + * Turn text (or array of text) into one or more vectors. + * @param texts string or array + * @param opts override default mean-pooling/normalize + */ + public static async embed( + texts: string | string[], + opts: FeatureExtractionPipelineOptions = { pooling: 'mean', normalize: true }, + ): Promise { + // Pipeline is loaded & cached by TransformerPipeline + const pipe = await this.getInstance(); + const input = Array.isArray(texts) ? texts : [texts]; + // call the pipeline + const raw = await pipe(input, opts); + const arrs = (Array.isArray(raw) ? raw : [raw]).map((tensor) => { + // each tensor.data is a Float32Array + const data = (tensor as any).data as Float32Array; + const vec = Array.from(data); + if (vec.length !== config.embeddingDim) { + throw new Error(`Expected embeddingDim=${config.embeddingDim}, got ${vec.length}`); + } + return vec; + }) as number[][]; + return arrs; + } +} diff --git a/src/competence-matcher/src/tasks/matching-pipeline.ts b/src/competence-matcher/src/tasks/matching-pipeline.ts new file mode 100644 index 000000000..c653a3515 --- /dev/null +++ b/src/competence-matcher/src/tasks/matching-pipeline.ts @@ -0,0 +1,49 @@ +import workerManager from '../worker/worker-manager'; +import { MatchingJob, MatchingTask, TaskEmbeddingJob } from '../utils/types'; +import { getLogger } from '../utils/logger'; +import { getDB } from '../utils/db'; + +/** + * Coordinates the two-step matching workflow by first embedding tasks and then + * handing the job over to the matcher worker pool. + */ +export function scheduleMatchingPipeline( + matchingJob: MatchingJob, + tasks: MatchingTask[], + requestId?: string, +): void { + const logger = getLogger(); + + const taskEmbeddingJob: TaskEmbeddingJob = { + jobId: matchingJob.jobId, + dbName: matchingJob.dbName, + mode: 'task', + tasks, + }; + + workerManager + .enqueue(taskEmbeddingJob, 'embedder') + .then(() => workerManager.enqueue(matchingJob, 'matcher')) + .catch((error) => { + logger.error( + 'system', + 'Task embedding stage failed before matcher enqueue', + error instanceof Error ? error : new Error(String(error)), + { jobId: matchingJob.jobId }, + requestId, + ); + + try { + const db = getDB(matchingJob.dbName); + db.updateJobStatus(matchingJob.jobId, 'failed'); + } catch (dbError) { + logger.error( + 'system', + 'Failed to mark job as failed after embedding stage error', + dbError instanceof Error ? dbError : new Error(String(dbError)), + { jobId: matchingJob.jobId }, + requestId, + ); + } + }); +} diff --git a/src/competence-matcher/src/tasks/reason.ts b/src/competence-matcher/src/tasks/reason.ts new file mode 100644 index 000000000..11480331f --- /dev/null +++ b/src/competence-matcher/src/tasks/reason.ts @@ -0,0 +1,89 @@ +import { ollama } from '../utils/ollama'; +import { config } from '../config'; +import { MATCH_REASON as intructPrompt } from '../utils/prompts'; +import type { Message } from 'ollama'; +import { Match } from '../utils/types'; +import { ReasoningError, OllamaConnectionError } from '../utils/errors'; +import { getLogger } from '../utils/logger'; + +const { reasonModel } = config; + +export async function addReason( + matches: T[], + targetText: string, +): Promise { + if (matches.length === 0) { + return matches; // No matches to reason about + } + + const logger = getLogger(); + + logger.debug('model', `Adding reasoning to ${matches.length} matches`, { + targetTextLength: targetText.length, + reasonModel, + }); + + const reasonMatches: T[] = await Promise.all( + matches.map(async (match, index) => { + const messages: Message[] = [ + ...intructPrompt, + { + role: 'user', + content: `Task: ${targetText}\nCompetence: ${match.text}\nSimilarity Score: ${match.distance}\nAlignment: ${match.alignment}`, + }, + ]; + try { + const response = await ollama.chat({ + model: reasonModel, + messages: messages, + }); + + // Extract the reason from the response + const reason = response.message.content.trim(); + + logger.debug('model', `Generated reasoning for match ${index + 1}/${matches.length}`, { + matchText: match.text.substring(0, 50) + (match.text.length > 50 ? '...' : ''), + reasonLength: reason.length, + }); + + return { + ...match, + reason, // Add the reason to the match + }; + } catch (error) { + const reasoningError = new ReasoningError( + 1, // Single match reasoning failure + error instanceof Error ? error : new Error(String(error)), + ); + + logger.error('model', 'Failed to generate reasoning for match', reasoningError, { + matchIndex: index, + totalMatches: matches.length, + targetTextLength: targetText.length, + matchText: match.text.substring(0, 100) + (match.text.length > 100 ? '...' : ''), + similarity: match.distance, + reasonModel, + }); + + // If there's an error, just keep the original match without a reason + return match; + } + }), + ); + + const successfulReasons = reasonMatches.filter( + (match) => 'reason' in match && match.reason, + ).length; + + logger.debug( + 'model', + `Reasoning completed: ${successfulReasons}/${matches.length} matches received reasons`, + { + successfulReasons, + totalMatches: matches.length, + reasonModel, + }, + ); + + return reasonMatches; +} diff --git a/src/competence-matcher/src/tasks/semantic-split.ts b/src/competence-matcher/src/tasks/semantic-split.ts new file mode 100644 index 000000000..16dc980a3 --- /dev/null +++ b/src/competence-matcher/src/tasks/semantic-split.ts @@ -0,0 +1,132 @@ +import { ollama } from '../utils/ollama'; +import { config } from '../config'; +import { SEMANTIC_SPLITTER as intructPrompt } from '../utils/prompts'; +import type { Message } from 'ollama'; +import { EmbeddingTask } from '../utils/types'; +import { SemanticSplittingError, OllamaConnectionError } from '../utils/errors'; +import { getLogger } from '../utils/logger'; + +const { + splittingModel, + splittingSymbol, + ollamaBatchSize, + splittingLength: MIN_TEXT_LENGTH, +} = config; + +export async function splitSemantically(tasks: EmbeddingTask[]): Promise { + const logger = getLogger(); + const splittedTasks: EmbeddingTask[] = []; + const toSplit: { task: EmbeddingTask; messages: Message[] }[] = []; + + logger.debug('system', `Processing ${tasks.length} tasks for semantic splitting`, { + taskCount: tasks.length, + minTextLength: MIN_TEXT_LENGTH, + }); + + for (const task of tasks) { + const messages: Message[] = [ + ...intructPrompt, + { + role: 'user', + content: task.text, + }, + ]; + + // Filter out empty, whitespace-only and too short messages + const filteredMessages = messages.filter((message) => { + const content = message.content + // Remove excessive whitespace + .replace(/\s+/g, ' ') + // Remove potential occurences of splittingSymbol in upper and lower case + .replace(new RegExp(splittingSymbol, 'gi'), ' ') + // Trim leading and trailing whitespace + .trim(); + return content !== '' && content.length > MIN_TEXT_LENGTH; + }); + + if (filteredMessages.length === 0 || MIN_TEXT_LENGTH === 0) { + splittedTasks.push({ ...task, text: task.text }); + } else { + toSplit.push({ task, messages: filteredMessages }); + } + } + + logger.debug('system', `[Semantic Split] ${toSplit.length} tasks require splitting`); + + // Process in batches + for (let i = 0; i < toSplit.length; i += ollamaBatchSize) { + const batch = toSplit.slice(i, i + ollamaBatchSize); + + logger.debug( + 'system', + `[Semantic Split] Processing batch ${Math.floor(i / ollamaBatchSize) + 1}/${Math.ceil(toSplit.length / ollamaBatchSize)} (${batch.length} tasks)`, + ); + + const promises = batch.map(async ({ task, messages }) => { + try { + const response = await ollama.chat({ + model: splittingModel, + messages, + }); + + const parts = response.message.content + .split(splittingSymbol) + .map((part: string) => part.trim()) + .filter((part: string) => part !== ''); + + if (parts.length === 0) { + logger.warn( + 'system', + `[Semantic Split] No valid parts found for task ${task.listId}/${task.resourceId}/${task.competenceId}, using original text`, + ); + splittedTasks.push({ ...task, text: task.text }); + } else { + logger.debug( + 'system', + `[Semantic Split] Split task ${task.listId}/${task.resourceId}/${task.competenceId} into ${parts.length} parts`, + ); + for (const part of parts) { + splittedTasks.push({ ...task, text: part }); + } + } + } catch (error) { + const semanticError = new SemanticSplittingError( + task.text.length, + error instanceof Error ? error : new Error(String(error)), + ); + + logger.error('system', 'Semantic splitting error', semanticError, { + taskId: `${task.listId}/${task.resourceId}/${task.competenceId}`, + textLength: task.text.length, + splittingModel, + }); + + // Fallback to original text + splittedTasks.push({ ...task, text: task.text }); + } + }); + + try { + await Promise.all(promises); + } catch (error) { + // This shouldn't happen since we catch errors in individual promises, + // but just in case there's an unexpected Promise.all failure + logger.error( + 'system', + 'Semantic splitting error', + new SemanticSplittingError( + batch.length, + error instanceof Error ? error : new Error(String(error)), + ), + { batchSize: batch.length, batchIndex: Math.floor(i / ollamaBatchSize) }, + ); + } + } + + logger.debug( + 'system', + `[Semantic Split] Completed: ${tasks.length} input tasks → ${splittedTasks.length} output tasks`, + ); + + return splittedTasks; +} diff --git a/src/competence-matcher/src/tasks/semantic-zeroshot.ts b/src/competence-matcher/src/tasks/semantic-zeroshot.ts new file mode 100644 index 000000000..662c7d63a --- /dev/null +++ b/src/competence-matcher/src/tasks/semantic-zeroshot.ts @@ -0,0 +1,229 @@ +import { + PipelineType, + TextClassificationPipeline, + ZeroShotClassificationPipeline, +} from '@huggingface/transformers'; +import { config } from '../config'; +import { TransformerPipeline } from '../utils/model'; +import { TransformerPipelineOptions } from '../utils/types'; + +export const labels = ['entailment', 'neutral statement', 'contradiction or not related']; + +export default class ZeroShot extends TransformerPipeline { + protected static override getPipelineOptions(): TransformerPipelineOptions { + return { + task: 'zero-shot-classification' as PipelineType, + model: config.nliModel, + options: { + model_file_name: 'model.onnx', + use_external_data_format: true, + local_files_only: true, + }, + }; + } + + /** + * Run zero-shot classification using explicit hypotheses. + * We keep the existing generic classify() but add a helper to compute entail/neutral/contradict + * probabilities given a premise and a hypothesis (text pair). + */ + public static async classify(text: string, labels?: string[], hypothesisTemplate?: string) { + const _labels = labels || [ + 'contradicting', + 'aligning, thus a good match', + 'neither aligning nor contradicting', + ]; + const hypothesis_template = + hypothesisTemplate || 'Task description and Skill/Capability descriptions are {}.'; + const pipe = await this.getInstance(); + return pipe(text, _labels, { hypothesis_template }); + } + + /** + * Run MNLI-style check in one direction: premise -> hypothesis sentence string. + * Returns object { entail, neutral, contradict } with values in [0,1]. + */ + private static async nliDirection(premise: string, hypothesis: string) { + const pipe = await this.getInstance(); + // Reuse zero-shot pipeline: pass premise as text and labels that (when inserted into template) + // produce a hypothesis that matches desired explicit hypothesis. + // Instead, easier: pass candidate_labels matching MNLI classes and set hypothesis_template to "{}" + const hypothesis_template = `${hypothesis}`; // pipeline will insert label into template; we want explicit hypothesis + // Some transformer-js wrappers may require candidate_labels to be "labels", and will return { labels: [...], scores: [...] } + const out = await pipe(premise, labels, { hypothesis_template }); + // out may be e.g. { labels: ['entailment','neutral','contradiction'], scores: [0.7,0.2,0.1] } + const mapping: Record = {}; + if ( + out && + typeof out === 'object' && + !Array.isArray(out) && + 'labels' in out && + 'scores' in out && + Array.isArray((out as any).labels) && + Array.isArray((out as any).scores) + ) { + (out as any).labels.forEach( + (lbl: string, i: number) => (mapping[lbl] = (out as any).scores[i]), + ); + } else if (Array.isArray(out) && out.length) { + // Some versions return array of {label, score} objects + out.forEach((item: any) => { + if (item.label && typeof item.score === 'number') mapping[item.label] = item.score; + }); + } else { + // fallback: return zeros + return { entail: 0, neutral: 0, contradict: 0 }; + } + + return { + entail: mapping[labels[0]] ?? 0, + neutral: mapping[labels[1]] ?? 0, + contradict: mapping[labels[2]] ?? 0, + }; + } + + /** + * Run NLI both directions (premise=capability -> hypothesis: task, + * and premise=task -> hypothesis: capability). Returns aggregated features. + */ + public static async nliBiDirectional(task: string, capability: string) { + // Build explicit hypothesis sentences + const h1 = `This is a/an {} to the task: ${task}.`; // capability as premise -> can they perform the task? + const h2 = `The described task is a/an {} to the capability: ${capability}.`; // task as premise -> does it require the capability? + + const h3 = `The task and capability are a/an {}.`; // symmetric + const mix = `Task: ${task}\nCapability: ${capability}`; + + const [dir1, dir2, dir3] = await Promise.all([ + this.nliDirection(capability, h1), + this.nliDirection(task, h2), + this.nliDirection(mix, h3), + ]); + + const result = { + entail: (dir1.entail + dir2.entail + dir3.entail) / 3, + neutral: (dir1.neutral + dir2.neutral + dir3.neutral) / 3, + contradict: Math.max(dir1.contradict, dir2.contradict, dir3.contradict), // max contradict + details: { 'competence on task': dir1, 'task on competence': dir2, combined: dir3 }, + }; + + // Order the labels by their score in descending order + const ranking = ['entail', 'neutral', 'contradict'].sort( + // @ts-ignore + (a, b) => result[b] - result[a], + ); + + return { + ...result, + ranking, + }; + } + + public static async contradictionCheck(task: string, capability: string) { + const labels = ['contradicting', 'not contradicting']; + const pipe = await this.getInstance(); + + const h1 = `The capability is {} to the task: ${task}.`; // capability as premise -> can they perform the task? + const h2 = `The task is {} to the capability: ${capability}.`; // task as premise -> does it require the capability? + + const h3 = `The task and capability are {}.`; // symmetric + const mix = `Task: ${task}\nCapability: ${capability}`; + + const [out1, out2, out3] = await Promise.all([ + pipe(capability, labels, { hypothesis_template: h1 }), + pipe(task, labels, { hypothesis_template: h2 }), + pipe(mix, labels, { hypothesis_template: h3 }), + ]); + + // console.log('________________________________________'); + // console.log('contradiction check results:'); + // console.log('task', task); + // console.log('capability', capability); + // console.log(out1, out2, out3); + // console.log('________________________________________'); + + const sortedOut = labels.reduce( + (acc: Record, label: string) => { + if (acc[label] === undefined) acc[label] = []; + [out1, out2, out3].forEach((out) => { + // Find index of label in out + const idx = ((out as any).labels as string[]).indexOf(label); + // Add corresponding score + if (idx >= 0) acc[label].push((out as any).scores[idx]); + }); + + return acc; + }, + {} as Record, + ); + + const result = { + max: Math.max(...(sortedOut[labels[0]] || [0])), + avg: + (sortedOut[labels[0]] || [0]).reduce((sum, val) => sum + val, 0) / + (sortedOut[labels[0]] || [0]).length, + details: { 'competence on task': out1, 'task on competence': out2, combined: out3 }, + }; + + return { + ...result, + contradicting: result.max > 0.5 || result.avg > 0.45, + }; + } + + public static async alignmentCheck(task: string, capability: string) { + const labels = ['somewhat sufficently', 'only partially', 'not at all']; + const pipe = await this.getInstance(); + + const h1 = `The capability is meeting the requirements of the task "${task}" {}.`; + const h2 = `The tasks requirements are met {} by the capability: "${capability}".`; + + const h3 = `The task and capability are {} matching.`; // symmetric + const mix = `Task: "${task}"\nCapability: "${capability}"`; + + const [out1, out2, out3] = await Promise.all([ + pipe(capability, labels, { hypothesis_template: h1 }), + pipe(task, labels, { hypothesis_template: h2 }), + pipe(mix, labels, { hypothesis_template: h3 }), + ]); + + // console.log('________________________________________'); + // console.log('alignment check results:'); + // console.log('task: ', task); + // console.log('capability: ', capability); + // console.log(out1, out2, out3); + // console.log('________________________________________'); + + const sortedOut = labels.reduce( + (acc: Record, label: string) => { + if (acc[label] === undefined) acc[label] = []; + [out1, out2, out3].forEach((out) => { + // Find index of label in out + const idx = ((out as any).labels as string[]).indexOf(label); + // Add corresponding score + if (idx >= 0) acc[label].push((out as any).scores[idx]); + }); + + return acc; + }, + {} as Record, + ); + + // console.log('________________________________________'); + // console.log('sortedOut: ', sortedOut); + // console.log('________________________________________'); + + const result = { + max: Math.max(...(sortedOut[labels[0]] || [0])), + avg: + (sortedOut[labels[0]] || [0]).reduce((sum, val) => sum + val, 0) / + (sortedOut[labels[0]] || [0]).length, + details: { 'competence on task': out1, 'task on competence': out2, combined: out3 }, + }; + + return { + ...result, + aligning: result.max > 0.65 && result.avg > 0.5, + }; + } +} diff --git a/src/competence-matcher/src/utils/db.ts b/src/competence-matcher/src/utils/db.ts new file mode 100644 index 000000000..5d02224b2 --- /dev/null +++ b/src/competence-matcher/src/utils/db.ts @@ -0,0 +1,7 @@ +import DBManager from '../db/db-manager'; + +export function getDB(name: string) { + const dbManager = DBManager.getInstance(); + DBManager.setActiveDB(name); + return dbManager.getDB(name); +} diff --git a/src/competence-matcher/src/utils/errors.ts b/src/competence-matcher/src/utils/errors.ts new file mode 100644 index 000000000..6e81ec5e7 --- /dev/null +++ b/src/competence-matcher/src/utils/errors.ts @@ -0,0 +1,161 @@ +/** + * Custom error classes for the competence matcher service + * These provide descriptive error messages with context + */ + +export class CompetenceMatcherError extends Error { + public readonly context: string; + public readonly statusCode: number; + public readonly requestId?: string; + public readonly details?: any; + + constructor( + message: string, + context: string, + statusCode: number = 500, + requestId?: string, + details?: any, + ) { + super(message); + this.name = this.constructor.name; + this.context = context; + this.statusCode = statusCode; + this.requestId = requestId; + this.details = details; + + // Maintains proper stack trace for where our error was thrown + Error.captureStackTrace(this, this.constructor); + } + + toJSON() { + return { + name: this.name, + message: this.message, + context: this.context, + statusCode: this.statusCode, + requestId: this.requestId, + details: this.details, + stack: this.stack, + }; + } +} + +export class ValidationError extends CompetenceMatcherError { + constructor(message: string, field: string, value: any, requestId?: string) { + super( + `Validation failed for field '${field}': ${message}`, + 'input_validation', + 400, + requestId, + { field, value }, + ); + } +} + +export class DatabaseError extends CompetenceMatcherError { + constructor(operation: string, error: Error, requestId?: string) { + super( + `Database operation '${operation}' failed: ${error.message}`, + 'database_operation', + 500, + requestId, + { operation, originalError: error.message }, + ); + } +} + +export class ModelError extends CompetenceMatcherError { + constructor(modelName: string, operation: string, error: Error, requestId?: string) { + super( + `Model '${modelName}' operation '${operation}' failed: ${error.message}`, + 'model_operation', + 500, + requestId, + { modelName, operation, originalError: error.message }, + ); + } +} + +export class ResourceNotFoundError extends CompetenceMatcherError { + constructor(resourceType: string, resourceId: string, requestId?: string) { + super( + `${resourceType} with ID '${resourceId}' was not found`, + 'resource_not_found', + 404, + requestId, + { resourceType, resourceId }, + ); + } +} + +export class WorkerError extends CompetenceMatcherError { + constructor(workerType: string, jobId: string, error: Error, requestId?: string) { + super( + `Worker '${workerType}' failed for job '${jobId}': ${error.message}`, + 'worker_execution', + 500, + requestId, + { workerType, jobId, originalError: error.message }, + ); + } +} + +export class SemanticSplittingError extends CompetenceMatcherError { + constructor(textLength: number, error: Error, requestId?: string) { + super( + `Semantic splitting failed for text of length ${textLength}: ${error.message}`, + 'semantic_splitting', + 500, + requestId, + { textLength, originalError: error.message }, + ); + } +} + +export class ReasoningError extends CompetenceMatcherError { + constructor(matchCount: number, error: Error, requestId?: string) { + super( + `Reasoning process failed for ${matchCount} matches: ${error.message}`, + 'reasoning', + 500, + requestId, + { matchCount, originalError: error.message }, + ); + } +} + +export class EmbeddingError extends CompetenceMatcherError { + constructor(modelName: string, taskCount: number, error: Error, requestId?: string) { + super( + `Embedding generation with model '${modelName}' failed for ${taskCount} tasks: ${error.message}`, + 'embedding_generation', + 500, + requestId, + { modelName, taskCount, originalError: error.message }, + ); + } +} + +export class OllamaConnectionError extends CompetenceMatcherError { + constructor(host: string, operation: string, error: Error, requestId?: string) { + super( + `Failed to connect to Ollama at '${host}' for operation '${operation}': ${error.message}`, + 'ollama_connection', + 503, + requestId, + { host, operation, originalError: error.message }, + ); + } +} + +export class HuggingFaceModelError extends CompetenceMatcherError { + constructor(modelName: string, operation: string, error: Error, requestId?: string) { + super( + `HuggingFace model '${modelName}' operation '${operation}' failed: ${error.message}`, + 'huggingface_model', + 500, + requestId, + { modelName, operation, originalError: error.message }, + ); + } +} diff --git a/src/competence-matcher/src/utils/huggingface.ts b/src/competence-matcher/src/utils/huggingface.ts new file mode 100644 index 000000000..0898b8609 --- /dev/null +++ b/src/competence-matcher/src/utils/huggingface.ts @@ -0,0 +1,44 @@ +import CrossEncoder from '../tasks/cross-encode'; +import Embedding from '../tasks/embedding'; +import ZeroShotSemanticOpposites from '../tasks/semantic-zeroshot'; +import { HuggingFaceModelError } from './errors'; +import { getLogger } from './logger'; + +const logger = getLogger(); + +/** + * Ensures that all required Hugging Face models are available by attempting to load them. + * If any model fails to load, an error is thrown. + * + * As this is meant to be used in the main thread, it will trigger the download and caching of models if not already present. + * Since the models inference, however, is meant to be run in worker threads, the model are loaded into ram in the main thread redundantly. + * To mitigate this, the model instances are deleted after the check, so they can be reloaded in the worker threads when needed. + */ +export async function ensureAllHuggingfaceModelsAreAvailable() { + logger.debug('model', 'Checking availability of required models...'); + + try { + logger.debug('model', 'Initialising embedding model...'); + await Embedding.getInstance(); + + logger.debug('model', 'Initialising zero-shot semantic opposites model...'); + await ZeroShotSemanticOpposites.getInstance(); + + // logger.debug('model', 'Initialising cross-encoder model...'); + // await CrossEncoder.getInstance(); + + logger.modelInfo('All HuggingFace models initialised successfully'); + + // Delete instances to free up memory as they will be reloaded in worker threads + Embedding.deleteInstance(); + ZeroShotSemanticOpposites.deleteInstance(); + // CrossEncoder.deleteInstance(); + } catch (error) { + throw new HuggingFaceModelError( + 'unknown', // We don't know which specific model failed - will maybe add later + 'initialisation', + error instanceof Error ? error : new Error(String(error)), + ); + } + logger.modelInfo('All required HuggingFace models are available'); +} diff --git a/src/competence-matcher/src/utils/logger.ts b/src/competence-matcher/src/utils/logger.ts new file mode 100644 index 000000000..598d9b019 --- /dev/null +++ b/src/competence-matcher/src/utils/logger.ts @@ -0,0 +1,349 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { config } from '../config'; +import { LogLevel, LogType, LogEntry, LoggerConfig } from './types'; + +export function createLoggerConfig(): LoggerConfig { + // Parse enabled log types from config + const enabledTypes = config.logTypes + .split(',') + .map((type) => type.trim()) + .filter((type) => + ['server', 'request', 'worker', 'database', 'model', 'system'].includes(type), + ) as LogType[]; + + // Convert string log level to enum + const logLevelMap: { [key: string]: LogLevel } = { + DEBUG: LogLevel.DEBUG, + INFO: LogLevel.INFO, + WARN: LogLevel.WARN, + ERROR: LogLevel.ERROR, + }; + + const mappedLevel = logLevelMap[config.logLevel.toUpperCase()]; + + return { + level: mappedLevel !== undefined ? mappedLevel : LogLevel.INFO, + enabledTypes, + enableConsole: config.logToConsole, + enableFile: config.logToFile, + logDir: config.logPath, + colorize: true, + }; +} + +// ANSI color codes for console output +const colors = { + DEBUG: '\x1b[36m', // Cyan + INFO: '\x1b[32m', // Green + WARN: '\x1b[33m', // Yellow + ERROR: '\x1b[31m', // Red + reset: '\x1b[0m', // Reset + dim: '\x1b[2m', // Dim + bold: '\x1b[1m', // Bold + gray: '\x1b[90m', // Gray + red: '\x1b[31m', // Red +}; + +// Type colors for better visual distinction +const typeColors = { + server: '\x1b[35m', // Magenta + request: '\x1b[34m', // Blue + worker: '\x1b[36m', // Cyan + database: '\x1b[33m', // Yellow + model: '\x1b[32m', // Green + system: '\x1b[37m', // White +}; + +export class Logger { + private static instance: Logger; + private config: LoggerConfig; + private logStreams: Map = new Map(); + private currentLogDate: string = ''; // Track current log date + + private constructor(config: LoggerConfig) { + this.config = config; + this.initialiseLogStreams(); + } + + public static getInstance(config?: LoggerConfig): Logger { + if (!Logger.instance) { + if (!config) { + throw new Error('Logger must be initialised with config on first call'); + } + Logger.instance = new Logger(config); + } + return Logger.instance; + } + + private initialiseLogStreams(): void { + if (!this.config.enableFile) return; + + const today = new Date().toISOString().split('T')[0]; + this.currentLogDate = today; + + this.createLogStreamsForDate(today); + } + + /** + * Create log streams for a specific date + */ + private createLogStreamsForDate(date: string): void { + const dayLogDir = path.join(this.config.logDir, date); + + // Ensure date-specific log directory exists + if (!fs.existsSync(dayLogDir)) { + fs.mkdirSync(dayLogDir, { recursive: true }); + } + + // Close existing streams if they exist + this.closeLogStreams(); + + // Main log file (all logs) - structured JSON + const mainLogPath = path.join(dayLogDir, 'competence-matcher.json'); + this.logStreams.set('main', fs.createWriteStream(mainLogPath, { flags: 'a' })); + + // Error-only log file + const errorLogPath = path.join(dayLogDir, 'errors.log'); + this.logStreams.set('error', fs.createWriteStream(errorLogPath, { flags: 'a' })); + + // Request-only log file + const requestLogPath = path.join(dayLogDir, 'requests.log'); + this.logStreams.set('request', fs.createWriteStream(requestLogPath, { flags: 'a' })); + } + + /** + * Check if we need to rotate logs to a new date and do so if necessary + */ + private checkAndRotateLogsIfNeeded(): void { + const today = new Date().toISOString().split('T')[0]; + + if (today !== this.currentLogDate) { + // Date has changed, rotate to new log files + this.currentLogDate = today; + this.createLogStreamsForDate(today); + } + } + + /** + * Close all log streams + */ + private closeLogStreams(): void { + for (const stream of this.logStreams.values()) { + stream.end(); + } + this.logStreams.clear(); + } + + private shouldLog(level: LogLevel, type: LogType): boolean { + return level >= this.config.level && this.config.enabledTypes.includes(type); + } + + private formatTimestamp(): string { + return new Date().toISOString().replace('T', ' ').replace('Z', ''); + } + + private formatConsoleMessage(entry: LogEntry): string { + if (!this.config.colorize) { + return this.formatPlainMessage(entry); + } + + const levelColor = colors[LogLevel[entry.level] as keyof typeof colors]; + const typeColor = typeColors[entry.type]; + const timestamp = `${colors.gray}[${entry.timestamp}]${colors.reset}`; + const level = `${levelColor}${entry.levelName.padEnd(5)}${colors.reset}`; + const type = `${typeColor}[${entry.type}${entry.requestId ? `:${String(entry.requestId).slice(-12)}` : ''}]${colors.reset}`; + const message = entry.message; + + let output = `${timestamp} ${level} ${type} ${message}`; + + // Add data if present + if (entry.data) { + const dataStr = + typeof entry.data === 'object' ? JSON.stringify(entry.data, null, 2) : String(entry.data); + output += `\n${colors.dim}${dataStr}${colors.reset}`; + } + + // Add error details if present + if (entry.error) { + output += `\n${colors.red}Error: ${entry.error.message}${colors.reset}`; + if (entry.error.stack) { + output += `\n${colors.dim}${entry.error.stack}${colors.reset}`; + } + } + + return output; + } + + private formatPlainMessage(entry: LogEntry): string { + const timestamp = `[${entry.timestamp}]`; + const level = entry.levelName.padEnd(5); + const type = `[${entry.type}${entry.requestId ? `:${String(entry.requestId).slice(-12)}` : ''}]`; + + let output = `${timestamp} ${level} ${type} ${entry.message}`; + + if (entry.data) { + const dataStr = + typeof entry.data === 'object' ? JSON.stringify(entry.data) : String(entry.data); + output += ` | ${dataStr}`; + } + + if (entry.error) { + output += ` | Error: ${entry.error.message}`; + } + + return output; + } + + private writeToFile(entry: LogEntry): void { + if (!this.config.enableFile) return; + + // Check if we need to rotate logs for a new date + this.checkAndRotateLogsIfNeeded(); + + // Write to main log file (structured JSON) - EXCEPT for requests + const mainStream = this.logStreams.get('main'); + if (mainStream && entry.type !== 'request') { + mainStream.write(JSON.stringify(entry) + '\n'); + } + + // Write to error log file + if (entry.level === LogLevel.ERROR) { + const errorStream = this.logStreams.get('error'); + if (errorStream) { + errorStream.write(this.formatPlainMessage(entry) + '\n'); + } + } + + // Write to request log file + if (entry.type === 'request') { + const requestStream = this.logStreams.get('request'); + if (requestStream) { + requestStream.write(this.formatPlainMessage(entry) + '\n'); + } + } + } + + private log( + level: LogLevel, + type: LogType, + message: string, + data?: any, + error?: Error, + requestId?: string, + ): void { + if (!this.shouldLog(level, type)) return; + + const entry: LogEntry = { + timestamp: this.formatTimestamp(), + level, + levelName: LogLevel[level], + type, + message, + requestId, + data, + error: error + ? { + message: error.message, + stack: error.stack, + name: error.name, + } + : undefined, + }; + + // Console output + if (this.config.enableConsole) { + console.log(this.formatConsoleMessage(entry)); + } + + // File output + this.writeToFile(entry); + } + + // Public logging methods + public debug(type: LogType, message: string, data?: any, requestId?: string): void { + this.log(LogLevel.DEBUG, type, message, data, undefined, requestId); + } + + public info(type: LogType, message: string, data?: any, requestId?: string): void { + this.log(LogLevel.INFO, type, message, data, undefined, requestId); + } + + public warn(type: LogType, message: string, data?: any, requestId?: string): void { + this.log(LogLevel.WARN, type, message, data, undefined, requestId); + } + + public error( + type: LogType, + message: string, + error?: Error, + data?: any, + requestId?: string, + ): void { + this.log(LogLevel.ERROR, type, message, data, error, requestId); + } + + // Specialised logging methods for common patterns + public request( + method: string, + path: string, + statusCode: number, + responseTime: number, + requestId: string, + additionalData?: any, + ): void { + const message = `${method} ${path} (${statusCode}) ${responseTime}ms`; + this.log(LogLevel.INFO, 'request', message, additionalData, undefined, requestId); + } + + public worker(message: string, data?: any, requestId?: string): void { + this.log(LogLevel.DEBUG, 'worker', message, data, undefined, requestId); + } + + public workerInfo(message: string, data?: any, requestId?: string): void { + this.log(LogLevel.INFO, 'worker', message, data, undefined, requestId); + } + + public workerError(message: string, error?: Error, data?: any, requestId?: string): void { + this.log(LogLevel.ERROR, 'worker', message, data, error, requestId); + } + + public database(message: string, data?: any, requestId?: string): void { + this.log(LogLevel.DEBUG, 'database', message, data, undefined, requestId); + } + + public databaseError(message: string, error?: Error, data?: any, requestId?: string): void { + this.log(LogLevel.ERROR, 'database', message, data, error, requestId); + } + + public model(message: string, data?: any, requestId?: string): void { + this.log(LogLevel.DEBUG, 'model', message, data, undefined, requestId); + } + + public modelInfo(message: string, data?: any, requestId?: string): void { + this.log(LogLevel.INFO, 'model', message, data, undefined, requestId); + } + + // Utility methods + public generateRequestId(): string { + return `req_${Date.now()}_${randomUUID().slice(0, 8)}`; + } + + public close(): void { + this.closeLogStreams(); + } + + // Update configuration + public updateConfig(newConfig: Partial): void { + this.config = { ...this.config, ...newConfig }; + + // Reinitialise streams if file logging config changed + if (newConfig.enableFile !== undefined || newConfig.logDir) { + this.initialiseLogStreams(); + } + } +} + +// Export singleton instance getter +export const getLogger = () => Logger.getInstance(); diff --git a/src/competence-matcher/src/utils/model.ts b/src/competence-matcher/src/utils/model.ts new file mode 100644 index 000000000..b0320beb2 --- /dev/null +++ b/src/competence-matcher/src/utils/model.ts @@ -0,0 +1,70 @@ +import { config } from '../config'; + +import { + pipeline, + env as huggingfaceEnv, + PipelineType, + PretrainedModelOptions, +} from '@huggingface/transformers'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { TransformerPipelineOptions } from './types'; + +import { isMainThread } from 'node:worker_threads'; + +/** + * Base class that handles: + * - singleton pipeline instance + * - model cache dir + * - on‑demand loading with optional progressCallback + */ +export abstract class TransformerPipeline { + protected static instance: any = null; + protected static loaded = false; + + protected static getPipelineOptions(): TransformerPipelineOptions { + throw new Error('getPipelineOptions must be implemented in subclasses'); + } + + private static async initEnv(cacheDir?: string) { + if (cacheDir) { + const abs = path.resolve(cacheDir); + if (!fs.existsSync(abs)) fs.mkdirSync(abs, { recursive: true }); + huggingfaceEnv.cacheDir = abs; + } + huggingfaceEnv.allowLocalModels = true; + } + + public static async getInstance(): Promise { + if (this.instance === null) { + const { task, model, options } = this.getPipelineOptions(); + await this.initEnv(options?.cache_dir || config.modelCache); + + const opts: PretrainedModelOptions = { + cache_dir: options?.cache_dir || config.modelCache, + use_external_data_format: options?.use_external_data_format ?? true, + device: options?.device || (config.useGPU ? 'cuda' : 'cpu'), + dtype: options?.dtype || 'fp32', + progress_callback: options?.progress_callback, + }; + + // actually load the pipeline + this.instance = await pipeline(task as PipelineType, model, opts); + + // mark it as loaded and log on first load + if (!this.loaded && isMainThread) { + const { getLogger } = require('./logger'); + const logger = getLogger(); + logger.modelInfo(`Model pipeline ${model} (${task}) is ready`); + this.loaded = true; + } + } + + return this.instance; + } + + public static deleteInstance() { + this.instance = null; + this.loaded = false; + } +} diff --git a/src/competence-matcher/src/utils/ollama.ts b/src/competence-matcher/src/utils/ollama.ts new file mode 100644 index 000000000..b1a9fe795 --- /dev/null +++ b/src/competence-matcher/src/utils/ollama.ts @@ -0,0 +1,261 @@ +import { Ollama } from 'ollama'; +import { config } from '../config'; +import { OllamaConnectionError } from './errors'; +import { getLogger } from './logger'; + +const { + ollamaPath, + splittingModel, + reasonModel, + ollamaBearerToken, + maxOllamaRetries, + ollamaRetryDelay, + ollamaRetryBackoff, +} = config; + +// Lazy logger initialization to avoid module loading order issues +let logger: ReturnType | null = null; +const getLoggerInstance = () => { + if (!logger) { + logger = getLogger(); + } + return logger; +}; + +export const ollama = new Ollama({ + host: ollamaPath, + headers: { + Authorization: `Bearer ${ollamaBearerToken}`, // https://github.com/ollama/ollama-js?tab=readme-ov-file#user-content-custom-headers + 'User-Agent': 'PROCEED Competence Matcher', + }, +}); + +/** + * Utility function to sleep for a given duration + */ +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Check if an error is likely a timeout error from nginx proxy + */ +function isTimeoutError(error: any): boolean { + if (error instanceof OllamaConnectionError) { + const originalError = error.cause; + // Check for common timeout/proxy error indicators + if (originalError && typeof originalError === 'object') { + const errorString = String(originalError).toLowerCase(); + const errorMessage = (originalError as any).message?.toLowerCase() || ''; + + return ( + errorString.includes('504') || // Gateway timeout + errorString.includes('timeout') || // General timeout + errorString.includes('etimedout') || // Node.js timeout + errorString.includes('econnreset') || // Connection reset + errorMessage.includes('504') || + errorMessage.includes('timeout') || + errorMessage.includes('gateway timeout') + ); + } + } + return false; +} + +/** + * Get error message safely from unknown error type + */ +function getErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + return String(error); +} + +/** + * Attempt to pull a model with retry logic for handling proxy timeouts + */ +async function pullModelWithRetry(modelName: string): Promise { + for (let attempt = 1; attempt <= maxOllamaRetries; attempt++) { + try { + getLoggerInstance().debug( + 'model', + `Attempting to pull Ollama model '${modelName}' (attempt ${attempt}/${maxOllamaRetries})`, + ); + + const modelpull = await ollama.pull({ + model: modelName, + insecure: false, + stream: false, + }); + + // Check if the model was successfully pulled + if (!modelpull || modelpull.status !== 'success') { + throw new OllamaConnectionError( + ollamaPath, + 'pull_model', + new Error(`Model pull failed: ${modelpull?.status || 'Unknown error'}`), + ); + } + + getLoggerInstance().info('model', `Successfully pulled Ollama model '${modelName}'`); + return true; + } catch (error) { + const isTimeout = isTimeoutError(error); + const isLastAttempt = attempt === maxOllamaRetries; + + if (isTimeout && !isLastAttempt) { + // Log as warning for timeout errors that we'll retry + const delay = ollamaRetryDelay * Math.pow(ollamaRetryBackoff, attempt - 1); + getLoggerInstance().warn( + 'model', + `Ollama model pull timeout for '${modelName}' (attempt ${attempt}/${maxOllamaRetries}), retrying in ${Math.round(delay / 1000)}s...`, + { + modelName, + attempt, + maxRetries: maxOllamaRetries, + retryDelay: delay, + error: getErrorMessage(error), + }, + ); + + // Wait before retrying with exponential backoff + await sleep(delay); + continue; + } else if (isLastAttempt) { + // Final attempt failed - escalate to error + getLoggerInstance().error( + 'model', + `Failed to pull Ollama model '${modelName}' after ${maxOllamaRetries} attempts`, + error instanceof Error ? error : new Error(String(error)), + { + modelName, + totalAttempts: maxOllamaRetries, + finalError: getErrorMessage(error), + }, + ); + throw error; + } else { + // Non-timeout error on non-final attempt + getLoggerInstance().warn( + 'model', + `Ollama model pull error for '${modelName}' (attempt ${attempt}/${maxOllamaRetries}): ${getErrorMessage(error)}`, + { + modelName, + attempt, + maxRetries: maxOllamaRetries, + error: getErrorMessage(error), + }, + ); + + // Wait before retrying (shorter delay for non-timeout errors) + await sleep(Math.min(ollamaRetryDelay, 5000)); + continue; + } + } + } + + return false; // Should never reach here due to throw in loop +} + +/** + * Ensures that all required models are available by checking their existence + * in the Ollama server. If a model is not available, it will be downloaded. + * If the model cannot be downloaded or is not available, an error will be thrown. + * (Ensures all needed models are actually available) + */ +export async function ensureAllOllamaModelsAreAvailable() { + const models = [splittingModel, reasonModel]; + + getLoggerInstance().debug( + 'model', + `Checking availability of Ollama models: ${models.join(', ')}`, + ); + + let availableModels: string[]; + try { + const modelList = await ollama.list(); + availableModels = modelList.models.map((model) => model.model); + + getLoggerInstance().debug('model', `Available Ollama models: ${availableModels.join(', ')}`); + } catch (error) { + throw new OllamaConnectionError( + ollamaPath, + 'list_models', + error instanceof Error ? error : new Error(String(error)), + ); + } + + for (const model of models) { + if (!availableModels.includes(model)) { + getLoggerInstance().info('model', `Ollama model '${model}' not found, attempting to pull...`); + + try { + await pullModelWithRetry(model); + + // After successful pull, re-check model availability to ensure it's actually there + getLoggerInstance().debug('model', `Re-checking availability of '${model}' after pull...`); + + try { + const updatedModelList = await ollama.list(); + const updatedAvailableModels = updatedModelList.models.map((m) => m.model); + + if (!updatedAvailableModels.includes(model)) { + throw new OllamaConnectionError( + ollamaPath, + 'verify_model', + new Error( + `Model '${model}' was reportedly pulled successfully but is not available in model list`, + ), + ); + } + + getLoggerInstance().debug('model', `Confirmed '${model}' is now available after pull`); + } catch (verifyError) { + getLoggerInstance().warn( + 'model', + `Failed to verify model '${model}' availability after pull: ${getErrorMessage(verifyError)}`, + { model, verifyError: getErrorMessage(verifyError) }, + ); + + // If verification fails, we still consider the pull successful if it didn't throw + // This handles cases where the model is actually available but listing fails + } + } catch (error) { + // Re-check one more time in case the model was actually pulled despite the error + getLoggerInstance().debug( + 'model', + `Pull failed for '${model}', performing final availability check...`, + ); + + try { + const finalModelList = await ollama.list(); + const finalAvailableModels = finalModelList.models.map((m) => m.model); + + if (finalAvailableModels.includes(model)) { + getLoggerInstance().info( + 'model', + `Model '${model}' is now available despite pull error - continuing`, + ); + continue; // Model is actually available, continue to next model + } + } catch (listError) { + getLoggerInstance().debug( + 'model', + `Final model list check failed: ${getErrorMessage(listError)}`, + ); + } + + // Model is definitely not available, propagate the original error + throw error; + } + } else { + getLoggerInstance().debug( + 'model', + `Ollama model '${model}' is available (already downloaded)`, + ); + } + } + + getLoggerInstance().modelInfo('All required Ollama models are available'); +} diff --git a/src/competence-matcher/src/utils/onnx-lock.ts b/src/competence-matcher/src/utils/onnx-lock.ts new file mode 100644 index 000000000..9804b3511 --- /dev/null +++ b/src/competence-matcher/src/utils/onnx-lock.ts @@ -0,0 +1,48 @@ +import { workerData } from 'worker_threads'; + +// WorkerData is provided when the worker is constructed. We expect an object +// with an `onnxLock` SharedArrayBuffer so that all workers can coordinate +// access to the ONNX runtime. If absent, locking is skipped. +const sharedBuffer: SharedArrayBuffer | undefined = + workerData && workerData.onnxLock instanceof SharedArrayBuffer + ? (workerData.onnxLock as SharedArrayBuffer) + : undefined; + +const lockView = sharedBuffer ? new Int32Array(sharedBuffer) : null; + +function acquire(): void { + if (!lockView) return; + + while (true) { + const prev = Atomics.compareExchange(lockView, 0, 0, 1); + if (prev === 0) { + return; // we acquired the lock + } + + Atomics.wait(lockView, 0, 1); + } +} + +function release(): void { + if (!lockView) return; + Atomics.store(lockView, 0, 0); + Atomics.notify(lockView, 0, 1); +} + +export async function withOnnxLock(operation: () => Promise): Promise { + acquire(); + try { + return await operation(); + } finally { + release(); + } +} + +export function withOnnxLockSync(operation: () => T): T { + acquire(); + try { + return operation(); + } finally { + release(); + } +} diff --git a/src/competence-matcher/src/utils/prompts.ts b/src/competence-matcher/src/utils/prompts.ts new file mode 100644 index 000000000..042997195 --- /dev/null +++ b/src/competence-matcher/src/utils/prompts.ts @@ -0,0 +1,695 @@ +import type { Message } from 'ollama'; +import { config } from '../config'; + +const { splittingSymbol } = config; + +const SEMANTIC_SPLITTER_INTRUCT: Message = { + role: 'system', + content: ` + Your task is to segment the following text (i.e. user input such as plain prose, bullet points or listings) into semantically independent parts. + Do not add, remove, or modify any words - under no circumstances should you ever add any additional text, comments, or explanations. + Preserve the original ordering of words within each group — but groups themselves need not follow the original sequence. + Separate each group only by the delimiter + ${splittingSymbol} + (i.e. exactly as shown, on a line by themselves, no additional whitespaces, just '${splittingSymbol}'). + If the entire input is already one coherent semantic unit, return it verbatim without any delimiter. + Grouping need do not be adjacent - just semantically related (i.e. two related text parts might be separated by other text parts). + If the input is empty, return an empty string. + If the input is a single word, return it verbatim without any delimiter. + If the input is a single sentence, return it verbatim without any delimiter. + If you are unsure about the grouping, return the entire input as a single group without any delimiters. + Under no circumstances should you ever add any additional text, comments, or explanations. + `, +}; +const SEMANTIC_SPLITTER_EXAMPLES: Message[] = [ + { + role: 'user', + content: ` + The job requires welding experience. Tick welding would be preferable if the person is familiar with it. The cage is designed for small pets like rabbits. Therefore, it must not contain any sharp edges that could harm them. Experience with welding small wires could be beneficial. + `, + }, + { + role: 'assistant', + content: ` + The job requires welding experience. Tick welding would be preferable if the person is familiar with it. Experience with welding small wires could be beneficial. + ${splittingSymbol} + The cage is designed for small pets like rabbits. Therefore, it must not contain any sharp edges that could harm them. + `, + }, + { + role: 'user', + content: ` + - Assemble circuit boards according to schematic diagrams + - Test each board for continuity and signal integrity + - Package finished units in protective casing + - Ship completed orders to customers worldwide + `, + }, + { + role: 'assistant', + content: ` + - Assemble circuit boards according to schematic diagrams + - Test each board for continuity and signal integrity + ${splittingSymbol} + - Package finished units in protective casing + - Ship completed orders to customers worldwide + `, + }, + { + role: 'user', + content: ` + 1. Prepare raw materials for production + 2. Record daily output and machine performance. + 3. Clean workstations and restock supplies + 4. Order order new materials when needed. + 5. Calibrate and maintain measurement instruments. + `, + }, + { + role: 'assistant', + content: ` + 1. Prepare raw materials for production + 4. Order order new materials when needed. + ${splittingSymbol} + 2. Record daily output and machine performance. + ${splittingSymbol} + 3. Clean workstations and restock supplies + ${splittingSymbol} + 5. Calibrate and maintain measurement instruments. + `, + }, + { + role: 'user', + content: ` + Operate CNC milling machines to produce precision metal parts. Perform quality inspections using calipers, micrometers, and gauges. Monitor machine operation and adjust feed rates, speeds, and tooling as needed. Maintain a clean and safe workspace, following all OSHA safety guidelines. Collaborate with engineers to troubleshoot design issues and implement improvements. Document production logs, inspection reports, and maintenance records daily. Assist in training new operators on standard operating procedures and best practices. + `, + }, + { + role: 'assistant', + content: ` + Operate CNC milling machines to produce precision metal parts. + Monitor machine operation and adjust feed rates, speeds, and tooling as needed. + ${splittingSymbol} + Perform quality inspections using calipers, micrometers, and gauges. + Collaborate with engineers to troubleshoot design issues and implement improvements. + ${splittingSymbol} + Maintain a clean and safe workspace, following all OSHA safety guidelines. + Document production logs, inspection reports, and maintenance records daily. + Assist in training new operators on standard operating procedures and best practices. + `, + }, +]; + +export const SEMANTIC_SPLITTER: Message[] = [ + SEMANTIC_SPLITTER_INTRUCT, + ...SEMANTIC_SPLITTER_EXAMPLES, +]; + +/** + * ------------------------------------------------------------- + */ + +// const MATCH_REASON_INTRUCT: Message = { +// role: 'system', +// content: ` +// You are an expert in generating reasons for matching scores between tasks and competences. +// Your task is to generate a reason for the matching score between a task and a competence. +// The reason should be one to three short, concise sentence that explain why the task and competence match as well as they did or why they did not match that well. +// Do not mention the similarity score in your response. +// The reason should be based on the text of the task and the competence and their estimated normalized similarity score. +// The similarity score is a number between 0 and 1, where 0 means no similarity and 1 means perfect similarity. +// Do not mention the similarity score in your response. +// `, +// }; + +// const MATCH_REASON_EXAMPLES: Message[] = [ +// { +// role: 'user', +// content: ` +// Task: Operate CNC milling machines to produce precision metal parts. +// Competence: Experience with CNC milling machines and precision machining. +// Similarity Score: 0.95 +// `, +// }, +// { +// role: 'assistant', +// content: ` +// The statements match very well because the task requires operating CNC milling machines, which is exactly what the competence is about. +// `, +// }, +// { +// role: 'user', +// content: ` +// Task: Assemble circuit boards according to schematic diagrams. +// Competence: Basic knowledge of electronics and soldering skills. +// Similarity Score: 0.65 +// `, +// }, +// { +// role: 'assistant', +// content: ` +// The the statements have a moderate match because while assembling circuit boards requires some knowledge of electronics, it does not specifically require advanced soldering skills. +// `, +// }, +// { +// role: 'user', +// content: ` +// Task: Prepare raw materials for production. +// Competence: Experience with inventory management and supply chain logistics. +// Similarity Score: 0.30 +// `, +// }, +// { +// role: 'assistant', +// content: ` +// The statements have a low match because preparing raw materials is a basic task that does not require advanced inventory management or supply chain logistics skills. +// `, +// }, +// ]; + +let MATCH_REASON_EXAMPLES: Message[] = [ + { + role: 'user', + content: ` + Task: Operate CNC milling machines to produce precision metal parts. + Competence: Experience with CNC milling machines and precision machining. + Similarity Score: 0.91 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The statements match very well because the task requires operating CNC milling machines, which is exactly what the competence is about. + `, + }, + { + role: 'user', + content: ` + Task: Delivering packages to customers on time. Driving a delivery van safely through city traffic. Loading and unloading packages efficiently. Communicating with customers professionally. Planning optimal delivery routes using GPS technology. + Competence: Has no drivers license and cannot operate vehicles. + Similarity Score: 0.0 + Alignment: contradicting + `, + }, + { + role: 'assistant', + content: ` + The statements do not match. The task requires driving a delivery van, but the competence indicates that the person cannot operate vehicles at all. + `, + }, + { + role: 'user', + content: ` + Task: Assemble circuit boards according to schematic diagrams. + Competence: Basic knowledge of electronics and soldering skills. + Similarity Score: 0.18 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + The the statements only slightly match because while assembling circuit boards requires some knowledge of electronics, it does not specifically require advanced soldering skills. + `, + }, + { + role: 'user', + content: ` + Task: Prepare raw materials for production. + Competence: Experience with inventory management and supply chain logistics. + Similarity Score: 0 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + The task and competence do not match because preparing raw materials is a basic task that does not require advanced inventory management or supply chain logistics skills. + `, + }, + //////////////////////////////////////////////////////////////// + { + role: 'user', + content: ` + Task: Lead Agile Scrum teams, facilitate sprint planning and retrospectives. + Competence: Certified Scrum Master with 5 years of experience running scrum ceremonies. + Similarity Score: 0.95 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence directly covers leading Scrum teams and facilitating ceremonies, which matches the task's requirements precisely. + `, + }, + { + role: 'user', + content: ` + Task: Develop native iOS applications in Swift and ship to the App Store. + Competence: Experienced Android engineer, Kotlin and Jetpack Compose. + Similarity Score: 0.40 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + The competence demonstrates mobile-app development experience but on a different platform and language, so it provides some transferable skills but not a direct match. + `, + }, + { + role: 'user', + content: ` + Task: Translate legal contracts from English to German ensuring legal terminology is correct. + Competence: Native German translator with certification in legal translation. + Similarity Score: 0.92 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence combines native German proficiency with a legal translation credential, matching the task's need for accurate legal terminology. + `, + }, + { + role: 'user', + content: ` + Task: Manage payroll, calculate taxes, and file payroll reports. + Competence: Certified public accountant (CPA) with payroll processing experience. + Similarity Score: 0.89 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence includes both the certification and hands-on payroll experience required to perform payroll calculation and filings. + `, + }, + { + role: 'user', + content: ` + Task: Operate hydraulic excavators on construction sites. + Competence: Forklift operator certified with several years of warehouse experience. + Similarity Score: 0.12 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + Both roles involve operating heavy equipment, but the specific machinery and skills differ, so the relevance is limited. + `, + }, + { + role: 'user', + content: ` + Task: Write unit tests in Python using pytest and maintain test coverage. + Competence: Experienced in writing unit tests in Java with JUnit and some general testing knowledge. + Similarity Score: 0.45 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + The competence shows solid testing experience but in a different language and framework, making it partly relevant but not a perfect fit. + `, + }, + { + role: 'user', + content: ` + Task: Handle inbound customer support calls with empathy and problem resolution. + Competence: Strong written communication skills and two years of live chat customer support. + Similarity Score: 0.55 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence demonstrates customer-facing communication and support experience that transfers well to voice support, though the channel differs. + `, + }, + { + role: 'user', + content: ` + Task: Perform blood draws (phlebotomy) and prepare specimens for lab testing. + Competence: Certified phlebotomist with clinical experience. + Similarity Score: 0.99 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence explicitly covers phlebotomy certification and clinical practice, which directly matches the task's duties. + `, + }, + { + role: 'user', + content: ` + Task: Conduct workplace hazard assessments and maintain safety documentation. + Competence: OSHA 30 certification and experience conducting safety audits. + Similarity Score: 0.87 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence shows the required safety certification and audit experience needed to perform hazard assessments and documentation. + `, + }, + { + role: 'user', + content: ` + Task: Build high-frequency trading systems in low-latency C++. + Competence: Strong Python data-analysis background and finance domain knowledge. + Similarity Score: 0.30 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + The competence provides valuable finance and data skills, but lacks the specific low-latency C++ systems expertise required for the task. + `, + }, + { + role: 'user', + content: ` + Task: Lead a team of graphic designers to create brand identity and assets. + Competence: Senior UX designer with a portfolio of visual branding and team leadership. + Similarity Score: 0.70 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence combines visual branding experience and leadership, which fits the task of leading designers to create brand assets. + `, + }, + { + role: 'user', + content: ` + Task: Translate complex medical research articles accurately into English. + Competence: Marketing copywriter with no medical background. + Similarity Score: 0.0 + Alignment: contradicting + `, + }, + { + role: 'assistant', + content: ` + The task requires specialized medical knowledge and terminology that the competence does not provide. + `, + }, + { + role: 'user', + content: ` + Task: Ensure PCI-level data protection and handle cardholder data securely. + Competence: History of incidents involving mishandling of sensitive customer data. + Similarity Score: 0.0 + Alignment: contradicting + `, + }, + { + role: 'assistant', + content: ` + The task requires strict data security practices, but the competence indicates past failures to protect sensitive information, creating a direct conflict. + `, + }, + /////////////////////// + { + role: 'user', + content: ` + Task: Install and configure Nginx as a reverse proxy with TLS termination for multiple web services. + Competence: Experience configuring Nginx, handling TLS certificates, and reverse proxy setups for web applications. + Similarity Score: 0.92 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence explicitly covers Nginx configuration, TLS certificate handling, and reverse-proxy setups, which directly match the task. That operational experience maps to the task's technical requirements. Overall this is a strong, practical fit. + `, + }, + { + role: 'user', + content: ` + Task: Write automated end-to-end tests for a React web application using Playwright. + Competence: Experienced in unit/integration testing in React and in end-to-end tests using Selenium. + Similarity Score: 0.63 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence shows solid testing experience and familiarity with React, which transfers to end-to-end testing concepts. While the specific test framework differs and Selenium's patterns are not identical to Playwright's modern APIs, the fit is definitely relevant although not a perfect match. + `, + }, + { + role: 'user', + content: ` + Task: Manage emergency room triage, including rapid patient assessment and prioritization. + Competence: Registered nurse with prior ER triage experience and ACLS certification. + Similarity Score: 1 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence explicitly describes ER triage experience and advanced life support certification, which match the task's clinical and triage responsibilities. Those credentials and practical duties map directly to rapid patient assessment and prioritization. This competence is therefore highly appropriate for the role. + `, + }, + { + role: 'user', + content: ` + Task: Operate retail point-of-sale (POS) systems and handle daily cash reconciliation. + Competence: Previous retail cashier experience including operating POS terminals and performing end-of-day cash counts. + Similarity Score: 0.74 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence lists the exact operational duties — running POS systems and reconciling cash — which align with the task. That hands-on experience covers most daily requirements and reduces training needs. Overall the candidate is well suited for this role. + `, + }, + { + role: 'user', + content: ` + Task: Ensure GDPR-compliant handling of personal data, manage consent records, and perform regular data protection impact assessments. + Competence: Proven track record implementing data protection policies and running DPIAs. + Similarity Score: 0.81 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence demonstrates direct experience implementing data-protection policies and performing impact assessments, which correspond to the task's key responsibilities. Those policy and assessment skills indicate strong familiarity with GDPR-related workflows and controls. The competence is therefore a very good fit for the task. + `, + }, + { + role: 'user', + content: ` + Task: Drive a passenger shuttle daily and secure vehicle inspections. + Competence: Medical restriction explicitly prohibiting driving. + Similarity Score: 0.00 + Alignment: contradicting + `, + }, + { + role: 'assistant', + content: ` + The task requires driving a passenger shuttle, but the competence indicates a medical restriction against driving, creating a direct conflict. + `, + }, + ///////////////// + { + role: 'user', + content: ` + Task: Translate engineering user manuals from Chinese to English, preserving technical terms and measurements. + Competence: Professional Chinese-to-English translator with experience in marketing and legal documents, no engineering background. + Similarity Score: 0.79 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence demonstrates strong translation skills between Chinese and English, which are essential for the task. However, the lack of engineering-specific experience means some technical terminology may require additional research. Overall, the candidate is quite well suited but may need to upskill on engineering concepts. + `, + }, + { + role: 'user', + content: ` + Task: Administer Microsoft SQL Server databases, tune queries and manage backups. + Competence: Extensive experience with MySQL and PostgreSQL administration. + Similarity Score: 0.45 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + The competence shows strong database administration skills that transfer conceptually to SQL Server. However, the specific platform differences mean the candidate will need to learn SQL Server's unique features and tools. The fit is partial: good foundational skills but not yet complete for SQL Server-specific tasks. + `, + }, + { + role: 'user', + content: ` + Task: Improve front-end performance for a React single-page app, including code-splitting and profiling render hotspots. + Competence: Experienced React developer who has optimized bundle sizes and refactored components but limited experience with advanced profiler workflows. + Similarity Score: 0.50 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + The competence brings directly relevant front-end and React experience, including bundle optimization and component improvements. However, the candidate's limited exposure to advanced profiling workflows means they may need to upskill for deep-render hotspot analysis. Overall, they are reasonably well placed to improve performance but may require some additional profiling experience. + `, + }, + { + role: 'user', + content: ` + Task: Build SQL analytics queries and ETL jobs in Amazon Redshift, optimize for large datasets. + Competence: Strong SQL and analytics experience with PostgreSQL and routine ETL pipelines, but no Redshift-specific deployments. + Similarity Score: 0.52 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence demonstrates solid SQL and ETL experience that transfers to analytics work, and understanding of query design is directly useful for Redshift. The lack of Redshift-specific tuning and cluster knowledge is a remaining gap that will affect large-scale optimizations. The candidate is generally well suited but will need platform-specific tuning skills. + `, + }, + { + role: 'user', + content: ` + Task: Onboard mid-market customers to a SaaS product and drive adoption through structured training and playbooks. + Competence: Account manager with experience in customer onboarding for software services, but mostly for on-premise deployments and manual training sessions. + Similarity Score: 0.48 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + The competence shows relevant onboarding and customer engagement experience that transfers to SaaS adoption activities. However, the difference between on-premise manual onboarding and recurring SaaS flows means playbook-driven, product-led onboarding skills may be missing. The candidate can perform many onboarding tasks but will need to adapt to SaaS-specific tooling and processes. + `, + }, + { + role: 'user', + content: ` + Task: Deploy machine-learning models to production with CI/CD, containerization, and monitoring. + Competence: ML researcher experienced in model development and validation, with limited DevOps/infrastructure deployment experience. + Similarity Score: 0.47 + Alignment: neutral + `, + }, + { + role: 'assistant', + content: ` + The competence demonstrates strong machine-learning model development skills, which are foundational for production deployment. However, the lack of DevOps and infrastructure experience means the candidate may struggle with CI/CD pipelines, containerization, and monitoring aspects. The fit is partial: good ML skills but significant gaps in deployment expertise. + `, + }, + { + role: 'user', + content: ` + Task: Provide German-language phone-based customer support for technical B2B software. + Competence: Native German speaker with one year of live chat support experience on a technical product. + Similarity Score: 0.55 + Alignment: aligning + `, + }, + { + role: 'assistant', + content: ` + The competence supplies native-level language skills and hands-on technical support experience, which transfer well to customer-facing roles. Phone-based support requires real-time verbal communication and some nuances compared with chat, but the underlying technical troubleshooting ability is present. Overall this competence is a good fit, with small channel-specific adaptation required. + `, + }, +]; + +// Add intermediate system instructions for more consistent reasoning +const systemIntermezzo: Message = { + role: 'system', + content: ` + Remember to base your reason on both the similarity score and the alignment label without mentioning them explicitly. + Scores above 0.6 generally indicate a good up to perfect suitability. + Scores between 0.45 and 0.6 generally indicate some up to a good degree of suitability. + Scores between 0.25 and 0.45 generally indicate some degree of suitability. + Scores below or equal to 0.25 generally indicate little to no suitability. + `, +}; + +// Add after every assistant message +MATCH_REASON_EXAMPLES = MATCH_REASON_EXAMPLES.flatMap((msg) => + msg.role === 'assistant' ? [msg, systemIntermezzo] : [msg], +); + +const MATCH_REASON_INSTRUCT: Message = { + role: 'system', + content: ` +You are an expert in generating concise reasons that explain why a Task and a Competence match to the degree they do. + +Core rules: +- Produce exactly 1–3 short, concise sentences that explain why the task and competence matched (or did not). Use active voice and concrete wording. +- Base the reason only on the text of the Task and the Competence and implicitly on the provided similarity score and alignment label. Do NOT invent facts. +- Do NOT mention the numeric similarity score or the alignment label in the reason. +- Avoid speculation, generic filler, or unrelated information. Keep reasons grounded in wording present in the two texts. +- If texts explicitly conflict (for example, task requires driving and competence states inability to drive), explain the conflict clearly and directly without naming the label. +- When the texts imply partial transferability (different platform/language/tool but overlapping concepts), explain which parts transfer and which gaps remain. +- Keep tone neutral and factual. + +Score interpretation (matches your semantics): +- 0.00 : Not suited — no suitability (either direct contradiction or no overlap). +- Greater than 0 up to 0.20 (0 < score ≤ 0.20) : Slightly suited — small degree of relevance or very limited transferability. +- Greater than 0.20 up to 0.50 (0.20 < score ≤ 0.50) : Somewhat suited — a noticeable but incomplete overlap; useful for partial tasks or with training. +- Greater than 0.50 up to 0.70 (0.50 < score ≤ 0.70) : Quite well suited — competence covers most requirements with small gaps. +- Greater than 0.70 up to 0.85 (0.70 < score ≤ 0.85) : Very well suited — strong coverage of task requirements. +- Greater than 0.85 up to 1.00 (0.85 < score ≤ 1.00) : Close to perfectly suited — competence largely or fully covers the task. + +Important handling of inconsistencies: +- If the numeric score and the alignment label conflict (for example, a nonzero high score but an alignment that indicates explicit contradiction), prefer the alignment label when the competence text clearly contradicts the task. In that case, produce a reason that highlights the explicit conflicting statements in the texts. +- Otherwise, use the score and label implicitly to set the tone and specificity of the reason. + +Output constraints: +- 1–3 short sentences only. +- Do NOT include numeric values or alignment labels in the output. +- Be specific: reference the precise skill, tool, domain, or limitation that explains the match degree. +`, +}; + +export const MATCH_REASON: Message[] = [MATCH_REASON_INSTRUCT, ...MATCH_REASON_EXAMPLES]; + +/** + * ------------------------------------------------------------- + */ +// """""""""" +// The warehouse must maintain ambient temperatures between 15°C and 25°C to protect sensitive goods. Humidity levels should not exceed 60% to prevent corrosion and mold growth. Inventory audits are scheduled weekly to ensure accuracy and compliance with safety standards. +// """""""""" + +// """""""""" +// Our catering service provides vegetarian, vegan, and gluten-free menu options to accommodate diverse dietary needs. All dishes are prepared fresh daily using locally sourced ingredients whenever possible. Orders must be placed at least 48 hours in advance to guarantee availability. Delivery times range from 8 AM to 6 PM on weekdays. +// """""""""" + +// """""""""" +// Employees are required to complete the annual cybersecurity training module before accessing the new intranet portal. The module covers password hygiene, phishing identification, and secure remote-access procedures. Failure to complete training by the deadline will result in temporary revocation of network privileges. +// """""""""" + +// """""""""" +// The production line must operate continuously in three shifts—morning, afternoon, and night—to meet daily target outputs of 5,000 units. All machinery, including conveyor belts and hydraulic presses, requires a thorough safety inspection at the start of each shift to ensure proper lubrication and guard alignment. Operators are responsible for logging any unusual vibrations, noise anomalies, or temperature spikes immediately in the maintenance ledger. Raw material deliveries arrive twice weekly and must be verified against purchase orders for quantity, grade, and certificate of analysis before being released to the staging area. Finished goods undergo a final quality check where dimensional tolerances, surface finish, and functional performance are recorded. Any nonconforming parts are quarantined and tagged, then reported to quality assurance for root‐cause analysis. At the end of each month, team leads compile production metrics, downtime reasons, and scrap rates into a summarized report for review at the management safety and efficiency meeting. +// """""""""" diff --git a/src/competence-matcher/src/utils/types.ts b/src/competence-matcher/src/utils/types.ts new file mode 100644 index 000000000..65eecb805 --- /dev/null +++ b/src/competence-matcher/src/utils/types.ts @@ -0,0 +1,204 @@ +import { PretrainedModelOptions } from '@huggingface/transformers'; + +// ===== LOGGING TYPES ===== + +// Logger levels in order of severity +export enum LogLevel { + DEBUG = 0, + INFO = 1, + WARN = 2, + ERROR = 3, +} + +// Logger types for different components +export type LogType = 'server' | 'request' | 'worker' | 'database' | 'model' | 'system'; + +// Log entry structure +export interface LogEntry { + timestamp: string; + level: LogLevel; + levelName: string; + type: LogType; + message: string; + context?: string; + requestId?: string; + data?: any; + error?: { + message: string; + stack?: string; + name?: string; + }; +} + +// Logger configuration +export interface LoggerConfig { + level: LogLevel; + enabledTypes: LogType[]; + enableConsole: boolean; + enableFile: boolean; + logDir: string; + colorize: boolean; +} + +// ===== COMPETENCE MATCHING TYPES ===== + +export type Competence = { + listId: string; // UUIDString + resourceId: string; // UUIDString + competenceId: string; // UUIDString + name?: string; // optional + description?: string; // optional but recommended to have content + externalQualificationNeeded?: boolean; // optional + renewTime?: number; // DaysAsInteger, optional + proficiencyLevel?: string; // optional + qualificationDates?: string[]; // ISO date strings, optional + lastUsages?: string[]; // ISO date strings, optional +}; + +export type CompetenceInput = { + competenceId?: string; + name?: string; + description?: string; + externalQualificationNeeded?: boolean; + renewTime?: number; + proficiencyLevel?: string; + qualificationDates?: string[]; + lastUsages?: string[]; +}; + +export type ResourceInput = { + resourceId?: string; + competencies: CompetenceInput[]; +}; + +export type ResourceListInput = ResourceInput[]; + +export type MatchingTask = { + taskId: string; // UUIDString + name?: string; // optional + description?: string; // optional but recommended to have content + executionInstructions?: string; // optional, e.g. HTML + requiredCompetencies?: string[] | CompetenceInput[]; // either array of competenceIds or array of CompetenceInput +}; + +export type Match = { + competenceId: string; + resourceId: string; + text: string; + type: string; + distance: number; + reason?: string; +}; + +export interface VectorDBOptions { + filePath?: string; // If undefined or ":memory:", use in-memory; else path to file - Note: memory will not work with workers!! + embeddingDim: number; +} + +export type CompetenceDBOutput = { + competence_id: string; + competence_name: string | null; + competence_description: string | null; + external_qualification_needed: number; // 0 or 1 + renew_time: number | null; + proficiency_level: string | null; + qualification_dates: string | null; // JSON string + last_usages: string | null; // JSON string +}; + +export type EmbeddingTask = { + listId: string; // UUIDString + resourceId: string; // UUIDString + competenceId: string; // UUIDString + text: string; // Text to embed + type: 'name' | 'description' | 'proficiencyLevel'; // Type of text +}; + +export interface Job { + jobId: string; + dbName: string; +} + +export interface ResourceEmbeddingJob extends Job { + mode?: 'resource'; + tasks: EmbeddingTask[]; +} + +export interface TaskEmbeddingJob extends Job { + mode: 'task'; + tasks: MatchingTask[]; +} + +export type EmbeddingJob = ResourceEmbeddingJob | TaskEmbeddingJob; + +export interface MatchingJob extends Job { + listId?: string; // Which List to match against + resourceId?: string; // Optional: If matching against a single resource + tasks: MatchingTask[]; // Tasks to match + taskEmbeddings?: Record; // Optional precomputed task embeddings keyed by taskId +} + +export type ResourceRanking = { + resourceId: string; + taskMatchings: { + taskId: string; // Which of the tasks this matching is referring to + competenceMatchings: { + competenceId: string; + matchings: { + text: string; + type: 'name' | 'description' | 'proficiencyLevel'; + // Sorted DESC by + matchProbability: number; // Normalised inverted distance (, where distance refers to the cosine similarity) + alignment: 'contradicting' | 'neutral' | 'aligning'; // Semantic opposite classification + reason?: string; // Reason for the match + }[]; // Array: Competence-Parts matched to task + // Sorted DESC by either: + avgMatchProbability: number; // Average matchProbability of all parts of this competence + avgBestFitMatchProbability: number; // Average of the parts that align well with the task, 0 means there is none + }[]; // Array: Competences matched to task + // Sorted DESC by either: + maxMatchProbability: number; // Best avgMatchingProbability of all competences for this task + maxBestFitMatchProbability: number; // Best avgBestFitMatchProbability of all competences for this task, 0 means there is none + }[]; // Array: Matching of the resource to each task, respectively + // Sorted DESC first by [not contradicting , contradicting] then by either: + avgTaskMatchProbability: number; // Average maxMatchProbability of all tasks for this resource + avgBestFitTaskMatchProbability: number; // Average maxBestFitMatchProbability of all tasks for this resource, 0 means there is none + + contradicting: boolean; // Whether there is a part in a competence of this resource that contradicts the task +}[]; + +export type TaskOverview = { + taskId: string; // UUIDString + taskText: string; // Text of the task +}[]; + +export type GroupedMatchResults = { + tasks: TaskOverview; + resourceRanking: ResourceRanking; +}; + +export type workerTypes = 'embedder' | 'matcher' | 'inference'; + +export interface WorkerQueue { + job: any; + workerScript: workerTypes; + options?: { + onOnline?: (job: any) => void; + onExit?: (job: any, code: number) => void; + onError?: (job: any, error: Error) => void; + onMessage?: (job: any, message: any) => void; + }; +} + +export interface TransformerPipelineOptions { + task: string; + model: string; + options?: PretrainedModelOptions; +} + +export interface JobQueueItem { + job: EmbeddingJob | MatchingJob; + resolve: (result: any) => void; + reject: (error: Error) => void; + retryCount: number; +} diff --git a/src/competence-matcher/src/utils/worker.ts b/src/competence-matcher/src/utils/worker.ts new file mode 100644 index 000000000..9a9367037 --- /dev/null +++ b/src/competence-matcher/src/utils/worker.ts @@ -0,0 +1,165 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { Worker, parentPort, threadId } from 'worker_threads'; +import VectorDataBase from '../db/db'; +import { getDB } from './db'; +import { config } from '../config'; + +const {} = config; + +export function createWorker(filename: string, workerData?: any): Worker { + const tsPath = path.resolve(__dirname, `../worker/${filename}.ts`); + const jsPath = path.resolve(__dirname, `../worker/${filename}.js`); + const isTs = fs.existsSync(tsPath); + + const workerFile = isTs ? tsPath : jsPath; + + const execArgv = isTs + ? [...process.execArgv, '-r', 'ts-node/register/transpile-only'] + : process.execArgv; + + const options = workerData ? { execArgv, workerData } : { execArgv }; + + const worker = new Worker(workerFile, options); + + return worker; +} + +export async function withJobUpdates( + job: { jobId: string; dbName: string }, + cb: (db: VectorDataBase, payload: T) => Promise, + options?: { + onStart?: () => void; + onDone?: () => void; + onError?: (error: Error) => void; + }, +) { + const db = getDB(job.dbName); + try { + if (options && options.onStart) { + options.onStart(); + } else { + db.updateJobStatus(job.jobId, 'running'); + parentPort!.postMessage({ type: 'status', jobId: job.jobId, status: 'running' }); + } + + await cb(db, job as any as T); + + if (options && options.onDone) { + options.onDone(); + } else { + db.updateJobStatus(job.jobId, 'completed'); + parentPort!.postMessage({ type: 'status', jobId: job.jobId, status: 'completed' }); + } + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + + if (options && options.onError) { + options.onError(error); + } else { + // Update job status in database + db.updateJobStatus(job.jobId, 'failed'); + + // Send error message to parent thread + parentPort!.postMessage({ + type: 'error', + jobId: job.jobId, + error: error.message, + }); + } + + // Always re-throw the error so the worker can handle it appropriately + throw error; + } finally { + // Don't close the database connection - let DBManager handle connection lifecycle + // Don't close parentPort or exit process for static worker pools + // Workers need to stay alive to process more jobs + } +} /** + * Utility to log from within worker threads + * This should be used in worker files to maintain context + */ +export function workerLogger( + type: string, + level: 'debug' | 'info' | 'warn' | 'error', + message: string, + data?: any, + error?: Error, +): void { + // In worker threads, we'll post messages back to main thread for logging + // This ensures all logs go through the central logger + const logData = { + type: 'log', + level, + logType: type, // Add type field for consistency with main logger + message, + data, + error: error + ? { + message: error.message, + stack: error.stack, + name: error.name, + } + : undefined, + timestamp: new Date().toISOString(), + threadId: process.env.WORKER_THREAD_ID || 'unknown', + }; + + // For Node.js worker threads, we use parentPort + if (parentPort) { + parentPort.postMessage(logData); + } +} + +/** + * Start sending heartbeat messages to the main thread + * This should be called once when a worker starts up + * + * @param workerType - Type of worker (e.g., 'embedder', 'matcher') + * @param intervalMs - Heartbeat interval in milliseconds (default: 20000ms = 20s) + * @returns Function to stop the heartbeat + */ +export function startHeartbeat(workerType: string, intervalMs: number = 20000): () => void { + if (!parentPort) { + throw new Error('startHeartbeat can only be called from worker threads'); + } + + const sendHeartbeat = () => { + workerLogger('system', 'debug', `${workerType} worker sending heartbeat`, { + workerType, + threadId, + }); + + parentPort!.postMessage({ + type: 'heartbeat', + workerType, + threadId, + timestamp: Date.now(), + }); + }; + + // Send initial heartbeat immediately + sendHeartbeat(); + + // Set up interval for regular heartbeats + const heartbeatInterval = setInterval(sendHeartbeat, intervalMs); + + // Return cleanup function + return () => { + clearInterval(heartbeatInterval); + }; +} + +export function sendHeartbeat(workerType: string): void { + if (!parentPort) { + throw new Error('sendHeartbeat can only be called from worker threads'); + } + + parentPort.postMessage({ + type: 'heartbeat', + workerType, + threadId, + timestamp: Date.now(), + source: 'manual', + }); +} diff --git a/src/competence-matcher/src/worker/inference.ts b/src/competence-matcher/src/worker/inference.ts new file mode 100644 index 000000000..8e67f290d --- /dev/null +++ b/src/competence-matcher/src/worker/inference.ts @@ -0,0 +1,274 @@ +import { parentPort, threadId } from 'worker_threads'; +import Embedding from '../tasks/embedding'; +import ZeroShot from '../tasks/semantic-zeroshot'; +import { withJobUpdates, workerLogger, startHeartbeat, sendHeartbeat } from '../utils/worker'; +import { + EmbeddingJob, + ResourceEmbeddingJob, + TaskEmbeddingJob, + MatchingJob, + Match, +} from '../utils/types'; +import { getDB } from '../utils/db'; +import { config } from '../config'; + +if (!parentPort) throw new Error('This file must be run as a Worker thread'); + +let embeddingInitialised = false; +let matchingInitialised = false; + +async function ensureEmbeddingInitialised() { + if (embeddingInitialised) return; + await Embedding.getInstance(); + embeddingInitialised = true; + workerLogger('system', 'info', 'Inference worker embedding pipeline ready', { threadId }); +} +async function ensureMatchingInitialised() { + if (matchingInitialised) return; + await ZeroShot.getInstance(); + matchingInitialised = true; + workerLogger('system', 'info', 'Inference worker matching pipeline ready', { threadId }); +} + +startHeartbeat('inference', config.workerHeartbeatInterval); + +parentPort.on('message', async (raw: any) => { + const possible = raw as EmbeddingJob & MatchingJob; + const tasks: any[] | undefined = (possible as any).tasks; + const mode = (possible as any).mode; + // Classification logic: explicit mode or task shape for embedding + const isEmbeddingJob = + mode === 'task' || + mode === 'resource' || + (Array.isArray(tasks) && tasks.length > 0 && 'competenceId' in tasks[0] && 'text' in tasks[0]); + const jobId = possible.jobId; + workerLogger(jobId || 'system', 'debug', 'Inference worker received job', { + threadId, + jobId, + inferredMode: mode || null, + kind: isEmbeddingJob ? 'embedding' : 'matching', + taskCount: tasks?.length || 0, + }); + try { + if (isEmbeddingJob) await ensureEmbeddingInitialised(); + else await ensureMatchingInitialised(); + } catch (err) { + workerLogger( + jobId || 'system', + 'error', + 'Inference worker failed model init', + { threadId, kind: isEmbeddingJob ? 'embedding' : 'matching' }, + err instanceof Error ? err : new Error(String(err)), + ); + parentPort!.postMessage({ + type: 'error', + jobId, + error: `Model initialisation failed: ${err instanceof Error ? err.message : String(err)}`, + }); + parentPort!.postMessage({ type: 'job_completed', jobId }); + return; + } + if (isEmbeddingJob) { + await handleEmbeddingJob(raw as EmbeddingJob); + } else { + await handleMatchingJob(raw as MatchingJob); + } +}); + +async function handleEmbeddingJob(job: EmbeddingJob) { + try { + if ((job as TaskEmbeddingJob).mode === 'task') + await processTaskEmbeddingJob(job as TaskEmbeddingJob); + else await processResourceEmbeddingJob(job as ResourceEmbeddingJob); + workerLogger(job.jobId, 'info', 'Embedding job completed', { threadId }); + } catch (error) { + workerLogger( + job.jobId, + 'error', + 'Embedding job failed', + { threadId }, + error instanceof Error ? error : new Error(String(error)), + ); + } finally { + parentPort!.postMessage({ type: 'job_completed', jobId: job.jobId }); + } +} + +async function processResourceEmbeddingJob(job: ResourceEmbeddingJob): Promise { + workerLogger(job.jobId, 'debug', `Resource embedding job start (${job.tasks.length} tasks)`, { + threadId, + }); + await withJobUpdates(job, async (db, { tasks, jobId }) => { + for (const { listId, resourceId, competenceId, text, type } of tasks) { + sendHeartbeat('inference'); + try { + const vectors = await Embedding.embed(text); + const [vector] = vectors; + sendHeartbeat('inference'); + db.upsertEmbedding({ listId, resourceId, competenceId, text, type, embedding: vector }); + } catch (err) { + workerLogger( + jobId, + 'error', + 'Failed embedding task', + { threadId, competenceId, type }, + err instanceof Error ? err : new Error(String(err)), + ); + parentPort!.postMessage({ + type: 'error', + jobId, + error: err instanceof Error ? err.message : String(err), + }); + } + } + }); +} + +async function processTaskEmbeddingJob(job: TaskEmbeddingJob): Promise { + workerLogger(job.jobId, 'debug', `Task embedding job start (${job.tasks.length} tasks)`, { + threadId, + }); + const dbInstance = getDB(job.dbName); + await withJobUpdates( + job, + async (db, { tasks, jobId }) => { + for (const task of tasks) { + const { taskId, description } = task; + if (!description) continue; + sendHeartbeat('inference'); + try { + const vectors = await Embedding.embed(description); + const [vector] = vectors; + db.upsertTaskEmbedding(jobId, taskId, vector); + sendHeartbeat('inference'); + } catch (err) { + workerLogger( + jobId, + 'error', + 'Failed task embedding', + { threadId, taskId }, + err instanceof Error ? err : new Error(String(err)), + ); + parentPort!.postMessage({ + type: 'error', + jobId, + error: err instanceof Error ? err.message : String(err), + }); + } + } + }, + { + onStart: () => { + dbInstance.updateJobStatus(job.jobId, 'preprocessing'); + parentPort!.postMessage({ type: 'status', jobId: job.jobId, status: 'preprocessing' }); + }, + onDone: () => { + dbInstance.updateJobStatus(job.jobId, 'pending'); + parentPort!.postMessage({ type: 'status', jobId: job.jobId, status: 'pending' }); + }, + }, + ); +} + +async function handleMatchingJob(job: MatchingJob) { + workerLogger(job.jobId, 'debug', `Matching job start (${job.tasks.length} tasks)`, { threadId }); + try { + const matchResults: Record = {}; + for (const t of job.tasks) if (t.description) matchResults[t.description] = []; + await withJobUpdates( + job, + async (db, { jobId, tasks, listId: listIdFilter, resourceId: resourceIdFilter }) => { + for (const task of tasks) { + const { taskId, description } = task; + if (!description) continue; + sendHeartbeat('inference'); + try { + const vector = job.taskEmbeddings?.[taskId] ?? db.getTaskEmbedding(jobId, taskId); + if (!vector) throw new Error(`No embedding stored for task ${taskId}`); + if (vector.length !== config.embeddingDim) + throw new Error(`Embedding length mismatch for task ${taskId}`); + sendHeartbeat('inference'); + const matches: Match[] = db.searchEmbedding(vector, { + filter: { listId: listIdFilter, resourceId: resourceIdFilter }, + }); + for (const match of matches) { + let flag = 'neutral'; + let newDistance = Math.min( + 1, + Math.max(0, match.distance - config.matchDistanceOffset) * + config.matchDistanceMultiplier, + ); + const sentiment = await ZeroShot.nliBiDirectional(description, match.text); + sendHeartbeat('inference'); + const contradiction = await ZeroShot.contradictionCheck(description, match.text); + const alignment = await ZeroShot.alignmentCheck(description, match.text); + sendHeartbeat('inference'); + if ( + sentiment.ranking[0] === 'contradict' || + sentiment.contradict > config.contradictionThreshold || + contradiction.contradicting + ) { + flag = 'contradicting'; + newDistance = 0.0; + } else if ( + sentiment.entail > config.entailmentThreshold && + match.distance > (config as any).alignmentDistanceThreshold && + alignment.aligning + ) { + newDistance = Math.min(1, newDistance * (config as any).alignmentBoostMultiplier); + flag = 'aligning'; + } else { + newDistance *= (config as any).neutralReductionMultiplier || 1; + } + matchResults[description].push({ + jobId, + taskId, + taskText: description, + competenceId: match.competenceId, + resourceId: match.resourceId, + text: match.text, + type: match.type as 'name' | 'description' | 'proficiencyLevel', + alignment: flag, + distance: newDistance, + reason: match.reason, + }); + } + } catch (err) { + workerLogger( + jobId, + 'error', + `Failed to process task ${taskId}`, + { threadId, taskId }, + err instanceof Error ? err : new Error(String(err)), + ); + parentPort!.postMessage({ + type: 'error', + jobId, + error: err instanceof Error ? err.message : String(err), + }); + } + } + }, + { + onDone: () => + parentPort!.postMessage({ type: 'job', job: 'reason', workload: matchResults }), + }, + ); + workerLogger(job.jobId, 'info', 'Matching job completed', { threadId }); + } catch (err) { + workerLogger( + job.jobId, + 'error', + 'Matching job failed', + { threadId }, + err instanceof Error ? err : new Error(String(err)), + ); + } finally { + try { + getDB(job.dbName).deleteTaskEmbeddings(job.jobId); + } catch {} + parentPort!.postMessage({ type: 'job_completed', jobId: job.jobId }); + } +} + +workerLogger('system', 'debug', 'Inference worker ready', { threadId }); diff --git a/src/competence-matcher/src/worker/multiworker/embedder.ts b/src/competence-matcher/src/worker/multiworker/embedder.ts new file mode 100644 index 000000000..cf9e11e2e --- /dev/null +++ b/src/competence-matcher/src/worker/multiworker/embedder.ts @@ -0,0 +1,172 @@ +// Backup original embedder worker (multi-worker mode) +import { parentPort, threadId } from 'worker_threads'; +import Embedding from '../../tasks/embedding'; +import { withJobUpdates, workerLogger, startHeartbeat, sendHeartbeat } from '../../utils/worker'; +import { EmbeddingJob, ResourceEmbeddingJob, TaskEmbeddingJob } from '../../utils/types'; +import { getDB } from '../../utils/db'; +import { config } from '../../config'; +import { withOnnxLock } from '../../utils/onnx-lock'; + +if (!parentPort) throw new Error('This file must be run as a Worker thread'); + +let modelsInitialised = false; +async function ensureModelsInitialised() { + if (modelsInitialised) return; + await withOnnxLock(() => Embedding.getInstance()); + modelsInitialised = true; + workerLogger('system', 'debug', 'Embedder worker online', { threadId }); +} + +startHeartbeat('embedder', config.workerHeartbeatInterval); + +parentPort.on('message', async (message: any) => { + const job = message as EmbeddingJob; + workerLogger(job.jobId || 'system', 'debug', 'Embedder worker received job', { + threadId, + jobId: job.jobId, + taskCount: job.tasks?.length || 0, + mode: job.mode ?? 'resource', + }); + + try { + await ensureModelsInitialised(); + } catch (err) { + workerLogger( + job.jobId || 'system', + 'debug', + 'Embedder worker failed to initialize models', + { threadId, jobId: job.jobId }, + err instanceof Error ? err : new Error(String(err)), + ); + parentPort!.postMessage({ + type: 'error', + jobId: job.jobId, + error: `Model initialisation failed: ${err instanceof Error ? err.message : String(err)}`, + }); + parentPort!.postMessage({ type: 'job_completed', jobId: job.jobId }); + return; + } + + try { + if (job.mode === 'task') { + await processTaskEmbeddingJob(job as TaskEmbeddingJob); + } else { + await processResourceEmbeddingJob(job as ResourceEmbeddingJob); + } + workerLogger(job.jobId, 'debug', 'Embedding job completed', { + threadId, + taskCount: job.tasks.length, + mode: job.mode ?? 'resource', + }); + } catch (error) { + workerLogger( + job.jobId, + 'debug', + 'Embedding job failed', + { threadId, mode: job.mode ?? 'resource' }, + error instanceof Error ? error : new Error(String(error)), + ); + } finally { + parentPort!.postMessage({ type: 'job_completed', jobId: job.jobId }); + } +}); + +workerLogger('system', 'debug', 'Embedder worker thread ready', { threadId }); + +async function processResourceEmbeddingJob(job: ResourceEmbeddingJob): Promise { + workerLogger( + job.jobId, + 'debug', + `Starting resource embedding job with ${job.tasks.length} tasks`, + { threadId, taskCount: job.tasks.length }, + ); + await withJobUpdates(job, async (db, { tasks, jobId }) => { + for (const { listId, resourceId, competenceId, text, type } of tasks) { + sendHeartbeat('embedder'); + try { + const vectors = await withOnnxLock(() => Embedding.embed(text)); + const [vector] = vectors; + sendHeartbeat('embedder'); + workerLogger(jobId, 'debug', `Generated embedding for ${type} text`, { + threadId, + competenceId, + textLength: text.length, + }); + db.upsertEmbedding({ listId, resourceId, competenceId, text, type, embedding: vector }); + sendHeartbeat('embedder'); + } catch (error) { + workerLogger( + jobId, + 'error', + 'Failed to process embedding task', + { threadId, competenceId, type }, + error instanceof Error ? error : new Error(String(error)), + ); + parentPort!.postMessage({ + type: 'error', + jobId, + error: `Failed to process embedding task: ${error instanceof Error ? error.message : String(error)}`, + }); + } + } + }); +} + +async function processTaskEmbeddingJob(job: TaskEmbeddingJob): Promise { + workerLogger(job.jobId, 'debug', `Starting task embedding job with ${job.tasks.length} tasks`, { + threadId, + taskCount: job.tasks.length, + }); + const dbInstance = getDB(job.dbName); + await withJobUpdates( + job, + async (db, { tasks, jobId }) => { + for (const task of tasks) { + const { taskId, description } = task; + if (!description) { + workerLogger(jobId, 'warn', 'Skipping task without description for embedding', { + threadId, + taskId, + }); + continue; + } + sendHeartbeat('embedder'); + try { + const vectors = await withOnnxLock(() => Embedding.embed(description)); + const [vector] = vectors; + sendHeartbeat('embedder'); + workerLogger(jobId, 'debug', 'Generated task embedding', { + threadId, + taskId, + textLength: description.length, + }); + db.upsertTaskEmbedding(jobId, taskId, vector); + sendHeartbeat('embedder'); + } catch (error) { + workerLogger( + jobId, + 'error', + 'Failed to generate task embedding', + { threadId, taskId }, + error instanceof Error ? error : new Error(String(error)), + ); + parentPort!.postMessage({ + type: 'error', + jobId, + error: `Failed to process task ${taskId}: ${error instanceof Error ? error.message : String(error)}`, + }); + } + } + }, + { + onStart: () => { + dbInstance.updateJobStatus(job.jobId, 'preprocessing'); + parentPort!.postMessage({ type: 'status', jobId: job.jobId, status: 'preprocessing' }); + }, + onDone: () => { + dbInstance.updateJobStatus(job.jobId, 'pending'); + parentPort!.postMessage({ type: 'status', jobId: job.jobId, status: 'pending' }); + }, + }, + ); +} diff --git a/src/competence-matcher/src/worker/multiworker/matcher.ts b/src/competence-matcher/src/worker/multiworker/matcher.ts new file mode 100644 index 000000000..e67b0f0e0 --- /dev/null +++ b/src/competence-matcher/src/worker/multiworker/matcher.ts @@ -0,0 +1,172 @@ +// Backup original matcher worker (multi-worker mode) +import { parentPort, threadId } from 'worker_threads'; +import { withJobUpdates, workerLogger, startHeartbeat, sendHeartbeat } from '../../utils/worker'; +import { addReason } from '../../tasks/reason'; +import { Match, MatchingJob } from '../../utils/types'; +import ZeroShot from '../../tasks/semantic-zeroshot'; +import { config } from '../../config'; +import { getDB } from '../../utils/db'; +import { withOnnxLock } from '../../utils/onnx-lock'; + +if (!parentPort) throw new Error('This file must be run as a Worker thread'); + +let modelsInitialised = false; +async function ensureModelsInitialised() { + if (modelsInitialised) return; + await withOnnxLock(() => ZeroShot.getInstance()); + modelsInitialised = true; + workerLogger('system', 'debug', 'Matcher worker online', { threadId }); +} + +startHeartbeat('matcher', config.workerHeartbeatInterval); + +parentPort.on('message', async (message: any) => { + const job = message as MatchingJob; + workerLogger(job.jobId || 'system', 'debug', 'Matcher worker received job', { + threadId, + jobId: job.jobId, + taskCount: job.tasks?.length || 0, + listId: job.listId, + }); + try { + await ensureModelsInitialised(); + } catch (err) { + workerLogger( + job.jobId || 'system', + 'debug', + 'Matcher worker failed to initialize models', + { threadId, jobId: job.jobId }, + err instanceof Error ? err : new Error(String(err)), + ); + parentPort!.postMessage({ + type: 'error', + jobId: job.jobId, + error: `Model initialisation failed: ${err instanceof Error ? err.message : String(err)}`, + }); + parentPort!.postMessage({ type: 'job_completed', jobId: job.jobId }); + return; + } + workerLogger(job.jobId, 'debug', `Starting matching job with ${job.tasks.length} tasks`, { + threadId, + taskCount: job.tasks.length, + }); + try { + const matchResults: { [description: string]: any[] } = {}; + for (const task of job.tasks) if (task.description) matchResults[task.description] = []; + await withJobUpdates( + job, + async (db, { jobId, tasks, listId: listIdFilter, resourceId: resourceIdFilter }) => { + for (const task of tasks) { + const { taskId, description } = task; + if (!description) continue; + sendHeartbeat('matcher'); + try { + const vector = job.taskEmbeddings?.[taskId] ?? db.getTaskEmbedding(jobId, taskId); + if (!vector) throw new Error(`No embedding stored for task ${taskId}`); + if (vector.length !== config.embeddingDim) + throw new Error( + `Embedding length mismatch for task ${taskId}: expected ${config.embeddingDim}, received ${vector.length}`, + ); + sendHeartbeat('matcher'); + const matches: Match[] = db.searchEmbedding(vector, { + filter: { listId: listIdFilter, resourceId: resourceIdFilter }, + }); + for (const match of matches) { + let flag = 'neutral'; + let newDistance = Math.min( + 1, + Math.max(0, match.distance - config.matchDistanceOffset) * + config.matchDistanceMultiplier, + ); + const sentiment = await withOnnxLock(() => + ZeroShot.nliBiDirectional(description, match.text), + ); + sendHeartbeat('matcher'); + const contradiction = await withOnnxLock(() => + ZeroShot.contradictionCheck(description, match.text), + ); + const alignment = await withOnnxLock(() => + ZeroShot.alignmentCheck(description, match.text), + ); + sendHeartbeat('matcher'); + if ( + sentiment.ranking[0] === 'contradict' || + sentiment.contradict > config.contradictionThreshold || + contradiction.contradicting + ) { + flag = 'contradicting'; + newDistance = 0.0; + } else if ( + sentiment.entail > config.entailmentThreshold && + match.distance > config.alignmentDistanceThreshold && + alignment.aligning + ) { + flag = 'aligning'; + newDistance = Math.min(1, newDistance * config.alignmentBoostMultiplier); + } else { + flag = 'neutral'; + newDistance *= config.neutralReductionMultiplier; + } + matchResults[description].push({ + jobId, + taskId, + taskText: description, + competenceId: match.competenceId, + resourceId: match.resourceId, + text: match.text, + type: match.type as 'name' | 'description' | 'proficiencyLevel', + alignment: flag, + distance: newDistance, + reason: match.reason, + }); + } + } catch (error) { + workerLogger( + jobId, + 'error', + `Failed to process task ${taskId}`, + { threadId, taskId }, + error instanceof Error ? error : new Error(String(error)), + ); + parentPort!.postMessage({ + type: 'error', + jobId, + error: `Failed to process task ${taskId}: ${error instanceof Error ? error.message : String(error)}`, + }); + } + } + }, + { + onDone: () => + parentPort!.postMessage({ type: 'job', job: 'reason', workload: matchResults }), + }, + ); + workerLogger(job.jobId, 'debug', 'Matching job completed', { + threadId, + taskCount: job.tasks.length, + }); + } catch (error) { + workerLogger( + job.jobId, + 'debug', + 'Matching job failed', + { threadId }, + error instanceof Error ? error : new Error(String(error)), + ); + } finally { + try { + getDB(job.dbName).deleteTaskEmbeddings(job.jobId); + } catch (error) { + workerLogger( + job.jobId, + 'warn', + 'Failed to clean up task embeddings after matching job', + { threadId }, + error instanceof Error ? error : new Error(String(error)), + ); + } + parentPort!.postMessage({ type: 'job_completed', jobId: job.jobId }); + } +}); + +workerLogger('system', 'debug', 'Matcher worker thread ready', { threadId }); diff --git a/src/competence-matcher/src/worker/multiworker/worker-manager.ts b/src/competence-matcher/src/worker/multiworker/worker-manager.ts new file mode 100644 index 000000000..6a1d87b87 --- /dev/null +++ b/src/competence-matcher/src/worker/multiworker/worker-manager.ts @@ -0,0 +1,235 @@ +// Backup original multi-worker manager +import { Worker } from 'worker_threads'; +import { config } from '../../config'; +import { createWorker } from '../../utils/worker'; +import { EmbeddingJob, JobQueueItem, MatchingJob, workerTypes } from '../../utils/types'; +import { WorkerError } from '../../utils/errors'; +import { getLogger } from '../../utils/logger'; +import { addReason } from '../../tasks/reason'; +import { getDB } from '../../utils/db'; + +const { embeddingWorkers, matchingWorkers, workerDeathTimeout, jobMaxRetries } = config; +const logger = getLogger(); + +class WorkerPool { + private workers: Worker[] = []; + private availableWorkers: Set = new Set(); + private busyWorkers: Map = new Map(); + private activeJobs: Map = new Map(); + private jobQueue: JobQueueItem[] = []; + private workerDeathTimers: Map = new Map(); + private workersBeingReplaced: Set = new Set(); + private readonly workerType: workerTypes; + private readonly poolSize: number; + private readonly maxJobRetries: number; + private readonly workerInitData?: Record; + constructor(workerType: workerTypes, poolSize: number, workerInitData?: Record) { + this.workerType = workerType; + this.poolSize = poolSize; + this.maxJobRetries = Math.max(0, jobMaxRetries); + this.workerInitData = workerInitData; + this.initialiseWorkers(); + } + private initialiseWorkers(): void { + for (let i = 0; i < this.poolSize; i++) this.createWorker(); + } + private createWorker(): void { + const worker = createWorker(`multiworker/${this.workerType}`, this.workerInitData); + this.workers.push(worker); + const deathTimer = setTimeout(() => this.killWorker(worker), workerDeathTimeout); + this.workerDeathTimers.set(worker, deathTimer); + worker.on('message', (m: any) => { + if (m.type === 'heartbeat') this.handleHeartbeat(worker); + else if (m.type === 'log') this.handleWorkerLog(m); + }); + worker.on('error', (e) => this.handleWorkerFailure(worker, e)); + worker.once('exit', () => this.handleWorkerFailure(worker, new Error('Worker exited'))); + } + private handleHeartbeat(worker: Worker): void { + const existing = this.workerDeathTimers.get(worker); + if (existing) clearTimeout(existing); + const timer = setTimeout(() => this.killWorker(worker), workerDeathTimeout); + this.workerDeathTimers.set(worker, timer); + if (!this.busyWorkers.has(worker)) { + this.availableWorkers.add(worker); + this.processNextJob(); + } + } + private handleWorkerLog(message: any): void { + const { level, logType, message: logMessage, data, error } = message; + if (error) { + const reconstructed = new Error(error.message); + reconstructed.stack = error.stack; + reconstructed.name = error.name; + getLogger()[level as 'debug' | 'info' | 'warn' | 'error']( + logType, + logMessage, + reconstructed, + data, + ); + } else { + getLogger()[level as 'debug' | 'info' | 'warn' | 'error'](logType, logMessage, data); + } + } + private killWorker(worker: Worker): void { + if (this.workersBeingReplaced.has(worker.threadId)) return; + this.workersBeingReplaced.add(worker.threadId); + const activeJob = this.activeJobs.get(worker); + if (activeJob) { + this.activeJobs.delete(worker); + this.busyWorkers.delete(worker); + this.handleJobFailure(activeJob, new Error('Worker heartbeat timeout')); + } + this.removeWorker(worker); + this.createWorker(); + this.workersBeingReplaced.delete(worker.threadId); + this.processNextJob(); + } + private handleWorkerFailure(worker: Worker, error: Error): void { + if (this.workersBeingReplaced.has(worker.threadId)) return; + this.workersBeingReplaced.add(worker.threadId); + const activeJob = this.activeJobs.get(worker); + if (activeJob) { + this.activeJobs.delete(worker); + this.busyWorkers.delete(worker); + this.handleJobFailure(activeJob, error); + } + this.removeWorker(worker); + this.createWorker(); + this.workersBeingReplaced.delete(worker.threadId); + if (activeJob) this.processNextJob(); + } + private removeWorker(worker: Worker): void { + this.availableWorkers.delete(worker); + this.busyWorkers.delete(worker); + this.activeJobs.delete(worker); + const timer = this.workerDeathTimers.get(worker); + if (timer) { + clearTimeout(timer); + this.workerDeathTimers.delete(worker); + } + const index = this.workers.indexOf(worker); + if (index > -1) this.workers.splice(index, 1); + worker.removeAllListeners(); + worker.terminate(); + this.workersBeingReplaced.delete(worker.threadId); + } + public async executeJob(job: EmbeddingJob | MatchingJob): Promise { + return new Promise((resolve, reject) => { + this.jobQueue.push({ job, resolve, reject, retryCount: 0 }); + this.processNextJob(); + }); + } + private processing = false; + private processNextJob(): void { + if (this.processing) return; + if (this.jobQueue.length === 0 || this.availableWorkers.size === 0) return; + this.processing = true; + const worker = this.availableWorkers.values().next().value as Worker; + const item = this.jobQueue.shift()!; + this.assignJobToWorker(worker, item); + this.processing = false; + } + private assignJobToWorker(worker: Worker, queueItem: JobQueueItem): void { + const { job, resolve, reject, retryCount } = queueItem; + this.availableWorkers.delete(worker); + this.busyWorkers.set(worker, job.jobId); + this.activeJobs.set(worker, queueItem); + const handler = (message: any) => { + if (message.type === 'job_completed' && message.jobId === job.jobId) { + worker.removeListener('message', handler); + this.activeJobs.delete(worker); + this.markWorkerAvailable(worker); + resolve(message.result || 'Job completed'); + this.processNextJob(); + } else if (message.type === 'error' && message.jobId === job.jobId) { + worker.removeListener('message', handler); + this.activeJobs.delete(worker); + this.markWorkerAvailable(worker); + this.handleJobFailure(queueItem, new Error(message.error)); + this.processNextJob(); + } else if (message.type === 'job' && message.job === 'reason') { + this.handleReasoning(job, message); + } + }; + worker.on('message', handler); + worker.postMessage(job); + } + private markWorkerAvailable(worker: Worker): void { + this.busyWorkers.delete(worker); + this.availableWorkers.add(worker); + } + private handleJobFailure(queueItem: JobQueueItem, error: Error): void { + const { job, resolve, reject, retryCount } = queueItem; + if (retryCount < this.maxJobRetries) { + this.jobQueue.unshift({ job, resolve, reject, retryCount: retryCount + 1 }); + } else { + reject(new WorkerError(this.workerType, job.jobId, error)); + } + } + private async handleReasoning(job: any, message: any): Promise { + const finalMatches = []; + for (const [task, matches] of Object.entries(message.workload)) { + try { + const taskMatches = await addReason(matches as any[], task); + finalMatches.push(...taskMatches); + } catch { + finalMatches.push(...(matches as any[])); + } + } + const db = getDB(job.dbName); + for (const match of finalMatches) { + try { + db.addMatchResult({ + jobId: match.jobId, + taskId: match.taskId, + taskText: match.taskText, + competenceId: match.competenceId, + resourceId: match.resourceId, + distance: match.distance, + text: match.text, + type: match.type, + alignment: match.alignment, + reason: match.reason, + }); + } catch {} + } + try { + db.updateJobStatus(job.jobId, 'completed'); + } catch {} + } + public async shutdown(): Promise { + this.jobQueue.forEach((i) => i.reject(new Error('Shutting down'))); + this.activeJobs.forEach((i) => i.reject(new Error('Shutting down'))); + this.workerDeathTimers.forEach((t) => clearTimeout(t)); + this.workers.forEach((w) => w.terminate()); + this.jobQueue.length = 0; + this.activeJobs.clear(); + this.workerDeathTimers.clear(); + this.workers.length = 0; + this.availableWorkers.clear(); + this.busyWorkers.clear(); + } +} +class WorkerManager { + private embeddingPool: WorkerPool; + private matchingPool: WorkerPool; + constructor() { + const onnxLockBuffer = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT); + const view = new Int32Array(onnxLockBuffer); + Atomics.store(view, 0, 0); + const sharedWorkerData = { onnxLock: onnxLockBuffer } as Record; + this.embeddingPool = new WorkerPool('embedder', embeddingWorkers, sharedWorkerData); + this.matchingPool = new WorkerPool('matcher', matchingWorkers, sharedWorkerData); + } + public async enqueue(job: EmbeddingJob | MatchingJob, workerType: workerTypes): Promise { + if (workerType === 'embedder') return this.embeddingPool.executeJob(job); + if (workerType === 'matcher') return this.matchingPool.executeJob(job); + throw new Error(`Unknown worker type: ${workerType}`); + } + public async shutdown(): Promise { + await Promise.all([this.embeddingPool.shutdown(), this.matchingPool.shutdown()]); + } +} +const manager = new WorkerManager(); +export default manager; diff --git a/src/competence-matcher/src/worker/worker-manager.ts b/src/competence-matcher/src/worker/worker-manager.ts new file mode 100644 index 000000000..9bf508639 --- /dev/null +++ b/src/competence-matcher/src/worker/worker-manager.ts @@ -0,0 +1,349 @@ +import { Worker } from 'worker_threads'; +import { config } from '../config'; +import { createWorker } from '../utils/worker'; +import { EmbeddingJob, MatchingJob, JobQueueItem, workerTypes } from '../utils/types'; +import { WorkerError } from '../utils/errors'; +import { getLogger } from '../utils/logger'; +import { addReason } from '../tasks/reason'; +import { getDB } from '../utils/db'; + +const { workerHeartbeatInterval, workerDeathTimeout, jobMaxRetries } = config; +const logger = getLogger(); + +// Recommended heartbeat strategy values (can be tuned via env): +// heartbeat interval: workerHeartbeatInterval (e.g. 30-60s) +// death timeout: workerDeathTimeout (e.g. 240s) => ~4-8x interval + +interface SingleJobItem extends JobQueueItem { + startedAt?: number; + queueDepthAtStart?: number; + meta?: { + kind: 'embedding' | 'matching'; + subtaskErrors: number; + reasonMatches: number; + reasonedCount: number; + }; +} + +/** + * SingleWorkerManager - streamlined mono-worker queue for inference. + * Keeps public API: enqueue(job, type) + shutdown(). All computation lives in one worker. + */ +class WorkerManager { + private worker: Worker | null = null; + private queue: SingleJobItem[] = []; + private current: SingleJobItem | null = null; + private lastHeartbeat = Date.now(); + private heartbeatMonitor: NodeJS.Timeout | null = null; + private shuttingDown = false; + + constructor() { + logger.info('worker', 'Initialising SingleWorkerManager (inference)', { + workerHeartbeatInterval, + workerDeathTimeout, + jobMaxRetries, + }); + this.spawnWorker(); + this.startHeartbeatMonitor(); + } + + private spawnWorker(): void { + this.worker = createWorker('inference'); + const threadId = this.worker.threadId; + logger.info('worker', 'Spawned inference worker', { threadId }); + + this.worker.on('message', (msg: any) => this.handleMessage(msg)); + this.worker.on('error', (err) => this.handleWorkerCrash(err)); + this.worker.on('exit', (code) => { + this.handleWorkerCrash(new Error(`Worker exited with code ${code}`)); + }); + } + + private startHeartbeatMonitor(): void { + if (this.heartbeatMonitor) clearInterval(this.heartbeatMonitor); + this.heartbeatMonitor = setInterval( + () => { + if (!this.worker) return; + const silenceMs = Date.now() - this.lastHeartbeat; + if (silenceMs > workerDeathTimeout) { + const activeJobId = this.current?.job.jobId; + logger.warn('worker', 'Heartbeat timeout – restarting worker', { + silenceMs, + workerDeathTimeout, + activeJobId, + queueLength: this.queue.length, + retryCount: this.current?.retryCount, + }); + this.respawnWorker(true); + } + }, + Math.max(10_000, workerHeartbeatInterval / 2), + ); + } + + private respawnWorker(dueToTimeoutOrCrash = false): void { + if (this.worker) { + this.worker.removeAllListeners(); + try { + this.worker.terminate(); + } catch {} + } + this.worker = null; + // Requeue current job if any (unless shutting down) + if (dueToTimeoutOrCrash && this.current && !this.shuttingDown) { + const item = this.current; + this.current = null; + this.retryOrFail(item, new Error('Worker unresponsive / crashed')); + } else { + this.current = null; + } + this.spawnWorker(); + // Continue processing remaining queue + this.processNext(); + } + + private handleMessage(msg: any): void { + // Any message counts as liveness indicator + this.lastHeartbeat = Date.now(); + if (msg.type === 'heartbeat') { + logger.debug('worker', 'Heartbeat received', { threadId: this.worker?.threadId }); + return; // already updated timestamp + } + if (msg.type === 'log') { + // Forward log (avoid duplicating error stack later) + const { level, logType, message, data, error } = msg; + const logFn = + typeof level === 'string' && ['debug', 'info', 'warn', 'error'].includes(level) + ? (logger as any)[level].bind(logger) + : (logger as any).info.bind(logger); + if (error) { + const e = new Error(error.message); + e.stack = error.stack; + e.name = error.name; + logFn(logType, message, e, data); + } else { + logFn(logType, message, data); + } + return; + } + if (!this.current) return; // Ignore job-specific messages if no active job + const jobId = this.current.job.jobId; + + if (msg.type === 'error' && msg.jobId === jobId) { + // Worker reported a job-level error + logger.info('worker', 'Job error received', { jobId, message: msg.error }); + this.retryOrFail(this.current, new Error(msg.error)); + return; + } + if (msg.type === 'job' && msg.job === 'reason') { + this.handleReasoning(this.current.job, msg.workload).catch((err) => { + logger.error('worker', 'Reasoning handling failed', err, { jobId }); + }); + return; + } + if (msg.type === 'job_completed' && msg.jobId === jobId) { + const finished = this.current; + this.current = null; + const durationMs = Date.now() - (finished.startedAt || Date.now()); + logger.info('worker', 'Job completed', { + jobId, + durationMs, + retries: finished.retryCount, + }); + const summary = this.buildSummary(finished, durationMs); + finished.resolve(summary); + this.processNext(); + return; + } + } + + private buildSummary(item: SingleJobItem, durationMs: number) { + const kind = item.meta?.kind; + return { + jobId: item.job.jobId, + kind, + retries: item.retryCount, + durationMs, + queueDepthAtStart: item.queueDepthAtStart, + workerThreadId: this.worker?.threadId, + matchStats: + kind === 'matching' + ? { + reasonedCount: item.meta?.reasonedCount || 0, + totalMatchesSeen: item.meta?.reasonMatches || 0, + } + : undefined, + embeddingStats: + kind === 'embedding' ? { tasks: (item.job as EmbeddingJob).tasks.length } : undefined, + subtaskErrors: item.meta?.subtaskErrors || 0, + completedAt: new Date().toISOString(), + }; + } + + private processNext(): void { + if (this.current || this.shuttingDown) return; + if (!this.worker) return; // will resume after respawn + const next = this.queue.shift(); + if (!next) return; + this.current = next; + next.startedAt = Date.now(); + next.meta = next.meta || { + kind: this.detectKind(next.job), + subtaskErrors: 0, + reasonMatches: 0, + reasonedCount: 0, + }; + logger.info('worker', 'Starting job', { + jobId: next.job.jobId, + kind: next.meta.kind, + retryCount: next.retryCount, + queueRemaining: this.queue.length, + }); + try { + this.worker.postMessage(next.job); + } catch (err) { + logger.error('worker', 'Failed to post job to worker (will retry)', err as Error, { + jobId: next.job.jobId, + }); + this.retryOrFail(next, err instanceof Error ? err : new Error(String(err))); + } + } + + private detectKind(job: EmbeddingJob | MatchingJob): 'embedding' | 'matching' { + // Explicit mode wins + const mode = (job as any).mode; + if (mode === 'task' || mode === 'resource') return 'embedding'; + // Heuristic: resource embedding tasks have competenceId & text + const tasks: any[] | undefined = (job as any).tasks; + if (Array.isArray(tasks) && tasks.length) { + const sample = tasks[0]; + if (sample && 'competenceId' in sample && 'text' in sample && 'type' in sample) { + return 'embedding'; + } + } + return 'matching'; + } + + public async enqueue(job: EmbeddingJob | MatchingJob, workerType: workerTypes): Promise { + if (this.shuttingDown) throw new Error('WorkerManager shutting down'); + const kind = this.detectKind(job); + logger.info('worker', 'Enqueue job', { + jobId: job.jobId, + workerType, + kind, + queueLengthBefore: this.queue.length, + }); + return new Promise((resolve, reject) => { + const item: SingleJobItem = { + job, + resolve, + reject, + retryCount: 0, + queueDepthAtStart: this.queue.length, + }; + this.queue.push(item); + this.processNext(); + }); + } + + private retryOrFail(item: SingleJobItem, err: Error): void { + const jobId = item.job.jobId; + if (item.retryCount < jobMaxRetries) { + item.retryCount += 1; + logger.info('worker', 'Retrying job', { jobId, retryCount: item.retryCount }); + // Put at front of queue + this.queue.unshift(item); + if (this.current === item) this.current = null; // ensure freed + this.processNext(); + } else { + logger.error('worker', 'Job failed permanently', new WorkerError('inference', jobId, err), { + jobId, + retries: item.retryCount, + }); + if (this.current === item) this.current = null; + item.reject(new WorkerError('inference', jobId, err)); + this.processNext(); + } + } + + private async handleReasoning(job: EmbeddingJob | MatchingJob, workload: Record) { + const finalMatches: any[] = []; + const taskCount = Object.keys(workload).length; + logger.debug('worker', 'Reasoning workload received', { + jobId: job.jobId, + taskCount, + }); + for (const [taskText, matches] of Object.entries(workload)) { + try { + const enriched = await addReason(matches as any[], taskText); + finalMatches.push(...enriched); + if (this.current?.meta) { + this.current.meta.reasonMatches += (matches as any[]).length; + this.current.meta.reasonedCount += enriched.filter((m) => m.reason).length; + } + } catch { + finalMatches.push(...(matches as any[])); + } + } + const db = getDB(job.dbName); + for (const match of finalMatches) { + try { + db.addMatchResult({ + jobId: match.jobId, + taskId: match.taskId, + taskText: match.taskText, + competenceId: match.competenceId, + resourceId: match.resourceId, + distance: match.distance, + text: match.text, + type: match.type, + alignment: match.alignment, + reason: match.reason, + }); + } catch {} + } + try { + db.updateJobStatus(job.jobId, 'completed'); + } catch {} + logger.info('worker', 'Reasoning stored', { + jobId: job.jobId, + taskCount, + rawMatchGroups: Object.values(workload).reduce((sum, arr) => sum + (arr as any[]).length, 0), + matchCount: finalMatches.length, + reasonedCount: finalMatches.filter((m) => m.reason).length, + }); + } + + private handleWorkerCrash(error: Error): void { + logger.error('worker', 'Worker crash detected – respawning', error, { + activeJobId: this.current?.job.jobId, + }); + this.respawnWorker(true); + } + + public async shutdown(): Promise { + this.shuttingDown = true; + logger.info('worker', 'Shutting down worker manager', { + inFlight: this.current?.job.jobId, + queueLength: this.queue.length, + }); + // Reject queued jobs + for (const item of this.queue) item.reject(new Error('Shutting down')); + this.queue.length = 0; + if (this.current) { + this.current.reject(new Error('Shutting down')); + this.current = null; + } + if (this.worker) { + this.worker.removeAllListeners(); + try { + await this.worker.terminate(); + } catch {} + this.worker = null; + } + if (this.heartbeatMonitor) clearInterval(this.heartbeatMonitor); + } +} + +const manager = new WorkerManager(); +export default manager; diff --git a/src/competence-matcher/tools/onnx-model-external-data.py b/src/competence-matcher/tools/onnx-model-external-data.py new file mode 100644 index 000000000..83e9c3ab2 --- /dev/null +++ b/src/competence-matcher/tools/onnx-model-external-data.py @@ -0,0 +1,34 @@ +import onnx +import argparse +import sys + +# optimum-cli export onnx --model + +def main(): + parser = argparse.ArgumentParser(description="Convert ONNX model weights to external data format.") + parser.add_argument('--input', '-i', required=True, help='Path to the input ONNX model') + parser.add_argument('--output', '-o', required=False, help='Path to save the modified ONNX model (defaults to input path)', default=None) + + # If only -h/--help is present, show help and exit + if len(sys.argv) == 2 and sys.argv[1] in ('-h', '--help'): + parser.print_help() + sys.exit(0) + + args = parser.parse_args() + + if args.output is None: + args.output = args.input + + model = onnx.load(args.input) + + onnx.external_data_helper.convert_model_to_external_data( + model, + convert_attribute=True, + all_tensors_to_one_file=True, + location="model.onnx_data" + ) + + onnx.save(model, args.output) + +if __name__ == "__main__": + main() diff --git a/src/competence-matcher/tsconfig.json b/src/competence-matcher/tsconfig.json new file mode 100644 index 000000000..c7bba6d11 --- /dev/null +++ b/src/competence-matcher/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "lib": ["esnext", "dom"], + "target": "ES2020", + "module": "commonjs", + "rootDir": "src", + "outDir": "dist", + "strict": true, + "esModuleInterop": true, + "moduleResolution": "node", + "skipLibCheck": true + }, + "include": ["src"] +}