From 25f799d6c027d0b3499ff341f10a28d440e50cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Thu, 19 Feb 2026 17:46:21 +0100 Subject: [PATCH] Align runner with tool-spec input/data model (#13) --- package.json | 2 +- src/api/tools/enpoints.ts | 49 ++++++--- src/index.ts | 6 +- src/models/ToolConfig.ts | 9 +- src/parameter.ts | 187 +++++++++++++++++++-------------- src/run.ts | 28 ++--- tests/spec-inputs-data.test.js | 158 ++++++++++++++++++++++++++++ 7 files changed, 328 insertions(+), 111 deletions(-) create mode 100644 tests/spec-inputs-data.test.js diff --git a/package.json b/package.json index 700b7b2..02c81e9 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", + "test": "npm run build && node tests/spec-inputs-data.test.js", "build": "tsc", "format": "prettier --write \"src/**/*.ts\" \"src/**/*.js\"", "lint": "tslint -p tsconfig.json" diff --git a/src/api/tools/enpoints.ts b/src/api/tools/enpoints.ts index 77655ac..84cf5ad 100644 --- a/src/api/tools/enpoints.ts +++ b/src/api/tools/enpoints.ts @@ -2,6 +2,7 @@ import * as e from 'express'; import { ReqTool, ReqTools } from './middleware'; import * as run from '../../run'; +import { InputValidationError } from '../../parameter'; export const addToolEndpoints = (app: e.Express, defaultResultPath?: string): e.Express => { @@ -46,13 +47,22 @@ export const addToolEndpoints = (app: e.Express, defaultResultPath?: string): e. } // run the tool - const response = await run.runTool(tool, opts, runArgs) - - // return - res.status(200).json({ - message: `Run of tool '${tool.name}' finished.`, - output: response - }) + try { + const response = await run.runTool(tool, opts, runArgs) + + // return + res.status(200).json({ + message: `Run of tool '${tool.name}' finished.`, + output: response + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + const status = error instanceof InputValidationError ? 400 : 500 + res.status(status).json({ + message: `Run of tool '${tool.name}' failed.`, + error: message + }) + } }) app.post('/tools/:toolName/run', async (req, res) => { @@ -74,16 +84,25 @@ export const addToolEndpoints = (app: e.Express, defaultResultPath?: string): e. } // run the tool - const response = await run.runTool(tool, opts, args) - - // return - res.status(200).json({ - message: `Run of tool '${tool.name}' finished.`, - output: response - }) + try { + const response = await run.runTool(tool, opts, args) + + // return + res.status(200).json({ + message: `Run of tool '${tool.name}' finished.`, + output: response + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + const status = error instanceof InputValidationError ? 400 : 500 + res.status(status).json({ + message: `Run of tool '${tool.name}' failed.`, + error: message + }) + } }) return app -} \ No newline at end of file +} diff --git a/src/index.ts b/src/index.ts index 3ecebb0..48a9223 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,9 +2,9 @@ export { healthz, DockerHealth } from './docker'; // Run container options -export { listTools, runTool, RunOptions } from './run'; -export { ToolConfig, ParameterConfig } from './models/ToolConfig'; +export { listTools, runTool, RunOptions, buildContainerOptions } from './run'; +export { ToolConfig, ParameterConfig, DataConfig } from './models/ToolConfig'; export { StepContent, StepPreview, ListStepFilter } from './step'; // api functions -export { runServer, RunServerOptions } from './api/api' \ No newline at end of file +export { runServer, RunServerOptions } from './api/api' diff --git a/src/models/ToolConfig.ts b/src/models/ToolConfig.ts index a24802b..4101c6c 100644 --- a/src/models/ToolConfig.ts +++ b/src/models/ToolConfig.ts @@ -9,6 +9,12 @@ export interface ParameterConfig { array?: boolean; } +export interface DataConfig { + extension?: string; + description?: string; + optional?: boolean; +} + export interface ToolConfig { image: string; name: string; @@ -16,5 +22,6 @@ export interface ToolConfig { description: string; version?: string; cmd?: string; - parameters: {[name: string]: ParameterConfig} + parameters: {[name: string]: ParameterConfig}; + data?: string[] | {[name: string]: DataConfig}; } diff --git a/src/parameter.ts b/src/parameter.ts index 0e329c1..09fbb18 100644 --- a/src/parameter.ts +++ b/src/parameter.ts @@ -1,80 +1,109 @@ import * as fs from 'fs'; -import { unparse } from 'papaparse'; - -import { ToolConfig } from "./models/ToolConfig" - -export const buildParameterFile = (args: any, inDir: string, toolConfig: ToolConfig): string => { - // build the params object - const params: any = {} - - // iterate through all arguments - Object.entries(args).forEach(([paramName, paramValue]) => { - // get the paramName - const paramConfig = toolConfig.parameters[paramName]; - - if (paramConfig.type === 'file') { - // matrix - if (Array.isArray(paramValue) && paramValue.every(r => Array.isArray(r))) { - // 2D matrix - if (paramValue.every(row => row.every((cell: any) => typeof cell === 'number'))) { - // create a dat matrix - const matrix_rows = paramValue.map((row: number[]) => row.map(cell => cell.toPrecision()).join(' ')) - const matrix = matrix_rows.join('\r\n') - - // write the file and save - fs.writeFileSync(`${inDir}/${paramName}.dat`, matrix) - params[paramName] = `/in/${paramName}.dat` - } - // CSV - else if (Array.isArray(paramValue) && paramValue.every(r => typeof r === 'object')) { - const csv = unparse(paramValue) - - // write - fs.writeFileSync(`${inDir}/${paramName}.csv`, csv) - params[paramName] = `/in/${paramName}.csv` - } - } else if (typeof paramValue === 'string') { - // this is an existing file ==> copy - if (fs.existsSync(paramValue)) { - // copy the file and save - fs.copyFileSync(paramValue, `${inDir}/${paramName}.${paramValue.split('.').pop()}`) - params[paramName] = `/in/${paramName}.${paramValue.split('.').pop()}` - } - - // this is base64 encoded payload - else if (paramValue.includes(';base64,')) { - // split and get the payload - const [mime, payload] = paramValue.substring(5).split(';base64,') - const fname = `${paramName}.${mime.split('/').pop()}` // here a lookup dict could be implemented - - // write the file - fs.writeFileSync(`${inDir}/${fname}`, payload, {encoding: 'base64'}) - params[paramName] = `/in/${fname}` - } - - // this is assumed to be a text file - else { - // plain text file is assumed - fs.writeFileSync(`${inDir}/${paramName}.txt`, paramValue) - params[paramName] = `/in/${paramName}.txt` - } - - } - else { - // just serialize it - fs.writeFileSync(`${inDir}/${paramName}.json`, JSON.stringify(paramValue)) - params[paramName] = `/in/${paramName}.json` - } - - } else { - params[paramName] = paramValue; - } - }) - - // write the file - const parameters: any = {} - parameters[toolConfig.name] = params - fs.writeFileSync(`${inDir}/parameters.json`, JSON.stringify(parameters, null, 4)) - - return `${inDir}/parameters.json` -} \ No newline at end of file +import * as path from 'path'; + +import { ToolConfig, DataConfig } from './models/ToolConfig'; + +export class InputValidationError extends Error { + constructor(message: string) { + super(message); + this.name = 'InputValidationError'; + Object.setPrototypeOf(this, InputValidationError.prototype); + } +} + +const getDataConfigMap = (toolConfig: ToolConfig): {[name: string]: DataConfig} => { + const dataSpec = toolConfig.data; + + if (!dataSpec) { + return {}; + } + + if (Array.isArray(dataSpec)) { + return dataSpec.reduce((acc, name) => { + acc[name] = {}; + return acc; + }, {} as {[name: string]: DataConfig}); + } + + return dataSpec; +}; + +const listAllowed = (names: string[]): string => { + return names.length > 0 ? names.join(', ') : '(none)'; +}; + +const mapDataInput = (name: string, value: any, inDir: string): string => { + if (typeof value !== 'string') { + throw new InputValidationError(`Invalid data input '${name}': expected a file path string or '/in/...' reference.`); + } + + if (value.startsWith('/in/')) { + return value; + } + + if (!fs.existsSync(value) || !fs.statSync(value).isFile()) { + throw new InputValidationError(`Invalid data input '${name}': path '${value}' does not exist or is not a file.`); + } + + const extension = path.extname(value); + const targetName = `${name}${extension}`; + const targetPath = path.join(inDir, targetName); + + fs.copyFileSync(value, targetPath); + return `/in/${targetName}`; +}; + +export const buildInputFile = (args: any, inDir: string, toolConfig: ToolConfig): string => { + const parameters: {[name: string]: any} = {}; + const data: {[name: string]: string} = {}; + + const parameterConfig = toolConfig.parameters || {}; + const dataConfig = getDataConfigMap(toolConfig); + + const allowedParameterNames = Object.keys(parameterConfig); + const allowedDataNames = Object.keys(dataConfig); + + const inputArgs = args || {}; + Object.entries(inputArgs).forEach(([name, value]) => { + if (Object.prototype.hasOwnProperty.call(parameterConfig, name)) { + parameters[name] = value; + return; + } + + if (Object.prototype.hasOwnProperty.call(dataConfig, name)) { + data[name] = mapDataInput(name, value, inDir); + return; + } + + throw new InputValidationError( + `Invalid input key '${name}'. Allowed parameters: ${listAllowed(allowedParameterNames)}. Allowed data: ${listAllowed(allowedDataNames)}.` + ); + }); + + Object.entries(parameterConfig).forEach(([name, config]) => { + const hasValue = Object.prototype.hasOwnProperty.call(parameters, name); + const hasDefault = config.default !== undefined; + const optional = config.optional === true; + + if (!hasValue && !hasDefault && !optional) { + throw new InputValidationError(`Missing required parameter '${name}'.`); + } + }); + + Object.entries(dataConfig).forEach(([name, config]) => { + const hasValue = Object.prototype.hasOwnProperty.call(data, name); + const optional = config.optional === true; + + if (!hasValue && !optional) { + throw new InputValidationError(`Missing required data input '${name}'.`); + } + }); + + const input: {[name: string]: {parameters: {[name: string]: any}; data: {[name: string]: string}}} = {}; + input[toolConfig.name] = { parameters, data }; + + const inputPath = `${inDir}/input.json`; + fs.writeFileSync(inputPath, JSON.stringify(input, null, 4)); + + return inputPath; +}; diff --git a/src/run.ts b/src/run.ts index 15ffab1..afc7446 100644 --- a/src/run.ts +++ b/src/run.ts @@ -10,7 +10,7 @@ import { performance } from 'perf_hooks'; import docker from './docker'; import { ToolConfig } from './models/ToolConfig'; -import { buildParameterFile } from './parameter'; +import { buildInputFile } from './parameter'; const getImageTags = async (prefix: string[] | string = 'tbr_'): Promise => { @@ -70,6 +70,19 @@ export interface RunOptions { resultPath?: string } +export const buildContainerOptions = (tool: ToolConfig, inDir: string, outDir: string): ContainerCreateOptions => { + return { + Env: ['PARAM_FILE=/in/input.json', `RUN_TOOL=${tool.name}`], + Tty: false, + HostConfig: { + Binds: [ + `${path.resolve(inDir)}:/in`, + `${path.resolve(outDir)}:/out` + ] + } + }; +}; + export const runTool = async (tool: ToolConfig, options: RunOptions= {}, args: {}= {}): Promise => { // handle the paths let basePath: string; @@ -86,23 +99,14 @@ export const runTool = async (tool: ToolConfig, options: RunOptions= {}, args: { if (!fs.existsSync(outDir)) fs.mkdirSync(outDir) // build the parameterization - const paramPath = buildParameterFile(args, inDir, tool) + buildInputFile(args, inDir, tool) // capture the streams const stdout = new WritableStream() const stderr = new WritableStream() // build the options - const opts: ContainerCreateOptions = { - Env: ['PARAM_FILE=/in/parameters.json', `RUN_TOOL=${tool.name}`], - Tty: false, - HostConfig: { - Binds: [ - `${path.resolve(inDir)}:/in`, - `${path.resolve(outDir)}:/out` - ] - } - } + const opts = buildContainerOptions(tool, inDir, outDir) // run and get the container const t1 = performance.now() diff --git a/tests/spec-inputs-data.test.js b/tests/spec-inputs-data.test.js new file mode 100644 index 0000000..d8789f1 --- /dev/null +++ b/tests/spec-inputs-data.test.js @@ -0,0 +1,158 @@ +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { buildInputFile, InputValidationError } = require('../lib/parameter'); +const { runTool, buildContainerOptions } = require('../lib/run'); + +const makeTmpDir = () => fs.mkdtempSync(path.join(os.tmpdir(), 'tool-runner-js-test-')); + +const readJson = (filePath) => JSON.parse(fs.readFileSync(filePath, 'utf8')); + +const testParametersOnly = () => { + const baseDir = makeTmpDir(); + const inDir = path.join(baseDir, 'in'); + fs.mkdirSync(inDir); + + const tool = { + name: 'demo', + image: 'demo:latest', + title: 'demo', + description: 'demo', + parameters: { alpha: { type: 'integer' } } + }; + + const file = buildInputFile({ alpha: 42 }, inDir, tool); + const payload = readJson(file); + + assert.strictEqual(path.basename(file), 'input.json'); + assert.deepStrictEqual(payload, { demo: { parameters: { alpha: 42 }, data: {} } }); +}; + +const testDataList = () => { + const baseDir = makeTmpDir(); + const inDir = path.join(baseDir, 'in'); + fs.mkdirSync(inDir); + + const sourcePath = path.join(baseDir, 'matrix.csv'); + fs.writeFileSync(sourcePath, 'a,b\n1,2\n'); + + const tool = { + name: 'demo', + image: 'demo:latest', + title: 'demo', + description: 'demo', + parameters: {}, + data: ['matrix'] + }; + + const file = buildInputFile({ matrix: sourcePath }, inDir, tool); + const payload = readJson(file); + + assert.strictEqual(payload.demo.data.matrix, '/in/matrix.csv'); + assert.ok(fs.existsSync(path.join(inDir, 'matrix.csv'))); +}; + +const testDataObjectValidation = () => { + const baseDir = makeTmpDir(); + const inDir = path.join(baseDir, 'in'); + fs.mkdirSync(inDir); + + const reqPath = path.join(baseDir, 'required.txt'); + fs.writeFileSync(reqPath, 'value'); + + const tool = { + name: 'demo', + image: 'demo:latest', + title: 'demo', + description: 'demo', + parameters: {}, + data: { + required_input: { extension: '.txt' }, + optional_input: { extension: '.txt', optional: true } + } + }; + + const file = buildInputFile({ required_input: reqPath }, inDir, tool); + const payload = readJson(file); + assert.strictEqual(payload.demo.data.required_input, '/in/required_input.txt'); + assert.strictEqual(payload.demo.data.optional_input, undefined); + + assert.throws( + () => buildInputFile({}, inDir, tool), + (error) => error instanceof InputValidationError && error.message.includes("Missing required data input 'required_input'") + ); +}; + +const testUnknownKeyValidation = () => { + const baseDir = makeTmpDir(); + const inDir = path.join(baseDir, 'in'); + fs.mkdirSync(inDir); + + const tool = { + name: 'demo', + image: 'demo:latest', + title: 'demo', + description: 'demo', + parameters: { alpha: { type: 'integer' } }, + data: ['file_input'] + }; + + assert.throws( + () => buildInputFile({ does_not_exist: true }, inDir, tool), + (error) => error instanceof InputValidationError && error.message.includes("Invalid input key 'does_not_exist'") + ); +}; + +const testContainerEnv = () => { + const tool = { + name: 'demo', + image: 'demo:latest', + title: 'demo', + description: 'demo', + parameters: {} + }; + + const opts = buildContainerOptions(tool, '/tmp/in', '/tmp/out'); + assert.ok(opts.Env.includes('PARAM_FILE=/in/input.json')); + assert.ok(opts.Env.includes('RUN_TOOL=demo')); +}; + +const testRunToolRejectsBeforeContainer = async () => { + const baseDir = makeTmpDir(); + const tool = { + name: 'demo', + image: 'demo:latest', + title: 'demo', + description: 'demo', + parameters: {}, + data: ['required_input'] + }; + + let failed = false; + try { + await runTool(tool, { mountPath: baseDir }, {}); + } catch (error) { + failed = true; + assert.ok(error instanceof InputValidationError); + assert.ok(error.message.includes("Missing required data input 'required_input'")); + } + + assert.ok(failed, 'runTool should reject when required data is missing'); +}; + +const run = async () => { + testParametersOnly(); + testDataList(); + testDataObjectValidation(); + testUnknownKeyValidation(); + testContainerEnv(); + await testRunToolRejectsBeforeContainer(); + console.log('spec-inputs-data tests passed'); +}; + +run().catch((error) => { + console.error(error); + process.exit(1); +});