Align with current tool-spec input/data model

package.json

@@ -5,7 +5,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "npm run build && node tests/spec-inputs-data.test.js",
     "build": "tsc",
     "format": "prettier --write \"src/**/*.ts\" \"src/**/*.js\"",
     "lint": "tslint -p tsconfig.json"

src/api/tools/enpoints.ts

@@ -2,6 +2,7 @@ import * as e from 'express';
 
 import { ReqTool, ReqTools } from './middleware';
 import * as run from '../../run';
+import { InputValidationError } from '../../parameter';
 
 
 export const addToolEndpoints = (app: e.Express, defaultResultPath?: string): e.Express => {
@@ -46,13 +47,22 @@ export const addToolEndpoints = (app: e.Express, defaultResultPath?: string): e.
         }
 
         // run the tool
-        const response = await run.runTool(tool, opts, runArgs)
-
-        // return 
-        res.status(200).json({
-            message: `Run of tool '${tool.name}' finished.`,
-            output: response
-        })
+        try {
+            const response = await run.runTool(tool, opts, runArgs)
+
+            // return 
+            res.status(200).json({
+                message: `Run of tool '${tool.name}' finished.`,
+                output: response
+            })
+        } catch (error) {
+            const message = error instanceof Error ? error.message : 'Unknown error'
+            const status = error instanceof InputValidationError ? 400 : 500
+            res.status(status).json({
+                message: `Run of tool '${tool.name}' failed.`,
+                error: message
+            })
+        }
     })
 
     app.post('/tools/:toolName/run', async (req, res) => {
@@ -74,16 +84,25 @@ export const addToolEndpoints = (app: e.Express, defaultResultPath?: string): e.
         }
 
         // run the tool
-        const response = await run.runTool(tool, opts, args)
-
-        // return 
-        res.status(200).json({
-            message: `Run of tool '${tool.name}' finished.`,
-            output: response
-        })
+        try {
+            const response = await run.runTool(tool, opts, args)
+
+            // return 
+            res.status(200).json({
+                message: `Run of tool '${tool.name}' finished.`,
+                output: response
+            })
+        } catch (error) {
+            const message = error instanceof Error ? error.message : 'Unknown error'
+            const status = error instanceof InputValidationError ? 400 : 500
+            res.status(status).json({
+                message: `Run of tool '${tool.name}' failed.`,
+                error: message
+            })
+        }
 
     })
 
     return app
 
-}
+}

src/index.ts

@@ -2,9 +2,9 @@
 export { healthz, DockerHealth } from './docker';
 
 // Run container options
-export { listTools, runTool, RunOptions } from './run';
-export { ToolConfig, ParameterConfig } from './models/ToolConfig';
+export { listTools, runTool, RunOptions, buildContainerOptions } from './run';
+export { ToolConfig, ParameterConfig, DataConfig } from './models/ToolConfig';
 export { StepContent, StepPreview, ListStepFilter } from './step';
 
 // api functions
-export { runServer, RunServerOptions } from './api/api'
+export { runServer, RunServerOptions } from './api/api'

src/models/ToolConfig.ts

@@ -9,12 +9,19 @@ export interface ParameterConfig {
     array?: boolean;
 }
 
+export interface DataConfig {
+    extension?: string;
+    description?: string;
+    optional?: boolean;
+}
+
 export interface ToolConfig {
     image: string;
     name: string;
     title: string;
     description: string;
     version?: string;
     cmd?: string;
-    parameters: {[name: string]: ParameterConfig}
+    parameters: {[name: string]: ParameterConfig};
+    data?: string[] | {[name: string]: DataConfig};
 }

src/parameter.ts

@@ -1,80 +1,109 @@
 import * as fs from 'fs';
-import { unparse } from 'papaparse';
-
-import { ToolConfig } from "./models/ToolConfig"
-
-export const buildParameterFile = (args: any, inDir: string, toolConfig: ToolConfig): string => {
-    // build the params object
-    const params: any = {}
-
-    // iterate through all arguments
-    Object.entries(args).forEach(([paramName, paramValue]) => {
-        // get the paramName
-        const paramConfig = toolConfig.parameters[paramName];
-
-        if (paramConfig.type === 'file') {
-            // matrix
-            if (Array.isArray(paramValue) && paramValue.every(r => Array.isArray(r))) {
-                // 2D matrix
-                if (paramValue.every(row => row.every((cell: any) => typeof cell === 'number'))) {
-                    // create a dat matrix
-                    const matrix_rows = paramValue.map((row: number[]) => row.map(cell => cell.toPrecision()).join(' '))
-                    const matrix = matrix_rows.join('\r\n')
-
-                    // write the file and save
-                    fs.writeFileSync(`${inDir}/${paramName}.dat`, matrix)
-                    params[paramName] = `/in/${paramName}.dat`
-                }
-                // CSV
-                else if (Array.isArray(paramValue) && paramValue.every(r => typeof r === 'object')) {
-                    const csv = unparse(paramValue)
-
-                    // write
-                    fs.writeFileSync(`${inDir}/${paramName}.csv`, csv)
-                    params[paramName] = `/in/${paramName}.csv`
-                }
-            } else if (typeof paramValue === 'string') {
-                // this is an existing file ==> copy
-                if (fs.existsSync(paramValue)) {
-                    // copy the file and save
-                    fs.copyFileSync(paramValue, `${inDir}/${paramName}.${paramValue.split('.').pop()}`)
-                    params[paramName] = `/in/${paramName}.${paramValue.split('.').pop()}`
-                } 
-
-                // this is base64 encoded payload
-                else if (paramValue.includes(';base64,')) {
-                    // split and get the payload
-                    const [mime, payload] = paramValue.substring(5).split(';base64,')
-                    const fname = `${paramName}.${mime.split('/').pop()}`    // here a lookup dict could be implemented
-
-                    // write the file
-                    fs.writeFileSync(`${inDir}/${fname}`, payload, {encoding: 'base64'})
-                    params[paramName] = `/in/${fname}`
-                } 
-
-                // this is assumed to be a text file
-                else {
-                    // plain text file is assumed
-                    fs.writeFileSync(`${inDir}/${paramName}.txt`, paramValue)
-                    params[paramName] = `/in/${paramName}.txt`
-                }
-
-            } 
-            else {
-                // just serialize it
-                fs.writeFileSync(`${inDir}/${paramName}.json`, JSON.stringify(paramValue))
-                params[paramName] = `/in/${paramName}.json`
-            }
-
-        } else {
-            params[paramName] = paramValue;
-        }    
-    })
-
-    // write the file
-    const parameters: any = {}
-    parameters[toolConfig.name] = params
-    fs.writeFileSync(`${inDir}/parameters.json`, JSON.stringify(parameters, null, 4))
-
-    return `${inDir}/parameters.json`
-}
+import * as path from 'path';
+
+import { ToolConfig, DataConfig } from './models/ToolConfig';
+
+export class InputValidationError extends Error {
+    constructor(message: string) {
+        super(message);
+        this.name = 'InputValidationError';
+        Object.setPrototypeOf(this, InputValidationError.prototype);
+    }
+}
+
+const getDataConfigMap = (toolConfig: ToolConfig): {[name: string]: DataConfig} => {
+    const dataSpec = toolConfig.data;
+
+    if (!dataSpec) {
+        return {};
+    }
+
+    if (Array.isArray(dataSpec)) {
+        return dataSpec.reduce((acc, name) => {
+            acc[name] = {};
+            return acc;
+        }, {} as {[name: string]: DataConfig});
+    }
+
+    return dataSpec;
+};
+
+const listAllowed = (names: string[]): string => {
+    return names.length > 0 ? names.join(', ') : '(none)';
+};
+
+const mapDataInput = (name: string, value: any, inDir: string): string => {
+    if (typeof value !== 'string') {
+        throw new InputValidationError(`Invalid data input '${name}': expected a file path string or '/in/...' reference.`);
+    }
+
+    if (value.startsWith('/in/')) {
+        return value;
+    }
+
+    if (!fs.existsSync(value) || !fs.statSync(value).isFile()) {
+        throw new InputValidationError(`Invalid data input '${name}': path '${value}' does not exist or is not a file.`);
+    }
+
+    const extension = path.extname(value);
+    const targetName = `${name}${extension}`;
+    const targetPath = path.join(inDir, targetName);
+
+    fs.copyFileSync(value, targetPath);
+    return `/in/${targetName}`;
+};
+
+export const buildInputFile = (args: any, inDir: string, toolConfig: ToolConfig): string => {
+    const parameters: {[name: string]: any} = {};
+    const data: {[name: string]: string} = {};
+
+    const parameterConfig = toolConfig.parameters || {};
+    const dataConfig = getDataConfigMap(toolConfig);
+
+    const allowedParameterNames = Object.keys(parameterConfig);
+    const allowedDataNames = Object.keys(dataConfig);
+
+    const inputArgs = args || {};
+    Object.entries(inputArgs).forEach(([name, value]) => {
+        if (Object.prototype.hasOwnProperty.call(parameterConfig, name)) {
+            parameters[name] = value;
+            return;
+        }
+
+        if (Object.prototype.hasOwnProperty.call(dataConfig, name)) {
+            data[name] = mapDataInput(name, value, inDir);
+            return;
+        }
+
+        throw new InputValidationError(
+            `Invalid input key '${name}'. Allowed parameters: ${listAllowed(allowedParameterNames)}. Allowed data: ${listAllowed(allowedDataNames)}.`
+        );
+    });
+
+    Object.entries(parameterConfig).forEach(([name, config]) => {
+        const hasValue = Object.prototype.hasOwnProperty.call(parameters, name);
+        const hasDefault = config.default !== undefined;
+        const optional = config.optional === true;
+
+        if (!hasValue && !hasDefault && !optional) {
+            throw new InputValidationError(`Missing required parameter '${name}'.`);
+        }
+    });
+
+    Object.entries(dataConfig).forEach(([name, config]) => {
+        const hasValue = Object.prototype.hasOwnProperty.call(data, name);
+        const optional = config.optional === true;
+
+        if (!hasValue && !optional) {
+            throw new InputValidationError(`Missing required data input '${name}'.`);
+        }
+    });
+
+    const input: {[name: string]: {parameters: {[name: string]: any}; data: {[name: string]: string}}} = {};
+    input[toolConfig.name] = { parameters, data };
+
+    const inputPath = `${inDir}/input.json`;
+    fs.writeFileSync(inputPath, JSON.stringify(input, null, 4));
+
+    return inputPath;
+};

src/run.ts

@@ -10,7 +10,7 @@ import { performance } from 'perf_hooks';
 
 import docker from './docker';
 import { ToolConfig } from './models/ToolConfig';
-import { buildParameterFile } from './parameter';
+import { buildInputFile } from './parameter';
 
 
 const getImageTags = async (prefix: string[] | string = 'tbr_'): Promise<string[]> => {
@@ -70,6 +70,19 @@ export interface RunOptions {
     resultPath?: string
 }
 
+export const buildContainerOptions = (tool: ToolConfig, inDir: string, outDir: string): ContainerCreateOptions => {
+    return {
+        Env: ['PARAM_FILE=/in/input.json', `RUN_TOOL=${tool.name}`],
+        Tty: false,
+        HostConfig: {
+            Binds: [
+                `${path.resolve(inDir)}:/in`,
+                `${path.resolve(outDir)}:/out`
+            ]
+        }
+    };
+};
+
 export const runTool = async (tool: ToolConfig, options: RunOptions= {}, args: {}= {}): Promise<string> => {
     // handle the paths
     let basePath: string;
@@ -86,23 +99,14 @@ export const runTool = async (tool: ToolConfig, options: RunOptions= {}, args: {
     if (!fs.existsSync(outDir)) fs.mkdirSync(outDir)
 
     // build the parameterization
-    const paramPath = buildParameterFile(args, inDir, tool)
+    buildInputFile(args, inDir, tool)
 
     // capture the streams
     const stdout = new WritableStream()
     const stderr = new WritableStream()
 
     // build the options
-    const opts: ContainerCreateOptions = {
-        Env: ['PARAM_FILE=/in/parameters.json', `RUN_TOOL=${tool.name}`],
-        Tty: false,
-        HostConfig: {
-            Binds: [
-                `${path.resolve(inDir)}:/in`,
-                `${path.resolve(outDir)}:/out`
-            ]
-        }
-    }
+    const opts = buildContainerOptions(tool, inDir, outDir)
 
     // run and get the container
     const t1 = performance.now()