Add local-runner for extraction testing and improve test loading

code/.env.example

@@ -0,0 +1,2 @@
+# TODO: add External system specific variables that will be replaced in the Fixture tests that you want to run.
+TODO_API_KEY=test-api-key

code/fixtures/start_extracting_data/event.json

@@ -0,0 +1,14 @@
+{
+  "connection_data": {
+    "org_id": "test_org",
+    "org_name": "Test Organization",
+    "key": "${TODO_API_KEY}",
+    "key_type": "pat"
+  },
+  "payload": {
+    "event_type": "START_EXTRACTING_DATA"
+  },
+  "event_context": {
+    "external_sync_unit_id": "test_external_sync_unit_id"
+  }
+}

code/fixtures/start_extracting_data/extraction_scope.json

@@ -0,0 +1,11 @@
+{
+  "todos": {
+    "extract": true
+  },
+  "users": {
+    "extract": true
+  },
+  "attachments": {
+    "extract": true
+  }
+}

code/fixtures/start_extracting_data/state.json

@@ -0,0 +1,5 @@
+{
+  "todos": { "completed": false },
+  "users": { "completed": false },
+  "attachments": { "completed": false }
+}

code/fixtures/start_extracting_external_sync_units/event.json

@@ -0,0 +1,12 @@
+{
+  "function_name": "extraction",
+  "payload": {
+    "event_type": "START_EXTRACTING_EXTERNAL_SYNC_UNITS"
+  },
+  "connection_data": {
+    "org_id": "test_org",
+    "org_name": "Test Organization",
+    "key": "test-api-key",
+    "key_type": "pat"
+  }
+}

code/src/main.ts

@@ -11,16 +11,16 @@ import { testRunner } from './test-runner/test-runner';
     },
     functionName: {
       type: 'string',
-      require: true,
+      require: false,
     },
   }).argv;
 
-  if (!argv.fixturePath || !argv.functionName) {
-    console.error('Please make sure you have passed fixturePath & functionName');
+  if (!argv.fixturePath) {
+    console.error('Please make sure you have fixturePath in your command');
   }
 
   await testRunner({
     fixturePath: argv.fixturePath,
-    functionName: argv.functionName as FunctionFactoryType,
+    functionName: argv.functionName as FunctionFactoryType | undefined,
   });
 })();

code/src/test-runner/test-runner.ts

@@ -1,45 +1,147 @@
-import { AirdropEvent } from '@devrev/ts-adaas';
 import * as dotenv from 'dotenv';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { AirdropEvent, createMockEvent, DeepPartial, EventType, MockServer } from '@devrev/ts-adaas';
+
 import { functionFactory, FunctionFactoryType } from '../function-factory';
 
-export interface TestRunnerProps {
-  functionName: FunctionFactoryType;
+export interface LocalRunnerProps {
   fixturePath: string;
+  functionName?: FunctionFactoryType;
 }
-export function addCredentials(events: AirdropEvent[], env: dotenv.DotenvParseOutput): AirdropEvent[] {
-  return events.map((event: AirdropEvent) => {
-    return {
-      ...event,
-      context: {
-        ...event.context,
-        secrets: {
-          ...event.context.secrets,
-          service_account_token: env['DEVREV_PAT'],
-        },
-      },
-    };
+
+/**
+ * Replaces `${VAR_NAME}` placeholders with values from `process.env`.
+ * Values are JSON-escaped so special characters don't break the JSON structure.
+ */
+function resolveEnvVariables(raw: string, filePath: string): string {
+  return raw.replace(/\$\{(\w+)\}/g, (_match, varName: string) => {
+    const value = process.env[varName];
+    if (value === undefined) {
+      throw new Error(
+        `Environment variable "${varName}" referenced in ${filePath} is not set. ` +
+        'Make sure it is defined in your .env file or exported in your shell.'
+      );
+    }
+    return JSON.stringify(value).slice(1, -1);
   });
 }
 
-export const testRunner = async ({ functionName, fixturePath }: TestRunnerProps) => {
-  const env = dotenv.config();
+function readFixtureFile<T>(filePath: string): T | undefined {
+  if (!fs.existsSync(filePath)) {
+    return undefined;
+  }
+  const raw = fs.readFileSync(filePath, 'utf-8').trim();
+  if (raw.length === 0) {
+    return undefined;
+  }
+  const resolved = resolveEnvVariables(raw, filePath);
+  return JSON.parse(resolved) as T;
+}
+
+function resolveEventType(input: string): EventType {
+  const match = Object.values(EventType).find((v) => v === input);
+  if (match) return match as EventType;
+
+  throw new Error(`Unknown event_type "${input}". Must be one of: ${Object.values(EventType).join(', ')}`);
+}
+
+export const testRunner = async ({ fixturePath, functionName }: LocalRunnerProps) => {
+  dotenv.config();
 
-  console.log('env:', env);
+  const fixturesDir = path.resolve(__dirname, '../../fixtures', fixturePath);
+  if (!fs.existsSync(fixturesDir)) {
+    throw new Error(`Fixture directory not found: ${fixturesDir}`);
+  }
+  return runWithFixtureDir(fixturesDir, functionName);
+};
 
-  if (!functionFactory[functionName]) {
-    console.error(`${functionName} is not found in the functionFactory`);
-    console.error('Add your function to the function-factory.ts file');
-    throw new Error('Function is not found in the functionFactory');
+function getFunctionName(event_type: string): FunctionFactoryType {
+  if (event_type.indexOf('EXTRACT') != -1) {
+    return 'extraction';
+  } else if (event_type.indexOf('LOAD') != -1) {
+    return 'loading';
   }
 
-  const run = functionFactory[functionName];
+  throw new Error(`No functionName found for event ${event_type}. Specify functionName using '--functionName' parameter.`);
+}
+
+async function runWithFixtureDir(fixturesDir: string, functionName?: FunctionFactoryType) {
+  const eventPath = path.join(fixturesDir, 'event.json');
+  const statePath = path.join(fixturesDir, 'state.json');
+  const extractionScopePath = path.join(fixturesDir, 'extraction_scope.json');
 
-  // eslint-disable-next-line @typescript-eslint/no-require-imports
-  const eventFixture = require(`../fixtures/${fixturePath}`);
+  const fixtureEvent = readFixtureFile<DeepPartial<AirdropEvent>>(eventPath);
+  const fixtureState = readFixtureFile<Record<string, unknown>>(statePath);
+  const fixtureExtractionScope = readFixtureFile<Record<string, unknown>>(extractionScopePath);
+
+  if (!fixtureEvent) {
+    throw new Error(
+      `Missing or empty event.json in fixture directory: ${eventPath}. ` +
+      'Every fixture must have an event.json with at least an "event_type" field.'
+    );
+  }
+
+  if (!fixtureEvent.payload?.event_type) {
+    throw new Error(
+      `event.json at ${eventPath} is missing the required "event_type" field. ` +
+      'Specify an event type such as "START_EXTRACTING_DATA" or "START_EXTRACTING_EXTERNAL_SYNC_UNITS".'
+    );
+  }
 
-  if (env.parsed) {
-    await run(addCredentials(eventFixture, env.parsed));
+  const resolvedFunctionName = functionName ?? getFunctionName(fixtureEvent.payload?.event_type);
+
+  if (!resolvedFunctionName) {
+    throw new Error(
+      'No function name provided. Either pass --functionName on the CLI ' + 'or set "function_name" in event.json.'
+    );
+  }
+
+  if (!(resolvedFunctionName in functionFactory)) {
+    throw new Error(
+      `Function "${resolvedFunctionName}" not found in functionFactory. ` +
+      `Available: ${Object.keys(functionFactory).join(', ')}`
+    );
+  }
+
+  const eventType = resolveEventType(fixtureEvent.payload?.event_type);
+
+  console.log(`[test-runner] Function : ${resolvedFunctionName}`);
+  console.log(`[test-runner] Event    : ${eventType}`);
+  console.log(`[test-runner] Fixture  : ${fixturesDir}`);
+
+  const mockServer = new MockServer(0);
+  await mockServer.start();
+
+  console.log(`[test-runner] MockServer started on ${mockServer.baseUrl}`);
+
+  mockServer.setRoute({
+    path: '/worker_data_url.get',
+    method: 'GET',
+    status: 200,
+    body: { state: JSON.stringify(fixtureState ?? {}), objects: JSON.stringify(fixtureExtractionScope ?? {}) },
+  });
+  if (fixtureState) {
+    console.log(`[test-runner] Injected state from state.json`);
   } else {
-    await run(eventFixture);
+    console.log(`[test-runner] No state.json found — using default empty state`);
   }
-};
+
+  const event = createMockEvent(mockServer.baseUrl, fixtureEvent);
+
+  process.argv.push("--local");
+
+  const run = functionFactory[resolvedFunctionName];
+
+  try {
+    await run([event]);
+    console.log(`[test-runner] Function completed successfully`);
+  } catch (err) {
+    console.error(`[test-runner] Function threw an error:`, err);
+    process.exitCode = 1;
+  } finally {
+    await mockServer.stop();
+    console.log(`[test-runner] MockServer stopped`);
+  }
+}