Fix/chunked upload for large syncs

src/reporter.js

@@ -4,6 +4,7 @@ const debug = require('debug')('testomatio:ids');
 const { request } = isHttps ? require('https') : require('http');
 const path = require('path');
 const fs = require('fs');
+const { TEST_ID_REGEX } = require('./updateIds/constants');
 
 class Reporter {
   constructor(apiKey, framework, workDir) {
@@ -18,24 +19,29 @@ class Reporter {
     this.workDir = workDir || process.cwd();
     this.tests = [];
     this.files = {};
+    this.maxChunkBytes = 1 * 1024 * 1024;
+    this.maxChunkFiles = 100;
+    this.maxChunkTests = 100;
   }
 
   addTests(tests) {
     this.tests = this.tests.concat(tests);
   }
 
-  attachFiles() {
-    this.files = {};
-
-    const uniqueFiles = [...new Set(this.tests.map(test => test.file).filter(f => !!f))];
+  attachFiles(tests = this.tests) {
+    const files = {};
+    const uniqueFiles = [...new Set(tests.map(test => test.file).filter(f => !!f))];
 
     for (const fileName of uniqueFiles) {
       try {
-        this.files[fileName] = fs.readFileSync(path.resolve(this.workDir, fileName), 'utf8');
+        files[fileName] = fs.readFileSync(path.resolve(this.workDir, fileName), 'utf8');
       } catch (err) {
         debug(`Error reading file ${fileName}: ${err.message}`);
       }
     }
+
+    this.files = files;
+    return files;
   }
 
   getFilesFromServer(exportAutomated, suiteIds) {
@@ -125,41 +131,297 @@ class Reporter {
     });
   }
 
-  send(opts = {}) {
-    return new Promise((resolve, reject) => {
-      console.log('\n 🚀 Sending data to testomat.io\n');
+  async send(opts = {}) {
+    console.log('\n 🚀 Sending data to testomat.io\n');
+
+    this.tests = this.prepareTests();
+    const payloadOpts = this.buildUploadOptions(opts);
+    const { newTests, existingTests } = this.splitTestsById(this.tests);
+
+    if (this.framework !== 'manual' && newTests.length > 0 && existingTests.length > 0) {
+      await this.sendInPhases(payloadOpts, newTests, existingTests);
+      return;
+    }
+
+    this.attachFiles();
+
+    const chunks = this.createUploadChunks(payloadOpts);
+    if (chunks.length > 1) {
+      this.logChunkedUploadStart(chunks.length);
+      await this.sendInChunks(payloadOpts, chunks);
+      return;
+    }
+
+    const data = this.buildPayload(payloadOpts, this.tests, this.files);
+    await this.sendRequest(data);
+  }
 
-      // Parse labels from environment variable (supports both TESTOMATIO_LABELS and TESTOMATIO_SYNC_LABELS)
-      const labelsFromEnv = this.parseLabels(process.env.TESTOMATIO_LABELS || process.env.TESTOMATIO_SYNC_LABELS);
+  prepareTests() {
+    const labelsFromEnv = this.parseLabels(process.env.TESTOMATIO_LABELS || process.env.TESTOMATIO_SYNC_LABELS);
 
-      const tests = this.tests.map(test => {
-        // make file path relative to TESTOMATIO_WORKDIR if provided
-        if (process.env.TESTOMATIO_WORKDIR && test.file) {
-          const workdir = path.resolve(process.env.TESTOMATIO_WORKDIR);
-          const absoluteTestPath = path.resolve(test.file);
-          test.file = path.relative(workdir, absoluteTestPath);
+    return this.tests.map(test => {
+      const nextTest = { ...test };
+
+      if (process.env.TESTOMATIO_WORKDIR && nextTest.file) {
+        const workdir = path.resolve(process.env.TESTOMATIO_WORKDIR);
+        const absoluteTestPath = path.resolve(nextTest.file);
+        nextTest.file = path.relative(workdir, absoluteTestPath);
+      }
+
+      nextTest.file = nextTest.file?.replace(/\\/g, '/');
+
+      if (labelsFromEnv.length > 0) {
+        nextTest.labels = labelsFromEnv;
+      }
+
+      return nextTest;
+    });
+  }
+
+  buildUploadOptions(opts = {}) {
+    const nextOpts = { ...opts };
+
+    if (process.env.TESTOMATIO_PREPEND_DIR) nextOpts.dir = process.env.TESTOMATIO_PREPEND_DIR;
+    if (process.env.TESTOMATIO_SUITE) nextOpts.suite = process.env.TESTOMATIO_SUITE;
+
+    return nextOpts;
+  }
+
+  buildPayload(opts = {}, tests = this.tests, files = this.files, extra = {}) {
+    return JSON.stringify({ ...opts, ...extra, tests, framework: this.framework, files });
+  }
+
+  splitTestsById(tests = this.tests) {
+    return tests.reduce(
+      (groups, test) => {
+        if (this.hasTestId(test)) {
+          groups.existingTests.push(test);
+        } else {
+          groups.newTests.push(test);
         }
 
-        // unify path to use slashes (prevent backslashes on windows)
-        test.file = test.file?.replace(/\\/g, '/');
+        return groups;
+      },
+      { newTests: [], existingTests: [] },
+    );
+  }
+
+  hasTestId(test) {
+    if (typeof test.id === 'string' && test.id.trim()) return true;
+    if (typeof test.name === 'string' && TEST_ID_REGEX.test(test.name)) return true;
+    return false;
+  }
+
+  createUploadChunks(opts = {}, tests = this.tests, files = this.files) {
+    if (tests.length === 0) {
+      return [{ tests, files }];
+    }
 
-        // Apply labels to each test
-        if (labelsFromEnv.length > 0) {
-          test.labels = labelsFromEnv;
+    const groups = this.groupTestsByFile(tests, files);
+    const chunks = [];
+    let currentChunk = { tests: [], files: {} };
+
+    for (const group of groups) {
+      const groupChunks = this.splitOversizedGroup(group, opts);
+
+      for (const groupChunk of groupChunks) {
+        const nextChunk = {
+          tests: currentChunk.tests.concat(groupChunk.tests),
+          files: { ...currentChunk.files, ...groupChunk.files },
+        };
+        const nextChunkFilesCount = Object.keys(nextChunk.files).length;
+        const nextChunkTestsCount = nextChunk.tests.length;
+
+        if (
+          currentChunk.tests.length > 0 &&
+          (this.getPayloadSize(opts, nextChunk.tests, nextChunk.files) > this.maxChunkBytes ||
+            nextChunkFilesCount > this.maxChunkFiles ||
+            nextChunkTestsCount > this.maxChunkTests)
+        ) {
+          chunks.push(currentChunk);
+          currentChunk = groupChunk;
+          continue;
         }
 
-        return test;
+        currentChunk = nextChunk;
+      }
+    }
+
+    if (currentChunk.tests.length > 0 || Object.keys(currentChunk.files).length > 0 || chunks.length === 0) {
+      chunks.push(currentChunk);
+    }
+
+    return chunks;
+  }
+
+  groupTestsByFile(tests = this.tests, files = this.files) {
+    const groups = [];
+    const fileGroups = new Map();
+
+    tests.forEach((test, index) => {
+      const key = test.file || `__no_file__${index}`;
+
+      if (!fileGroups.has(key)) {
+        const group = {
+          tests: [],
+          files: test.file && files[test.file] !== undefined ? { [test.file]: files[test.file] } : {},
+        };
+        fileGroups.set(key, group);
+        groups.push(group);
+      }
+
+      fileGroups.get(key).tests.push(test);
+    });
+
+    return groups;
+  }
+
+  splitOversizedGroup(group, opts = {}) {
+    if (
+      (this.getPayloadSize(opts, group.tests, group.files) <= this.maxChunkBytes &&
+        group.tests.length <= this.maxChunkTests) ||
+      group.tests.length <= 1
+    ) {
+      return [group];
+    }
+
+    const splitGroups = [];
+    let currentGroup = { tests: [], files: group.files };
+
+    for (const test of group.tests) {
+      const nextGroup = {
+        tests: currentGroup.tests.concat(test),
+        files: group.files,
+      };
+
+      if (
+        currentGroup.tests.length > 0 &&
+        (this.getPayloadSize(opts, nextGroup.tests, nextGroup.files) > this.maxChunkBytes ||
+          nextGroup.tests.length > this.maxChunkTests)
+      ) {
+        splitGroups.push(currentGroup);
+        currentGroup = {
+          tests: [test],
+          files: group.files,
+        };
+        continue;
+      }
+
+      currentGroup = nextGroup;
+    }
+
+    if (currentGroup.tests.length > 0) {
+      splitGroups.push(currentGroup);
+    }
+
+    return splitGroups;
+  }
+
+  getPayloadSize(opts = {}, tests = this.tests, files = this.files, extra = {}) {
+    return Buffer.byteLength(this.buildPayload(opts, tests, files, extra));
+  }
+
+  logChunkedUploadStart(totalChunks) {
+    console.log(`Chunked upload enabled: ${totalChunks} chunks`);
+    console.log(
+      `Chunk limits: ${this.formatChunkBytes(this.maxChunkBytes)}, ${this.maxChunkFiles} files, ${
+        this.maxChunkTests
+      } tests per chunk`,
+    );
+  }
+
+  logChunkedUploadProgress(index, totalChunks) {
+    console.log(`Uploading chunk ${index}/${totalChunks}...`);
+  }
+
+  logChunkedUploadComplete(totalChunks) {
+    console.log(`🎉 Chunked upload completed: ${totalChunks}/${totalChunks} chunks sent`);
+  }
+
+  formatChunkBytes(bytes) {
+    if (bytes < 1024 * 1024) {
+      return `${(bytes / 1024).toFixed(1)} KB`;
+    }
+
+    if (bytes % (1024 * 1024) === 0) {
+      return `${bytes / (1024 * 1024)}.0 MB`;
+    }
+
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+  }
+
+  async sendInPhases(opts, newTests, existingTests) {
+    console.log(`Two-phase import enabled: ${newTests.length} new tests, ${existingTests.length} existing tests`);
+
+    const newFiles = this.attachFiles(newTests);
+    const newChunks = this.createUploadChunks(opts, newTests, newFiles);
+    console.log('Phase 1/2: uploading new tests without ids');
+    this.logChunkedUploadStart(newChunks.length);
+    const importId = await this.sendInChunks(opts, newChunks, {
+      finishLast: false,
+      requireImportId: true,
+    });
+
+    const existingFiles = this.attachFiles(existingTests);
+    const existingChunks = [{ tests: existingTests, files: existingFiles }];
+    console.log('Phase 2/2: uploading existing tests with ids');
+    this.logChunkedUploadStart(existingChunks.length);
+    await this.sendInChunks(opts, existingChunks, {
+      startImportId: importId,
+      finishLast: true,
+      requireImportId: false,
+    });
+  }
+
+  async sendInChunks(opts, chunks, sendOpts = {}) {
+    const { startImportId = null, finishLast = true, requireImportId = chunks.length > 1 } = sendOpts;
+    let importId = startImportId;
+
+    for (let index = 0; index < chunks.length; index += 1) {
+      const chunk = chunks[index];
+      this.logChunkedUploadProgress(index + 1, chunks.length);
+      const extra = {
+        chunk_upload: true,
+        finish: finishLast && index === chunks.length - 1,
+      };
+
+      if (importId) extra.import_id = importId;
+
+      const response = await this.sendRequest(this.buildPayload(opts, chunk.tests, chunk.files, extra), {
+        quietSuccessLog: true,
       });
-      this.tests = tests;
 
-      if (process.env.TESTOMATIO_PREPEND_DIR) opts.dir = process.env.TESTOMATIO_PREPEND_DIR;
-      if (process.env.TESTOMATIO_SUITE) opts.suite = process.env.TESTOMATIO_SUITE;
+      if (response.statusCode >= 400) {
+        throw new Error(response.body || `Chunk upload failed (${response.statusCode}: ${response.statusMessage})`);
+      }
 
-      this.attachFiles();
+      if (!importId) {
+        importId = this.extractImportId(response.body);
+        if (!importId && requireImportId) {
+          throw new Error('Chunk upload failed: import_id was not returned after the first chunk');
+        }
+      }
+    }
+
+    this.logChunkedUploadComplete(chunks.length);
+    return importId;
+  }
 
-      const data = JSON.stringify({ ...opts, tests: this.tests, framework: this.framework, files: this.files });
+  extractImportId(message) {
+    if (!message) return null;
 
-      debug('Sending test data to Testomat.io', data);
+    try {
+      const parsed = JSON.parse(message);
+      return parsed.import_id || null;
+    } catch (err) {
+      return null;
+    }
+  }
+
+  sendRequest(data, requestOpts = {}) {
+    debug('Sending test data to Testomat.io', data);
+
+    return new Promise((resolve, reject) => {
       const req = request(
         `${URL.trim()}/api/load?api_key=${this.apiKey}`,
         {
@@ -177,10 +439,15 @@ class Reporter {
             if (resp.statusCode >= 400) {
               console.log(' ✖️ ', message, `(${resp.statusCode}: ${resp.statusMessage})`);
               process.exitCode = 1;
-            } else {
+            } else if (!requestOpts.quietSuccessLog) {
               console.log(' 🎉 Data received at Testomat.io');
             }
-            resolve();
+
+            resolve({
+              statusCode: resp.statusCode,
+              statusMessage: resp.statusMessage,
+              body: message,
+            });
           });
 
           resp.on('data', chunk => {