From 5c440cb7bd2223c4a879c2ec32a019e1a8a46271 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Wed, 9 Jul 2025 03:51:18 +0900
Subject: [PATCH 01/12] refactor: split RTCStatsServer into multiple modules

---
 src/RTCStatsServer.js | 173 ++++--------------------------------------
 src/ServerSetup.js    | 117 ++++++++++++++++++++++++++++
 src/app.js            |  73 +++++-------------
 src/services.js       | 117 ++++++++++++++++++++++++++++
 4 files changed, 266 insertions(+), 214 deletions(-)
 create mode 100644 src/ServerSetup.js
 create mode 100644 src/services.js

diff --git a/src/RTCStatsServer.js b/src/RTCStatsServer.js
index 25e7b66..dd681d3 100644
--- a/src/RTCStatsServer.js
+++ b/src/RTCStatsServer.js
@@ -2,18 +2,16 @@ const JSONStream = require('JSONStream');
 const assert = require('assert').strict;
 const config = require('config');
 const fs = require('fs');
-const http = require('http');
-const https = require('https');
 const path = require('path');
 const { pipeline } = require('stream');
 const WebSocket = require('ws');
 
 const { name: appName, version: appVersion } = require('../package');
 
+const { setupMetricsServer, setupWebServer } = require('./ServerSetup');
 const DemuxSink = require('./demux');
 const logger = require('./logging');
 const PromCollector = require('./metrics/PromCollector');
-const S3Manager = require('./store/S3Manager');
 const { ConnectionInformation, ClientType } = require('./utils/ConnectionInformation');
 const { asyncDeleteFile,
     getEnvName,
@@ -23,15 +21,12 @@ const { asyncDeleteFile,
     obfuscatePII,
     isSessionOngoing,
     isSessionReconnect } = require('./utils/utils');
-const AwsSecretManager = require('./webhooks/AwsSecretManager');
-const WebhookSender = require('./webhooks/WebhookSender');
 const WorkerPool = require('./worker-pool/WorkerPool');
 
 let featPublisher;
 let metadataStorage;
 let tempPath;
 let webhookSender;
-let secretManager;
 let store;
 
 /**
@@ -152,18 +147,6 @@ workerPool.on(ResponseType.ERROR, body => {
     }
 });
 
-/**
- * Initialize the service which will persist the dump files.
- *
- */
-function setupDumpStorage() {
-    if (config.s3?.region && config.s3?.bucket) {
-        store = new S3Manager(config.s3);
-    } else {
-        logger.warn('[App] S3 is not configured!');
-    }
-}
-
 /**
  * Initialize the directory where temporary dump files will be stored.
  */
@@ -211,37 +194,6 @@ function setupWorkDirectory() {
     }
 }
 
-/**
- * Initialize http server exposing prometheus statistics.
- */
-function setupMetricsServer() {
-    const { metrics: port } = config.get('server');
-
-    if (!port) {
-        logger.warn('[App] Metrics server is not configured!');
-
-        return;
-    }
-
-    const metricsServer = http
-        .createServer((request, response) => {
-            switch (request.url) {
-            case '/metrics':
-                PromCollector.queueSize.set(workerPool.getTaskQueueSize());
-                PromCollector.collectDefaultMetrics();
-                response.writeHead(200, { 'Content-Type': PromCollector.getPromContentType() });
-                response.end(PromCollector.metrics());
-                break;
-            default:
-                response.writeHead(404);
-                response.end();
-            }
-        })
-        .listen(port);
-
-    return metricsServer;
-}
-
 /**
  * Main handler for web socket connections.
  * Messages are sent through a node stream which saves them to a dump file.
@@ -367,108 +319,6 @@ function setupWebSocketsServer(wsServer) {
     wss.on('connection', wsConnectionHandler);
 }
 
-/**
- * Handler used for basic availability checks.
- *
- * @param {*} request
- * @param {*} response
- */
-function serverHandler(request, response) {
-    switch (request.url) {
-    case '/healthcheck':
-        response.writeHead(200);
-        response.end();
-        break;
-    case '/bindcheck':
-        logger.info('Accessing bind check!');
-        response.writeHead(200);
-        response.end();
-        break;
-    default:
-        response.writeHead(404);
-        response.end();
-    }
-}
-
-/**
- * In case one wants to run the server locally, https is required, as browsers normally won't allow non
- * secure web sockets on a https domain, so something like the bello
- * server instead of http.
- *
- * @param {number} port
- */
-function setupHttpsServer(port) {
-    const { keyPath, certPath } = config.get('server');
-
-    if (!(keyPath && certPath)) {
-        throw new Error('[App] Please provide certificates for the https server!');
-    }
-
-    const options = {
-        key: fs.readFileSync(keyPath),
-        cert: fs.readFileSync(certPath)
-    };
-
-    return https.createServer(options, serverHandler).listen(port);
-}
-
-/**
- *
- */
-function setupHttpServer(port) {
-    return http.createServer(serverHandler).listen(port);
-}
-
-
-/**
- * Initialize the http or https server used for websocket connections.
- */
-function setupWebServer() {
-    const { useHTTPS, port } = config.get('server');
-
-    if (!port) {
-        throw new Error('[App] Please provide a server port!');
-    }
-
-    let server;
-
-    if (useHTTPS) {
-        server = setupHttpsServer(port);
-    } else {
-        server = setupHttpServer(port);
-    }
-
-    setupWebSocketsServer(server);
-}
-
-/**
- * Initialize service that sends webhooks through the JaaS Webhook API.
- */
-async function setupWebhookSender() {
-    const { webhooks: { apiEndpoint } } = config;
-
-    // If an endpoint is configured enable the webhook sender.
-    if (apiEndpoint && secretManager) {
-        webhookSender = new WebhookSender(config, secretManager);
-        await webhookSender.init();
-    } else {
-        logger.warn('[App] Webhook sender is not configured');
-    }
-}
-
-/**
- * Initialize service responsible with retrieving required secrets..
- */
-function setupSecretManager() {
-    const { secretmanager: { region } = {} } = config;
-
-    if (region) {
-        secretManager = new AwsSecretManager(config);
-    } else {
-        logger.warn('[App] Secret manager is not configured');
-    }
-}
-
 /**
  * Set the services used by the server.
  * @param {FeaturePublisher} featPublisherParam - The feature publisher instance.
@@ -483,21 +333,28 @@ function setServices(featPublisherParam, metadataStorageParam) {
  * Start the RTCStatsServer.
  *
  * @param {FeaturePublisher} featurePublisherParam - The feature publisher instance.
+ * @param {MetadataStorage} metadataStorageParam - The metadata storage instance.
+ * @param {WebhookSender} webhookSenderParam - The webhook sender instance.
+ * @param {Store} storeParam - The store instance.
  */
-async function start(featurePublisherParam, metadataStorageParam) {
+function start({
+    featurePublisherParam,
+    metadataStorageParam,
+    webhookSenderParam,
+    storeParam
+}) {
     logger.info('[App] Initializing: %s; version: %s; env: %s ...', appName, appVersion, getEnvName());
 
     tempPath = config.server.tempPath;
 
     // TODO All dependencies should be injected, this is a temporary solution.
+    featPublisher = featurePublisherParam; // Pass the feature publisher instance
     metadataStorage = metadataStorageParam;
-    setupSecretManager();
-    await setupWebhookSender();
+    webhookSender = webhookSenderParam;
     setupWorkDirectory();
-    setupDumpStorage();
-    featPublisher = featurePublisherParam; // Pass the feature publisher instance
-    setupMetricsServer();
-    setupWebServer();
+    store = storeParam;
+    setupMetricsServer(workerPool);
+    setupWebServer(setupWebSocketsServer);
 
     logger.info('[App] Initialization complete.');
 }
diff --git a/src/ServerSetup.js b/src/ServerSetup.js
new file mode 100644
index 0000000..e834972
--- /dev/null
+++ b/src/ServerSetup.js
@@ -0,0 +1,117 @@
+const config = require('config');
+const fs = require('fs');
+const http = require('http');
+const https = require('https');
+
+const logger = require('./logging');
+const PromCollector = require('./metrics/PromCollector');
+
+
+/**
+ * Initialize http server exposing prometheus statistics.
+ */
+function setupMetricsServer(workerPool) {
+    const { metrics: port } = config.get('server');
+
+    if (!port) {
+        logger.warn('[App] Metrics server is not configured!');
+
+        return;
+    }
+
+    const metricsServer = http
+        .createServer((request, response) => {
+            switch (request.url) {
+            case '/metrics':
+                PromCollector.queueSize.set(workerPool.getTaskQueueSize());
+                PromCollector.collectDefaultMetrics();
+                response.writeHead(200, { 'Content-Type': PromCollector.getPromContentType() });
+                response.end(PromCollector.metrics());
+                break;
+            default:
+                response.writeHead(404);
+                response.end();
+            }
+        })
+        .listen(port);
+
+    return metricsServer;
+}
+
+/**
+ * Handler used for basic availability checks.
+ *
+ * @param {*} request
+ * @param {*} response
+ */
+function serverHandler(request, response) {
+    switch (request.url) {
+    case '/healthcheck':
+        response.writeHead(200);
+        response.end();
+        break;
+    case '/bindcheck':
+        logger.info('Accessing bind check!');
+        response.writeHead(200);
+        response.end();
+        break;
+    default:
+        response.writeHead(404);
+        response.end();
+    }
+}
+
+/**
+ * In case one wants to run the server locally, https is required, as browsers normally won't allow non
+ * secure web sockets on a https domain, so something like the bello
+ * server instead of http.
+ *
+ * @param {number} port
+ */
+function setupHttpsServer(port) {
+    const { keyPath, certPath } = config.get('server');
+
+    if (!(keyPath && certPath)) {
+        throw new Error('[App] Please provide certificates for the https server!');
+    }
+
+    const options = {
+        key: fs.readFileSync(keyPath),
+        cert: fs.readFileSync(certPath)
+    };
+
+    return https.createServer(options, serverHandler).listen(port);
+}
+
+/**
+ *
+ */
+function setupHttpServer(port) {
+    return http.createServer(serverHandler).listen(port);
+}
+
+/**
+ * Initialize the http or https server used for websocket connections.
+ */
+function setupWebServer(setupWebSocketsServer) {
+    const { useHTTPS, port } = config.get('server');
+
+    if (!port) {
+        throw new Error('[App] Please provide a server port!');
+    }
+
+    let server;
+
+    if (useHTTPS) {
+        server = setupHttpsServer(port);
+    } else {
+        server = setupHttpServer(port);
+    }
+
+    setupWebSocketsServer(server);
+}
+
+module.exports = {
+    setupMetricsServer,
+    setupWebServer
+};
diff --git a/src/app.js b/src/app.js
index d9f4c11..d681287 100644
--- a/src/app.js
+++ b/src/app.js
@@ -1,11 +1,12 @@
-const config = require('config');
-
 const { start, stop } = require('./RTCStatsServer');
-const FeaturesPublisher = require('./database/FeaturesPublisher');
-const FirehoseConnector = require('./database/FirehoseConnector');
 const logger = require('./logging');
-const DynamoDataSender = require('./store/DynamoDataSender');
-const MetadataStorageHandler = require('./store/MetadataStorageHandler');
+const {
+    setupFeaturesPublisher,
+    setupMetadataStorageHandler,
+    setupSecretManager,
+    setupWebhookSender,
+    setupDumpStorage
+} = require('./services');
 const { exitAfterLogFlush } = require('./utils/utils');
 
 /**
@@ -28,56 +29,16 @@ process.on('unhandledRejection', async reason => {
     await closeServerAndExit(logger, 1);
 });
 
-/**
- * Initialize the service that will send extracted features to the configured database.
- */
-function setupFeaturesPublisher() {
-    const {
-        firehose = {},
-        server: {
-            appEnvironment
-        }
-    } = config;
-
-    // We use the `region` as a sort of enabled/disabled flag, if this config is set then so to must all other
-    // parameters in the firehose config section, invariant check will fail otherwise and the server
-    // will fail to start.
-    if (firehose.region) {
-        const dbConnector = new FirehoseConnector(firehose);
-
-        const featPublisher = new FeaturesPublisher(dbConnector, appEnvironment);
-
-        return featPublisher;
-    }
-
-    logger.warn('[App] Firehose is not configured!');
-}
-
-
-/**
- * Initialize the service that will handle the storage of metadata entries.
- */
-function setupMetadataStorageHandler() {
-    const {
-        dynamo: {
-            tableName,
-            endpoint
-        } = {},
-        s3: {
-            region
-        } = {}
-    } = config;
-
-    if (tableName && region) {
-        const storageInterface = new DynamoDataSender(region, tableName, endpoint);
-
-        return new MetadataStorageHandler(storageInterface);
-    }
-
-    logger.warn('[App] DynamoDB is not configured!');
-}
 
 const featuresPublisher = setupFeaturesPublisher();
 const metadataStorageHandler = setupMetadataStorageHandler();
-
-start(featuresPublisher, metadataStorageHandler);
+const secretManager = setupSecretManager();
+const webhookSender = setupWebhookSender(secretManager);
+const dumpStorage = setupDumpStorage();
+
+start({
+    featurePublisherParam: featuresPublisher,
+    metadataStorageParam: metadataStorageHandler,
+    webhookSenderParam: webhookSender,
+    storeParam: dumpStorage
+});
diff --git a/src/services.js b/src/services.js
new file mode 100644
index 0000000..08ebec4
--- /dev/null
+++ b/src/services.js
@@ -0,0 +1,117 @@
+const config = require('config');
+
+const FeaturesPublisher = require('./database/FeaturesPublisher');
+const FirehoseConnector = require('./database/FirehoseConnector');
+const logger = require('./logging');
+const DynamoDataSender = require('./store/DynamoDataSender');
+const MetadataStorageHandler = require('./store/MetadataStorageHandler');
+const S3Manager = require('./store/S3Manager');
+const AwsSecretManager = require('./webhooks/AwsSecretManager');
+const WebhookSender = require('./webhooks/WebhookSender');
+
+/**
+ * Initialize the service that will send extracted features to the configured database.
+ */
+function setupFeaturesPublisher() {
+    const {
+        firehose = {},
+        server: {
+            appEnvironment
+        }
+    } = config;
+
+    // We use the `region` as a sort of enabled/disabled flag, if this config is set then so to must all other
+    // parameters in the firehose config section, invariant check will fail otherwise and the server
+    // will fail to start.
+    if (firehose.region) {
+        const dbConnector = new FirehoseConnector(firehose);
+
+        const featPublisher = new FeaturesPublisher(dbConnector, appEnvironment);
+
+        return featPublisher;
+    }
+
+    logger.warn('[App] Firehose is not configured!');
+}
+
+
+/**
+ * Initialize the service that will handle the storage of metadata entries.
+ */
+function setupMetadataStorageHandler() {
+    const {
+        dynamo: {
+            tableName,
+            endpoint
+        } = {},
+        s3: {
+            region
+        } = {}
+    } = config;
+
+    if (tableName && region) {
+        const storageInterface = new DynamoDataSender(region, tableName, endpoint);
+
+        return new MetadataStorageHandler(storageInterface);
+    }
+
+    logger.warn('[App] DynamoDB is not configured!');
+}
+
+/**
+ * Initialize the service which will persist the dump files.
+ *
+ */
+function setupDumpStorage() {
+    let store;
+
+    if (config.s3?.region && config.s3?.bucket) {
+        store = new S3Manager(config.s3);
+    } else {
+        logger.warn('[App] S3 is not configured!');
+    }
+
+    return store;
+}
+
+/**
+ * Initialize service that sends webhooks through the JaaS Webhook API.
+ */
+async function setupWebhookSender(secretManager) {
+    const { webhooks: { apiEndpoint } } = config;
+    let webhookSender;
+
+    // If an endpoint is configured enable the webhook sender.
+    if (apiEndpoint && secretManager) {
+        webhookSender = new WebhookSender(config, secretManager);
+        await webhookSender.init();
+    } else {
+        logger.warn('[App] Webhook sender is not configured');
+    }
+
+    return webhookSender;
+}
+
+/**
+ * Initialize service responsible with retrieving required secrets..
+ */
+function setupSecretManager() {
+    const { secretmanager: { region } = {} } = config;
+    let secretManager;
+
+    if (region) {
+        secretManager = new AwsSecretManager(config);
+    } else {
+        logger.warn('[App] Secret manager is not configured');
+    }
+
+    return secretManager;
+}
+
+module.exports = {
+    setupFeaturesPublisher,
+    setupMetadataStorageHandler,
+    setupDumpStorage,
+    setupWebhookSender,
+    setupSecretManager
+};

From 735fb3106beb860177c64ba28a054b292fe56ce0 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Wed, 23 Jul 2025 02:07:51 +0900
Subject: [PATCH 02/12] fix: fix RTCStatsServer.start() parameters to pass
 integration tests

---
 src/RTCStatsServer.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/RTCStatsServer.js b/src/RTCStatsServer.js
index dd681d3..4d93afb 100644
--- a/src/RTCStatsServer.js
+++ b/src/RTCStatsServer.js
@@ -337,12 +337,12 @@ function setServices(featPublisherParam, metadataStorageParam) {
  * @param {WebhookSender} webhookSenderParam - The webhook sender instance.
  * @param {Store} storeParam - The store instance.
  */
-function start({
-    featurePublisherParam,
-    metadataStorageParam,
-    webhookSenderParam,
-    storeParam
-}) {
+function start(
+        featurePublisherParam,
+        metadataStorageParam,
+        webhookSenderParam,
+        storeParam
+) {
     logger.info('[App] Initializing: %s; version: %s; env: %s ...', appName, appVersion, getEnvName());
 
     tempPath = config.server.tempPath;

From ce5e67a2f7b861e10dd4a88139ef502b23c83480 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Wed, 23 Jul 2025 02:10:26 +0900
Subject: [PATCH 03/12] feat: Add mongodb and mongoose packages

---
 package-lock.json | 327 ++++++++++++++++++++++++++++++++++++++++++++--
 package.json      |   2 +
 2 files changed, 319 insertions(+), 10 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 63952c2..b0ea11c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -17,6 +17,8 @@
         "get-folder-size": "^2.0.1",
         "JSONStream": "^1.3.5",
         "jsonwebtoken": "^8.5.1",
+        "mongodb": "^6.17.0",
+        "mongoose": "^8.16.4",
         "object-sizeof": "^1.6.1",
         "platform": "^1.3.6",
         "prom-client": "^11.3.0",
@@ -1451,6 +1453,14 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@mongodb-js/saslprep": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.3.0.tgz",
+      "integrity": "sha512-zlayKCsIjYb7/IdfqxorK5+xUMyi4vOKcFy10wKJYc63NSdKI8mNME+uJqfatkPmOSMMUiojrL58IePKBm3gvQ==",
+      "dependencies": {
+        "sparse-bitfield": "^3.0.3"
+      }
+    },
     "node_modules/@sinclair/typebox": {
       "version": "0.23.5",
       "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.23.5.tgz",
@@ -1573,6 +1583,19 @@
       "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==",
       "dev": true
     },
+    "node_modules/@types/webidl-conversions": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz",
+      "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA=="
+    },
+    "node_modules/@types/whatwg-url": {
+      "version": "11.0.5",
+      "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz",
+      "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==",
+      "dependencies": {
+        "@types/webidl-conversions": "*"
+      }
+    },
     "node_modules/@types/yargs": {
       "version": "17.0.10",
       "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.10.tgz",
@@ -2058,6 +2081,14 @@
         "node-int64": "^0.4.0"
       }
     },
+    "node_modules/bson": {
+      "version": "6.10.4",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz",
+      "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==",
+      "engines": {
+        "node": ">=16.20.1"
+      }
+    },
     "node_modules/buffer": {
       "version": "4.9.2",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz",
@@ -5677,6 +5708,14 @@
         "safe-buffer": "^5.0.1"
       }
     },
+    "node_modules/kareem": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.6.3.tgz",
+      "integrity": "sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==",
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/kleur": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
@@ -5831,6 +5870,11 @@
         "tmpl": "1.0.5"
       }
     },
+    "node_modules/memory-pager": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
+      "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg=="
+    },
     "node_modules/merge-stream": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@@ -5922,6 +5966,105 @@
         "node": "*"
       }
     },
+    "node_modules/mongodb": {
+      "version": "6.17.0",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.17.0.tgz",
+      "integrity": "sha512-neerUzg/8U26cgruLysKEjJvoNSXhyID3RvzvdcpsIi2COYM3FS3o9nlH7fxFtefTb942dX3W9i37oPfCVj4wA==",
+      "dependencies": {
+        "@mongodb-js/saslprep": "^1.1.9",
+        "bson": "^6.10.4",
+        "mongodb-connection-string-url": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=16.20.1"
+      },
+      "peerDependencies": {
+        "@aws-sdk/credential-providers": "^3.188.0",
+        "@mongodb-js/zstd": "^1.1.0 || ^2.0.0",
+        "gcp-metadata": "^5.2.0",
+        "kerberos": "^2.0.1",
+        "mongodb-client-encryption": ">=6.0.0 <7",
+        "snappy": "^7.2.2",
+        "socks": "^2.7.1"
+      },
+      "peerDependenciesMeta": {
+        "@aws-sdk/credential-providers": {
+          "optional": true
+        },
+        "@mongodb-js/zstd": {
+          "optional": true
+        },
+        "gcp-metadata": {
+          "optional": true
+        },
+        "kerberos": {
+          "optional": true
+        },
+        "mongodb-client-encryption": {
+          "optional": true
+        },
+        "snappy": {
+          "optional": true
+        },
+        "socks": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/mongodb-connection-string-url": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz",
+      "integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==",
+      "dependencies": {
+        "@types/whatwg-url": "^11.0.2",
+        "whatwg-url": "^14.1.0 || ^13.0.0"
+      }
+    },
+    "node_modules/mongoose": {
+      "version": "8.16.4",
+      "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.16.4.tgz",
+      "integrity": "sha512-jslgdQ8pY2vcNSKPv3Dbi5ogo/NT8zcvf6kPDyD8Sdsjsa1at3AFAF0F5PT+jySPGSPbvlNaQ49nT9h+Kx2UDA==",
+      "dependencies": {
+        "bson": "^6.10.4",
+        "kareem": "2.6.3",
+        "mongodb": "~6.17.0",
+        "mpath": "0.9.0",
+        "mquery": "5.0.0",
+        "ms": "2.1.3",
+        "sift": "17.1.3"
+      },
+      "engines": {
+        "node": ">=16.20.1"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/mongoose"
+      }
+    },
+    "node_modules/mongoose/node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+    },
+    "node_modules/mpath": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz",
+      "integrity": "sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/mquery": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/mquery/-/mquery-5.0.0.tgz",
+      "integrity": "sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==",
+      "dependencies": {
+        "debug": "4.x"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
     "node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -6453,11 +6596,9 @@
       "dev": true
     },
     "node_modules/punycode": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
-      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
-      "dev": true,
-      "peer": true,
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
+      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
       "engines": {
         "node": ">=6"
       }
@@ -6693,6 +6834,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/sift": {
+      "version": "17.1.3",
+      "resolved": "https://registry.npmjs.org/sift/-/sift-17.1.3.tgz",
+      "integrity": "sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ=="
+    },
     "node_modules/signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -6765,6 +6911,14 @@
         "source-map": "^0.6.0"
       }
     },
+    "node_modules/sparse-bitfield": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
+      "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+      "dependencies": {
+        "memory-pager": "^1.0.2"
+      }
+    },
     "node_modules/sprintf-js": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@@ -7123,6 +7277,17 @@
         "nodetouch": "bin/nodetouch.js"
       }
     },
+    "node_modules/tr46": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz",
+      "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==",
+      "dependencies": {
+        "punycode": "^2.3.1"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/triple-beam": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz",
@@ -7305,6 +7470,26 @@
         "makeerror": "1.0.12"
       }
     },
+    "node_modules/webidl-conversions": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+      "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/whatwg-url": {
+      "version": "14.2.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz",
+      "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==",
+      "dependencies": {
+        "tr46": "^5.1.0",
+        "webidl-conversions": "^7.0.0"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@@ -8601,6 +8786,14 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "@mongodb-js/saslprep": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.3.0.tgz",
+      "integrity": "sha512-zlayKCsIjYb7/IdfqxorK5+xUMyi4vOKcFy10wKJYc63NSdKI8mNME+uJqfatkPmOSMMUiojrL58IePKBm3gvQ==",
+      "requires": {
+        "sparse-bitfield": "^3.0.3"
+      }
+    },
     "@sinclair/typebox": {
       "version": "0.23.5",
       "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.23.5.tgz",
@@ -8723,6 +8916,19 @@
       "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==",
       "dev": true
     },
+    "@types/webidl-conversions": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz",
+      "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA=="
+    },
+    "@types/whatwg-url": {
+      "version": "11.0.5",
+      "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz",
+      "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==",
+      "requires": {
+        "@types/webidl-conversions": "*"
+      }
+    },
     "@types/yargs": {
       "version": "17.0.10",
       "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.10.tgz",
@@ -9087,6 +9293,11 @@
         "node-int64": "^0.4.0"
       }
     },
+    "bson": {
+      "version": "6.10.4",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz",
+      "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng=="
+    },
     "buffer": {
       "version": "4.9.2",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz",
@@ -11790,6 +12001,11 @@
         "safe-buffer": "^5.0.1"
       }
     },
+    "kareem": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.6.3.tgz",
+      "integrity": "sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q=="
+    },
     "kleur": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz",
@@ -11920,6 +12136,11 @@
         "tmpl": "1.0.5"
       }
     },
+    "memory-pager": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
+      "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg=="
+    },
     "merge-stream": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@@ -11984,6 +12205,59 @@
       "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
       "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w=="
     },
+    "mongodb": {
+      "version": "6.17.0",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.17.0.tgz",
+      "integrity": "sha512-neerUzg/8U26cgruLysKEjJvoNSXhyID3RvzvdcpsIi2COYM3FS3o9nlH7fxFtefTb942dX3W9i37oPfCVj4wA==",
+      "requires": {
+        "@mongodb-js/saslprep": "^1.1.9",
+        "bson": "^6.10.4",
+        "mongodb-connection-string-url": "^3.0.0"
+      }
+    },
+    "mongodb-connection-string-url": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz",
+      "integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==",
+      "requires": {
+        "@types/whatwg-url": "^11.0.2",
+        "whatwg-url": "^14.1.0 || ^13.0.0"
+      }
+    },
+    "mongoose": {
+      "version": "8.16.4",
+      "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.16.4.tgz",
+      "integrity": "sha512-jslgdQ8pY2vcNSKPv3Dbi5ogo/NT8zcvf6kPDyD8Sdsjsa1at3AFAF0F5PT+jySPGSPbvlNaQ49nT9h+Kx2UDA==",
+      "requires": {
+        "bson": "^6.10.4",
+        "kareem": "2.6.3",
+        "mongodb": "~6.17.0",
+        "mpath": "0.9.0",
+        "mquery": "5.0.0",
+        "ms": "2.1.3",
+        "sift": "17.1.3"
+      },
+      "dependencies": {
+        "ms": {
+          "version": "2.1.3",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+          "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+        }
+      }
+    },
+    "mpath": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz",
+      "integrity": "sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew=="
+    },
+    "mquery": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/mquery/-/mquery-5.0.0.tgz",
+      "integrity": "sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==",
+      "requires": {
+        "debug": "4.x"
+      }
+    },
     "ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -12375,11 +12649,9 @@
       "dev": true
     },
     "punycode": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
-      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
-      "dev": true,
-      "peer": true
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
+      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="
     },
     "qs": {
       "version": "6.10.3",
@@ -12544,6 +12816,11 @@
         "object-inspect": "^1.9.0"
       }
     },
+    "sift": {
+      "version": "17.1.3",
+      "resolved": "https://registry.npmjs.org/sift/-/sift-17.1.3.tgz",
+      "integrity": "sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ=="
+    },
     "signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -12608,6 +12885,14 @@
         "source-map": "^0.6.0"
       }
     },
+    "sparse-bitfield": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
+      "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+      "requires": {
+        "memory-pager": "^1.0.2"
+      }
+    },
     "sprintf-js": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@@ -12878,6 +13163,14 @@
         "nopt": "~1.0.10"
       }
     },
+    "tr46": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz",
+      "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==",
+      "requires": {
+        "punycode": "^2.3.1"
+      }
+    },
     "triple-beam": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz",
@@ -13020,6 +13313,20 @@
         "makeerror": "1.0.12"
       }
     },
+    "webidl-conversions": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+      "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="
+    },
+    "whatwg-url": {
+      "version": "14.2.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz",
+      "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==",
+      "requires": {
+        "tr46": "^5.1.0",
+        "webidl-conversions": "^7.0.0"
+      }
+    },
     "which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
diff --git a/package.json b/package.json
index 0c7ff7c..40726e3 100644
--- a/package.json
+++ b/package.json
@@ -34,6 +34,8 @@
     "get-folder-size": "^2.0.1",
     "JSONStream": "^1.3.5",
     "jsonwebtoken": "^8.5.1",
+    "mongodb": "^6.17.0",
+    "mongoose": "^8.16.4",
     "object-sizeof": "^1.6.1",
     "platform": "^1.3.6",
     "prom-client": "^11.3.0",

From 5ed0421106f93be0f7cbd58376d6cde49efca176 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Wed, 23 Jul 2025 02:52:18 +0900
Subject: [PATCH 04/12] feat: Add MongoDB configuration variables

---
 config/custom-environment-variables.yaml |  9 ++++++++-
 config/default.yaml                      | 10 +++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/config/custom-environment-variables.yaml b/config/custom-environment-variables.yaml
index 5132402..2073e96 100644
--- a/config/custom-environment-variables.yaml
+++ b/config/custom-environment-variables.yaml
@@ -1,6 +1,7 @@
 server:
     logLevel: RTCSTATS_LOG_LEVEL
     appEnvironment: RTCSTATS_ENVIRONMENT
+    serviceType: RTCSTATS_SERVICE_TYPE
 
 amplitude:
     key: RTCSTATS_AMPLITUDE_KEY
@@ -30,4 +31,10 @@ webhooks:
 
 secretmanager:
     region: RTCSTATS_AWS_SECRET_REGION
-    jwtSecretId: RTCSTATS_JWT_SECRET_ID
\ No newline at end of file
+    jwtSecretId: RTCSTATS_JWT_SECRET_ID
+
+mongodb:
+    uri: RTCSTATS_MONGODB_URI
+    dbName: RTCSTATS_MONGODB_NAME
+    collectionName: RTCSTATS_METADATA_COLLECTION
+    gridfsBucketName: RTCSTATS_GRIDFS_BUCKET
\ No newline at end of file
diff --git a/config/default.yaml b/config/default.yaml
index 71d4d5c..6e3ca6c 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -12,6 +12,8 @@ server:
     certPath: './certs/cert.pem'
     tempPath : 'temp'
     appEnvironment: dev
+    # Set the service type. Can be "AWS" or "MongoDB".
+    serviceType: AWS
 
 features:
     disableFeatExtraction: false
@@ -66,4 +68,10 @@ webhooks:
 
 secretmanager:
     region:
-    jwtSecretId:
\ No newline at end of file
+    jwtSecretId:
+
+mongodb:
+    uri:
+    dbName:
+    collectionName:
+    gridfsBucketName:
\ No newline at end of file

From f37a85bdd9ec6644a7772ed30748750b9e44a5f2 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Wed, 23 Jul 2025 02:58:23 +0900
Subject: [PATCH 05/12] feat: add Prometheus counter for MongoDB error count

---
 src/metrics/PromCollector.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/metrics/PromCollector.js b/src/metrics/PromCollector.js
index 555d375..75158c1 100644
--- a/src/metrics/PromCollector.js
+++ b/src/metrics/PromCollector.js
@@ -40,6 +40,11 @@ const PromCollector = {
         help: 'number of firehose put fails'
     }),
 
+    mongodbErrorCount: new prom.Counter({
+        name: 'rtcstats_mongodb_error_count',
+        help: 'number of mongodb inserts failed'
+    }),
+
     processErrorCount: new prom.Counter({
         name: 'rtcstats_process_error_count',
         help: 'number of files with errors during processing'

From 41549f7a8c202f8987923cf57cf83dbd0a3a93e1 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Wed, 23 Jul 2025 15:30:58 +0900
Subject: [PATCH 06/12] fix: properly await setupWebhookSender initialization

---
 src/app.js | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/app.js b/src/app.js
index d681287..e1da060 100644
--- a/src/app.js
+++ b/src/app.js
@@ -30,15 +30,23 @@ process.on('unhandledRejection', async reason => {
 });
 
 
-const featuresPublisher = setupFeaturesPublisher();
-const metadataStorageHandler = setupMetadataStorageHandler();
-const secretManager = setupSecretManager();
-const webhookSender = setupWebhookSender(secretManager);
-const dumpStorage = setupDumpStorage();
-
-start({
-    featurePublisherParam: featuresPublisher,
-    metadataStorageParam: metadataStorageHandler,
-    webhookSenderParam: webhookSender,
-    storeParam: dumpStorage
-});
+/**
+ * Starts the RTCStats server and initializes the services.
+ */
+async function main() {
+
+    const featuresPublisher = setupFeaturesPublisher();
+    const metadataStorageHandler = setupMetadataStorageHandler();
+    const secretManager = setupSecretManager();
+    const webhookSender = await setupWebhookSender(secretManager);
+    const dumpStorage = setupDumpStorage();
+
+    start({
+        featuresPublisherParam: featuresPublisher,
+        metadataStorageParam: metadataStorageHandler,
+        webhookSenderParam: webhookSender,
+        storeParam: dumpStorage
+    });
+}
+
+main();

From 91696ee8fbd1df174d0b7091d0c2f01de5e20e8d Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Thu, 24 Jul 2025 19:45:45 +0900
Subject: [PATCH 07/12] feat: add mongodb connector

---
 src/RTCStatsServer.js                |   2 +-
 src/app.js                           |  34 ++++-----
 src/{services.js => services/AWS.js} |  16 ++---
 src/services/MongoDB.js              |  78 ++++++++++++++++++++
 src/services/index.js                |  59 +++++++++++++++
 src/store/GridFSManager.js           |  67 +++++++++++++++++
 src/store/MongoDataSender.js         | 104 +++++++++++++++++++++++++++
 7 files changed, 331 insertions(+), 29 deletions(-)
 rename src/{services.js => services/AWS.js} (84%)
 create mode 100644 src/services/MongoDB.js
 create mode 100644 src/services/index.js
 create mode 100644 src/store/GridFSManager.js
 create mode 100644 src/store/MongoDataSender.js

diff --git a/src/RTCStatsServer.js b/src/RTCStatsServer.js
index 4d93afb..cbf31e4 100644
--- a/src/RTCStatsServer.js
+++ b/src/RTCStatsServer.js
@@ -48,7 +48,7 @@ async function storeDump(sinkMeta, uniqueClientId) {
 
     try {
 
-        logger.info(`[S3] Storing dump ${uniqueClientId} with path ${dumpPath}`);
+        logger.info(`[App] Storing dump ${uniqueClientId} with path ${dumpPath}`);
 
         await store?.put(uniqueClientId, dumpPath);
 
diff --git a/src/app.js b/src/app.js
index e1da060..03ad699 100644
--- a/src/app.js
+++ b/src/app.js
@@ -1,12 +1,6 @@
 const { start, stop } = require('./RTCStatsServer');
 const logger = require('./logging');
-const {
-    setupFeaturesPublisher,
-    setupMetadataStorageHandler,
-    setupSecretManager,
-    setupWebhookSender,
-    setupDumpStorage
-} = require('./services');
+const { setupServices } = require('./services');
 const { exitAfterLogFlush } = require('./utils/utils');
 
 /**
@@ -34,19 +28,19 @@ process.on('unhandledRejection', async reason => {
  * Starts the RTCStats server and initializes the services.
  */
 async function main() {
-
-    const featuresPublisher = setupFeaturesPublisher();
-    const metadataStorageHandler = setupMetadataStorageHandler();
-    const secretManager = setupSecretManager();
-    const webhookSender = await setupWebhookSender(secretManager);
-    const dumpStorage = setupDumpStorage();
-
-    start({
-        featuresPublisherParam: featuresPublisher,
-        metadataStorageParam: metadataStorageHandler,
-        webhookSenderParam: webhookSender,
-        storeParam: dumpStorage
-    });
+    const {
+        featuresPublisher,
+        metadataStorageHandler,
+        webhookSender,
+        dumpStorage
+    } = await setupServices();
+
+    start(
+        featuresPublisher,
+        metadataStorageHandler,
+        webhookSender,
+        dumpStorage
+    );
 }
 
 main();
diff --git a/src/services.js b/src/services/AWS.js
similarity index 84%
rename from src/services.js
rename to src/services/AWS.js
index 08ebec4..2076ae6 100644
--- a/src/services.js
+++ b/src/services/AWS.js
@@ -1,13 +1,13 @@
 const config = require('config');
 
-const FeaturesPublisher = require('./database/FeaturesPublisher');
-const FirehoseConnector = require('./database/FirehoseConnector');
-const logger = require('./logging');
-const DynamoDataSender = require('./store/DynamoDataSender');
-const MetadataStorageHandler = require('./store/MetadataStorageHandler');
-const S3Manager = require('./store/S3Manager');
-const AwsSecretManager = require('./webhooks/AwsSecretManager');
-const WebhookSender = require('./webhooks/WebhookSender');
+const FeaturesPublisher = require('../database/FeaturesPublisher');
+const FirehoseConnector = require('../database/FirehoseConnector');
+const logger = require('../logging');
+const DynamoDataSender = require('../store/DynamoDataSender');
+const MetadataStorageHandler = require('../store/MetadataStorageHandler');
+const S3Manager = require('../store/S3Manager');
+const AwsSecretManager = require('../webhooks/AwsSecretManager');
+const WebhookSender = require('../webhooks/WebhookSender');
 
 /**
  * Initialize the service that will send extracted features to the configured database.
diff --git a/src/services/MongoDB.js b/src/services/MongoDB.js
new file mode 100644
index 0000000..f643c12
--- /dev/null
+++ b/src/services/MongoDB.js
@@ -0,0 +1,78 @@
+const assert = require('assert').strict;
+const config = require('config');
+const mongoose = require('mongoose');
+
+const logger = require('../logging');
+const GridFSManager = require('../store/GridFSManager');
+const MetadataStorageHandler = require('../store/MetadataStorageHandler');
+const MongoDataSender = require('../store/MongoDataSender');
+
+/**
+ * Connects to a MongoDB database using Mongoose.
+ */
+async function connectToMongoDB() {
+    const {
+        mongodb: {
+            uri,
+            dbName
+        } = {}
+    } = config;
+
+    assert(uri, 'MongoDB URI is required when initializing MongoDB');
+    assert(dbName, 'Database name is required when initializing MongoDB');
+
+    try {
+        await mongoose.connect(uri, { dbName });
+        logger.info('[App] MongoDB connected');
+    } catch (err) {
+        logger.error('[App] MongoDB connection error:', err);
+        throw err;
+    }
+}
+
+/**
+ * Initialize the service that will handle the storage of metadata entries.
+ */
+function setupMetadataStorageHandler() {
+    const {
+        mongodb: {
+            collectionName
+        } = {}
+    } = config;
+
+    if (collectionName) {
+        const storageInterface = new MongoDataSender(collectionName);
+
+        return new MetadataStorageHandler(storageInterface);
+    }
+
+    logger.warn('[App] MongoDB is not configured!');
+}
+
+/**
+ * Initialize the service which will persist the dump files.
+ *
+ */
+function setupDumpStorage() {
+    let store;
+
+    const {
+        mongodb: {
+            gridfsBucketName
+        } = {}
+    } = config;
+
+    if (gridfsBucketName) {
+        store = new GridFSManager(gridfsBucketName);
+    } else {
+        logger.warn('[App] MongoDB GridFS is not configured!');
+    }
+
+    return store;
+}
+
+module.exports = {
+    connectToMongoDB,
+    setupMetadataStorageHandler,
+    setupDumpStorage
+};
diff --git a/src/services/index.js b/src/services/index.js
new file mode 100644
index 0000000..c60b63a
--- /dev/null
+++ b/src/services/index.js
@@ -0,0 +1,59 @@
+const config = require('config');
+
+const logger = require('../logging');
+
+const aws = require('./AWS');
+const mongodb = require('./MongoDB');
+
+/**
+ * @returns {Object} An object containing the initialized services.
+ * This includes:
+ * - featuresPublisher: The service that publishes features.
+ * - metadataStorageHandler: The service that handles metadata storage.
+ * - webhookSender: The service that sends webhooks.
+ * - dumpStorage: The service that stores dumps.
+ */
+async function setupServices() {
+
+    const serviceType = config.server.serviceType;
+
+    switch (serviceType) {
+    case 'AWS': {
+        logger.info('[App] Initializing AWS services...');
+
+        const featuresPublisher = aws.setupFeaturesPublisher();
+        const metadataStorageHandler = aws.setupMetadataStorageHandler();
+        const secretManager = aws.setupSecretManager();
+        const webhookSender = await aws.setupWebhookSender(secretManager);
+        const dumpStorage = aws.setupDumpStorage();
+
+        return {
+            featuresPublisher,
+            metadataStorageHandler,
+            webhookSender,
+            dumpStorage
+        };
+    }
+
+    case 'MongoDB': {
+        logger.info('[App] Initializing MongoDB services...');
+
+        await mongodb.connectToMongoDB();
+        const metadataStorageHandler = mongodb.setupMetadataStorageHandler();
+        const dumpStorage = mongodb.setupDumpStorage();
+
+        return {
+            metadataStorageHandler,
+            dumpStorage
+        };
+    }
+    default:
+        logger.warn(`[App] Unknown service type: ${serviceType}`);
+        throw new Error(`Unknown service type: ${serviceType}`);
+    }
+
+}
+
+module.exports = {
+    setupServices
+};
diff --git a/src/store/GridFSManager.js b/src/store/GridFSManager.js
new file mode 100644
index 0000000..c1e9173
--- /dev/null
+++ b/src/store/GridFSManager.js
@@ -0,0 +1,67 @@
+const { assert } = require('console');
+const fs = require('fs');
+const { GridFSBucket } = require('mongodb');
+const mongoose = require('mongoose');
+const zlib = require('zlib');
+
+const logger = require('../logging');
+
+/**
+ * GridFsManager is a class that wraps the MongoDB GridFS functionality
+ *
+ * @class GridFSManager
+ */
+class GridFSManager {
+
+    /**
+     * C'tor
+     * @param {string} bucketName - The name of the GridFS bucket
+     */
+    constructor(bucketName) {
+
+        assert(bucketName, 'GridFS bucket name is required when initializing GridFSManager');
+
+        // Connection needs to be established before using GridFSManager
+        if (mongoose.connection.readyState !== 1) {
+            throw new Error('[GridFS] MongoDB connection is not established.');
+        }
+
+        const db = mongoose.connection.db;
+
+        this._bucket = new GridFSBucket(db, { bucketName });
+
+    }
+
+    /**
+     * Puts a file to an GridFS bucket and compresses it using gzip.
+     *
+     * @param {string} key - the key to be used to store the file in GridFS bucket
+     * @param {string} filename - path of the file that needs to be uploaded to the GridFS bucket
+     * @returns {Promise} - A promise that is resolved when the file is successfully uploaded to the GridFS bucket.
+     */
+    put(key, filename) {
+
+        return new Promise((resolve, reject) => {
+            const readStream = fs.createReadStream(filename);
+            const gzipStream = zlib.createGzip();
+
+            const uploadStream = this._bucket.openUploadStream(key, {
+                contentType: 'application/gzip'
+            });
+
+            readStream.pipe(gzipStream).pipe(uploadStream);
+
+            uploadStream.on('finish', () => {
+                logger.info(`[GridFS] Successfully uploaded ${key}.`);
+                resolve();
+            });
+
+            uploadStream.on('error', err => {
+                logger.error(`[GridFS] Error uploading ${key}. Error: %o`, err);
+                reject(err);
+            });
+        });
+    }
+}
+
+module.exports = GridFSManager;
diff --git a/src/store/MongoDataSender.js b/src/store/MongoDataSender.js
new file mode 100644
index 0000000..73e12c3
--- /dev/null
+++ b/src/store/MongoDataSender.js
@@ -0,0 +1,104 @@
+const assert = require('assert').strict;
+const mongoose = require('mongoose');
+
+const logger = require('../logging');
+const PromCollector = require('../metrics/PromCollector');
+
+
+/**
+ * Class representing a MongoDataSender.
+ * @class
+ */
+class MongoDataSender {
+
+    /**
+      * Represents a MongoDataSender object that is responsible for sending data to MongoDB.
+      * @param {string} collectionName - The name of the collection to store metadata.
+      * @constructor
+      */
+    constructor(collectionName) {
+
+        assert(collectionName, 'Collection Name is required when initializing MongoDB');
+
+        // Connection needs to be established before using MongoDataSender
+        if (mongoose.connection.readyState !== 1) {
+            throw new Error('[MongoDB] Connection is not established.');
+        }
+
+        const metadataSchema = new mongoose.Schema({
+            conferenceId: {
+                type: String,
+                index: true
+            },
+            conferenceUrl: {
+                type: String,
+                index: true
+            },
+            dumpId: String,
+            userId: String,
+            app: String,
+            sessionId: {
+                type: String,
+                index: true
+            },
+            logsId: String,
+            startDate: {
+                type: Number,
+                index: true
+            },
+            endDate: Number
+        }, {
+            collection: collectionName,
+            versionKey: false
+        });
+
+        this._model = mongoose.models.Metadata || mongoose.model('Metadata', metadataSchema);
+
+    }
+
+    /**
+     *
+     * Saves an entry to MongoDB.
+     * @param {Object} entry - The entry to be saved.
+     * @param {string} entry.dumpId - The client ID.
+     * @param {string} entry.conferenceId - The conference ID.
+     * @param {string} entry.conferenceUrl - The conference URL.
+     * @param {string} entry.userId - The user ID.
+     * @param {string} entry.app - The app.
+     * @param {string} entry.sessionId - The session ID.
+     * @param {string} entry.baseDumpId - The base dump ID.
+     * @param {number} entry.startDate - The start date.
+     * @param {number} entry.endDate - The end date.
+     * @returns {Promise<boolean>} - A promise that resolves to true if the entry is saved successfully,
+     * or false if there is a duplicate entry.
+     */
+    async saveEntry(entry) {
+        const { dumpId } = entry;
+
+        logger.info('[MongoDB] Saving metadata for statsSessionId:', dumpId);
+
+        try {
+            const document = new this._model(entry);
+
+            logger.info('[MongoDB] Saving metadata for statsSessionId:', dumpId, 'with data:', entry);
+
+            await document.save();
+            logger.info('[MongoDB] Saved metadata for statsSessionId:', dumpId);
+
+            return true;
+        } catch (error) {
+            // MongoDB returns this error code in case there is a duplicate entry
+            if (error.code === 11000) {
+                logger.warn('[MongoDB] duplicate entry for statsSessionId: %s; error: %o', dumpId, error);
+
+                return false;
+            }
+
+            PromCollector.mongodbErrorCount.inc();
+
+            throw error;
+        }
+    }
+}
+
+module.exports = MongoDataSender;

From 1abae6b51e6878a1eb1e0273a7f6a20f9ee84cc9 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Thu, 21 Aug 2025 15:41:50 +0900
Subject: [PATCH 08/12] fix: use assert module instead of console.assert

---
 src/store/GridFSManager.js | 2 +-
 src/store/S3Manager.js     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/store/GridFSManager.js b/src/store/GridFSManager.js
index c1e9173..1d274e2 100644
--- a/src/store/GridFSManager.js
+++ b/src/store/GridFSManager.js
@@ -1,4 +1,4 @@
-const { assert } = require('console');
+const assert = require('assert');
 const fs = require('fs');
 const { GridFSBucket } = require('mongodb');
 const mongoose = require('mongoose');
diff --git a/src/store/S3Manager.js b/src/store/S3Manager.js
index d23da97..d6488c5 100644
--- a/src/store/S3Manager.js
+++ b/src/store/S3Manager.js
@@ -1,5 +1,5 @@
+const assert = require('assert');
 const AWS = require('aws-sdk');
-const { assert } = require('console');
 const fs = require('fs');
 const zlib = require('zlib');
 

From 97af5118aff69b2fb238bc7906cf9d6c0a1b1f36 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Thu, 21 Aug 2025 15:44:42 +0900
Subject: [PATCH 09/12] chore: add jest env to .eslintrc.js

---
 .eslintrc.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.eslintrc.js b/.eslintrc.js
index ba7ba9d..280edf4 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -8,5 +8,8 @@ module.exports = {
     globals: {
         'process': true,
         '__dirname': true
+    },
+    env: {
+        jest: true
     }
 };

From 4869d34dcf1c4a9376e665080e8a426c5ed97c62 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Thu, 21 Aug 2025 15:53:30 +0900
Subject: [PATCH 10/12] test: add unit tests for store

---
 src/store/test/jest/DynamoDataSender.test.js | 77 ++++++++++++++++++
 src/store/test/jest/GridFSManager.test.js    | 73 +++++++++++++++++
 src/store/test/jest/MongoDataSender.test.js  | 69 ++++++++++++++++
 src/store/test/jest/S3Manager.test.js        | 83 ++++++++++++++++++++
 src/store/test/mock/PromCollector.js         |  8 ++
 src/store/test/mock/aws-sdk.js               | 24 ++++++
 src/store/test/mock/dynamoose.js             | 27 +++++++
 src/store/test/mock/fs.js                    | 10 +++
 src/store/test/mock/logging.js               |  5 ++
 src/store/test/mock/mongodb.js               | 15 ++++
 src/store/test/mock/mongoose.js              | 19 +++++
 src/store/test/mock/zlib.js                  | 10 +++
 12 files changed, 420 insertions(+)
 create mode 100644 src/store/test/jest/DynamoDataSender.test.js
 create mode 100644 src/store/test/jest/GridFSManager.test.js
 create mode 100644 src/store/test/jest/MongoDataSender.test.js
 create mode 100644 src/store/test/jest/S3Manager.test.js
 create mode 100644 src/store/test/mock/PromCollector.js
 create mode 100644 src/store/test/mock/aws-sdk.js
 create mode 100644 src/store/test/mock/dynamoose.js
 create mode 100644 src/store/test/mock/fs.js
 create mode 100644 src/store/test/mock/logging.js
 create mode 100644 src/store/test/mock/mongodb.js
 create mode 100644 src/store/test/mock/mongoose.js
 create mode 100644 src/store/test/mock/zlib.js

diff --git a/src/store/test/jest/DynamoDataSender.test.js b/src/store/test/jest/DynamoDataSender.test.js
new file mode 100644
index 0000000..e24cb27
--- /dev/null
+++ b/src/store/test/jest/DynamoDataSender.test.js
@@ -0,0 +1,77 @@
+const DynamoDataSender = require('../../DynamoDataSender');
+const PromCollector = require('../mock/PromCollector');
+const { mockSave, mockDocument } = require('../mock/dynamoose');
+
+jest.mock('dynamoose', () => require('../mock/dynamoose'));
+jest.mock('../../../metrics/PromCollector', () => require('../mock/PromCollector'));
+
+describe('DynamoDataSender', () => {
+    const REGION = 'test-region';
+    const TABLE_NAME = 'test-table';
+    const ENDPOINT = 'http://localhost:8000';
+
+    beforeEach(() => {
+        jest.clearAllMocks();
+    });
+
+    describe('constructor', () => {
+        it('should throw an error if region is not provided', () => {
+            expect(() => new DynamoDataSender(null, TABLE_NAME)).toThrow('Region is required');
+        });
+
+        it('should throw an error if table name is not provided', () => {
+            expect(() => new DynamoDataSender(REGION, null)).toThrow('Table name is required');
+        });
+
+        it('should configure dynamoose with local endpoint if provided', () => {
+            const dynamoose = require('dynamoose');
+
+            // eslint-disable-next-line no-new
+            new DynamoDataSender(REGION, TABLE_NAME, ENDPOINT);
+            expect(dynamoose.aws.ddb.local).toHaveBeenCalledWith(ENDPOINT);
+        });
+
+        it('should not configure local endpoint if not provided', () => {
+            const dynamoose = require('dynamoose');
+
+            // eslint-disable-next-line no-new
+            new DynamoDataSender(REGION, TABLE_NAME);
+            expect(dynamoose.aws.ddb.local).not.toHaveBeenCalled();
+        });
+    });
+
+    describe('saveEntry', () => {
+        let sender;
+        const entry = { dumpId: 'test-dump-id' };
+
+        beforeEach(() => {
+            sender = new DynamoDataSender(REGION, TABLE_NAME);
+        });
+
+        it('should return true on successful save', async () => {
+            mockSave.mockResolvedValue(true);
+
+            await expect(sender.saveEntry(entry)).resolves.toBe(true);
+            expect(mockDocument).toHaveBeenCalledWith(entry);
+            expect(mockSave).toHaveBeenCalledWith({ overwrite: false });
+        });
+
+        it('should return false if a ConditionalCheckFailedException occurs', async () => {
+            const duplicateError = new Error('Duplicate entry');
+
+            duplicateError.code = 'ConditionalCheckFailedException';
+            mockSave.mockRejectedValue(duplicateError);
+
+            await expect(sender.saveEntry(entry)).resolves.toBe(false);
+        });
+
+        it('should re-throw other errors and increment metric', async () => {
+            const genericError = new Error('Something went wrong');
+
+            mockSave.mockRejectedValue(genericError);
+
+            await expect(sender.saveEntry(entry)).rejects.toThrow('Something went wrong');
+            expect(PromCollector.dynamoErrorCount.inc).toHaveBeenCalledTimes(1);
+        });
+    });
+});
diff --git a/src/store/test/jest/GridFSManager.test.js b/src/store/test/jest/GridFSManager.test.js
new file mode 100644
index 0000000..b1fa087
--- /dev/null
+++ b/src/store/test/jest/GridFSManager.test.js
@@ -0,0 +1,73 @@
+const GridFSManager = require('../../GridFSManager');
+const fs = require('../mock/fs');
+const { GridFSBucket, mockUploadStream, mockBucketInstance } = require('../mock/mongodb');
+const mongoose = require('../mock/mongoose');
+const zlib = require('../mock/zlib');
+
+jest.mock('fs', () => require('../mock/fs'));
+jest.mock('mongodb', () => require('../mock/mongodb'));
+jest.mock('mongoose', () => require('../mock/mongoose'));
+jest.mock('zlib', () => require('../mock/zlib'));
+jest.mock('../../../logging', () => require('../mock/logging'));
+
+describe('GridFSManager', () => {
+    const BUCKET_NAME = 'test-bucket';
+
+    beforeEach(() => {
+        jest.clearAllMocks();
+        mongoose.connection.readyState = 1;
+        mongoose.connection.db = {};
+    });
+
+    describe('constructor', () => {
+        it('should throw an error if bucketName is not provided', () => {
+            expect(() => new GridFSManager()).toThrow('GridFS bucket name is required');
+        });
+
+        it('should throw an error if mongoose connection is not ready', () => {
+            mongoose.connection.readyState = 0;
+            expect(() => new GridFSManager(BUCKET_NAME)).toThrow('[GridFS] MongoDB connection is not established.');
+        });
+
+        it('should create a GridFSBucket instance on success', () => {
+            // eslint-disable-next-line no-new
+            new GridFSManager(BUCKET_NAME);
+            expect(GridFSBucket).toHaveBeenCalledWith(mongoose.connection.db, { bucketName: BUCKET_NAME });
+        });
+    });
+
+    describe('put', () => {
+        it('should resolve the promise when uploadStream emits "finish"', async () => {
+            const manager = new GridFSManager(BUCKET_NAME);
+
+            mockUploadStream.on.mockImplementation((event, callback) => {
+                if (event === 'finish') {
+                    callback();
+                }
+            });
+
+            const putPromise = manager.put('test-key', '/path/to/file');
+
+            await expect(putPromise).resolves.toBeUndefined();
+
+            expect(fs.createReadStream).toHaveBeenCalledWith('/path/to/file');
+            expect(zlib.createGzip).toHaveBeenCalled();
+            expect(mockBucketInstance.openUploadStream).toHaveBeenCalledWith('test-key', expect.any(Object));
+        });
+
+        it('should reject the promise when uploadStream emits "error"', async () => {
+            const manager = new GridFSManager(BUCKET_NAME);
+            const mockError = new Error('Upload failed');
+
+            mockUploadStream.on.mockImplementation((event, callback) => {
+                if (event === 'error') {
+                    callback(mockError);
+                }
+            });
+
+            const putPromise = manager.put('test-key', '/path/to/file');
+
+            await expect(putPromise).rejects.toThrow('Upload failed');
+        });
+    });
+});
diff --git a/src/store/test/jest/MongoDataSender.test.js b/src/store/test/jest/MongoDataSender.test.js
new file mode 100644
index 0000000..f55ee66
--- /dev/null
+++ b/src/store/test/jest/MongoDataSender.test.js
@@ -0,0 +1,69 @@
+const MongoDataSender = require('../../MongoDataSender');
+const PromCollector = require('../mock/PromCollector');
+const mongoose = require('../mock/mongoose');
+const { mockSave, mockModel } = require('../mock/mongoose');
+
+jest.mock('mongoose', () => require('../mock/mongoose'));
+jest.mock('../../../metrics/PromCollector', () => require('../mock/PromCollector'));
+
+describe('MongoDataSender', () => {
+    const COLLECTION_NAME = 'test-collection';
+
+    beforeEach(() => {
+        jest.clearAllMocks();
+        mongoose.connection.readyState = 1;
+    });
+
+    describe('constructor', () => {
+        it('should throw an error if collection name is not provided', () => {
+            expect(() => new MongoDataSender(null)).toThrow('Collection Name is required');
+        });
+
+        it('should throw an error if mongoose connection is not ready', () => {
+            mongoose.connection.readyState = 0;
+
+            expect(() => new MongoDataSender(COLLECTION_NAME)).toThrow('[MongoDB] Connection is not established.');
+        });
+
+        it('should successfully create an instance if connection is ready', () => {
+            mongoose.connection.readyState = 1;
+
+            expect(() => new MongoDataSender(COLLECTION_NAME)).not.toThrow();
+        });
+    });
+
+    describe('saveEntry', () => {
+        let sender;
+        const entry = { dumpId: 'test-dump-id' };
+
+        beforeEach(() => {
+            sender = new MongoDataSender(COLLECTION_NAME);
+        });
+
+        it('should return true on successful save', async () => {
+            mockSave.mockResolvedValue(true);
+
+            await expect(sender.saveEntry(entry)).resolves.toBe(true);
+            expect(mockModel).toHaveBeenCalledWith(entry);
+            expect(mockSave).toHaveBeenCalled();
+        });
+
+        it('should return false if a duplicate key error (11000) occurs', async () => {
+            const duplicateError = new Error('Duplicate entry');
+
+            duplicateError.code = 11000;
+            mockSave.mockRejectedValue(duplicateError);
+
+            await expect(sender.saveEntry(entry)).resolves.toBe(false);
+        });
+
+        it('should re-throw other errors and increment metric', async () => {
+            const genericError = new Error('Something went wrong');
+
+            mockSave.mockRejectedValue(genericError);
+
+            await expect(sender.saveEntry(entry)).rejects.toThrow('Something went wrong');
+            expect(PromCollector.mongodbErrorCount.inc).toHaveBeenCalledTimes(1);
+        });
+    });
+});
diff --git a/src/store/test/jest/S3Manager.test.js b/src/store/test/jest/S3Manager.test.js
new file mode 100644
index 0000000..b853483
--- /dev/null
+++ b/src/store/test/jest/S3Manager.test.js
@@ -0,0 +1,83 @@
+const S3Manager = require('../../S3Manager');
+const AWS = require('../mock/aws-sdk');
+const { mockUploadPromise, mockGetSignedUrlPromise, mockS3Instance, mockUpdate } = require('../mock/aws-sdk');
+const fs = require('../mock/fs');
+const { mockReadStream } = require('../mock/fs');
+
+jest.mock('aws-sdk', () => require('../mock/aws-sdk'));
+jest.mock('fs', () => require('../mock/fs'));
+jest.mock('zlib', () => require('../mock/zlib'));
+
+describe('S3Manager', () => {
+    const baseConfig = {
+        region: 'test-region',
+        bucket: 'test-bucket',
+        signedLinkExpirationSec: 3600
+    };
+
+    beforeEach(() => {
+        jest.clearAllMocks();
+        AWS.config = { update: mockUpdate };
+    });
+
+    describe('constructor', () => {
+        it('should throw an error if region is missing', () => {
+            expect(() => new S3Manager({ bucket: 'b' })).toThrow();
+        });
+
+        it('should throw an error if bucket is missing', () => {
+            expect(() => new S3Manager({ region: 'r' })).toThrow();
+        });
+
+        it('should configure AWS SDK and create an S3 instance', () => {
+            // eslint-disable-next-line no-new
+            new S3Manager(baseConfig);
+            expect(mockUpdate).toHaveBeenCalledWith(baseConfig.region);
+            expect(AWS.S3).toHaveBeenCalled();
+        });
+    });
+
+    describe('put', () => {
+        it('should upload a gzipped stream and resolve on success', async () => {
+            const manager = new S3Manager(baseConfig);
+
+            mockUploadPromise.mockResolvedValue({ Location: 's3://location' });
+
+            await expect(manager.put('test-key', '/path/to/file')).resolves.toEqual({ Location: 's3://location' });
+
+            expect(fs.createReadStream).toHaveBeenCalledWith('/path/to/file', { encoding: 'utf-8' });
+            expect(mockS3Instance.upload).toHaveBeenCalledWith({
+                Bucket: baseConfig.bucket,
+                Key: 'test-key',
+                Body: mockReadStream.pipe()
+            });
+        });
+
+        it('should reject if upload fails', async () => {
+            const manager = new S3Manager(baseConfig);
+            const mockError = new Error('S3 Upload Failed');
+
+            mockUploadPromise.mockRejectedValue(mockError);
+
+            await expect(manager.put('test-key', '/path/to/file')).rejects.toThrow('S3 Upload Failed');
+        });
+    });
+
+    describe('getSignedUrl', () => {
+        it('should call getSignedUrlPromise with correct parameters', async () => {
+            const manager = new S3Manager(baseConfig);
+            const signedUrl = 'https://s3.amazonaws.com/signed-url';
+
+            mockGetSignedUrlPromise.mockResolvedValue(signedUrl);
+
+            const result = await manager.getSignedUrl('test-key');
+
+            expect(result).toBe(signedUrl);
+            expect(mockS3Instance.getSignedUrlPromise).toHaveBeenCalledWith('getObject', {
+                Bucket: baseConfig.bucket,
+                Key: 'test-key',
+                Expires: baseConfig.signedLinkExpirationSec
+            });
+        });
+    });
+});
diff --git a/src/store/test/mock/PromCollector.js b/src/store/test/mock/PromCollector.js
new file mode 100644
index 0000000..844f803
--- /dev/null
+++ b/src/store/test/mock/PromCollector.js
@@ -0,0 +1,8 @@
+module.exports = {
+    dynamoErrorCount: {
+        inc: jest.fn()
+    },
+    mongodbErrorCount: {
+        inc: jest.fn()
+    }
+};
diff --git a/src/store/test/mock/aws-sdk.js b/src/store/test/mock/aws-sdk.js
new file mode 100644
index 0000000..f92888f
--- /dev/null
+++ b/src/store/test/mock/aws-sdk.js
@@ -0,0 +1,24 @@
+const mockUploadPromise = jest.fn();
+const mockGetSignedUrlPromise = jest.fn();
+const mockUpdate = jest.fn();
+
+const mockS3Instance = {
+    upload: jest.fn().mockReturnValue({ promise: mockUploadPromise }),
+    getSignedUrlPromise: mockGetSignedUrlPromise
+};
+const mockS3 = jest.fn().mockImplementation(() => mockS3Instance);
+
+const mockAWS = {
+    config: {
+        update: mockUpdate
+    },
+    S3: mockS3
+};
+
+module.exports = {
+    ...mockAWS,
+    mockUploadPromise,
+    mockGetSignedUrlPromise,
+    mockS3Instance,
+    mockUpdate
+};
diff --git a/src/store/test/mock/dynamoose.js b/src/store/test/mock/dynamoose.js
new file mode 100644
index 0000000..0531a54
--- /dev/null
+++ b/src/store/test/mock/dynamoose.js
@@ -0,0 +1,27 @@
+const mockSave = jest.fn();
+
+const mockDocumentInstance = {
+    save: mockSave
+};
+
+const mockDocument = jest.fn().mockImplementation(() => mockDocumentInstance);
+
+const mockDynamoose = {
+    aws: {
+        sdk: {
+            config: {
+                update: jest.fn()
+            }
+        },
+        ddb: {
+            local: jest.fn()
+        }
+    },
+    model: jest.fn().mockReturnValue(mockDocument)
+};
+
+module.exports = {
+    ...mockDynamoose,
+    mockSave,
+    mockDocument
+};
diff --git a/src/store/test/mock/fs.js b/src/store/test/mock/fs.js
new file mode 100644
index 0000000..20fc1de
--- /dev/null
+++ b/src/store/test/mock/fs.js
@@ -0,0 +1,10 @@
+const mockReadStream = {
+    pipe: jest.fn()
+};
+
+mockReadStream.pipe.mockReturnThis();
+
+module.exports = {
+    createReadStream: jest.fn().mockReturnValue(mockReadStream),
+    mockReadStream
+};
diff --git a/src/store/test/mock/logging.js b/src/store/test/mock/logging.js
new file mode 100644
index 0000000..5e4029b
--- /dev/null
+++ b/src/store/test/mock/logging.js
@@ -0,0 +1,5 @@
+module.exports = {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn()
+};
diff --git a/src/store/test/mock/mongodb.js b/src/store/test/mock/mongodb.js
new file mode 100644
index 0000000..4c931eb
--- /dev/null
+++ b/src/store/test/mock/mongodb.js
@@ -0,0 +1,15 @@
+const mockUploadStream = {
+    on: jest.fn()
+};
+
+const mockBucketInstance = {
+    openUploadStream: jest.fn().mockReturnValue(mockUploadStream)
+};
+
+const GridFSBucket = jest.fn().mockImplementation(() => mockBucketInstance);
+
+module.exports = {
+    GridFSBucket,
+    mockUploadStream,
+    mockBucketInstance
+};
diff --git a/src/store/test/mock/mongoose.js b/src/store/test/mock/mongoose.js
new file mode 100644
index 0000000..f7844ed
--- /dev/null
+++ b/src/store/test/mock/mongoose.js
@@ -0,0 +1,19 @@
+const mockSave = jest.fn();
+const mockModelInstance = {
+    save: mockSave
+};
+const mockModel = jest.fn().mockImplementation(() => mockModelInstance);
+const mockMongoose = {
+    connection: {
+        readyState: 0
+    },
+    Schema: jest.fn(),
+    models: {},
+    model: jest.fn().mockReturnValue(mockModel)
+};
+
+module.exports = {
+    ...mockMongoose,
+    mockSave,
+    mockModel
+};
diff --git a/src/store/test/mock/zlib.js b/src/store/test/mock/zlib.js
new file mode 100644
index 0000000..a219885
--- /dev/null
+++ b/src/store/test/mock/zlib.js
@@ -0,0 +1,10 @@
+const mockGzipStream = {
+    pipe: jest.fn()
+};
+
+mockGzipStream.pipe.mockReturnThis();
+
+module.exports = {
+    createGzip: jest.fn().mockReturnValue(mockGzipStream),
+    mockGzipStream
+};

From c7e5892c831863d59198cd73410397749ab6448d Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Fri, 22 Aug 2025 19:31:56 +0900
Subject: [PATCH 11/12] fix: make error message consistent with parameter name

---
 src/store/MongoDataSender.js                | 2 +-
 src/store/test/jest/MongoDataSender.test.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/store/MongoDataSender.js b/src/store/MongoDataSender.js
index 73e12c3..9b5be39 100644
--- a/src/store/MongoDataSender.js
+++ b/src/store/MongoDataSender.js
@@ -18,7 +18,7 @@ class MongoDataSender {
       */
     constructor(collectionName) {
 
-        assert(collectionName, 'Collection Name is required when initializing MongoDB');
+        assert(collectionName, '\'collectionName\' is required when initializing MongoDB');
 
         // Connection needs to be established before using MongoDataSender
         if (mongoose.connection.readyState !== 1) {
diff --git a/src/store/test/jest/MongoDataSender.test.js b/src/store/test/jest/MongoDataSender.test.js
index f55ee66..f33ffea 100644
--- a/src/store/test/jest/MongoDataSender.test.js
+++ b/src/store/test/jest/MongoDataSender.test.js
@@ -16,7 +16,7 @@ describe('MongoDataSender', () => {
 
     describe('constructor', () => {
         it('should throw an error if collection name is not provided', () => {
-            expect(() => new MongoDataSender(null)).toThrow('Collection Name is required');
+            expect(() => new MongoDataSender(null)).toThrow('\'collectionName\' is required when initializing MongoDB');
         });
 
         it('should throw an error if mongoose connection is not ready', () => {

From 76e8b8cfedd7396525b47ecbd7226180770ade33 Mon Sep 17 00:00:00 2001
From: Taukon <83957880+Taukon@users.noreply.github.com>
Date: Fri, 22 Aug 2025 19:36:05 +0900
Subject: [PATCH 12/12] fix: avoid hardcoded model name to prevent conflicts

---
 src/store/MongoDataSender.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/store/MongoDataSender.js b/src/store/MongoDataSender.js
index 9b5be39..74875e4 100644
--- a/src/store/MongoDataSender.js
+++ b/src/store/MongoDataSender.js
@@ -52,7 +52,10 @@ class MongoDataSender {
             versionKey: false
         });
 
-        this._model = mongoose.models.Metadata || mongoose.model('Metadata', metadataSchema);
+        // Derive a unique model name from the collection name to avoid conflicts
+        const modelName = `Metadata_${collectionName.replace(/[^a-zA-Z0-9]/g, '_')}`;
+
+        this._model = mongoose.models[modelName] || mongoose.model(modelName, metadataSchema);
 
     }