diff --git a/bot.js b/bot.js index d97112e..7f674af 100644 --- a/bot.js +++ b/bot.js @@ -1,6 +1,9 @@ // bot.js - Main Discord bot application with integrated webserver and initialization require('dotenv').config(); +const { loadConfig } = require('./src/config'); +const { applyMigrations } = require('./src/db/migrations'); +const { normalizeIncomingCall } = require('./src/ingestion/normalizeCall'); // Get environment variables first, before any usage const { @@ -69,6 +72,15 @@ const { TONE_TIME_RESOLUTION_MS } = process.env; +const startupConfig = loadConfig(process.env); +if (!startupConfig.isValid) { + console.error('FATAL: Invalid configuration:'); + for (const error of startupConfig.errors) { + console.error(`- ${error.key}: ${error.message}`); + } + process.exit(1); +} + // --- VALIDATE AI-RELATED ENV VARS --- if (!AI_PROVIDER) { console.error("FATAL: AI_PROVIDER is not set in the .env file. Please specify 'ollama' or 'openai'."); @@ -944,92 +956,17 @@ function ensureApiKey() { } // Function to initialize database tables -function initializeDatabase() { - return new Promise((resolve, reject) => { - logger.info('Initializing database tables...'); - - db.serialize(() => { - let tablesCreated = 0; - let totalTables = ENABLE_AUTH?.toLowerCase() === 'true' ? 7 : 5; - - const tableCreated = (err, tableName) => { - if (err) { - logger.error(`Error creating ${tableName} table:`, err); - reject(err); - return; - } - tablesCreated++; - if (tablesCreated === totalTables) { - logger.info('Database tables initialized successfully.'); - resolve(); - } - }; - - db.run(`CREATE TABLE IF NOT EXISTS transcriptions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - talk_group_id TEXT, - timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, - transcription TEXT, - audio_file_path TEXT, - address TEXT, - lat REAL, - lon REAL, - category TEXT - )`, (err) => tableCreated(err, 'transcriptions')); - - db.run(`CREATE TABLE IF NOT EXISTS global_keywords ( - keyword TEXT UNIQUE, - talk_group_id TEXT - )`, (err) => tableCreated(err, 'global_keywords')); - - db.run(`CREATE TABLE IF NOT EXISTS talk_groups ( - id TEXT PRIMARY KEY, - hex TEXT, - alpha_tag TEXT, - mode TEXT, - description TEXT, - tag TEXT, - county TEXT - )`, (err) => tableCreated(err, 'talk_groups')); - - db.run(`CREATE TABLE IF NOT EXISTS frequencies ( - id INTEGER PRIMARY KEY, - frequency TEXT, - description TEXT - )`, (err) => tableCreated(err, 'frequencies')); - - db.run(`CREATE TABLE IF NOT EXISTS audio_files ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transcription_id INTEGER, - audio_data BLOB, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY(transcription_id) REFERENCES transcriptions(id) - )`, (err) => tableCreated(err, 'audio_files')); - - // Authentication tables (if auth is enabled) - if (ENABLE_AUTH?.toLowerCase() === 'true') { - db.run(`CREATE TABLE IF NOT EXISTS users ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - username TEXT UNIQUE NOT NULL, - password_hash TEXT NOT NULL, - salt TEXT NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP - )`, (err) => tableCreated(err, 'users')); - - db.run(`CREATE TABLE IF NOT EXISTS sessions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id INTEGER NOT NULL, - token TEXT UNIQUE NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - expires_at DATETIME NOT NULL, - last_activity DATETIME DEFAULT CURRENT_TIMESTAMP, - ip_address TEXT, - user_agent TEXT, - FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE - )`, (err) => tableCreated(err, 'sessions')); - } - }); +async function initializeDatabase() { + logger.info('Initializing database tables...'); + const applied = await applyMigrations(db, { + enableAuth: ENABLE_AUTH?.toLowerCase() === 'true' }); + + if (applied.length > 0) { + logger.info(`Applied database migrations: ${applied.join(', ')}`); + } else { + logger.info('Database schema already up to date.'); + } } // Function to create admin user if authentication is enabled @@ -1456,18 +1393,24 @@ app.post('/api/call-upload', (req, res) => { logger.info(`Received SDRTrunk audio: ${customFilename}`); + const normalizedCall = normalizeIncomingCall({ + source: 'sdrtrunk', + fields, + fileInfo + }); + handleNewAudio({ filename: customFilename, path: saveTo, - talkGroupID: fields.talkgroup, - systemName: fields.systemLabel, - talkGroupName: fields.talkgroupLabel, - dateTime: fields.dateTime, // Pass the original fields.dateTime for SDRTrunk - source: fields.source, - talkerAlias: fields.talkerAlias, // Add talkerAlias field from SDRTrunk - frequency: fields.frequency, - talkGroupGroup: fields.talkgroupGroup, - isTrunkRecorder: false + talkGroupID: normalizedCall.talkGroupID, + systemName: normalizedCall.systemName, + talkGroupName: normalizedCall.talkGroupName, + dateTime: normalizedCall.dateTime, + source: normalizedCall.source, + talkerAlias: normalizedCall.talkerAlias, + frequency: normalizedCall.frequency, + talkGroupGroup: normalizedCall.talkGroupGroup, + isTrunkRecorder: normalizedCall.isTrunkRecorder }); return sendResponse(200, 'Call imported successfully.'); @@ -1781,17 +1724,26 @@ app.post('/api/call-upload', (req, res) => { // Log fields before passing to handleNewAudio logger.info(`[UPLOAD] Preparing to call handleNewAudio, fields.srcList=${fields.srcList ? (typeof fields.srcList === 'string' ? `string(${fields.srcList.length} chars)` : `object`) : 'null/undefined'}, fields.freqList=${fields.freqList ? 'exists' : 'null/undefined'}`); + const normalizedCall = normalizeIncomingCall({ + source: inferredSourceSystem === 'rdio-scanner' ? 'rdio-scanner' : 'trunk-recorder', + fields: { + ...fields, + dateTime: Math.floor(callDateTime.getTime() / 1000) + }, + fileInfo + }); + handleNewAudio({ filename: customFilename, path: saveTo, - talkGroupID: fields.talkgroup, - systemName: fields.systemLabel, - talkGroupName: fields.talkgroupLabel, + talkGroupID: normalizedCall.talkGroupID, + systemName: normalizedCall.systemName, + talkGroupName: normalizedCall.talkGroupName, dateTime: Math.floor(callDateTime.getTime() / 1000), // Pass Unix timestamp (seconds) - source: fields.source, - talkerAlias: fields.talkerAlias, // <-- OTA alias from Trunk Recorder - frequency: fields.frequency, - talkGroupGroup: fields.talkgroupGroup, + source: normalizedCall.source, + talkerAlias: normalizedCall.talkerAlias, // <-- OTA alias from Trunk Recorder + frequency: normalizedCall.frequency, + talkGroupGroup: normalizedCall.talkGroupGroup, // Detect TrunkRecorder more reliably: check for TrunkRecorder-specific fields isTrunkRecorder: inferredSourceSystem === 'TrunkRecorder' || (fields.srcList && fields.srcList.trim() !== '' && fields.srcList.trim() !== '[]') || @@ -6497,4 +6449,4 @@ process.on('SIGINT', () => { process.exit(0); }); }); -}); \ No newline at end of file +}); diff --git a/src/ingestion/normalizeCall.js b/src/ingestion/normalizeCall.js index 9fecb7c..c77d97d 100644 --- a/src/ingestion/normalizeCall.js +++ b/src/ingestion/normalizeCall.js @@ -78,11 +78,12 @@ function enrichTrunkRecorderFields(fields = {}) { return enriched; } -function normalizeTrunkRecorderCall(fields = {}, fileInfo = {}) { +function normalizeTrunkRecorderCall(fields = {}, fileInfo = {}, options = {}) { const enriched = enrichTrunkRecorderFields(fields); + const provider = options.provider || 'trunk-recorder'; return { - provider: 'trunk-recorder', + provider, filename: fileInfo.originalFilename || enriched.filename || '', talkGroupID: enriched.talkgroup || enriched.talk_group_id || enriched.talkGroupID || '', systemName: enriched.system || enriched.systemName || enriched.systemLabel || '', @@ -93,16 +94,20 @@ function normalizeTrunkRecorderCall(fields = {}, fileInfo = {}) { talkerAlias: enriched.talkerAlias || '', frequency: enriched.frequency || enriched.freq || '', metadata: enriched, - isTrunkRecorder: true + isTrunkRecorder: provider === 'trunk-recorder' }; } function normalizeIncomingCall({ source, fields = {}, fileInfo = {} } = {}) { if (source === 'sdrtrunk') return normalizeSdrTrunkCall(fields, fileInfo); - if (source === 'trunk-recorder' || source === 'rdio-scanner') { + if (source === 'trunk-recorder') { return normalizeTrunkRecorderCall(fields, fileInfo); } + if (source === 'rdio-scanner') { + return normalizeTrunkRecorderCall(fields, fileInfo, { provider: 'rdio-scanner' }); + } + return normalizeTrunkRecorderCall(fields, fileInfo); } diff --git a/test/ingestion.test.js b/test/ingestion.test.js index ad5315b..04e094b 100644 --- a/test/ingestion.test.js +++ b/test/ingestion.test.js @@ -47,3 +47,16 @@ test('normalizeTrunkRecorderCall extracts source and alias from meta srcList', ( assert.equal(call.talkerAlias, 'Unit 12'); assert.equal(call.frequency, 853000000); }); + +test('normalizeIncomingCall preserves rdio-scanner as a non-TrunkRecorder provider', () => { + const call = normalizeIncomingCall({ + source: 'rdio-scanner', + fields: { + talkgroup: '3001', + dateTime: '2026-05-17T12:00:00Z' + } + }); + + assert.equal(call.provider, 'rdio-scanner'); + assert.equal(call.isTrunkRecorder, false); +}); diff --git a/webserver.js b/webserver.js index 0e628f8..e24e14f 100644 --- a/webserver.js +++ b/webserver.js @@ -1,7 +1,8 @@ // webserver.js - Web interface for viewing and managing calls with optional authentication -require('dotenv').config(); -const AWS = require('aws-sdk'); // Add AWS SDK +require('dotenv').config(); +const { loadConfig } = require('./src/config'); +const AWS = require('aws-sdk'); // Add AWS SDK const express = require('express'); const sqlite3 = require('sqlite3').verbose(); @@ -40,7 +41,16 @@ const { OPENAI_MODEL = 'gpt-4o-mini', // A good, fast, and cheap model for this task OLLAMA_URL = 'http://localhost:11434', OLLAMA_MODEL = 'llama3.1:8b' -} = process.env; +} = process.env; + +const startupConfig = loadConfig(process.env); +if (!startupConfig.isValid) { + console.error('ERROR: Invalid configuration:'); + for (const error of startupConfig.errors) { + console.error(`- ${error.key}: ${error.message}`); + } + process.exit(1); +} // Validate required environment variables const requiredVars = ['WEBSERVER_PORT', 'PUBLIC_DOMAIN']; @@ -1804,4 +1814,4 @@ process.on('SIGINT', () => { process.exit(0); }); }); -}); \ No newline at end of file +});