From 24362f23b21d05a0bfbe356a70bb2b97a37d0226 Mon Sep 17 00:00:00 2001 From: Dadud Date: Sun, 17 May 2026 19:05:52 -0500 Subject: [PATCH] Add architecture foundation modules --- .github/workflows/ci.yml | 30 +++++ demo/sample-calls.json | 35 ++++++ docs/modernization-roadmap.md | 58 ++++++++++ package.json | 6 +- scripts/check-config.js | 20 ++++ scripts/generate-demo-data.js | 46 ++++++++ src/config/index.js | 197 +++++++++++++++++++++++++++++++++ src/db/migrations.js | 123 ++++++++++++++++++++ src/ingestion/normalizeCall.js | 116 +++++++++++++++++++ src/permissions/roles.js | 28 +++++ test/config.test.js | 41 +++++++ test/ingestion.test.js | 49 ++++++++ test/migrations.test.js | 18 +++ test/permissions.test.js | 16 +++ 14 files changed, 782 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 demo/sample-calls.json create mode 100644 docs/modernization-roadmap.md create mode 100644 scripts/check-config.js create mode 100644 scripts/generate-demo-data.js create mode 100644 src/config/index.js create mode 100644 src/db/migrations.js create mode 100644 src/ingestion/normalizeCall.js create mode 100644 src/permissions/roles.js create mode 100644 test/config.test.js create mode 100644 test/ingestion.test.js create mode 100644 test/migrations.test.js create mode 100644 test/permissions.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7798712 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + smoke: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + run: npm ci --ignore-scripts + + - name: Syntax check + run: npm run check:syntax + + - name: Unit tests + run: npm test diff --git a/demo/sample-calls.json b/demo/sample-calls.json new file mode 100644 index 0000000..3dcc716 --- /dev/null +++ b/demo/sample-calls.json @@ -0,0 +1,35 @@ +[ + { + "id": 1, + "talk_group_id": "1001", + "timestamp": 1779033180, + "transcription": "Engine 12 responding to a medical call near Main Street and Oak Avenue.", + "audio_file_path": "", + "address": "Main Street and Oak Avenue", + "lat": 39.083997, + "lon": -77.152758, + "category": "Medical Call" + }, + { + "id": 2, + "talk_group_id": "2001", + "timestamp": 1779033360, + "transcription": "Units checking a vehicle collision near the northbound ramp.", + "audio_file_path": "", + "address": "Northbound ramp", + "lat": 39.099721, + "lon": -77.184516, + "category": "Vehicle Collision" + }, + { + "id": 3, + "talk_group_id": "3001", + "timestamp": 1779033540, + "transcription": "Police responding for a disturbance at the shopping center.", + "audio_file_path": "", + "address": "Shopping center", + "lat": 39.045753, + "lon": -77.118741, + "category": "Disturbance" + } +] diff --git a/docs/modernization-roadmap.md b/docs/modernization-roadmap.md new file mode 100644 index 0000000..b5d0dea --- /dev/null +++ b/docs/modernization-roadmap.md @@ -0,0 +1,58 @@ +# Scanner Map Modernization Roadmap + +This roadmap breaks the larger architecture work into reviewable PRs. Each phase should preserve current behavior while creating room for deeper changes. + +## Phase 1: Foundations + +- Add shared config parsing and validation. +- Add a migration module that can replace scattered table creation over time. +- Add ingestion normalization helpers for SDRTrunk, TrunkRecorder, and rdio-scanner compatible uploads. +- Add a local demo data generator. +- Add smoke tests and CI. +- Add role and permission primitives that can back future RBAC. + +## Phase 2: Runtime Integration + +- Replace scattered `process.env` reads in `bot.js`, `webserver.js`, and `geocoding.js` with the shared config module. +- Move database initialization to the migration runner. +- Route upload handling through the ingestion normalization helpers. +- Keep the old endpoint behavior intact while shrinking request-handler complexity. + +## Phase 3: Reliable Processing Queue + +- Persist call processing jobs in SQLite or a dedicated queue backend. +- Track job state: pending, processing, failed, complete, and retryable. +- Retry transcription, geocoding, categorization, Discord publishing, and storage steps independently. +- Add admin visibility for queue depth, failed jobs, and processing latency. + +## Phase 4: Local Demo And Developer Mode + +- Add a demo server mode that serves sample calls without SDRTrunk, TrunkRecorder, Discord, geocoding keys, or audio hardware. +- Add sample talkgroups, categories, and map markers. +- Make frontend work possible with one command. + +## Phase 5: Frontend Modules + +- Split `public/app.js` into modules for map setup, markers, audio playback, live feed, auth, talkgroup modal, purge modal, and geocoding search. +- Gate verbose browser logging behind a debug flag. +- Add targeted browser smoke tests once the local demo mode exists. + +## Phase 6: Data Model And Retention + +- Add schema versioning and repeatable migrations. +- Add indexes for common call history, talkgroup, timestamp, and category queries. +- Add configurable retention rules for calls and audio. +- Add database maintenance docs for long-running deployments. + +## Phase 7: Roles And Permissions + +- Add a `role` column for users and migrate existing admin users. +- Replace ad hoc admin checks with permission checks. +- Introduce viewer, editor, moderator, and admin roles. +- Add UI controls only when the current user has the matching permission. + +## Phase 8: Adapter Architecture + +- Formalize ingestion adapters for SDRTrunk, TrunkRecorder, and rdio-scanner compatible uploads. +- Add adapter tests with real-world fixture payloads. +- Make future upload sources additive instead of route-handler rewrites. diff --git a/package.json b/package.json index 615c553..9d729a7 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,11 @@ "scripts": { "start": "node bot.js", "web": "node webserver.js", - "import-talkgroups": "node import_csv.js" + "import-talkgroups": "node import_csv.js", + "check:config": "node scripts/check-config.js", + "check:syntax": "node --check bot.js && node --check webserver.js && node --check geocoding.js && node --check import_csv.js && node --check public/app.js", + "demo:data": "node scripts/generate-demo-data.js", + "test": "node --test test/*.test.js" }, "engines": { "node": ">=18" diff --git a/scripts/check-config.js b/scripts/check-config.js new file mode 100644 index 0000000..bea382d --- /dev/null +++ b/scripts/check-config.js @@ -0,0 +1,20 @@ +try { + require('dotenv').config(); +} catch { + // Allows this checker to run before npm install; CI still installs dependencies. +} + +const { loadConfig, redactConfig } = require('../src/config'); + +const result = loadConfig(process.env); + +if (!result.isValid) { + console.error('Configuration validation failed:'); + for (const error of result.errors) { + console.error(`- ${error.key}: ${error.message}`); + } + process.exit(1); +} + +console.log('Configuration looks valid.'); +console.log(JSON.stringify(redactConfig(result.config), null, 2)); diff --git a/scripts/generate-demo-data.js b/scripts/generate-demo-data.js new file mode 100644 index 0000000..968bbed --- /dev/null +++ b/scripts/generate-demo-data.js @@ -0,0 +1,46 @@ +const fs = require('fs'); +const path = require('path'); + +const outputDir = path.join(__dirname, '..', 'data'); +const outputFile = path.join(outputDir, 'demo-calls.json'); + +const now = Math.floor(Date.now() / 1000); +const calls = [ + { + id: 1, + talk_group_id: '1001', + timestamp: now - 420, + transcription: 'Engine 12 responding to a medical call near Main Street and Oak Avenue.', + audio_file_path: '', + address: 'Main Street and Oak Avenue', + lat: 39.083997, + lon: -77.152758, + category: 'Medical Call' + }, + { + id: 2, + talk_group_id: '2001', + timestamp: now - 240, + transcription: 'Units checking a vehicle collision near the northbound ramp.', + audio_file_path: '', + address: 'Northbound ramp', + lat: 39.099721, + lon: -77.184516, + category: 'Vehicle Collision' + }, + { + id: 3, + talk_group_id: '3001', + timestamp: now - 60, + transcription: 'Police responding for a disturbance at the shopping center.', + audio_file_path: '', + address: 'Shopping center', + lat: 39.045753, + lon: -77.118741, + category: 'Disturbance' + } +]; + +fs.mkdirSync(outputDir, { recursive: true }); +fs.writeFileSync(outputFile, `${JSON.stringify(calls, null, 2)}\n`); +console.log(`Wrote ${calls.length} demo calls to ${outputFile}`); diff --git a/src/config/index.js b/src/config/index.js new file mode 100644 index 0000000..cb603ab --- /dev/null +++ b/src/config/index.js @@ -0,0 +1,197 @@ +const DEFAULTS = { + botPort: 3306, + webserverPort: 3001, + publicDomain: 'localhost', + timezone: 'US/Eastern', + apiKeyFile: 'data/apikeys.json', + enableAuth: false, + sessionDurationDays: 7, + maxSessionsPerUser: 5, + storageMode: 'local', + aiProvider: 'ollama', + openaiModel: 'gpt-4o-mini', + ollamaUrl: 'http://localhost:11434', + ollamaModel: 'llama3.1:8b', + transcriptionMode: 'local', + whisperModel: 'large-v3', + transcriptionDevice: 'cpu', + pythonCommand: 'python', + autoUpdatePythonPackages: true, + summaryLookbackHours: 1, + askAiLookbackHours: 8, + maxConcurrentTranscriptions: 3, + enableMappedTalkGroups: true, + enableTwoToneMode: false, + twoToneQueueSize: 1 +}; + +const SECRET_KEYS = new Set([ + 'discordToken', + 'googleMapsApiKey', + 'locationIqApiKey', + 's3AccessKeyId', + 's3SecretAccessKey', + 'openaiApiKey', + 'icadApiKey', + 'webserverPassword' +]); + +function parseBoolean(value, fallback = false) { + if (value === undefined || value === null || value === '') return fallback; + return ['1', 'true', 'yes', 'on'].includes(String(value).trim().toLowerCase()); +} + +function parseNumber(value, fallback, { integer = false, min = undefined } = {}) { + if (value === undefined || value === null || value === '') return fallback; + const parsed = integer ? parseInt(value, 10) : parseFloat(value); + if (Number.isNaN(parsed)) return fallback; + if (min !== undefined && parsed < min) return fallback; + return parsed; +} + +function parseList(value) { + if (!value) return []; + return String(value) + .split(',') + .map((item) => item.trim()) + .filter(Boolean); +} + +function requireWhen(errors, condition, key, message) { + if (condition) errors.push({ key, message }); +} + +function loadConfig(env = process.env) { + const config = { + discordToken: env.DISCORD_TOKEN || '', + clientId: env.CLIENT_ID || '', + botPort: parseNumber(env.BOT_PORT, DEFAULTS.botPort, { integer: true, min: 1 }), + webserverPort: parseNumber(env.WEBSERVER_PORT, DEFAULTS.webserverPort, { integer: true, min: 1 }), + publicDomain: env.PUBLIC_DOMAIN || DEFAULTS.publicDomain, + timezone: env.TIMEZONE || DEFAULTS.timezone, + apiKeyFile: env.API_KEY_FILE || DEFAULTS.apiKeyFile, + enableAuth: parseBoolean(env.ENABLE_AUTH, DEFAULTS.enableAuth), + webserverPassword: env.WEBSERVER_PASSWORD || '', + sessionDurationDays: parseNumber(env.SESSION_DURATION_DAYS, DEFAULTS.sessionDurationDays, { integer: true, min: 1 }), + maxSessionsPerUser: parseNumber(env.MAX_SESSIONS_PER_USER, DEFAULTS.maxSessionsPerUser, { integer: true, min: 1 }), + googleMapsApiKey: env.GOOGLE_MAPS_API_KEY || '', + locationIqApiKey: env.LOCATIONIQ_API_KEY || '', + storageMode: (env.STORAGE_MODE || DEFAULTS.storageMode).toLowerCase(), + s3Endpoint: env.S3_ENDPOINT || '', + s3BucketName: env.S3_BUCKET_NAME || '', + s3AccessKeyId: env.S3_ACCESS_KEY_ID || '', + s3SecretAccessKey: env.S3_SECRET_ACCESS_KEY || '', + aiProvider: (env.AI_PROVIDER || DEFAULTS.aiProvider).toLowerCase(), + openaiApiKey: env.OPENAI_API_KEY || '', + openaiModel: env.OPENAI_MODEL || DEFAULTS.openaiModel, + ollamaUrl: env.OLLAMA_URL || DEFAULTS.ollamaUrl, + ollamaModel: env.OLLAMA_MODEL || DEFAULTS.ollamaModel, + transcriptionMode: (env.TRANSCRIPTION_MODE || DEFAULTS.transcriptionMode).toLowerCase(), + fasterWhisperServerUrl: env.FASTER_WHISPER_SERVER_URL || '', + whisperModel: env.WHISPER_MODEL || DEFAULTS.whisperModel, + transcriptionDevice: (env.TRANSCRIPTION_DEVICE || DEFAULTS.transcriptionDevice).toLowerCase(), + pythonCommand: env.PYTHON_COMMAND || DEFAULTS.pythonCommand, + autoUpdatePythonPackages: parseBoolean(env.AUTO_UPDATE_PYTHON_PACKAGES, DEFAULTS.autoUpdatePythonPackages), + icadUrl: env.ICAD_URL || '', + icadProfile: env.ICAD_PROFILE || '', + icadApiKey: env.ICAD_API_KEY || '', + openaiTranscriptionPrompt: env.OPENAI_TRANSCRIPTION_PROMPT || '', + openaiTranscriptionModel: env.OPENAI_TRANSCRIPTION_MODEL || '', + openaiTranscriptionTemperature: env.OPENAI_TRANSCRIPTION_TEMPERATURE || '', + mappedTalkGroups: parseList(env.MAPPED_TALK_GROUPS), + enableMappedTalkGroups: parseBoolean(env.ENABLE_MAPPED_TALK_GROUPS, DEFAULTS.enableMappedTalkGroups), + summaryLookbackHours: parseNumber(env.SUMMARY_LOOKBACK_HOURS, DEFAULTS.summaryLookbackHours, { min: 0 }), + askAiLookbackHours: parseNumber(env.ASK_AI_LOOKBACK_HOURS, DEFAULTS.askAiLookbackHours, { min: 0 }), + maxConcurrentTranscriptions: parseNumber(env.MAX_CONCURRENT_TRANSCRIPTIONS, DEFAULTS.maxConcurrentTranscriptions, { integer: true, min: 1 }), + enableTwoToneMode: parseBoolean(env.ENABLE_TWO_TONE_MODE, DEFAULTS.enableTwoToneMode), + twoToneTalkGroups: parseList(env.TWO_TONE_TALK_GROUPS), + twoToneQueueSize: parseNumber(env.TWO_TONE_QUEUE_SIZE, DEFAULTS.twoToneQueueSize, { integer: true, min: 1 }), + toneDetectionType: env.TONE_DETECTION_TYPE || '', + twoToneMinToneLength: env.TWO_TONE_MIN_TONE_LENGTH || '', + twoToneMaxToneLength: env.TWO_TONE_MAX_TONE_LENGTH || '', + pulsedMinCycles: env.PULSED_MIN_CYCLES || '', + pulsedMinOnMs: env.PULSED_MIN_ON_MS || '', + pulsedMaxOnMs: env.PULSED_MAX_ON_MS || '', + pulsedMinOffMs: env.PULSED_MIN_OFF_MS || '', + pulsedMaxOffMs: env.PULSED_MAX_OFF_MS || '', + pulsedBandwidthHz: env.PULSED_BANDWIDTH_HZ || '', + longToneMinLength: env.LONG_TONE_MIN_LENGTH || '', + longToneBandwidthHz: env.LONG_TONE_BANDWIDTH_HZ || '', + toneDetectionThreshold: env.TONE_DETECTION_THRESHOLD || '', + toneFrequencyBand: env.TONE_FREQUENCY_BAND || '', + toneTimeResolutionMs: env.TONE_TIME_RESOLUTION_MS || '' + }; + + const errors = validateConfig(config); + return { config, errors, isValid: errors.length === 0 }; +} + +function validateConfig(config) { + const errors = []; + const storageModes = new Set(['local', 's3']); + const aiProviders = new Set(['ollama', 'openai']); + const transcriptionModes = new Set(['local', 'remote', 'openai', 'icad']); + const transcriptionDevices = new Set(['cpu', 'cuda']); + + requireWhen(errors, !storageModes.has(config.storageMode), 'STORAGE_MODE', 'Must be local or s3.'); + requireWhen(errors, !aiProviders.has(config.aiProvider), 'AI_PROVIDER', 'Must be ollama or openai.'); + requireWhen(errors, !transcriptionModes.has(config.transcriptionMode), 'TRANSCRIPTION_MODE', 'Must be local, remote, openai, or icad.'); + requireWhen(errors, !transcriptionDevices.has(config.transcriptionDevice), 'TRANSCRIPTION_DEVICE', 'Must be cpu or cuda.'); + + requireWhen(errors, config.enableAuth && !config.webserverPassword, 'WEBSERVER_PASSWORD', 'Required when ENABLE_AUTH=true.'); + requireWhen(errors, config.storageMode === 's3' && !config.s3Endpoint, 'S3_ENDPOINT', 'Required when STORAGE_MODE=s3.'); + requireWhen(errors, config.storageMode === 's3' && !config.s3BucketName, 'S3_BUCKET_NAME', 'Required when STORAGE_MODE=s3.'); + requireWhen(errors, config.storageMode === 's3' && !config.s3AccessKeyId, 'S3_ACCESS_KEY_ID', 'Required when STORAGE_MODE=s3.'); + requireWhen(errors, config.storageMode === 's3' && !config.s3SecretAccessKey, 'S3_SECRET_ACCESS_KEY', 'Required when STORAGE_MODE=s3.'); + requireWhen(errors, config.aiProvider === 'openai' && !config.openaiApiKey, 'OPENAI_API_KEY', 'Required when AI_PROVIDER=openai.'); + requireWhen(errors, config.aiProvider === 'ollama' && !config.ollamaUrl, 'OLLAMA_URL', 'Required when AI_PROVIDER=ollama.'); + requireWhen(errors, config.aiProvider === 'ollama' && !config.ollamaModel, 'OLLAMA_MODEL', 'Required when AI_PROVIDER=ollama.'); + requireWhen(errors, config.transcriptionMode === 'remote' && !config.fasterWhisperServerUrl, 'FASTER_WHISPER_SERVER_URL', 'Required when TRANSCRIPTION_MODE=remote.'); + requireWhen(errors, config.transcriptionMode === 'openai' && !config.openaiApiKey, 'OPENAI_API_KEY', 'Required when TRANSCRIPTION_MODE=openai.'); + requireWhen(errors, config.transcriptionMode === 'icad' && !config.icadUrl, 'ICAD_URL', 'Required when TRANSCRIPTION_MODE=icad.'); + + const toneKeys = [ + ['TWO_TONE_TALK_GROUPS', config.twoToneTalkGroups.length > 0], + ['TONE_DETECTION_TYPE', config.toneDetectionType], + ['TWO_TONE_MIN_TONE_LENGTH', config.twoToneMinToneLength], + ['TWO_TONE_MAX_TONE_LENGTH', config.twoToneMaxToneLength], + ['PULSED_MIN_CYCLES', config.pulsedMinCycles], + ['PULSED_MIN_ON_MS', config.pulsedMinOnMs], + ['PULSED_MAX_ON_MS', config.pulsedMaxOnMs], + ['PULSED_MIN_OFF_MS', config.pulsedMinOffMs], + ['PULSED_MAX_OFF_MS', config.pulsedMaxOffMs], + ['PULSED_BANDWIDTH_HZ', config.pulsedBandwidthHz], + ['LONG_TONE_MIN_LENGTH', config.longToneMinLength], + ['LONG_TONE_BANDWIDTH_HZ', config.longToneBandwidthHz], + ['TONE_DETECTION_THRESHOLD', config.toneDetectionThreshold], + ['TONE_FREQUENCY_BAND', config.toneFrequencyBand], + ['TONE_TIME_RESOLUTION_MS', config.toneTimeResolutionMs] + ]; + + if (config.enableTwoToneMode) { + for (const [key, value] of toneKeys) { + requireWhen(errors, !value, key, 'Required when ENABLE_TWO_TONE_MODE=true.'); + } + } + + return errors; +} + +function redactConfig(config) { + return Object.fromEntries( + Object.entries(config).map(([key, value]) => { + if (SECRET_KEYS.has(key) && value) return [key, '[redacted]']; + return [key, value]; + }) + ); +} + +module.exports = { + DEFAULTS, + loadConfig, + parseBoolean, + parseList, + parseNumber, + redactConfig, + validateConfig +}; diff --git a/src/db/migrations.js b/src/db/migrations.js new file mode 100644 index 0000000..45e97b0 --- /dev/null +++ b/src/db/migrations.js @@ -0,0 +1,123 @@ +const BASE_MIGRATIONS = [ + { + id: '001_create_core_tables', + statements: [ + `CREATE TABLE IF NOT EXISTS transcriptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + talk_group_id TEXT, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, + transcription TEXT, + audio_file_path TEXT, + address TEXT, + lat REAL, + lon REAL, + category TEXT + )`, + `CREATE TABLE IF NOT EXISTS global_keywords ( + keyword TEXT UNIQUE, + talk_group_id TEXT + )`, + `CREATE TABLE IF NOT EXISTS talk_groups ( + id TEXT PRIMARY KEY, + hex TEXT, + alpha_tag TEXT, + mode TEXT, + description TEXT, + tag TEXT, + county TEXT + )`, + `CREATE TABLE IF NOT EXISTS frequencies ( + id INTEGER PRIMARY KEY, + frequency TEXT, + description TEXT + )`, + `CREATE TABLE IF NOT EXISTS audio_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transcription_id INTEGER, + audio_data BLOB, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(transcription_id) REFERENCES transcriptions(id) + )` + ] + }, + { + id: '002_create_auth_tables', + requires: ({ enableAuth }) => enableAuth, + statements: [ + `CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT UNIQUE NOT NULL, + password_hash TEXT NOT NULL, + salt TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'admin', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`, + `CREATE TABLE IF NOT EXISTS sessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + token TEXT UNIQUE NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + expires_at DATETIME NOT NULL, + last_activity DATETIME DEFAULT CURRENT_TIMESTAMP, + ip_address TEXT, + user_agent TEXT, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + )` + ] + } +]; + +function getMigrationPlan(options = {}) { + return BASE_MIGRATIONS.filter((migration) => { + if (!migration.requires) return true; + return migration.requires(options); + }); +} + +function run(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.run(sql, params, function onRun(err) { + if (err) reject(err); + else resolve(this); + }); + }); +} + +function all(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { + if (err) reject(err); + else resolve(rows); + }); + }); +} + +async function applyMigrations(db, options = {}) { + await run(db, `CREATE TABLE IF NOT EXISTS schema_migrations ( + id TEXT PRIMARY KEY, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP + )`); + + const appliedRows = await all(db, 'SELECT id FROM schema_migrations'); + const applied = new Set(appliedRows.map((row) => row.id)); + const appliedNow = []; + + for (const migration of getMigrationPlan(options)) { + if (applied.has(migration.id)) continue; + + for (const statement of migration.statements) { + await run(db, statement); + } + + await run(db, 'INSERT INTO schema_migrations (id) VALUES (?)', [migration.id]); + appliedNow.push(migration.id); + } + + return appliedNow; +} + +module.exports = { + BASE_MIGRATIONS, + applyMigrations, + getMigrationPlan +}; diff --git a/src/ingestion/normalizeCall.js b/src/ingestion/normalizeCall.js new file mode 100644 index 0000000..9fecb7c --- /dev/null +++ b/src/ingestion/normalizeCall.js @@ -0,0 +1,116 @@ +function parseJsonField(value, fallback = null) { + if (!value || typeof value !== 'string') return fallback; + try { + return JSON.parse(value); + } catch { + return fallback; + } +} + +function extractSourceFromFilename(filename) { + if (!filename) return undefined; + const match = filename.match(/FROM_(\d+)/); + return match ? match[1] : undefined; +} + +function normalizeSdrTrunkCall(fields = {}, fileInfo = {}) { + const filenameSource = extractSourceFromFilename(fileInfo.originalFilename); + + return { + provider: 'sdrtrunk', + filename: fileInfo.originalFilename || '', + talkGroupID: fields.talkgroup || fields.talk_group_id || '', + systemName: fields.systemLabel || fields.system || '', + talkGroupName: fields.talkgroupLabel || fields.talkgroupName || '', + talkGroupGroup: fields.talkgroupGroup || '', + dateTime: fields.dateTime || fields.start_time || '', + source: fields.source || filenameSource || '', + talkerAlias: fields.talkerAlias || '', + frequency: fields.frequency || '', + metadata: { ...fields }, + isTrunkRecorder: false + }; +} + +function enrichTrunkRecorderFields(fields = {}) { + const enriched = { ...fields }; + const metaData = parseJsonField(fields.meta, {}); + + if (metaData && typeof metaData === 'object') { + const directCopies = [ + 'freq', + 'freq_error', + 'signal', + 'noise', + 'emergency', + 'priority', + 'encrypted', + 'call_length', + 'start_time', + 'stop_time', + 'tdma_slot', + 'phase2_tdma', + 'color_code' + ]; + + for (const key of directCopies) { + if (metaData[key] !== undefined && enriched[key] === undefined) { + enriched[key === 'freq' ? 'frequency' : key] = metaData[key]; + } + } + + if (Array.isArray(metaData.srcList) && metaData.srcList.length > 0) { + const validSource = metaData.srcList.find((src) => src.src && src.src !== -1); + if (validSource) { + enriched.source = enriched.source || String(validSource.src); + if (validSource.tag && String(validSource.tag).trim()) { + enriched.talkerAlias = enriched.talkerAlias || String(validSource.tag).trim(); + } + } + enriched.srcList = enriched.srcList || JSON.stringify(metaData.srcList); + } + + if (Array.isArray(metaData.freqList)) { + enriched.freqList = enriched.freqList || JSON.stringify(metaData.freqList); + } + } + + return enriched; +} + +function normalizeTrunkRecorderCall(fields = {}, fileInfo = {}) { + const enriched = enrichTrunkRecorderFields(fields); + + return { + provider: 'trunk-recorder', + filename: fileInfo.originalFilename || enriched.filename || '', + talkGroupID: enriched.talkgroup || enriched.talk_group_id || enriched.talkGroupID || '', + systemName: enriched.system || enriched.systemName || enriched.systemLabel || '', + talkGroupName: enriched.talkgroupLabel || enriched.talkgroupName || enriched.talkGroupName || '', + talkGroupGroup: enriched.talkgroupGroup || '', + dateTime: enriched.dateTime || enriched.start_time || '', + source: enriched.source || '', + talkerAlias: enriched.talkerAlias || '', + frequency: enriched.frequency || enriched.freq || '', + metadata: enriched, + isTrunkRecorder: true + }; +} + +function normalizeIncomingCall({ source, fields = {}, fileInfo = {} } = {}) { + if (source === 'sdrtrunk') return normalizeSdrTrunkCall(fields, fileInfo); + if (source === 'trunk-recorder' || source === 'rdio-scanner') { + return normalizeTrunkRecorderCall(fields, fileInfo); + } + + return normalizeTrunkRecorderCall(fields, fileInfo); +} + +module.exports = { + enrichTrunkRecorderFields, + extractSourceFromFilename, + normalizeIncomingCall, + normalizeSdrTrunkCall, + normalizeTrunkRecorderCall, + parseJsonField +}; diff --git a/src/permissions/roles.js b/src/permissions/roles.js new file mode 100644 index 0000000..6fc7c44 --- /dev/null +++ b/src/permissions/roles.js @@ -0,0 +1,28 @@ +const ROLES = { + VIEWER: 'viewer', + EDITOR: 'editor', + MODERATOR: 'moderator', + ADMIN: 'admin' +}; + +const ROLE_PERMISSIONS = { + [ROLES.VIEWER]: ['calls:read', 'audio:read'], + [ROLES.EDITOR]: ['calls:read', 'audio:read', 'markers:update'], + [ROLES.MODERATOR]: ['calls:read', 'audio:read', 'markers:update', 'calls:purge'], + [ROLES.ADMIN]: ['calls:read', 'audio:read', 'markers:update', 'calls:purge', 'users:manage', 'sessions:manage'] +}; + +function permissionsForRole(role) { + return ROLE_PERMISSIONS[role] || ROLE_PERMISSIONS[ROLES.VIEWER]; +} + +function hasPermission(role, permission) { + return permissionsForRole(role).includes(permission); +} + +module.exports = { + ROLES, + ROLE_PERMISSIONS, + hasPermission, + permissionsForRole +}; diff --git a/test/config.test.js b/test/config.test.js new file mode 100644 index 0000000..9ef6d3e --- /dev/null +++ b/test/config.test.js @@ -0,0 +1,41 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { loadConfig, parseBoolean, parseList, redactConfig } = require('../src/config'); + +test('parseBoolean accepts common truthy values', () => { + assert.equal(parseBoolean('true'), true); + assert.equal(parseBoolean('1'), true); + assert.equal(parseBoolean('yes'), true); + assert.equal(parseBoolean('false'), false); +}); + +test('parseList trims and drops empty entries', () => { + assert.deepEqual(parseList('1001, 1002, ,2001'), ['1001', '1002', '2001']); +}); + +test('loadConfig reports conditional validation errors together', () => { + const result = loadConfig({ + STORAGE_MODE: 's3', + AI_PROVIDER: 'openai', + TRANSCRIPTION_MODE: 'remote' + }); + + assert.equal(result.isValid, false); + assert.deepEqual( + result.errors.map((error) => error.key), + ['S3_ENDPOINT', 'S3_BUCKET_NAME', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'OPENAI_API_KEY', 'FASTER_WHISPER_SERVER_URL'] + ); +}); + +test('redactConfig hides secret values', () => { + const redacted = redactConfig({ + discordToken: 'secret', + openaiApiKey: 'secret', + publicDomain: 'localhost' + }); + + assert.equal(redacted.discordToken, '[redacted]'); + assert.equal(redacted.openaiApiKey, '[redacted]'); + assert.equal(redacted.publicDomain, 'localhost'); +}); diff --git a/test/ingestion.test.js b/test/ingestion.test.js new file mode 100644 index 0000000..ad5315b --- /dev/null +++ b/test/ingestion.test.js @@ -0,0 +1,49 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { + extractSourceFromFilename, + normalizeIncomingCall, + normalizeTrunkRecorderCall +} = require('../src/ingestion/normalizeCall'); + +test('extractSourceFromFilename reads SDRTrunk FROM source IDs', () => { + assert.equal(extractSourceFromFilename('CALL_FROM_123456_TO_1001.mp3'), '123456'); + assert.equal(extractSourceFromFilename('call.mp3'), undefined); +}); + +test('normalizeIncomingCall maps SDRTrunk fields to the internal call shape', () => { + const call = normalizeIncomingCall({ + source: 'sdrtrunk', + fileInfo: { originalFilename: 'CALL_FROM_55_TO_1001.mp3' }, + fields: { + talkgroup: '1001', + systemLabel: 'County', + talkgroupLabel: 'Fire Dispatch', + dateTime: '2026-05-17T12:00:00Z' + } + }); + + assert.equal(call.provider, 'sdrtrunk'); + assert.equal(call.talkGroupID, '1001'); + assert.equal(call.source, '55'); + assert.equal(call.isTrunkRecorder, false); +}); + +test('normalizeTrunkRecorderCall extracts source and alias from meta srcList', () => { + const call = normalizeTrunkRecorderCall({ + talkgroup: '2001', + meta: JSON.stringify({ + start_time: 1779030000, + freq: 853000000, + srcList: [{ src: -1 }, { src: 9901, tag: 'Unit 12' }], + freqList: [{ freq: 853000000 }] + }) + }); + + assert.equal(call.provider, 'trunk-recorder'); + assert.equal(call.talkGroupID, '2001'); + assert.equal(call.source, '9901'); + assert.equal(call.talkerAlias, 'Unit 12'); + assert.equal(call.frequency, 853000000); +}); diff --git a/test/migrations.test.js b/test/migrations.test.js new file mode 100644 index 0000000..5bbe56b --- /dev/null +++ b/test/migrations.test.js @@ -0,0 +1,18 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { getMigrationPlan } = require('../src/db/migrations'); + +test('migration plan includes core tables by default', () => { + assert.deepEqual( + getMigrationPlan({ enableAuth: false }).map((migration) => migration.id), + ['001_create_core_tables'] + ); +}); + +test('migration plan includes auth tables when auth is enabled', () => { + assert.deepEqual( + getMigrationPlan({ enableAuth: true }).map((migration) => migration.id), + ['001_create_core_tables', '002_create_auth_tables'] + ); +}); diff --git a/test/permissions.test.js b/test/permissions.test.js new file mode 100644 index 0000000..4669ebc --- /dev/null +++ b/test/permissions.test.js @@ -0,0 +1,16 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { ROLES, hasPermission, permissionsForRole } = require('../src/permissions/roles'); + +test('admin can manage users', () => { + assert.equal(hasPermission(ROLES.ADMIN, 'users:manage'), true); +}); + +test('viewer cannot update markers', () => { + assert.equal(hasPermission(ROLES.VIEWER, 'markers:update'), false); +}); + +test('unknown roles fall back to viewer permissions', () => { + assert.deepEqual(permissionsForRole('unknown'), ['calls:read', 'audio:read']); +});