From d0d7ca67864a509988d595eece5f9649b9b7c499 Mon Sep 17 00:00:00 2001 From: Dadud Date: Sun, 17 May 2026 22:44:09 -0500 Subject: [PATCH 1/4] Resolve webserver runtime settings from SQLite --- src/settings/settingsService.js | 42 ++++++++++ src/setup/checks.js | 15 ++-- test/settingsService.test.js | 52 +++++++++++++ webserver.js | 131 +++++++++++++++++++------------- 4 files changed, 180 insertions(+), 60 deletions(-) diff --git a/src/settings/settingsService.js b/src/settings/settingsService.js index fa37e34..b61d46d 100644 --- a/src/settings/settingsService.js +++ b/src/settings/settingsService.js @@ -175,6 +175,45 @@ async function resolveSettings(db, env = process.env) { return { settings, secrets }; } +async function getRuntimeSetting(db, key, env = process.env) { + const definition = SETTING_DEFINITIONS[key]; + if (!definition) return undefined; + + const row = await get(db, 'SELECT value FROM app_settings WHERE key = ? AND is_secret = 0', [key]); + if (row && row.value !== undefined && row.value !== null) return row.value; + if (env[definition.envKey] !== undefined && env[definition.envKey] !== '') return env[definition.envKey]; + return definition.defaultValue; +} + +async function getRuntimeSecret(db, key, options = {}) { + const definition = SECRET_DEFINITIONS[key]; + if (!definition) return undefined; + + const env = options.env || process.env; + const row = await get(db, 'SELECT value FROM app_settings WHERE key = ? AND is_secret = 1', [key]); + if (row && row.value) { + const instanceSecret = getInstanceSecret({ env }); + return decryptSecret(row.value, instanceSecret); + } + + return env[definition.envKey] || ''; +} + +async function getRuntimeConfig(db, env = process.env) { + const settings = {}; + const secrets = {}; + + for (const key of Object.keys(SETTING_DEFINITIONS)) { + settings[key] = await getRuntimeSetting(db, key, env); + } + + for (const key of Object.keys(SECRET_DEFINITIONS)) { + secrets[key] = await getRuntimeSecret(db, key, { env }); + } + + return { settings, secrets }; +} + async function saveSettings(db, values, actor = 'admin') { const results = {}; @@ -269,6 +308,9 @@ module.exports = { decryptSecret, encryptSecret, getInstanceSecret, + getRuntimeConfig, + getRuntimeSecret, + getRuntimeSetting, getSetupStatus, resolveSettings, saveSecret, diff --git a/src/setup/checks.js b/src/setup/checks.js index 8beaa4b..9760a4f 100644 --- a/src/setup/checks.js +++ b/src/setup/checks.js @@ -41,6 +41,7 @@ function checkWritableDir(dirPath) { async function runSetupChecks(options = {}) { const rootDir = options.rootDir || path.join(__dirname, '..', '..'); const env = options.env || process.env; + const runtime = options.runtime || { settings: {}, secrets: {} }; const [node, python, ffmpeg, ollama] = await Promise.all([ checkCommand(process.execPath, ['--version']), checkCommand(env.PYTHON_COMMAND || (process.platform === 'win32' ? 'py' : 'python3'), ['--version']), @@ -57,19 +58,19 @@ async function runSetupChecks(options = {}) { dataDir: checkWritableDir(path.join(rootDir, 'data')), audioDir: checkWritableDir(path.join(rootDir, 'audio')), geocodingProvider: { - ok: Boolean(env.GOOGLE_MAPS_API_KEY || env.LOCATIONIQ_API_KEY), + ok: Boolean(runtime.secrets.googleMapsApiKey || runtime.secrets.locationIqApiKey || env.GOOGLE_MAPS_API_KEY || env.LOCATIONIQ_API_KEY), configuredProviders: { - google: Boolean(env.GOOGLE_MAPS_API_KEY), - locationiq: Boolean(env.LOCATIONIQ_API_KEY) + google: Boolean(runtime.secrets.googleMapsApiKey || env.GOOGLE_MAPS_API_KEY), + locationiq: Boolean(runtime.secrets.locationIqApiKey || env.LOCATIONIQ_API_KEY) } }, transcriptionProvider: { - ok: Boolean(env.TRANSCRIPTION_MODE || 'local'), - mode: env.TRANSCRIPTION_MODE || 'local' + ok: Boolean(runtime.settings.transcriptionMode || env.TRANSCRIPTION_MODE || 'local'), + mode: runtime.settings.transcriptionMode || env.TRANSCRIPTION_MODE || 'local' }, aiProvider: { - ok: Boolean(env.AI_PROVIDER || 'ollama'), - provider: env.AI_PROVIDER || 'ollama' + ok: Boolean(runtime.settings.aiProvider || env.AI_PROVIDER || 'ollama'), + provider: runtime.settings.aiProvider || env.AI_PROVIDER || 'ollama' }, uploadEndpoint: { ok: true, diff --git a/test/settingsService.test.js b/test/settingsService.test.js index ce572b4..1bd6cb2 100644 --- a/test/settingsService.test.js +++ b/test/settingsService.test.js @@ -4,6 +4,9 @@ const assert = require('node:assert/strict'); const { decryptSecret, encryptSecret, + getRuntimeConfig, + getRuntimeSecret, + getRuntimeSetting, getSetupStatus, resolveSettings } = require('../src/settings/settingsService'); @@ -14,6 +17,10 @@ function createFakeDb({ settingsRows = [], setupComplete = false, adminCount = 0 callback(null, settingsRows); }, get(sql, params, callback) { + if (sql.includes('app_settings')) { + callback(null, settingsRows.find((row) => row.key === params[0])); + return; + } if (sql.includes('setup_state')) { callback(null, setupComplete ? { value: 'true' } : undefined); return; @@ -47,6 +54,51 @@ test('resolveSettings prefers SQLite settings over env and defaults', async () = assert.equal(resolved.settings.storageMode.source, 'default'); }); +test('runtime setting reads SQLite before env before default', async () => { + const db = createFakeDb({ + settingsRows: [ + { key: 'storageMode', value: 's3', is_secret: 0, requires_restart: 1, updated_at: 'now' } + ] + }); + + assert.equal(await getRuntimeSetting(db, 'storageMode', { STORAGE_MODE: 'local' }), 's3'); + assert.equal(await getRuntimeSetting(createFakeDb(), 'storageMode', { STORAGE_MODE: 'local' }), 'local'); + assert.equal(await getRuntimeSetting(createFakeDb(), 'storageMode', {}), 'local'); +}); + +test('runtime secret decrypts SQLite values before env fallback', async () => { + const secret = 'test-instance-secret'; + const encrypted = encryptSecret('stored-openai-key', secret); + const db = createFakeDb({ + settingsRows: [ + { key: 'openaiApiKey', value: encrypted, is_secret: 1, requires_restart: 0, updated_at: 'now' } + ] + }); + + const value = await getRuntimeSecret(db, 'openaiApiKey', { + env: { + SETTINGS_ENCRYPTION_KEY: secret, + OPENAI_API_KEY: 'env-openai-key' + } + }); + + assert.equal(value, 'stored-openai-key'); + assert.equal(await getRuntimeSecret(createFakeDb(), 'openaiApiKey', { env: { OPENAI_API_KEY: 'env-key' } }), 'env-key'); +}); + +test('runtime config includes resolved settings and decrypted secrets', async () => { + const db = createFakeDb({ + settingsRows: [ + { key: 'timezone', value: 'America/Chicago', is_secret: 0, requires_restart: 0, updated_at: 'now' } + ] + }); + + const config = await getRuntimeConfig(db, { OPENAI_API_KEY: 'env-key' }); + + assert.equal(config.settings.timezone, 'America/Chicago'); + assert.equal(config.secrets.openaiApiKey, 'env-key'); +}); + test('resolveSettings redacts write-only secret values', async () => { const db = createFakeDb({ settingsRows: [ diff --git a/webserver.js b/webserver.js index 587a4ed..1631162 100644 --- a/webserver.js +++ b/webserver.js @@ -6,6 +6,7 @@ const { applyMigrations } = require('./src/db/migrations'); const { getJobSummary, getRecentJobs } = require('./src/jobs/processingJobs'); const { getSetupStatus, + getRuntimeConfig, markSetupComplete, resolveSettings, saveSecret, @@ -91,28 +92,33 @@ if (LOCATIONIQ_API_KEY) { const app = express(); app.use(express.json()); // Add this line to parse JSON bodies -app.get('/api/config/google-api-key', (req, res) => { - res.json({ apiKey: GOOGLE_MAPS_API_KEY }); -}); - -// Add endpoint to serve LocationIQ API key -app.get('/api/config/locationiq-api-key', (req, res) => { - res.json({ apiKey: LOCATIONIQ_API_KEY }); -}); - -// Add endpoint to serve all geocoding configuration -app.get('/api/config/geocoding', (req, res) => { - res.json({ - google: { - available: !!GOOGLE_MAPS_API_KEY, - apiKey: GOOGLE_MAPS_API_KEY - }, - locationiq: { - available: !!LOCATIONIQ_API_KEY, - apiKey: LOCATIONIQ_API_KEY - } - }); -}); +app.get('/api/config/google-api-key', async (req, res) => { + const runtime = await getResolvedRuntimeConfig(); + res.json({ apiKey: runtime.secrets.googleMapsApiKey }); +}); + +// Add endpoint to serve LocationIQ API key +app.get('/api/config/locationiq-api-key', async (req, res) => { + const runtime = await getResolvedRuntimeConfig(); + res.json({ apiKey: runtime.secrets.locationIqApiKey }); +}); + +// Add endpoint to serve all geocoding configuration +app.get('/api/config/geocoding', async (req, res) => { + const runtime = await getResolvedRuntimeConfig(); + const googleMapsApiKey = runtime.secrets.googleMapsApiKey; + const locationIqApiKey = runtime.secrets.locationIqApiKey; + res.json({ + google: { + available: !!googleMapsApiKey, + apiKey: googleMapsApiKey + }, + locationiq: { + available: !!locationIqApiKey, + apiKey: locationIqApiKey + } + }); +}); // Add endpoint to check if current user is admin app.get('/api/auth/is-admin', async (req, res) => { @@ -188,6 +194,11 @@ const dbReady = applyMigrations(db, { enableAuth: true }) .catch((err) => { console.error('[Webserver] Error initializing database schema:', err); }); + +async function getResolvedRuntimeConfig() { + await dbReady; + return getRuntimeConfig(db, process.env); +} // Helper Functions for Authentication function hashPassword(password, salt) { @@ -270,30 +281,36 @@ Transmission: "${transcript}" Category:`; - let category = 'OTHER'; // Default value - - const controller = new AbortController(); + let category = 'OTHER'; // Default value + const runtime = await getResolvedRuntimeConfig(); + const aiProvider = runtime.settings.aiProvider || AI_PROVIDER; + const openaiApiKey = runtime.secrets.openaiApiKey || OPENAI_API_KEY; + const openaiModel = runtime.settings.openaiModel || OPENAI_MODEL; + const ollamaUrl = runtime.settings.ollamaUrl || OLLAMA_URL; + const ollamaModel = runtime.settings.ollamaModel || OLLAMA_MODEL; + + const controller = new AbortController(); const timeoutId = setTimeout(() => { console.warn(`[Webserver] AI request timed out after 10 seconds during categorization.`); controller.abort(); - }, 10000); // 10-second timeout - - // --- AI Provider Logic --- - if (AI_PROVIDER.toLowerCase() === 'openai') { - if (!OPENAI_API_KEY) { - console.error('[Webserver] FATAL: AI_PROVIDER is set to openai, but OPENAI_API_KEY is not configured!'); - return 'OTHER'; // Fallback if key is missing - } - console.log(`[Webserver] Categorizing with OpenAI model: ${OPENAI_MODEL}`); - - const response = await fetch('https://api.openai.com/v1/chat/completions', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${OPENAI_API_KEY}` - }, - body: JSON.stringify({ - model: OPENAI_MODEL, + }, 10000); // 10-second timeout + + // --- AI Provider Logic --- + if (aiProvider.toLowerCase() === 'openai') { + if (!openaiApiKey) { + console.error('[Webserver] FATAL: AI_PROVIDER is set to openai, but OPENAI_API_KEY is not configured!'); + return 'OTHER'; // Fallback if key is missing + } + console.log(`[Webserver] Categorizing with OpenAI model: ${openaiModel}`); + + const response = await fetch('https://api.openai.com/v1/chat/completions', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${openaiApiKey}` + }, + body: JSON.stringify({ + model: openaiModel, messages: [{ role: 'user', content: commonPrompt }], temperature: 0.2, // Lower temp for more deterministic category max_tokens: 20 // A category name is short @@ -313,15 +330,15 @@ Category:`; if (result.choices && result.choices.length > 0 && result.choices[0].message) { category = result.choices[0].message.content.trim(); } - - } else { // Default to Ollama - console.log(`[Webserver] Categorizing with Ollama model: ${OLLAMA_MODEL}`); - - const response = await fetch(`${OLLAMA_URL}/api/generate`, { + + } else { // Default to Ollama + console.log(`[Webserver] Categorizing with Ollama model: ${ollamaModel}`); + + const response = await fetch(`${ollamaUrl}/api/generate`, { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ - model: OLLAMA_MODEL, + model: ollamaModel, prompt: commonPrompt, // The prompt is compatible stream: false, options: { @@ -741,7 +758,8 @@ app.get('/api/setup/status', async (req, res) => { app.get('/api/setup/checks', async (req, res) => { await dbReady; try { - const checks = await runSetupChecks({ rootDir: __dirname, env: process.env }); + const runtime = await getResolvedRuntimeConfig(); + const checks = await runSetupChecks({ rootDir: __dirname, env: process.env, runtime }); res.json(checks); } catch (err) { console.error('Error running setup checks:', err); @@ -811,7 +829,8 @@ app.post('/api/setup/test-provider', async (req, res) => { await dbReady; const { provider } = req.body || {}; try { - const checks = await runSetupChecks({ rootDir: __dirname, env: process.env }); + const runtime = await getResolvedRuntimeConfig(); + const checks = await runSetupChecks({ rootDir: __dirname, env: process.env, runtime }); const providerMap = { geocoding: checks.geocodingProvider, transcription: checks.transcriptionProvider, @@ -972,7 +991,12 @@ app.put('/api/settings', adminAuth, async (req, res) => { try { const result = await saveSettings(db, req.body || {}, req.user?.username || 'admin'); const requiresRestart = Object.values(result).some((item) => item.requiresRestart); - res.json({ ok: true, result, requiresRestart }); + res.json({ + ok: true, + result, + requiresRestart, + hotAppliedByWebserver: ['aiProvider', 'ollamaUrl', 'ollamaModel', 'openaiModel'] + }); } catch (err) { console.error('Error updating settings:', err); res.status(500).json({ error: 'Internal server error' }); @@ -997,7 +1021,8 @@ app.put('/api/settings/secrets/:key', adminAuth, async (req, res) => { app.get('/api/settings/checks', adminAuth, async (req, res) => { await dbReady; try { - const checks = await runSetupChecks({ rootDir: __dirname, env: process.env }); + const runtime = await getResolvedRuntimeConfig(); + const checks = await runSetupChecks({ rootDir: __dirname, env: process.env, runtime }); res.json(checks); } catch (err) { console.error('Error running settings checks:', err); From 0de44cf9c4fb79f4e17ca0b7ed75275b9863e49a Mon Sep 17 00:00:00 2001 From: Dadud Date: Sun, 17 May 2026 22:59:41 -0500 Subject: [PATCH 2/4] Integrate bot runtime settings --- bot.js | 162 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 83 insertions(+), 79 deletions(-) diff --git a/bot.js b/bot.js index 987ea4f..5b73dd6 100644 --- a/bot.js +++ b/bot.js @@ -4,6 +4,7 @@ require('dotenv').config(); const { loadConfig } = require('./src/config'); const { applyMigrations } = require('./src/db/migrations'); const { normalizeIncomingCall } = require('./src/ingestion/normalizeCall'); +const { getRuntimeConfig, getSetupStatus } = require('./src/settings/settingsService'); const { JOB_TYPES, createProcessingJob, @@ -81,56 +82,28 @@ const { const startupConfig = loadConfig(process.env); if (!startupConfig.isValid) { - console.error('FATAL: Invalid configuration:'); + console.warn('WARNING: Configuration has issues. Setup mode will remain available:'); for (const error of startupConfig.errors) { - console.error(`- ${error.key}: ${error.message}`); + console.warn(`- ${error.key}: ${error.message}`); } - process.exit(1); } -// --- VALIDATE AI-RELATED ENV VARS --- -if (!AI_PROVIDER) { - console.error("FATAL: AI_PROVIDER is not set in the .env file. Please specify 'ollama' or 'openai'."); - process.exit(1); -} - -if (AI_PROVIDER.toLowerCase() === 'openai') { - if (!OPENAI_API_KEY || !OPENAI_MODEL) { - console.error("FATAL: AI_PROVIDER is 'openai', but OPENAI_API_KEY or OPENAI_MODEL is missing in the .env file."); - process.exit(1); - } -} else if (AI_PROVIDER.toLowerCase() === 'ollama') { - if (!OLLAMA_URL || !OLLAMA_MODEL) { - console.error("FATAL: AI_PROVIDER is 'ollama', but OLLAMA_URL or OLLAMA_MODEL is missing in the .env file."); - process.exit(1); - } -} else { - console.error(`FATAL: Invalid AI_PROVIDER specified in .env file: '${AI_PROVIDER}'. Must be 'openai' or 'ollama'.`); - process.exit(1); -} -// --- END VALIDATION --- - // --- VALIDATE TRANSCRIPTION-RELATED ENV VARS --- const effectiveTranscriptionMode = TRANSCRIPTION_MODE || 'local'; // Keep this to ensure a default if (!['local', 'remote', 'openai', 'icad'].includes(effectiveTranscriptionMode)) { - console.error(`FATAL: Invalid TRANSCRIPTION_MODE specified in .env file: '${TRANSCRIPTION_MODE}'. Must be 'local', 'remote', 'openai', or 'icad'.`); - process.exit(1); + console.warn(`WARNING: Invalid TRANSCRIPTION_MODE specified in .env file: '${TRANSCRIPTION_MODE}'. Use /setup to choose local, remote, openai, or icad.`); } if (effectiveTranscriptionMode === 'local' && !TRANSCRIPTION_DEVICE) { - console.error("FATAL: TRANSCRIPTION_MODE is 'local', but TRANSCRIPTION_DEVICE is missing in the .env file. Please set it to 'cuda' for a GPU or 'cpu' for CPU."); - process.exit(1); + console.warn("WARNING: TRANSCRIPTION_MODE is 'local', but TRANSCRIPTION_DEVICE is missing. Use /setup to choose cpu or cuda."); } if (effectiveTranscriptionMode === 'remote' && !FASTER_WHISPER_SERVER_URL) { - console.error("FATAL: TRANSCRIPTION_MODE is 'remote', but FASTER_WHISPER_SERVER_URL is missing in the .env file."); - process.exit(1); + console.warn("WARNING: TRANSCRIPTION_MODE is 'remote', but FASTER_WHISPER_SERVER_URL is missing. Use /setup to configure it."); } if (effectiveTranscriptionMode === 'openai' && !OPENAI_API_KEY) { - console.error("FATAL: TRANSCRIPTION_MODE is 'openai', but OPENAI_API_KEY is missing in the .env file. This is required for OpenAI transcriptions."); - process.exit(1); + console.warn("WARNING: TRANSCRIPTION_MODE is 'openai', but OPENAI_API_KEY is missing. Use /setup to configure it."); } if (effectiveTranscriptionMode === 'icad' && !ICAD_URL) { - console.error("FATAL: TRANSCRIPTION_MODE is 'icad', but ICAD_URL is missing in the .env file. Please set it to your ICAD API endpoint URL."); - process.exit(1); + console.warn("WARNING: TRANSCRIPTION_MODE is 'icad', but ICAD_URL is missing. Use /setup to configure it."); } // --- END VALIDATION --- @@ -189,9 +162,7 @@ if (ENABLE_TWO_TONE_MODE && ENABLE_TWO_TONE_MODE.toLowerCase() === 'true') { const missingVars = requiredTwoToneVars.filter(varName => !process.env[varName]); if (missingVars.length > 0) { - console.error(`FATAL: Two-tone mode is enabled but missing required environment variables: ${missingVars.join(', ')}`); - console.error('Please add these variables to your .env file. See TWO_TONE_ENV_ADDITIONS.txt for the complete list.'); - process.exit(1); + console.warn(`WARNING: Two-tone mode is enabled but missing required environment variables: ${missingVars.join(', ')}. Use /setup or .env to complete this before enabling bot services.`); } } @@ -840,18 +811,18 @@ const AWS = require('aws-sdk'); let s3 = null; if (STORAGE_MODE === 's3') { if (!S3_ENDPOINT || !S3_BUCKET_NAME || !S3_ACCESS_KEY_ID || !S3_SECRET_ACCESS_KEY) { - logger.error('FATAL: STORAGE_MODE is s3, but required S3 environment variables are missing! Check bot .env'); - process.exit(1); // Exit if S3 config is incomplete + logger.warn('WARNING: STORAGE_MODE is s3, but required S3 environment variables are missing. Audio serving from S3 will be unavailable until setup is completed.'); + } else { + AWS.config.update({ + accessKeyId: S3_ACCESS_KEY_ID, + secretAccessKey: S3_SECRET_ACCESS_KEY, + endpoint: S3_ENDPOINT, + s3ForcePathStyle: true, // Necessary for MinIO/non-AWS S3 + signatureVersion: 'v4' + }); + s3 = new AWS.S3(); + logger.info(`[Bot] Storage mode set to S3. Endpoint: ${S3_ENDPOINT}, Bucket: ${S3_BUCKET_NAME}`); } - AWS.config.update({ - accessKeyId: S3_ACCESS_KEY_ID, - secretAccessKey: S3_SECRET_ACCESS_KEY, - endpoint: S3_ENDPOINT, - s3ForcePathStyle: true, // Necessary for MinIO/non-AWS S3 - signatureVersion: 'v4' - }); - s3 = new AWS.S3(); - logger.info(`[Bot] Storage mode set to S3. Endpoint: ${S3_ENDPOINT}, Bucket: ${S3_BUCKET_NAME}`); } else { logger.info('[Bot] Storage mode set to local.'); } @@ -976,6 +947,21 @@ async function initializeDatabase() { } } +async function getBotRuntimeConfig() { + return getRuntimeConfig(db, process.env); +} + +async function getPublicAudioUrl(audioId) { + let publicDomain = PUBLIC_DOMAIN || 'localhost'; + try { + const runtime = await getBotRuntimeConfig(); + publicDomain = runtime.settings.publicDomain || publicDomain; + } catch (error) { + logger.warn(`Could not load runtime public domain; falling back to startup config: ${error.message}`); + } + return `http://${publicDomain}/audio/${audioId}`; +} + // Function to create admin user if authentication is enabled function createAdminUser() { return new Promise((resolve, reject) => { @@ -1091,16 +1077,23 @@ async function initializeBot() { // Step 6: Create admin user for webserver if auth is enabled await createAdminUser(); - // Step 7: Start bot services (Discord and Express API) - await startBotServices(); - - // Step 8: Start webserver last - if (WEBSERVER_PORT && (GOOGLE_MAPS_API_KEY || LOCATIONIQ_API_KEY)) { + // Step 7: Start webserver before Discord so setup can run even when bot settings are incomplete + if (WEBSERVER_PORT) { await startWebserver(); } else { - logger.warn('Webserver not started: WEBSERVER_PORT or geocoding API key (GOOGLE_MAPS_API_KEY or LOCATIONIQ_API_KEY) not configured'); + logger.warn('Webserver not started: WEBSERVER_PORT is not configured'); + } + + // Step 8: In setup mode, keep the browser console available without forcing Discord login + const setupStatus = await getSetupStatus(db, process.env); + if (setupStatus.setupRequired) { + logger.warn(`Setup is incomplete (${setupStatus.missing.join(', ') || 'unknown requirements'}). Discord bot services will start after setup is completed and the app is restarted.`); + return true; } + // Step 9: Start bot services (Discord and Express API) + await startBotServices(); + logger.info('Bot initialization completed successfully!'); return true; } catch (error) { @@ -3920,7 +3913,7 @@ async function processMergedCallSegments( .trim(); // Build combined transcription lines for Discord (all segments in one message) - const audioUrl = `http://${PUBLIC_DOMAIN}/audio/${transcriptionId}`; + const audioUrl = await getPublicAudioUrl(transcriptionId); const transcriptionLines = []; for (const segment of sortedSegments) { @@ -4274,12 +4267,12 @@ function sendAlertMessage( callback ) { // Look up the audio_id from the database for this transcription - db.get('SELECT id FROM audio_files WHERE transcription_id = ?', [audioID], (err, row) => { + db.get('SELECT id FROM audio_files WHERE transcription_id = ?', [audioID], async (err, row) => { // Use transcription ID as fallback if audio ID not found const actualAudioID = (err || !row) ? audioID : row.id; // Create a URL for the audio file - const audioUrl = `http://${PUBLIC_DOMAIN}/audio/${actualAudioID}`; + const audioUrl = await getPublicAudioUrl(actualAudioID); // Log the IDs for debugging logger.info(`Alert - Transcription ID: ${audioID}, Audio ID: ${actualAudioID}, URL: ${audioUrl}`); @@ -4514,7 +4507,7 @@ function sendTranscriptionMessage( } // Get or create the channel within the category - getOrCreateChannel(channelName, category.id, (channel) => { + getOrCreateChannel(channelName, category.id, async (channel) => { if (!channel) { logger.error('Failed to get or create channel.'); if (callback) callback(); // Ensure callback is called even on error @@ -4525,7 +4518,7 @@ function sendTranscriptionMessage( // Note: We use transcription ID (`audioID` parameter) for the URL now // as audio_files might get cleaned up. // The audio server route /audio/:id expects the transcription ID. - const audioUrl = `http://${PUBLIC_DOMAIN}/audio/${audioID}`; + const audioUrl = await getPublicAudioUrl(audioID); // Log the ID and URL for debugging logger.info(`Creating link for Transcription ID: ${audioID}, Audio URL: ${audioUrl}`); @@ -5045,26 +5038,32 @@ Focus on providing insightful analysis of each transmission. The "description" f Include no other text besides this JSON.`; // Call the AI provider with a timeout + const runtime = await getBotRuntimeConfig(); + const aiProvider = (runtime.settings.aiProvider || AI_PROVIDER || 'ollama').toLowerCase(); + const openaiApiKey = runtime.secrets.openaiApiKey || OPENAI_API_KEY || ''; + const openaiModel = runtime.settings.openaiModel || OPENAI_MODEL || 'gpt-4o-mini'; + const ollamaUrl = runtime.settings.ollamaUrl || OLLAMA_URL || 'http://localhost:11434'; + const ollamaModel = runtime.settings.ollamaModel || OLLAMA_MODEL || 'llama3.1:8b'; const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 30000); // 30 second timeout let resultText = ''; try { - if (AI_PROVIDER.toLowerCase() === 'openai') { - if (!OPENAI_API_KEY) { + if (aiProvider === 'openai') { + if (!openaiApiKey) { logger.error("[Bot] FATAL: AI_PROVIDER is set to openai, but OPENAI_API_KEY is not configured!"); throw new Error("OpenAI API key is not configured."); } - logger.info(`[Bot] Generating summary with OpenAI model: ${OPENAI_MODEL}`); + logger.info(`[Bot] Generating summary with OpenAI model: ${openaiModel}`); const response = await fetch('https://api.openai.com/v1/chat/completions', { method: 'POST', headers: { 'Content-Type': 'application/json', - 'Authorization': `Bearer ${OPENAI_API_KEY}` + 'Authorization': `Bearer ${openaiApiKey}` }, body: JSON.stringify({ - model: OPENAI_MODEL, + model: openaiModel, messages: [{ role: 'user', content: commonPrompt }], temperature: 0.3, response_format: { type: "json_object" } // Request JSON output @@ -5082,13 +5081,13 @@ Include no other text besides this JSON.`; } } else { // Default to Ollama - logger.info(`[Bot] Generating summary with Ollama model: ${OLLAMA_MODEL}`); + logger.info(`[Bot] Generating summary with Ollama model: ${ollamaModel}`); - const response = await fetch(`${OLLAMA_URL}/api/generate`, { + const response = await fetch(`${ollamaUrl}/api/generate`, { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ - model: OLLAMA_MODEL, + model: ollamaModel, prompt: commonPrompt, stream: false }), @@ -5347,7 +5346,7 @@ async function updateSummaryEmbed() { if (summary.highlights && summary.highlights.length > 0) { for (const highlight of summary.highlights) { // Get audio URL - const audioUrl = `http://${PUBLIC_DOMAIN}/audio/${highlight.id}`; + const audioUrl = await getPublicAudioUrl(highlight.id); // Fix timestamp display - use timestamp directly from database let timestampDisplay; @@ -6226,8 +6225,13 @@ client.on('interactionCreate', async (interaction) => { const userQuestion = interaction.fields.getTextInputValue('ai_question'); try { - // --- Read lookback from .env, default to 8 hours --- - const askAiLookbackHours = parseFloat(ASK_AI_LOOKBACK_HOURS) || 8; + const runtime = await getBotRuntimeConfig(); + const askAiLookbackHours = parseFloat(runtime.settings.askAiLookbackHours || ASK_AI_LOOKBACK_HOURS) || 8; + const aiProvider = (runtime.settings.aiProvider || AI_PROVIDER || 'ollama').toLowerCase(); + const openaiApiKey = runtime.secrets.openaiApiKey || OPENAI_API_KEY || ''; + const openaiModel = runtime.settings.openaiModel || OPENAI_MODEL || 'gpt-4o-mini'; + const ollamaUrl = runtime.settings.ollamaUrl || OLLAMA_URL || 'http://localhost:11434'; + const ollamaModel = runtime.settings.ollamaModel || OLLAMA_MODEL || 'llama3.1:8b'; const now = new Date(); const queryStartDate = new Date(now.getTime() - askAiLookbackHours * 60 * 60 * 1000); // Convert start date to Unix seconds for the query @@ -6328,20 +6332,20 @@ User Question: ${userQuestion} let aiResponseText = 'Error: Could not get response from AI.'; try { - if (AI_PROVIDER.toLowerCase() === 'openai') { - if (!OPENAI_API_KEY) { + if (aiProvider === 'openai') { + if (!openaiApiKey) { throw new Error("OpenAI API key is not configured."); } - logger.info(`[Bot] Answering question with OpenAI model: ${OPENAI_MODEL}`); + logger.info(`[Bot] Answering question with OpenAI model: ${openaiModel}`); const response = await fetch('https://api.openai.com/v1/chat/completions', { method: 'POST', headers: { 'Content-Type': 'application/json', - 'Authorization': `Bearer ${OPENAI_API_KEY}` + 'Authorization': `Bearer ${openaiApiKey}` }, body: JSON.stringify({ - model: OPENAI_MODEL, + model: openaiModel, messages: [{ role: 'user', content: commonPrompt }], temperature: 0.5, max_tokens: 500 @@ -6358,13 +6362,13 @@ User Question: ${userQuestion} } } else { // Default to Ollama - logger.info(`[Bot] Answering question with Ollama model: ${OLLAMA_MODEL}`); + logger.info(`[Bot] Answering question with Ollama model: ${ollamaModel}`); - const response = await fetch(`${OLLAMA_URL}/api/generate`, { + const response = await fetch(`${ollamaUrl}/api/generate`, { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ - model: OLLAMA_MODEL, + model: ollamaModel, prompt: commonPrompt, stream: false, options: { num_ctx: 35000 } From 619ab35906c316cf9e0303d90f923dc96dc74e4c Mon Sep 17 00:00:00 2001 From: Dadud Date: Sun, 17 May 2026 23:04:40 -0500 Subject: [PATCH 3/4] Resolve transcription settings at runtime --- bot.js | 102 ++++++++++++++++++++------------ public/settings.html | 7 +++ public/settings.js | 5 +- src/settings/settingsService.js | 5 ++ 4 files changed, 80 insertions(+), 39 deletions(-) diff --git a/bot.js b/bot.js index 5b73dd6..f315194 100644 --- a/bot.js +++ b/bot.js @@ -382,8 +382,8 @@ function cleanStaleQueueEntries() { function detectTwoTone(audioFilePath, transcriptionId, talkGroupID, callback) { // For non-local modes, use a separate Python process for tone detection - if (effectiveTranscriptionMode !== 'local') { - logger.info(`Using standalone tone detection for ${effectiveTranscriptionMode} mode`); + if (activeTranscriptionMode !== 'local') { + logger.info(`Using standalone tone detection for ${activeTranscriptionMode} mode`); return detectTwoToneStandalone(audioFilePath, transcriptionId, talkGroupID, callback); } @@ -951,6 +951,25 @@ async function getBotRuntimeConfig() { return getRuntimeConfig(db, process.env); } +async function getBotTranscriptionConfig() { + const runtime = await getBotRuntimeConfig(); + const mode = (runtime.settings.transcriptionMode || TRANSCRIPTION_MODE || 'local').toLowerCase(); + + return { + mode: ['local', 'remote', 'openai', 'icad'].includes(mode) ? mode : 'local', + device: (runtime.settings.transcriptionDevice || TRANSCRIPTION_DEVICE || 'cpu').toLowerCase(), + fasterWhisperServerUrl: runtime.settings.fasterWhisperServerUrl || FASTER_WHISPER_SERVER_URL || '', + whisperModel: runtime.settings.whisperModel || WHISPER_MODEL || 'large-v3', + openaiApiKey: runtime.secrets.openaiApiKey || OPENAI_API_KEY || '', + openaiTranscriptionPrompt: runtime.settings.openaiTranscriptionPrompt || OPENAI_TRANSCRIPTION_PROMPT || '', + openaiTranscriptionModel: runtime.settings.openaiTranscriptionModel || OPENAI_TRANSCRIPTION_MODEL || 'whisper-1', + openaiTranscriptionTemperature: runtime.settings.openaiTranscriptionTemperature || OPENAI_TRANSCRIPTION_TEMPERATURE || '0.0', + icadUrl: runtime.settings.icadUrl || ICAD_URL || '', + icadProfile: runtime.settings.icadProfile || ICAD_PROFILE || 'whisper-1', + icadApiKey: runtime.secrets.icadApiKey || ICAD_API_KEY || '' + }; +} + async function getPublicAudioUrl(audioId) { let publicDomain = PUBLIC_DOMAIN || 'localhost'; try { @@ -1159,6 +1178,7 @@ let isBootComplete = false; const messageCache = new Map(); // Stores the latest message for each channel const MESSAGE_COOLDOWN = 15000; // 15 seconds in milliseconds let transcriptionProcess = null; +let activeTranscriptionMode = effectiveTranscriptionMode; let isProcessingTranscription = false; let currentTranscriptionId = null; // Track current transcription for timeout let transcriptionTimeout = null; // Timeout for current transcription @@ -1811,12 +1831,12 @@ app.get('/audio/:id', (req, res) => { // Function to start the transcription process // Function to start the transcription process async function startTranscriptionProcess() { - // *** ADD THIS CHECK AT THE TOP *** - if (effectiveTranscriptionMode !== 'local') { + const transcriptionConfig = await getBotTranscriptionConfig(); + activeTranscriptionMode = transcriptionConfig.mode; + if (transcriptionConfig.mode !== 'local') { logger.info('Transcription mode is not local, skipping Python process start.'); - return; // Don't start if mode is remote + return; } - // *** END ADDED CHECK *** // Clean up existing process if it exists if (transcriptionProcess) { @@ -2162,7 +2182,7 @@ async function startTranscriptionProcess() { transcriptionProcess.on('error', (err) => { logger.error(`Failed to start local transcription process: ${err.message}`); cleanupTranscriptionProcess(); - if (effectiveTranscriptionMode === 'local') { + if (activeTranscriptionMode === 'local') { logger.info('Will attempt to restart local transcription process in 10 seconds due to spawn error...'); setTimeout(startTranscriptionProcess, 10000); } @@ -2340,7 +2360,7 @@ async function startTranscriptionProcess() { cleanupTranscriptionProcess(); // Only restart if not too many recent failures - if (effectiveTranscriptionMode === 'local') { + if (activeTranscriptionMode === 'local') { if (code === null) { // For null exit codes (startup crashes), wait longer and provide guidance logger.error('STARTUP CRASH DETECTED - Will NOT automatically restart to prevent loop'); @@ -2459,14 +2479,14 @@ function startProcessHealthCheck() { if (timeSinceActivity > 600000 && queueSize > 0) { // 10 minutes + queue items = real problem logger.error(`Transcription process appears stuck (no activity for 10 minutes with ${queueSize} items queued). Restarting...`); cleanupTranscriptionProcess(); - if (effectiveTranscriptionMode === 'local') { + if (activeTranscriptionMode === 'local') { setTimeout(startTranscriptionProcess, 5000); } return; } else if (timeSinceActivity > 1800000) { // 30 minutes with no activity at all (safety net) logger.warn(`Very long radio silence detected (30+ minutes). Performing health check restart as precaution...`); cleanupTranscriptionProcess(); - if (effectiveTranscriptionMode === 'local') { + if (activeTranscriptionMode === 'local') { setTimeout(startTranscriptionProcess, 5000); } return; @@ -2492,7 +2512,7 @@ function startProcessHealthCheck() { if (queueSize > 15 && !isProcessingTranscription && timeSinceActivity > 300000) { // 5 minutes + 15+ items = real stuck logger.error(`Queue definitely stuck with ${queueSize} items and no processing for 5 minutes. Force restarting transcription process...`); cleanupTranscriptionProcess(); - if (effectiveTranscriptionMode === 'local') { + if (activeTranscriptionMode === 'local') { setTimeout(startTranscriptionProcess, 2000); } } @@ -2559,7 +2579,7 @@ function processNextTranscription() { logger.error(`Transcription timeout for ID ${currentTranscriptionId}. Restarting process...`); // Force restart the process on timeout cleanupTranscriptionProcess(); - if (effectiveTranscriptionMode === 'local') { + if (activeTranscriptionMode === 'local') { setTimeout(startTranscriptionProcess, 5000); } }, TRANSCRIPTION_TIMEOUT_MS); @@ -2593,8 +2613,10 @@ function processNextTranscription() { // *** NEW FUNCTION for Remote Transcription *** async function transcribeAudioRemotely(filePath, callback) { + const transcriptionConfig = await getBotTranscriptionConfig(); + // Ensure URL is configured for remote mode - if (!FASTER_WHISPER_SERVER_URL) { + if (!transcriptionConfig.fasterWhisperServerUrl) { logger.error('FATAL: FASTER_WHISPER_SERVER_URL is not configured for remote mode.'); if (callback) callback(""); // Fail gracefully return; @@ -2626,14 +2648,14 @@ async function transcribeAudioRemotely(filePath, callback) { const form = new FormData(); form.append('file', fs.createReadStream(filePath)); // Append model if specified in environment - if (WHISPER_MODEL) { - form.append('model', WHISPER_MODEL); - logger.info(`Requesting remote model: ${WHISPER_MODEL}`); + if (transcriptionConfig.whisperModel) { + form.append('model', transcriptionConfig.whisperModel); + logger.info(`Requesting remote model: ${transcriptionConfig.whisperModel}`); } // Add language parameter if needed // form.append('language', 'en'); - const apiEndpoint = `${FASTER_WHISPER_SERVER_URL}/v1/audio/transcriptions`; + const apiEndpoint = `${transcriptionConfig.fasterWhisperServerUrl}/v1/audio/transcriptions`; const filenameForLog = path.basename(filePath); logger.info(`Sending remote transcription request for ${filenameForLog} to ${apiEndpoint}`); @@ -2695,8 +2717,10 @@ async function transcribeAudioRemotely(filePath, callback) { } async function transcribeWithOpenAIAPI(filePath, callback) { + const transcriptionConfig = await getBotTranscriptionConfig(); + // Check for API Key - if (!OPENAI_API_KEY) { + if (!transcriptionConfig.openaiApiKey) { logger.error('FATAL: TRANSCRIPTION_MODE is openai, but OPENAI_API_KEY is not configured.'); if (callback) callback(""); // Fail gracefully return; @@ -2714,21 +2738,21 @@ async function transcribeWithOpenAIAPI(filePath, callback) { form.append('file', fs.createReadStream(filePath)); // Use the model from environment variable, fallback to whisper-1 if not set - const modelToUse = OPENAI_TRANSCRIPTION_MODEL || 'whisper-1'; + const modelToUse = transcriptionConfig.openaiTranscriptionModel; form.append('model', modelToUse); // Force language to English for better scanner audio transcription form.append('language', 'en'); // Add temperature parameter for transcription consistency (if supported) - const temperature = OPENAI_TRANSCRIPTION_TEMPERATURE || '0.0'; + const temperature = transcriptionConfig.openaiTranscriptionTemperature; form.append('temperature', temperature); const filenameForLog = path.basename(filePath); // Add custom prompt if configured to improve scanner audio transcription - if (OPENAI_TRANSCRIPTION_PROMPT) { - form.append('prompt', OPENAI_TRANSCRIPTION_PROMPT); + if (transcriptionConfig.openaiTranscriptionPrompt) { + form.append('prompt', transcriptionConfig.openaiTranscriptionPrompt); logger.info(`Using custom OpenAI transcription prompt for ${filenameForLog}`); } @@ -2747,7 +2771,7 @@ async function transcribeWithOpenAIAPI(filePath, callback) { method: 'POST', body: form, headers: { - 'Authorization': `Bearer ${OPENAI_API_KEY}`, + 'Authorization': `Bearer ${transcriptionConfig.openaiApiKey}`, ...form.getHeaders() }, signal: controller.signal @@ -2785,8 +2809,10 @@ async function transcribeWithOpenAIAPI(filePath, callback) { } async function transcribeWithICADAPI(filePath, callback) { + const transcriptionConfig = await getBotTranscriptionConfig(); + // Check for ICAD URL - if (!ICAD_URL) { + if (!transcriptionConfig.icadUrl) { logger.error('FATAL: TRANSCRIPTION_MODE is icad, but ICAD_URL is not configured.'); if (callback) callback(""); // Fail gracefully return; @@ -2804,7 +2830,7 @@ async function transcribeWithICADAPI(filePath, callback) { form.append('file', fs.createReadStream(filePath)); // Set model based on ICAD_PROFILE if provided, otherwise use default - const modelToUse = ICAD_PROFILE || 'whisper-1'; + const modelToUse = transcriptionConfig.icadProfile; form.append('model', modelToUse); // Add standard OpenAI Whisper API parameters that ICAD should understand @@ -2814,9 +2840,9 @@ async function transcribeWithICADAPI(filePath, callback) { // Explicitly disable clip_timestamps to override any profile settings form.append('clip_timestamps', ''); - const apiEndpoint = `${ICAD_URL}/v1/audio/transcriptions`; + const apiEndpoint = `${transcriptionConfig.icadUrl}/v1/audio/transcriptions`; const filenameForLog = path.basename(filePath); - const authStatus = ICAD_API_KEY ? 'with authentication' : 'without authentication'; + const authStatus = transcriptionConfig.icadApiKey ? 'with authentication' : 'without authentication'; logger.info(`Sending ICAD transcription request for ${filenameForLog} to ${apiEndpoint} using model/profile: ${modelToUse} (${authStatus})`); const controller = new AbortController(); @@ -2830,8 +2856,8 @@ async function transcribeWithICADAPI(filePath, callback) { }; // Add authorization header if ICAD_API_KEY is provided - if (ICAD_API_KEY) { - headers['Authorization'] = `Bearer ${ICAD_API_KEY}`; + if (transcriptionConfig.icadApiKey) { + headers['Authorization'] = `Bearer ${transcriptionConfig.icadApiKey}`; } const response = await fetch(apiEndpoint, { @@ -3029,6 +3055,7 @@ function handleNewAudio(audioData) { const transcriptionId = this.lastID; // Get the ID from the database insert let transcriptionJobId = null; + const transcriptionConfig = await getBotTranscriptionConfig(); logger.info(`Created transcription record ID ${transcriptionId} using storage path: ${storagePath}`); try { @@ -3038,7 +3065,7 @@ function handleNewAudio(audioData) { payload: { filename, talkGroupID, - transcriptionMode: effectiveTranscriptionMode, + transcriptionMode: transcriptionConfig.mode, storageMode: STORAGE_MODE } }); @@ -3147,7 +3174,7 @@ function handleNewAudio(audioData) { }; // Transcribe based on mode (use the same mode as the main call) - const segmentTranscriptionMode = effectiveTranscriptionMode || 'local'; + const segmentTranscriptionMode = transcriptionConfig.mode; if (segmentTranscriptionMode === 'openai') { transcribeWithOpenAIAPI(segment.audioPath, segmentCallback); } else if (segmentTranscriptionMode === 'remote') { @@ -3275,20 +3302,20 @@ function handleNewAudio(audioData) { // --- End common callback definition --- // --- Choose transcription method based on mode --- - logger.info(`Initiating transcription for ID ${transcriptionId} using mode: ${effectiveTranscriptionMode}`); + logger.info(`Initiating transcription for ID ${transcriptionId} using mode: ${transcriptionConfig.mode}`); if (transcriptionJobId) { safelyUpdateProcessingJob('mark transcription job processing', () => markJobProcessing(db, transcriptionJobId)); } - if (effectiveTranscriptionMode === 'openai') { + if (transcriptionConfig.mode === 'openai') { // OpenAI API transcription mode const pathToUse = (STORAGE_MODE === 'local') ? finalPathIfLocal : tempPath; transcribeWithOpenAIAPI(pathToUse, processingCallback); - } else if (effectiveTranscriptionMode === 'remote') { + } else if (transcriptionConfig.mode === 'remote') { // Use the remote function for faster-whisper server const pathToUseForRemote = (STORAGE_MODE === 'local') ? finalPathIfLocal : tempPath; transcribeAudioRemotely(pathToUseForRemote, processingCallback); - } else if (effectiveTranscriptionMode === 'icad') { + } else if (transcriptionConfig.mode === 'icad') { // ICAD API transcription mode (OpenAI-compatible interface) const pathToUse = (STORAGE_MODE === 'local') ? finalPathIfLocal : tempPath; transcribeWithICADAPI(pathToUse, processingCallback); @@ -5976,11 +6003,12 @@ client.once('ready', async () => { startSummaryScheduler(); // Start transcription process if needed - if (effectiveTranscriptionMode === 'local') { + const transcriptionConfig = await getBotTranscriptionConfig(); + if (transcriptionConfig.mode === 'local') { logger.info('Initializing local transcription process...'); startTranscriptionProcess(); } else { - logger.info(`Transcription mode set to '${effectiveTranscriptionMode}'. Local Python process will not be started.`); + logger.info(`Transcription mode set to '${transcriptionConfig.mode}'. Local Python process will not be started.`); } isBootComplete = true; diff --git a/public/settings.html b/public/settings.html index 89224b5..e10a199 100644 --- a/public/settings.html +++ b/public/settings.html @@ -55,9 +55,16 @@

Providers

+
+
+
+
+
+
+
diff --git a/public/settings.js b/public/settings.js index 47d24ae..eafc149 100644 --- a/public/settings.js +++ b/public/settings.js @@ -2,9 +2,10 @@ const normalKeys = [ 'publicDomain', 'timezone', 'summaryLookbackHours', 'askAiLookbackHours', 'mappedTalkGroups', 'enableMappedTalkGroups', 'storageMode', 'transcriptionMode', 'transcriptionDevice', 'aiProvider', 'ollamaUrl', 'ollamaModel', 'openaiModel', - 'fasterWhisperServerUrl' + 'fasterWhisperServerUrl', 'whisperModel', 'openaiTranscriptionPrompt', + 'openaiTranscriptionModel', 'openaiTranscriptionTemperature', 'icadUrl', 'icadProfile' ]; -const secretKeys = ['uploadApiKey', 'googleMapsApiKey', 'locationIqApiKey', 'openaiApiKey']; +const secretKeys = ['uploadApiKey', 'googleMapsApiKey', 'locationIqApiKey', 'openaiApiKey', 'icadApiKey']; function showStep(id) { document.querySelectorAll('.section').forEach((section) => section.classList.toggle('active', section.id === id)); diff --git a/src/settings/settingsService.js b/src/settings/settingsService.js index b61d46d..a5c17fa 100644 --- a/src/settings/settingsService.js +++ b/src/settings/settingsService.js @@ -14,6 +14,11 @@ const SETTING_DEFINITIONS = { openaiModel: { envKey: 'OPENAI_MODEL', defaultValue: 'gpt-4o-mini', requiresRestart: false }, fasterWhisperServerUrl: { envKey: 'FASTER_WHISPER_SERVER_URL', defaultValue: '', requiresRestart: false }, whisperModel: { envKey: 'WHISPER_MODEL', defaultValue: 'large-v3', requiresRestart: false }, + openaiTranscriptionPrompt: { envKey: 'OPENAI_TRANSCRIPTION_PROMPT', defaultValue: '', requiresRestart: false }, + openaiTranscriptionModel: { envKey: 'OPENAI_TRANSCRIPTION_MODEL', defaultValue: 'whisper-1', requiresRestart: false }, + openaiTranscriptionTemperature: { envKey: 'OPENAI_TRANSCRIPTION_TEMPERATURE', defaultValue: '0.0', requiresRestart: false }, + icadUrl: { envKey: 'ICAD_URL', defaultValue: '', requiresRestart: false }, + icadProfile: { envKey: 'ICAD_PROFILE', defaultValue: 'whisper-1', requiresRestart: false }, mappedTalkGroups: { envKey: 'MAPPED_TALK_GROUPS', defaultValue: '', requiresRestart: false }, enableMappedTalkGroups: { envKey: 'ENABLE_MAPPED_TALK_GROUPS', defaultValue: 'true', requiresRestart: false }, summaryLookbackHours: { envKey: 'SUMMARY_LOOKBACK_HOURS', defaultValue: '1', requiresRestart: false }, From 5432931b15138d12c7871c9739bddd8cb2cb0b06 Mon Sep 17 00:00:00 2001 From: Dadud Date: Sun, 17 May 2026 23:11:09 -0500 Subject: [PATCH 4/4] Harden runtime storage setup --- bot.js | 140 +++++++++++++++++++++----------- public/settings.html | 4 + public/settings.js | 4 +- public/setup.html | 16 ++++ public/setup.js | 18 ++++ src/settings/settingsService.js | 2 + src/setup/checks.js | 32 ++++++-- test/setupChecks.test.js | 56 ++++++++++++- webserver.js | 92 ++++++++++++++------- 9 files changed, 280 insertions(+), 84 deletions(-) diff --git a/bot.js b/bot.js index f315194..3b70b82 100644 --- a/bot.js +++ b/bot.js @@ -808,24 +808,7 @@ const logger = winston.createLogger({ // --- NEW: Add S3 Client Setup --- const AWS = require('aws-sdk'); -let s3 = null; -if (STORAGE_MODE === 's3') { - if (!S3_ENDPOINT || !S3_BUCKET_NAME || !S3_ACCESS_KEY_ID || !S3_SECRET_ACCESS_KEY) { - logger.warn('WARNING: STORAGE_MODE is s3, but required S3 environment variables are missing. Audio serving from S3 will be unavailable until setup is completed.'); - } else { - AWS.config.update({ - accessKeyId: S3_ACCESS_KEY_ID, - secretAccessKey: S3_SECRET_ACCESS_KEY, - endpoint: S3_ENDPOINT, - s3ForcePathStyle: true, // Necessary for MinIO/non-AWS S3 - signatureVersion: 'v4' - }); - s3 = new AWS.S3(); - logger.info(`[Bot] Storage mode set to S3. Endpoint: ${S3_ENDPOINT}, Bucket: ${S3_BUCKET_NAME}`); - } -} else { - logger.info('[Bot] Storage mode set to local.'); -} +logger.info(`[Bot] Startup storage mode from .env: ${STORAGE_MODE || 'local'}. Runtime settings may override this after database initialization.`); // --- END S3 Client Setup --- // --- INITIALIZATION FUNCTIONS --- @@ -970,6 +953,46 @@ async function getBotTranscriptionConfig() { }; } +async function getBotStorageConfig() { + const runtime = await getBotRuntimeConfig(); + const mode = (runtime.settings.storageMode || STORAGE_MODE || 'local').toLowerCase(); + + return { + mode: mode === 's3' ? 's3' : 'local', + s3Endpoint: runtime.settings.s3Endpoint || S3_ENDPOINT || '', + s3BucketName: runtime.settings.s3BucketName || S3_BUCKET_NAME || '', + s3AccessKeyId: runtime.secrets.s3AccessKeyId || S3_ACCESS_KEY_ID || '', + s3SecretAccessKey: runtime.secrets.s3SecretAccessKey || S3_SECRET_ACCESS_KEY || '' + }; +} + +function createS3Client(storageConfig) { + return new AWS.S3({ + accessKeyId: storageConfig.s3AccessKeyId, + secretAccessKey: storageConfig.s3SecretAccessKey, + endpoint: storageConfig.s3Endpoint, + s3ForcePathStyle: true, + signatureVersion: 'v4' + }); +} + +function isS3Ready(storageConfig) { + return Boolean( + storageConfig.mode === 's3' && + storageConfig.s3Endpoint && + storageConfig.s3BucketName && + storageConfig.s3AccessKeyId && + storageConfig.s3SecretAccessKey + ); +} + +function getToneAudioPath(storageConfig, audioFilePath) { + if (storageConfig.mode === 's3' && storageConfig.s3Endpoint && storageConfig.s3BucketName) { + return `https://${storageConfig.s3Endpoint.replace('https://', '').replace('http://', '')}/${storageConfig.s3BucketName}/${audioFilePath}`; + } + return path.join(__dirname, 'audio', audioFilePath); +} + async function getPublicAudioUrl(audioId) { let publicDomain = PUBLIC_DOMAIN || 'localhost'; try { @@ -3005,7 +3028,7 @@ function handleNewAudio(audioData) { } // Read file into buffer (This is needed for DB blob AND for S3->Local transcription) - fs.readFile(tempPath, (err, fileBuffer) => { + fs.readFile(tempPath, async (err, fileBuffer) => { if (err) { logger.error(`Error reading audio file ${tempPath}:`, err); // Clean up temp file if read fails @@ -3015,10 +3038,12 @@ function handleNewAudio(audioData) { return; } + const storageConfig = await getBotStorageConfig(); + // --- Start DB Operations --- Miminized changes here // Determine the storage path/key based on STORAGE_MODE let storagePath; - if (STORAGE_MODE === 's3') { + if (storageConfig.mode === 's3') { // For S3, we store the filename as the key (assuming it's unique enough) // You might want a more structured path like 'audio/YYYY/MM/DD/filename' storagePath = filename; @@ -3066,7 +3091,7 @@ function handleNewAudio(audioData) { filename, talkGroupID, transcriptionMode: transcriptionConfig.mode, - storageMode: STORAGE_MODE + storageMode: storageConfig.mode } }); logger.info(`Created transcription job ${transcriptionJobId} for transcription ID ${transcriptionId}`); @@ -3075,7 +3100,7 @@ function handleNewAudio(audioData) { } // Conditionally insert audio blob for Listen Live feature (local storage only) - if (STORAGE_MODE !== 's3') { + if (storageConfig.mode !== 's3') { db.run( `INSERT INTO audio_files (transcription_id, audio_data) VALUES (?, ?)`, [transcriptionId, fileBuffer], @@ -3168,7 +3193,8 @@ function handleNewAudio(audioData) { storagePath, audioPathForSplitting, tempPath, - finalPathIfLocal + finalPathIfLocal, + storageConfig.mode ); } }; @@ -3227,9 +3253,9 @@ function handleNewAudio(audioData) { logger.info(`Checking for two-tone in talk group ${talkGroupID} (ID: ${transcriptionId}) - empty transcription`); // Use the audio file path for tone detection - const audioPathForTones = STORAGE_MODE === 's3' ? - `https://${S3_ENDPOINT.replace('https://', '').replace('http://', '')}/${S3_BUCKET_NAME}/${filename}` : - (finalPathIfLocal || path.join(__dirname, 'audio', filename)); + const audioPathForTones = storageConfig.mode === 's3' + ? getToneAudioPath(storageConfig, filename) + : (finalPathIfLocal || path.join(__dirname, 'audio', filename)); // Wait for tone detection to complete before continuing await new Promise((resolve) => { @@ -3243,7 +3269,7 @@ function handleNewAudio(audioData) { } // Clean up temp file only if storage was S3 - if (STORAGE_MODE === 's3') { + if (storageConfig.mode === 's3') { // Use setImmediate to avoid file handle race conditions setImmediate(() => { fs.unlink(tempPath, (errUnlink) => { @@ -3274,11 +3300,12 @@ function handleNewAudio(audioData) { emergency, priority, encrypted, call_length, // <-- Pass call metadata freq_error, signalQuality, // <-- Pass signal quality frequency, start_time, stop_time, // <-- Pass timing/frequency - tdma_slot, phase2_tdma, color_code // <-- Pass TDMA/color code + tdma_slot, phase2_tdma, color_code, // <-- Pass TDMA/color code + storageConfig ); // Clean up temp file only if storage was S3 - if (STORAGE_MODE === 's3') { + if (storageConfig.mode === 's3') { // Use setImmediate to avoid file handle race conditions setImmediate(() => { fs.unlink(tempPath, (errUnlink) => { @@ -3309,21 +3336,21 @@ function handleNewAudio(audioData) { if (transcriptionConfig.mode === 'openai') { // OpenAI API transcription mode - const pathToUse = (STORAGE_MODE === 'local') ? finalPathIfLocal : tempPath; + const pathToUse = (storageConfig.mode === 'local') ? finalPathIfLocal : tempPath; transcribeWithOpenAIAPI(pathToUse, processingCallback); } else if (transcriptionConfig.mode === 'remote') { // Use the remote function for faster-whisper server - const pathToUseForRemote = (STORAGE_MODE === 'local') ? finalPathIfLocal : tempPath; + const pathToUseForRemote = (storageConfig.mode === 'local') ? finalPathIfLocal : tempPath; transcribeAudioRemotely(pathToUseForRemote, processingCallback); } else if (transcriptionConfig.mode === 'icad') { // ICAD API transcription mode (OpenAI-compatible interface) - const pathToUse = (STORAGE_MODE === 'local') ? finalPathIfLocal : tempPath; + const pathToUse = (storageConfig.mode === 'local') ? finalPathIfLocal : tempPath; transcribeWithICADAPI(pathToUse, processingCallback); } else { // 'local' transcription mode const localRequestId = uuidv4(); let payload; - if (STORAGE_MODE === 's3') { + if (storageConfig.mode === 's3') { // S3 Storage + Local Transcription: Send buffer logger.info(`Queueing local transcription (ID: ${localRequestId}) for DB ID ${transcriptionId} using BASE64 BUFFER`); @@ -3417,22 +3444,35 @@ function handleNewAudio(audioData) { // --- End afterStorageComplete function definition --- // --- Handle Audio Storage based on Mode --- - if (STORAGE_MODE === 's3') { + if (storageConfig.mode === 's3') { + if (!isS3Ready(storageConfig)) { + const error = new Error('S3 storage mode is selected, but S3 endpoint, bucket, or credentials are incomplete.'); + logger.error(error.message); + if (transcriptionJobId) { + safelyUpdateProcessingJob('mark transcription job failed', () => markJobFailed(db, transcriptionJobId, error)); + } + db.run('DELETE FROM transcriptions WHERE id = ?', [transcriptionId], () => {}); + fs.unlink(tempPath, (errUnlink) => { + if (errUnlink) logger.error(`Error deleting temp file after incomplete S3 config ${tempPath}:`, errUnlink); + }); + return; + } + const s3Client = createS3Client(storageConfig); // Upload the buffer to S3 const s3Params = { - Bucket: S3_BUCKET_NAME, + Bucket: storageConfig.s3BucketName, Key: storagePath, // Use the determined S3 key Body: fileBuffer, // ContentType: 'audio/mpeg', // Or determine dynamically }; - s3.upload(s3Params, (s3Err, data) => { + s3Client.upload(s3Params, (s3Err, data) => { if (s3Err) { // Check for specific MinIO storage threshold error const errorMessage = s3Err.message || s3Err.toString() || ''; if (errorMessage.includes('minimum free drive threshold') || errorMessage.includes('free drive threshold')) { logger.error(`[MINIO STORAGE FULL] MinIO server has reached its minimum free drive threshold.`); logger.error(`[MINIO STORAGE FULL] Transcription ID ${transcriptionId} could not be uploaded.`); - logger.error(`[MINIO STORAGE FULL] Action required: Free up disk space on MinIO server or delete old objects from bucket: ${S3_BUCKET_NAME}`); + logger.error(`[MINIO STORAGE FULL] Action required: Free up disk space on MinIO server or delete old objects from bucket: ${storageConfig.s3BucketName}`); logger.error(`[MINIO STORAGE FULL] Full error: ${errorMessage}`); } else { logger.error(`Error uploading audio to S3 for transcription ID ${transcriptionId} (key: ${storagePath}):`, s3Err); @@ -3924,7 +3964,8 @@ async function processMergedCallSegments( storagePath, audioPathForSplitting, tempPath, - finalPathIfLocal + finalPathIfLocal, + storageMode = STORAGE_MODE ) { logger.info(`Processing ${segmentTranscriptions.length} segments for merged call ID ${transcriptionId}`); @@ -3990,7 +4031,7 @@ async function processMergedCallSegments( ); // Clean up temp file only if storage was S3 - if (STORAGE_MODE === 's3') { + if (storageMode === 's3') { setImmediate(() => { fs.unlink(tempPath, (errUnlink) => { if (errUnlink && errUnlink.code !== 'ENOENT') { @@ -4029,11 +4070,13 @@ async function handleNewTranscription( stop_time, tdma_slot, phase2_tdma, - color_code + color_code, + storageConfig = null ) { logger.info(`Starting handleNewTranscription for ID ${id}`); logger.info(`Transcription text length: ${transcriptionText.length} characters`); logger.info(`Talk Group: ${talkGroupID} - ${talkGroupName}`); + const resolvedStorageConfig = storageConfig || await getBotStorageConfig(); // Auto-queue calls after two-tone detection (if in two-tone mode) if (IS_TWO_TONE_MODE_ENABLED && lastTwoToneTime > 0 && lastDetectedToneGroup) { @@ -4081,9 +4124,7 @@ async function handleNewTranscription( logger.info(`Checking for two-tone in talk group ${talkGroupID} (ID: ${id})`); // Construct the proper audio path for tone detection - const audioPathForTones = STORAGE_MODE === 's3' ? - `https://${S3_ENDPOINT.replace('https://', '').replace('http://', '')}/${S3_BUCKET_NAME}/${audioFilePath}` : - path.join(__dirname, 'audio', audioFilePath); // Construct full local path + const audioPathForTones = getToneAudioPath(resolvedStorageConfig, audioFilePath); // Wait for tone detection to complete before continuing await new Promise((resolve) => { @@ -5602,7 +5643,7 @@ function playAudioForTalkGroup(talkGroupID, transcriptionId) { } } -function processAudioQueue(talkGroupID) { +async function processAudioQueue(talkGroupID) { talkGroupID = talkGroupID.toString(); const talkGroupData = activeVoiceChannels.get(talkGroupID); if (!talkGroupData || !talkGroupData.player || !talkGroupData.queue) { @@ -5648,14 +5689,21 @@ function processAudioQueue(talkGroupID) { }); }; - if (STORAGE_MODE === 's3') { + const storageConfig = await getBotStorageConfig(); + if (storageConfig.mode === 's3') { + if (!isS3Ready(storageConfig)) { + logger.error(`S3 Mode: storage configuration incomplete for Discord playback (ID ${transcriptionId})`); + processAudioQueue(talkGroupID); + return; + } + const s3Client = createS3Client(storageConfig); db.get('SELECT audio_file_path FROM transcriptions WHERE id = ?', [transcriptionId], (err, row) => { if (err || !row || !row.audio_file_path) { logger.error(`S3 Mode: Could not find audio_file_path for transcription ID ${transcriptionId}`, err); processAudioQueue(talkGroupID); return; } - const s3Stream = s3.getObject({ Bucket: S3_BUCKET_NAME, Key: row.audio_file_path }).createReadStream(); + const s3Stream = s3Client.getObject({ Bucket: storageConfig.s3BucketName, Key: row.audio_file_path }).createReadStream(); s3Stream.on('error', s3Err => { logger.error(`Error streaming from S3 for Discord playback (ID ${transcriptionId}):`, s3Err); processAudioQueue(talkGroupID); diff --git a/public/settings.html b/public/settings.html index e10a199..ac2c6f0 100644 --- a/public/settings.html +++ b/public/settings.html @@ -48,6 +48,10 @@

Ingestion

Providers

+
+
+
+
diff --git a/public/settings.js b/public/settings.js index eafc149..df9b4d9 100644 --- a/public/settings.js +++ b/public/settings.js @@ -1,11 +1,11 @@ const normalKeys = [ 'publicDomain', 'timezone', 'summaryLookbackHours', 'askAiLookbackHours', 'mappedTalkGroups', 'enableMappedTalkGroups', 'storageMode', 'transcriptionMode', - 'transcriptionDevice', 'aiProvider', 'ollamaUrl', 'ollamaModel', 'openaiModel', + 's3Endpoint', 's3BucketName', 'transcriptionDevice', 'aiProvider', 'ollamaUrl', 'ollamaModel', 'openaiModel', 'fasterWhisperServerUrl', 'whisperModel', 'openaiTranscriptionPrompt', 'openaiTranscriptionModel', 'openaiTranscriptionTemperature', 'icadUrl', 'icadProfile' ]; -const secretKeys = ['uploadApiKey', 'googleMapsApiKey', 'locationIqApiKey', 'openaiApiKey', 'icadApiKey']; +const secretKeys = ['uploadApiKey', 'googleMapsApiKey', 'locationIqApiKey', 'openaiApiKey', 'icadApiKey', 's3AccessKeyId', 's3SecretAccessKey']; function showStep(id) { document.querySelectorAll('.section').forEach((section) => section.classList.toggle('active', section.id === id)); diff --git a/public/setup.html b/public/setup.html index abc6824..4098efe 100644 --- a/public/setup.html +++ b/public/setup.html @@ -84,6 +84,22 @@

Providers

+
+ + +
+
+ + +
+
+ + +
+
+ + +
diff --git a/public/setup.js b/public/setup.js index 30a1540..55a1874 100644 --- a/public/setup.js +++ b/public/setup.js @@ -73,6 +73,8 @@ document.getElementById('save-providers').addEventListener('click', async () => method: 'POST', body: JSON.stringify({ storageMode: document.getElementById('storage-mode').value, + s3Endpoint: document.getElementById('s3-endpoint').value, + s3BucketName: document.getElementById('s3-bucket').value, transcriptionMode: document.getElementById('transcription-mode').value, aiProvider: document.getElementById('ai-provider').value, timezone: document.getElementById('timezone').value @@ -95,6 +97,22 @@ document.getElementById('save-providers').addEventListener('click', async () => }); } + const s3AccessKey = document.getElementById('s3-access-key').value; + if (s3AccessKey) { + await jsonFetch('/api/setup/secrets', { + method: 'POST', + body: JSON.stringify({ key: 's3AccessKeyId', value: s3AccessKey }) + }); + } + + const s3SecretKey = document.getElementById('s3-secret-key').value; + if (s3SecretKey) { + await jsonFetch('/api/setup/secrets', { + method: 'POST', + body: JSON.stringify({ key: 's3SecretAccessKey', value: s3SecretKey }) + }); + } + renderMessage('provider-result', 'Provider settings saved. Restart may be required for some settings.'); await loadStatus(); } catch (error) { diff --git a/src/settings/settingsService.js b/src/settings/settingsService.js index a5c17fa..7fdb2f7 100644 --- a/src/settings/settingsService.js +++ b/src/settings/settingsService.js @@ -6,6 +6,8 @@ const SETTING_DEFINITIONS = { publicDomain: { envKey: 'PUBLIC_DOMAIN', defaultValue: 'localhost', requiresRestart: true }, timezone: { envKey: 'TIMEZONE', defaultValue: 'US/Eastern', requiresRestart: false }, storageMode: { envKey: 'STORAGE_MODE', defaultValue: 'local', requiresRestart: true }, + s3Endpoint: { envKey: 'S3_ENDPOINT', defaultValue: '', requiresRestart: true }, + s3BucketName: { envKey: 'S3_BUCKET_NAME', defaultValue: '', requiresRestart: true }, transcriptionMode: { envKey: 'TRANSCRIPTION_MODE', defaultValue: 'local', requiresRestart: true }, transcriptionDevice: { envKey: 'TRANSCRIPTION_DEVICE', defaultValue: 'cpu', requiresRestart: true }, aiProvider: { envKey: 'AI_PROVIDER', defaultValue: 'ollama', requiresRestart: false }, diff --git a/src/setup/checks.js b/src/setup/checks.js index 9760a4f..900cfb5 100644 --- a/src/setup/checks.js +++ b/src/setup/checks.js @@ -42,6 +42,24 @@ async function runSetupChecks(options = {}) { const rootDir = options.rootDir || path.join(__dirname, '..', '..'); const env = options.env || process.env; const runtime = options.runtime || { settings: {}, secrets: {} }; + const storageMode = (runtime.settings.storageMode || env.STORAGE_MODE || 'local').toLowerCase(); + const transcriptionMode = (runtime.settings.transcriptionMode || env.TRANSCRIPTION_MODE || 'local').toLowerCase(); + const aiProvider = (runtime.settings.aiProvider || env.AI_PROVIDER || 'ollama').toLowerCase(); + const hasS3Config = Boolean( + runtime.settings.s3Endpoint || env.S3_ENDPOINT + ) && Boolean( + runtime.settings.s3BucketName || env.S3_BUCKET_NAME + ) && Boolean( + runtime.secrets.s3AccessKeyId || env.S3_ACCESS_KEY_ID + ) && Boolean( + runtime.secrets.s3SecretAccessKey || env.S3_SECRET_ACCESS_KEY + ); + const transcriptionReady = + transcriptionMode === 'local' || + (transcriptionMode === 'remote' && Boolean(runtime.settings.fasterWhisperServerUrl || env.FASTER_WHISPER_SERVER_URL)) || + (transcriptionMode === 'openai' && Boolean(runtime.secrets.openaiApiKey || env.OPENAI_API_KEY)) || + (transcriptionMode === 'icad' && Boolean(runtime.settings.icadUrl || env.ICAD_URL)); + const aiReady = aiProvider !== 'openai' || Boolean(runtime.secrets.openaiApiKey || env.OPENAI_API_KEY); const [node, python, ffmpeg, ollama] = await Promise.all([ checkCommand(process.execPath, ['--version']), checkCommand(env.PYTHON_COMMAND || (process.platform === 'win32' ? 'py' : 'python3'), ['--version']), @@ -65,15 +83,19 @@ async function runSetupChecks(options = {}) { } }, transcriptionProvider: { - ok: Boolean(runtime.settings.transcriptionMode || env.TRANSCRIPTION_MODE || 'local'), - mode: runtime.settings.transcriptionMode || env.TRANSCRIPTION_MODE || 'local' + ok: transcriptionReady, + mode: transcriptionMode }, aiProvider: { - ok: Boolean(runtime.settings.aiProvider || env.AI_PROVIDER || 'ollama'), - provider: runtime.settings.aiProvider || env.AI_PROVIDER || 'ollama' + ok: aiReady, + provider: aiProvider + }, + storageProvider: { + ok: storageMode === 'local' || hasS3Config, + mode: storageMode }, uploadEndpoint: { - ok: true, + ok: Boolean(runtime.secrets.uploadApiKey || env.SCANNER_MAP_UPLOAD_API_KEY), url: `/api/call-upload` } }; diff --git a/test/setupChecks.test.js b/test/setupChecks.test.js index d24ad02..a514567 100644 --- a/test/setupChecks.test.js +++ b/test/setupChecks.test.js @@ -4,7 +4,7 @@ const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); -const { checkWritableDir } = require('../src/setup/checks'); +const { checkWritableDir, runSetupChecks } = require('../src/setup/checks'); test('checkWritableDir creates and verifies writable directories', () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-map-check-')); @@ -16,3 +16,57 @@ test('checkWritableDir creates and verifies writable directories', () => { assert.equal(fs.existsSync(nested), true); fs.rmSync(tempDir, { recursive: true, force: true }); }); + +test('runSetupChecks validates provider-specific readiness from runtime config', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-map-check-')); + + const checks = await runSetupChecks({ + rootDir: tempDir, + env: {}, + runtime: { + settings: { + storageMode: 's3', + transcriptionMode: 'remote', + aiProvider: 'openai' + }, + secrets: {} + } + }); + + assert.equal(checks.storageProvider.ok, false); + assert.equal(checks.transcriptionProvider.ok, false); + assert.equal(checks.aiProvider.ok, false); + assert.equal(checks.uploadEndpoint.ok, false); + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +test('runSetupChecks accepts configured S3 and provider secrets', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scanner-map-check-')); + + const checks = await runSetupChecks({ + rootDir: tempDir, + env: {}, + runtime: { + settings: { + storageMode: 's3', + s3Endpoint: 'http://localhost:9000', + s3BucketName: 'scanner-audio', + transcriptionMode: 'remote', + fasterWhisperServerUrl: 'http://localhost:8000', + aiProvider: 'openai' + }, + secrets: { + s3AccessKeyId: 'key', + s3SecretAccessKey: 'secret', + openaiApiKey: 'openai', + uploadApiKey: 'upload' + } + } + }); + + assert.equal(checks.storageProvider.ok, true); + assert.equal(checks.transcriptionProvider.ok, true); + assert.equal(checks.aiProvider.ok, true); + assert.equal(checks.uploadEndpoint.ok, true); + fs.rmSync(tempDir, { recursive: true, force: true }); +}); diff --git a/webserver.js b/webserver.js index 1631162..03c9c9f 100644 --- a/webserver.js +++ b/webserver.js @@ -136,25 +136,7 @@ app.get('/api/test', (req, res) => { res.json({ message: 'Server is working', timestamp: Date.now() }); }); -// --- NEW: S3 Client Setup --- -let s3 = null; -if (STORAGE_MODE === 's3') { - if (!S3_ENDPOINT || !S3_BUCKET_NAME || !S3_ACCESS_KEY_ID || !S3_SECRET_ACCESS_KEY) { - console.warn('WARNING: STORAGE_MODE=s3, but S3 configuration is incomplete. Audio serving from S3 will be unavailable until setup is completed.'); - } else { - AWS.config.update({ - accessKeyId: S3_ACCESS_KEY_ID, - secretAccessKey: S3_SECRET_ACCESS_KEY, - endpoint: S3_ENDPOINT, - s3ForcePathStyle: true, // Necessary for MinIO/non-AWS S3 - signatureVersion: 'v4' - }); - s3 = new AWS.S3(); - console.log(`[Webserver] Storage mode set to S3. Endpoint: ${S3_ENDPOINT}, Bucket: ${S3_BUCKET_NAME}`); - } -} else { - console.log('[Webserver] Storage mode set to local.'); -} +console.log(`[Webserver] Startup storage mode from .env: ${STORAGE_MODE || 'local'}. Runtime settings may override this after database initialization.`); // Authentication is enabled if ENABLE_AUTH=true const authEnabled = ENABLE_AUTH?.toLowerCase() === 'true'; @@ -199,8 +181,40 @@ async function getResolvedRuntimeConfig() { await dbReady; return getRuntimeConfig(db, process.env); } - -// Helper Functions for Authentication + +async function getWebserverStorageConfig() { + const runtime = await getResolvedRuntimeConfig(); + const mode = (runtime.settings.storageMode || STORAGE_MODE || 'local').toLowerCase(); + return { + mode: mode === 's3' ? 's3' : 'local', + s3Endpoint: runtime.settings.s3Endpoint || S3_ENDPOINT || '', + s3BucketName: runtime.settings.s3BucketName || S3_BUCKET_NAME || '', + s3AccessKeyId: runtime.secrets.s3AccessKeyId || S3_ACCESS_KEY_ID || '', + s3SecretAccessKey: runtime.secrets.s3SecretAccessKey || S3_SECRET_ACCESS_KEY || '' + }; +} + +function isS3Ready(storageConfig) { + return Boolean( + storageConfig.mode === 's3' && + storageConfig.s3Endpoint && + storageConfig.s3BucketName && + storageConfig.s3AccessKeyId && + storageConfig.s3SecretAccessKey + ); +} + +function createS3Client(storageConfig) { + return new AWS.S3({ + accessKeyId: storageConfig.s3AccessKeyId, + secretAccessKey: storageConfig.s3SecretAccessKey, + endpoint: storageConfig.s3Endpoint, + s3ForcePathStyle: true, + signatureVersion: 'v4' + }); +} + +// Helper Functions for Authentication function hashPassword(password, salt) { return crypto .pbkdf2Sync(password, salt, 10000, 64, 'sha512') @@ -700,14 +714,21 @@ app.get('/audio/:id', async (req, res) => { }); }); - if (transcriptionRow && transcriptionRow.audio_file_path) { - const audioStoragePath = transcriptionRow.audio_file_path; - const extension = path.extname(audioStoragePath).toLowerCase(); - const contentType = extension === '.m4a' ? 'audio/mp4' : 'audio/mpeg'; - - if (STORAGE_MODE === 's3') { - const params = { Bucket: S3_BUCKET_NAME, Key: audioStoragePath }; - const s3Stream = s3.getObject(params).createReadStream(); + if (transcriptionRow && transcriptionRow.audio_file_path) { + const audioStoragePath = transcriptionRow.audio_file_path; + const extension = path.extname(audioStoragePath).toLowerCase(); + const contentType = extension === '.m4a' ? 'audio/mp4' : 'audio/mpeg'; + const storageConfig = await getWebserverStorageConfig(); + + if (storageConfig.mode === 's3') { + if (!isS3Ready(storageConfig)) { + console.warn(`[Audio S3] S3 runtime settings are incomplete. Falling back to DB for transcription ${transcriptionId}.`); + serveAudioFromDb(res, transcriptionId); + return; + } + const s3Client = createS3Client(storageConfig); + const params = { Bucket: storageConfig.s3BucketName, Key: audioStoragePath }; + const s3Stream = s3Client.getObject(params).createReadStream(); s3Stream.on('error', (s3Err) => { console.warn(`[Audio S3] S3 stream error for key ${audioStoragePath}: ${s3Err.code}. Falling back to DB.`); serveAudioFromDb(res, transcriptionId); @@ -835,7 +856,7 @@ app.post('/api/setup/test-provider', async (req, res) => { geocoding: checks.geocodingProvider, transcription: checks.transcriptionProvider, ai: checks.aiProvider, - storage: checks.dataDir, + storage: checks.storageProvider, upload: checks.uploadEndpoint }; res.json(providerMap[provider] || { ok: false, error: 'Unknown provider test' }); @@ -852,6 +873,17 @@ app.post('/api/setup/complete', async (req, res) => { if (status.missing.length > 0) { return res.status(400).json({ error: 'Setup is incomplete', missing: status.missing }); } + const runtime = await getResolvedRuntimeConfig(); + const checks = await runSetupChecks({ rootDir: __dirname, env: process.env, runtime }); + const requiredChecks = ['node', 'python', 'ffmpeg', 'dataDir', 'audioDir', 'geocodingProvider', 'transcriptionProvider', 'aiProvider', 'storageProvider', 'uploadEndpoint']; + const failedChecks = requiredChecks.filter((key) => !checks[key] || !checks[key].ok); + if (failedChecks.length > 0) { + return res.status(400).json({ + error: 'Setup readiness checks failed', + failedChecks, + checks + }); + } await markSetupComplete(db, 'setup'); res.json({ ok: true, setupComplete: true }); } catch (err) {