diff --git a/src/__tests__/alerts.test.ts b/src/__tests__/alerts.test.ts new file mode 100644 index 0000000..fc3b87f --- /dev/null +++ b/src/__tests__/alerts.test.ts @@ -0,0 +1,87 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { triggerAlert } from '../monitor/alerts'; +import { logger } from '../utils/logger'; +import { getConfig } from '../utils/config'; + +// Mock config +vi.mock('../utils/config', () => ({ + getConfig: vi.fn(() => ({ + alert_cooldown: 300, + discord_webhook: 'https://discord.com/api/webhooks/dummy', + })), + getSMTPSettings: vi.fn(() => ({ + host: '', + port: 587, + auth: { user: '', pass: '' }, + to: '', + })), +})); + +// Mock logger +vi.mock('../utils/logger', () => ({ + logger: { + info: vi.fn(), + error: vi.fn(), + }, +})); + +// Mock fetch globally +const mockFetch = vi.fn().mockResolvedValue({ ok: true }); + +describe('alerts monitoring', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal('fetch', mockFetch); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('should trigger alert on first invocation', async () => { + const alertId = 'container:test-first:failure'; + await triggerAlert({ + id: alertId, + message: 'Container crashed', + type: 'container', + severity: 'critical', + }); + + expect(logger.info).toHaveBeenCalledWith(expect.stringContaining(`Triggering alert for ${alertId}`)); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('should respect cooldown but bypass it if the force option is true', async () => { + const alertId = 'container:test-flow:failure'; + + // 1. First trigger - should send + await triggerAlert({ + id: alertId, + message: 'Container crashed first time', + type: 'container', + severity: 'critical', + }); + expect(mockFetch).toHaveBeenCalledTimes(1); + + // 2. Second trigger within cooldown without force - should be suppressed + await triggerAlert({ + id: alertId, + message: 'Container crashed second time', + type: 'container', + severity: 'critical', + }); + // Call count should still be 1 + expect(mockFetch).toHaveBeenCalledTimes(1); + + // 3. Third trigger within cooldown with force: true - should send regardless of cooldown + await triggerAlert({ + id: alertId, + message: 'Container crashed third time', + type: 'container', + severity: 'critical', + }, { force: true }); + + // Call count should be 2 now + expect(mockFetch).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/__tests__/health.test.ts b/src/__tests__/health.test.ts index 5ee784b..dc82be2 100644 --- a/src/__tests__/health.test.ts +++ b/src/__tests__/health.test.ts @@ -85,6 +85,8 @@ describe('health command', () => { await program.parseAsync(['node', 'test', 'health', 'all']); + expect(getRunningContainers).toHaveBeenCalledWith({ forceAlert: true }); + expect(getRunningPods).toHaveBeenCalledWith({ forceAlert: true }); expect(logger.info).toHaveBeenCalledWith('Showing health for all...'); expect(tableUtils.renderTable).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/src/commands/health.ts b/src/commands/health.ts index c49b871..72b24e4 100644 --- a/src/commands/health.ts +++ b/src/commands/health.ts @@ -26,8 +26,8 @@ import { Command } from 'commander'; import chalk from 'chalk'; -import { getRunningContainers } from '../docker/containers'; -import { getRunningPods } from '../kubernetes/pods'; +import { getRunningContainers, type ContainerData } from '../docker/containers'; +import { getRunningPods, type PodData } from '../kubernetes/pods'; import { logger } from '../utils/logger'; import { createSpinner } from '../ui/spinner'; import { renderTable } from '../ui/table'; @@ -48,53 +48,59 @@ const healthColor = (status: string): string => { return chalk.yellow(status); }; -const fetchHealthRows = async (target: string): Promise<(string | number)[][]> => { - const rows: (string | number)[][] = []; +const getContainerRows = ( + result: PromiseSettledResult, + shouldFetch: boolean +): (string | number)[][] => { + if (!shouldFetch) return []; + if (result.status === 'fulfilled') { + return result.value.map((container) => [ + 'container', + container.name, + healthColor(container.state), + container.status, + ]); + } + const message = result.reason instanceof Error + ? result.reason.message + : String(result.reason); + logger.warn?.(`Docker unavailable: ${message}`); + return []; +}; + +const getPodRows = ( + result: PromiseSettledResult, + shouldFetch: boolean +): (string | number)[][] => { + if (!shouldFetch) return []; + if (result.status === 'fulfilled') { + return result.value.map((pod) => [ + 'pod', + pod.name, + healthColor(pod.status), + `namespace: ${pod.namespace}, restarts: ${pod.restarts}`, + ]); + } + const message = result.reason instanceof Error + ? result.reason.message + : String(result.reason); + logger.warn?.(`Kubernetes unavailable: ${message}`); + return []; +}; + +const fetchHealthRows = async (target: string, options?: { forceAlert?: boolean }): Promise<(string | number)[][]> => { const shouldFetchContainers = target === 'all' || target === 'containers'; const shouldFetchPods = target === 'all' || target === 'pods'; const [containerResult, podResult] = await Promise.allSettled([ - shouldFetchContainers ? getRunningContainers() : Promise.resolve([]), - shouldFetchPods ? getRunningPods() : Promise.resolve([]), + shouldFetchContainers ? getRunningContainers({ forceAlert: options?.forceAlert }) : Promise.resolve([]), + shouldFetchPods ? getRunningPods({ forceAlert: options?.forceAlert }) : Promise.resolve([]), ]); - if (shouldFetchContainers) { - if (containerResult.status === 'fulfilled') { - rows.push( - ...containerResult.value.map((container) => [ - 'container', - container.name, - healthColor(container.state), - container.status, - ]), - ); - } else { - const message = containerResult.reason instanceof Error - ? containerResult.reason.message - : String(containerResult.reason); - logger.warn?.(`Docker unavailable: ${message}`); - } - } - - if (shouldFetchPods) { - if (podResult.status === 'fulfilled') { - rows.push( - ...podResult.value.map((pod) => [ - 'pod', - pod.name, - healthColor(pod.status), - `namespace: ${pod.namespace}, restarts: ${pod.restarts}`, - ]), - ); - } else { - const message = podResult.reason instanceof Error - ? podResult.reason.message - : String(podResult.reason); - logger.warn?.(`Kubernetes unavailable: ${message}`); - } - } - - return rows; + return [ + ...getContainerRows(containerResult, shouldFetchContainers), + ...getPodRows(podResult, shouldFetchPods), + ]; }; export const showHealth = async (target: string, options: HealthOptions = {}): Promise => { @@ -150,7 +156,7 @@ export const showHealth = async (target: string, options: HealthOptions = {}): P return; } - const rows = await fetchHealthRows(target); + const rows = await fetchHealthRows(target, { forceAlert: true }); // Clear terminal screen process.stdout.write('\x1Bc'); @@ -180,7 +186,7 @@ export const showHealth = async (target: string, options: HealthOptions = {}): P await poll(); } else { const spinner = createSpinner(`Checking ${target} health...`).start(); - const rows = await fetchHealthRows(target); + const rows = await fetchHealthRows(target, { forceAlert: true }); spinner.stop(); if (rows.length === 0) { diff --git a/src/docker/containers.ts b/src/docker/containers.ts index eb3486d..3a403d7 100644 --- a/src/docker/containers.ts +++ b/src/docker/containers.ts @@ -10,7 +10,7 @@ export interface ContainerData { status: string; } -export const getRunningContainers = async (): Promise => { +export const getRunningContainers = async (options?: { forceAlert?: boolean }): Promise => { const docker = getDockerClient(); try { // Try to list containers, use a timeout if possible or just catch common connection errors @@ -27,7 +27,7 @@ export const getRunningContainers = async (): Promise => { type: 'container', severity: 'warning', message: `Docker container ${name} (${id}) is restarting.`, - }); + }, { force: options?.forceAlert }); } else if (c.State === 'exited') { const match = c.Status.match(/Exited \((\d+)\)/); const exitCode = match ? parseInt(match[1], 10) : 0; @@ -38,7 +38,7 @@ export const getRunningContainers = async (): Promise => { type: 'container', severity: 'critical', message: `Docker container ${name} (${id}) exited with code ${exitCode}.`, - }); + }, { force: options?.forceAlert }); } } diff --git a/src/kubernetes/pods.ts b/src/kubernetes/pods.ts index d2fe714..41b885a 100644 --- a/src/kubernetes/pods.ts +++ b/src/kubernetes/pods.ts @@ -11,7 +11,7 @@ export interface PodData { node: string; } -export const getRunningPods = async (): Promise => { +export const getRunningPods = async (options?: { forceAlert?: boolean }): Promise => { const api = getK8sApi(); try { const res = await api.listPodForAllNamespaces(); @@ -43,7 +43,7 @@ export const getRunningPods = async (): Promise => { type: 'pod', severity: 'critical', message: `Pod ${name} in namespace ${pod.metadata?.namespace} failed: ${failureReason}`, - }); + }, { force: options?.forceAlert }); } return { diff --git a/src/monitor/alerts.ts b/src/monitor/alerts.ts index 15da89f..731bd38 100644 --- a/src/monitor/alerts.ts +++ b/src/monitor/alerts.ts @@ -70,13 +70,13 @@ const sendEmailNotification = async (alert: Alert) => { } }; -export const triggerAlert = async (alert: Alert) => { +export const triggerAlert = async (alert: Alert, options?: { force?: boolean }) => { const now = Date.now(); const { alert_cooldown = 300 } = getConfig(); // Default 5 minutes const cooldownMs = alert_cooldown * 1000; const lastAlert = cooldownTracker.get(alert.id); - if (lastAlert && (now - lastAlert) < cooldownMs) { + if (!options?.force && lastAlert && (now - lastAlert) < cooldownMs) { return; // Still in cooldown }