diff --git a/src/server/lib/kubernetes/__tests__/getDeploymentPods.test.ts b/src/server/lib/kubernetes/__tests__/getDeploymentPods.test.ts index e2a0b3f..50dee5a 100644 --- a/src/server/lib/kubernetes/__tests__/getDeploymentPods.test.ts +++ b/src/server/lib/kubernetes/__tests__/getDeploymentPods.test.ts @@ -16,6 +16,8 @@ var mockListNamespacedDeployment: jest.Mock; var mockListNamespacedStatefulSet: jest.Mock; +var mockListNamespacedJob: jest.Mock; +var mockListNamespacedCronJob: jest.Mock; var mockListNamespacedPod: jest.Mock; var mockBuildFindOne: jest.Mock; @@ -23,12 +25,18 @@ jest.mock('@kubernetes/client-node', () => { const actual = jest.requireActual('@kubernetes/client-node'); mockListNamespacedDeployment = jest.fn(); mockListNamespacedStatefulSet = jest.fn(); + mockListNamespacedJob = jest.fn(); + mockListNamespacedCronJob = jest.fn(); mockListNamespacedPod = jest.fn(); const appsClient = { listNamespacedDeployment: mockListNamespacedDeployment, listNamespacedStatefulSet: mockListNamespacedStatefulSet, }; + const batchClient = { + listNamespacedJob: mockListNamespacedJob, + listNamespacedCronJob: mockListNamespacedCronJob, + }; const coreClient = { listNamespacedPod: mockListNamespacedPod, }; @@ -43,6 +51,10 @@ jest.mock('@kubernetes/client-node', () => { return appsClient; } + if (client === actual.BatchV1Api) { + return batchClient; + } + if (client === actual.CoreV1Api) { return coreClient; } @@ -113,6 +125,28 @@ function buildPod({ }; } +function buildJob({ + name, + matchLabels = { 'batch.kubernetes.io/controller-uid': `${name}-uid` }, + ownerReferences, +}: { + name: string; + matchLabels?: Record; + ownerReferences?: Array>; +}) { + return { + metadata: { + name, + ownerReferences, + }, + spec: { + selector: { + matchLabels, + }, + }, + }; +} + describe('getDeploymentPods', () => { beforeEach(() => { jest.clearAllMocks(); @@ -139,6 +173,12 @@ describe('getDeploymentPods', () => { mockListNamespacedStatefulSet.mockResolvedValue({ body: { items: [] }, }); + mockListNamespacedJob.mockResolvedValue({ + body: { items: [] }, + }); + mockListNamespacedCronJob.mockResolvedValue({ + body: { items: [] }, + }); }); it('filters terminated pods and keeps newest active pods first', async () => { @@ -261,4 +301,263 @@ describe('getDeploymentPods', () => { await expect(getDeploymentPods('sample-service', 'sample-env')).resolves.toEqual([]); }); + + it('uses a StatefulSet selector when no Deployment exists', async () => { + mockListNamespacedDeployment.mockResolvedValue({ + body: { items: [] }, + }); + mockListNamespacedStatefulSet.mockResolvedValue({ + body: { + items: [ + { + spec: { + selector: { + matchLabels: { + app: 'sample-stateful-service', + }, + }, + }, + }, + ], + }, + }); + mockListNamespacedPod.mockResolvedValue({ + body: { + items: [ + buildPod({ + name: 'stateful-active', + createdAt: '2026-03-27T19:00:00.000Z', + }), + ], + }, + }); + + const pods = await getDeploymentPods('sample-service', 'sample-env'); + + expect(pods.map((pod) => pod.podName)).toEqual(['stateful-active']); + expect(mockListNamespacedPod).toHaveBeenCalledWith( + 'env-sample-env', + undefined, + undefined, + undefined, + undefined, + 'app=sample-stateful-service' + ); + expect(mockListNamespacedJob).not.toHaveBeenCalled(); + }); + + it('falls back to Job pods and includes terminal job pods', async () => { + mockListNamespacedDeployment.mockResolvedValue({ + body: { items: [] }, + }); + mockListNamespacedStatefulSet.mockResolvedValue({ + body: { items: [] }, + }); + mockListNamespacedJob.mockResolvedValue({ + body: { + items: [buildJob({ name: 'sample-service-job' })], + }, + }); + mockListNamespacedPod.mockResolvedValue({ + body: { + items: [ + buildPod({ + name: 'job-succeeded', + createdAt: '2026-03-27T19:00:00.000Z', + phase: 'Succeeded', + containerStatuses: [ + { + name: 'app', + ready: false, + restartCount: 0, + state: { terminated: { reason: 'Completed' } }, + }, + ], + }), + buildPod({ + name: 'job-failed', + createdAt: '2026-03-27T18:00:00.000Z', + phase: 'Failed', + containerStatuses: [ + { + name: 'app', + ready: false, + restartCount: 1, + state: { terminated: { reason: 'Error' } }, + }, + ], + }), + buildPod({ + name: 'job-deleting', + createdAt: '2026-03-27T17:00:00.000Z', + deletionTimestamp: '2026-03-27T19:01:00.000Z', + }), + ], + }, + }); + + const pods = await getDeploymentPods('sample-service', 'sample-env'); + + expect(pods.map((pod) => pod.podName)).toEqual(['job-succeeded', 'job-failed']); + expect(pods.map((pod) => pod.status)).toEqual(['Completed', 'Error']); + expect(mockListNamespacedJob).toHaveBeenCalledWith( + 'env-sample-env', + undefined, + undefined, + undefined, + undefined, + 'app.kubernetes.io/instance=sample-service-sample-env' + ); + expect(mockListNamespacedPod).toHaveBeenCalledWith( + 'env-sample-env', + undefined, + undefined, + undefined, + undefined, + 'batch.kubernetes.io/controller-uid=sample-service-job-uid' + ); + }); + + it('falls back to job-name when a Job selector is unavailable', async () => { + mockListNamespacedDeployment.mockResolvedValue({ + body: { items: [] }, + }); + mockListNamespacedStatefulSet.mockResolvedValue({ + body: { items: [] }, + }); + mockListNamespacedJob.mockResolvedValue({ + body: { + items: [ + { + metadata: { + name: 'sample-service-job', + }, + }, + ], + }, + }); + mockListNamespacedPod.mockResolvedValue({ + body: { + items: [ + buildPod({ + name: 'job-active', + createdAt: '2026-03-27T19:00:00.000Z', + }), + ], + }, + }); + + await getDeploymentPods('sample-service', 'sample-env'); + + expect(mockListNamespacedPod).toHaveBeenCalledWith( + 'env-sample-env', + undefined, + undefined, + undefined, + undefined, + 'job-name=sample-service-job' + ); + }); + + it('returns CronJob child Job pods when no direct workload exists', async () => { + mockListNamespacedDeployment.mockResolvedValue({ + body: { items: [] }, + }); + mockListNamespacedStatefulSet.mockResolvedValue({ + body: { items: [] }, + }); + mockListNamespacedCronJob.mockResolvedValue({ + body: { + items: [ + { + metadata: { + name: 'sample-service-cron', + uid: 'cron-uid', + }, + }, + ], + }, + }); + mockListNamespacedJob + .mockResolvedValueOnce({ + body: { items: [] }, + }) + .mockResolvedValueOnce({ + body: { + items: [ + buildJob({ + name: 'sample-service-cron-123', + ownerReferences: [ + { + kind: 'CronJob', + name: 'sample-service-cron', + uid: 'cron-uid', + }, + ], + }), + buildJob({ + name: 'unrelated-job', + ownerReferences: [ + { + kind: 'CronJob', + name: 'unrelated-cron', + uid: 'unrelated-uid', + }, + ], + }), + ], + }, + }); + mockListNamespacedPod.mockResolvedValue({ + body: { + items: [ + buildPod({ + name: 'cronjob-succeeded', + createdAt: '2026-03-27T19:00:00.000Z', + phase: 'Succeeded', + containerStatuses: [ + { + name: 'app', + ready: false, + restartCount: 0, + state: { terminated: { reason: 'Completed' } }, + }, + ], + }), + ], + }, + }); + + const pods = await getDeploymentPods('sample-service', 'sample-env'); + + expect(pods.map((pod) => pod.podName)).toEqual(['cronjob-succeeded']); + expect(mockListNamespacedCronJob).toHaveBeenCalledWith( + 'env-sample-env', + undefined, + undefined, + undefined, + undefined, + 'app.kubernetes.io/instance=sample-service-sample-env' + ); + expect(mockListNamespacedJob).toHaveBeenLastCalledWith('env-sample-env'); + expect(mockListNamespacedPod).toHaveBeenCalledWith( + 'env-sample-env', + undefined, + undefined, + undefined, + undefined, + 'batch.kubernetes.io/controller-uid=sample-service-cron-123-uid' + ); + }); + + it('returns an empty list when no supported workload exists', async () => { + mockListNamespacedDeployment.mockResolvedValue({ + body: { items: [] }, + }); + mockListNamespacedStatefulSet.mockResolvedValue({ + body: { items: [] }, + }); + + await expect(getDeploymentPods('sample-service', 'sample-env')).resolves.toEqual([]); + }); }); diff --git a/src/server/lib/kubernetes/getDeploymentPods.ts b/src/server/lib/kubernetes/getDeploymentPods.ts index 3b49cb4..0319496 100644 --- a/src/server/lib/kubernetes/getDeploymentPods.ts +++ b/src/server/lib/kubernetes/getDeploymentPods.ts @@ -118,6 +118,10 @@ function isTerminalPod(pod: k8s.V1Pod): boolean { return appContainerStatuses.length > 0 && appContainerStatuses.every((status) => Boolean(status.state?.terminated)); } +function isDeletingPod(pod: k8s.V1Pod): boolean { + return Boolean(pod.metadata?.deletionTimestamp); +} + function containerState(cs?: k8s.V1ContainerStatus): { state: ContainerState; reason?: string } { if (!cs) return { state: 'Unknown' }; @@ -174,9 +178,76 @@ export function extractContainers(pod: k8s.V1Pod): ContainerInfo[] { return containers; } +function toPodInfo(pod: k8s.V1Pod): PodInfo { + const ageSeconds = podAgeSeconds(pod); + const containers = extractContainers(pod); + + return { + podName: pod.metadata?.name ?? '', + status: podStatus(pod), + restarts: podRestarts(pod), + ageSeconds, + age: formatAge(ageSeconds), + ready: podReady(pod), + containers, + }; +} + +function sortPodsByAge(pods: PodInfo[]): PodInfo[] { + return pods.sort((left, right) => left.ageSeconds - right.ageSeconds); +} + +function getJobPodSelector(job: k8s.V1Job): string | undefined { + const matchLabels = job.spec?.selector?.matchLabels; + + if (matchLabels && Object.keys(matchLabels).length > 0) { + return buildLabelSelector(matchLabels); + } + + const jobName = job.metadata?.name; + return jobName ? `job-name=${jobName}` : undefined; +} + +function isOwnedByCronJob(job: k8s.V1Job, cronJob: k8s.V1CronJob): boolean { + const cronJobName = cronJob.metadata?.name; + const cronJobUid = cronJob.metadata?.uid; + + return (job.metadata?.ownerReferences ?? []).some( + (owner) => + owner.kind === 'CronJob' && + ((cronJobUid && owner.uid === cronJobUid) || (!cronJobUid && cronJobName && owner.name === cronJobName)) + ); +} + +async function listPodsBySelectors( + coreV1: k8s.CoreV1Api, + namespace: string, + selectors: string[], + includeTerminalPods: boolean +): Promise { + const podsByName = new Map(); + + for (const selector of selectors) { + const podResp = await coreV1.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, selector); + + for (const pod of podResp.body.items ?? []) { + const podName = pod.metadata?.name; + if (!podName) continue; + podsByName.set(podName, pod); + } + } + + const pods = Array.from(podsByName.values()).filter((pod) => + includeTerminalPods ? !isDeletingPod(pod) : !isTerminalPod(pod) + ); + + return sortPodsByAge(pods.map(toPodInfo)); +} + export async function getDeploymentPods(deploymentName: string, uuid: string): Promise { const kc = loadKubeConfig(); const appsV1 = kc.makeApiClient(k8s.AppsV1Api); + const batchV1 = kc.makeApiClient(k8s.BatchV1Api); const coreV1 = kc.makeApiClient(k8s.CoreV1Api); try { @@ -184,7 +255,7 @@ export async function getDeploymentPods(deploymentName: string, uuid: string): P const fullDeploymentName = `${deploymentName}-${uuid}`; const workloadSelector = `app.kubernetes.io/instance=${fullDeploymentName}`; - let matchLabels: Record | undefined; + let workloadPodSelector: string | undefined; // Try to find a Deployment using the label selector const deployResp = await appsV1.listNamespacedDeployment( @@ -197,7 +268,9 @@ export async function getDeploymentPods(deploymentName: string, uuid: string): P ); if (deployResp.body.items.length > 0) { - matchLabels = deployResp.body.items[0].spec?.selector?.matchLabels; + const matchLabels = deployResp.body.items[0].spec?.selector?.matchLabels; + workloadPodSelector = + matchLabels && Object.keys(matchLabels).length > 0 ? buildLabelSelector(matchLabels) : undefined; } else { // if no Deployment found, try to find a StatefulSet const stsResp = await appsV1.listNamespacedStatefulSet( @@ -210,48 +283,57 @@ export async function getDeploymentPods(deploymentName: string, uuid: string): P ); if (stsResp.body.items.length > 0) { - matchLabels = stsResp.body.items[0].spec?.selector?.matchLabels; + const matchLabels = stsResp.body.items[0].spec?.selector?.matchLabels; + workloadPodSelector = + matchLabels && Object.keys(matchLabels).length > 0 ? buildLabelSelector(matchLabels) : undefined; } } - // If neither found or no labels to match, return empty - if (!matchLabels || Object.keys(matchLabels).length === 0) { - return []; + if (workloadPodSelector) { + return listPodsBySelectors(coreV1, namespace, [workloadPodSelector], false); } - const labelSelector = buildLabelSelector(matchLabels); + const jobResp = await batchV1.listNamespacedJob( + namespace, + undefined, + undefined, + undefined, + undefined, + workloadSelector + ); + const jobPodSelectors = (jobResp.body.items ?? []) + .map((job) => getJobPodSelector(job)) + .filter((selector): selector is string => Boolean(selector)); + + if (jobPodSelectors.length > 0) { + return listPodsBySelectors(coreV1, namespace, jobPodSelectors, true); + } - const podResp = await coreV1.listNamespacedPod( + const cronJobResp = await batchV1.listNamespacedCronJob( namespace, undefined, undefined, undefined, undefined, - labelSelector + workloadSelector ); + const cronJobs = cronJobResp.body.items ?? []; + + if (cronJobs.length === 0) { + return []; + } - const pods = (podResp.body.items ?? []).filter((pod) => !isTerminalPod(pod)); + const allJobsResp = await batchV1.listNamespacedJob(namespace); + const cronJobPodSelectors = (allJobsResp.body.items ?? []) + .filter((job) => cronJobs.some((cronJob) => isOwnedByCronJob(job, cronJob))) + .map((job) => getJobPodSelector(job)) + .filter((selector): selector is string => Boolean(selector)); - if (pods.length === 0) { + if (cronJobPodSelectors.length === 0) { return []; } - return pods - .map((pod) => { - const ageSeconds = podAgeSeconds(pod); - const containers = extractContainers(pod); - - return { - podName: pod.metadata?.name ?? '', - status: podStatus(pod), - restarts: podRestarts(pod), - ageSeconds, - age: formatAge(ageSeconds), - ready: podReady(pod), - containers, - }; - }) - .sort((left, right) => left.ageSeconds - right.ageSeconds); + return listPodsBySelectors(coreV1, namespace, cronJobPodSelectors, true); } catch (error) { getLogger().error({ error }, `K8s: failed to list workload pods service=${deploymentName}`); throw error;