Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions src/server/lib/deploymentManager/deploymentManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,10 @@ export class DeploymentManager {
const jobId = generateJobId();
const deployService = new DeployService();
const runUUID = deploy.runUUID || nanoid();
if (deploy.runUUID !== runUUID) {
await deploy.$query().patch({ runUUID });
deploy.runUUID = runUUID;
}

try {
await deployService.patchAndUpdateActivityFeed(
Expand Down Expand Up @@ -239,14 +243,11 @@ export class DeploymentManager {
throw new Error('Pods failed to become ready within timeout');
}
} catch (error) {
await deployService.patchAndUpdateActivityFeed(
deploy,
{
status: DeployStatus.DEPLOY_FAILED,
statusMessage: `Deployment failed for ${deploy.uuid}. Check deploy logs in Console > Deploy tab for details.`,
},
runUUID
);
await deployService.recordDeployFailure(deploy, runUUID, {
status: DeployStatus.DEPLOY_FAILED,
error,
fallbackMessage: `Deployment failed for ${deploy.uuid}. Check deploy logs in Console > Deploy tab for details.`,
});
throw error;
}
});
Expand Down
24 changes: 12 additions & 12 deletions src/server/lib/nativeHelm/helm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,10 @@ export async function deployHelm(deploys: Deploy[]): Promise<void> {
const startTime = Date.now();
const runUUID = deploy.runUUID ?? nanoid();
const deployService = new DeployService();
if (deploy.runUUID !== runUUID) {
await deploy.$query().patch({ runUUID });
deploy.runUUID = runUUID;
}

try {
const useNative = await shouldUseNativeHelm(deploy);
Expand Down Expand Up @@ -503,18 +507,14 @@ export async function deployHelm(deploys: Deploy[]): Promise<void> {

await trackHelmDeploymentMetrics(deploy, 'success', Date.now() - startTime);
} catch (error) {
await trackHelmDeploymentMetrics(deploy, 'failure', Date.now() - startTime, error.message);

await deployService.patchAndUpdateActivityFeed(
deploy,
{
status: DeployStatus.DEPLOY_FAILED,
statusMessage: error.message.includes('timed out')
? error.message
: `${error.message}. Check deploy logs in Console > Deploy tab for details.`,
},
runUUID
);
const errorMessage = error instanceof Error ? error.message : String(error);
await trackHelmDeploymentMetrics(deploy, 'failure', Date.now() - startTime, errorMessage);

await deployService.recordDeployFailure(deploy, runUUID, {
status: DeployStatus.DEPLOY_FAILED,
error,
fallbackMessage: `Helm deployment failed for ${deploy.uuid}. Check deploy logs in Console > Deploy tab for details.`,
});

throw error;
}
Expand Down
33 changes: 33 additions & 0 deletions src/server/lib/terminalFailure.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { DeployStatus } from 'shared/constants';

const MAX_STATUS_MESSAGE_LENGTH = 1000;

export function compactStatusMessage(message: string): string {
const compact = message.replace(/\s+/g, ' ').trim();
return compact.length > MAX_STATUS_MESSAGE_LENGTH ? `${compact.slice(0, MAX_STATUS_MESSAGE_LENGTH - 3)}...` : compact;
}

export function statusMessageFromError(error: unknown, fallbackMessage: string): string {
if (error instanceof Error && error.message) {
return compactStatusMessage(error.message);
}

if (typeof error === 'string' && error.trim().length > 0) {
return compactStatusMessage(error);
}

return compactStatusMessage(fallbackMessage);
}

export function fallbackDeployStatusMessage(status: DeployStatus): string {
switch (status) {
case DeployStatus.BUILD_FAILED:
return 'Build failed. Check build logs for details.';
case DeployStatus.DEPLOY_FAILED:
return 'Deployment failed. Check deploy logs for details.';
case DeployStatus.ERROR:
return 'Deploy failed unexpectedly.';
default:
return '';
}
}
171 changes: 171 additions & 0 deletions src/server/services/__tests__/build.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
/**
* Copyright 2026 GoodRx, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

const mockDeployQuery = jest.fn();
const mockGenerateManifest = jest.fn();
const mockApplyManifests = jest.fn();
const mockWaitForPodReady = jest.fn();
const mockGetAllConfigs = jest.fn();
const mockQueueAdd = jest.fn();

jest.mock('server/lib/dependencies', () => ({
defaultDb: {},
defaultRedis: {},
defaultRedlock: {},
defaultQueueManager: {},
redisClient: {
getConnection: jest.fn(),
},
}));

jest.mock('server/lib/tracer', () => ({
Tracer: {
getInstance: jest.fn(() => ({
initialize: jest.fn(),
})),
},
}));

jest.mock('server/lib/logger', () => ({
getLogger: jest.fn(() => ({
error: jest.fn(),
info: jest.fn(),
warn: jest.fn(),
debug: jest.fn(),
})),
withLogContext: jest.fn((ctx, fn) => fn()),
extractContextForQueue: jest.fn(() => ({})),
updateLogContext: jest.fn(),
LogStage: {},
}));

jest.mock('server/models', () => ({
Build: class {},
Deploy: {
query: () => mockDeployQuery(),
},
Environment: class {},
Service: class {},
BuildServiceOverride: class {},
}));

jest.mock('server/lib/kubernetes', () => ({
generateManifest: (...args: any[]) => mockGenerateManifest(...args),
applyManifests: (...args: any[]) => mockApplyManifests(...args),
waitForPodReady: (...args: any[]) => mockWaitForPodReady(...args),
createOrUpdateNamespace: jest.fn(),
createOrUpdateServiceAccount: jest.fn(),
}));

jest.mock('server/services/globalConfig', () => ({
__esModule: true,
default: {
getInstance: jest.fn(() => ({
getAllConfigs: (...args: any[]) => mockGetAllConfigs(...args),
})),
},
}));

jest.mock('server/lib/fastly', () =>
jest.fn().mockImplementation(() => ({
getServiceDashboardUrl: jest.fn(),
}))
);

import BuildService from '../build';
import { DeployStatus, DeployTypes } from 'shared/constants';

describe('BuildService failure boundaries', () => {
let buildService: BuildService;
let recordDeployFailure: jest.Mock;

beforeEach(() => {
jest.clearAllMocks();
recordDeployFailure = jest.fn();
const queueManager = {
registerQueue: jest.fn(() => ({
add: mockQueueAdd,
process: jest.fn(),
on: jest.fn(),
})),
};
buildService = new BuildService(
{
services: {
Deploy: {
recordDeployFailure,
},
},
} as any,
{} as any,
{} as any,
queueManager as any
);
(buildService as any).ingressService = {
ingressManifestQueue: {
add: mockQueueAdd,
},
};
mockGetAllConfigs.mockResolvedValue({ serviceAccount: { name: 'sample-service-account' } });
});

test('classic manifest failures stay build-scoped when no deploy can be identified', async () => {
const deploys = [
{
id: 1,
active: true,
uuid: 'sample-api',
status: DeployStatus.BUILT,
service: { type: DeployTypes.GITHUB, name: 'sample-api' },
},
{
id: 2,
active: true,
uuid: 'sample-worker',
status: DeployStatus.BUILT,
service: { type: DeployTypes.DOCKER, name: 'sample-worker' },
},
];
const query = {
where: jest.fn().mockReturnThis(),
withGraphFetched: jest.fn().mockResolvedValue(deploys),
};
const rolloutError = new Error('Pods for build not ready after 15 minutes');
mockDeployQuery.mockReturnValue(query);
mockGenerateManifest.mockReturnValue('apiVersion: apps/v1\nkind: Deployment\n');
mockApplyManifests.mockResolvedValue([]);
mockWaitForPodReady.mockRejectedValue(rolloutError);

await expect(
buildService.generateAndApplyManifests({
build: {
id: 123,
uuid: 'sample-build',
runUUID: 'run-1',
namespace: 'sample-namespace',
enableFullYaml: false,
$query: jest.fn(() => ({
patch: jest.fn().mockResolvedValue(undefined),
})),
} as any,
githubRepositoryId: null,
namespace: 'sample-namespace',
})
).rejects.toThrow(rolloutError);

expect(recordDeployFailure).not.toHaveBeenCalled();
});
});
Loading
Loading