diff --git a/src/server/lib/deploymentManager/deploymentManager.ts b/src/server/lib/deploymentManager/deploymentManager.ts index e30f6031..840dd2ca 100644 --- a/src/server/lib/deploymentManager/deploymentManager.ts +++ b/src/server/lib/deploymentManager/deploymentManager.ts @@ -176,6 +176,10 @@ export class DeploymentManager { const jobId = generateJobId(); const deployService = new DeployService(); const runUUID = deploy.runUUID || nanoid(); + if (deploy.runUUID !== runUUID) { + await deploy.$query().patch({ runUUID }); + deploy.runUUID = runUUID; + } try { await deployService.patchAndUpdateActivityFeed( @@ -239,14 +243,11 @@ export class DeploymentManager { throw new Error('Pods failed to become ready within timeout'); } } catch (error) { - await deployService.patchAndUpdateActivityFeed( - deploy, - { - status: DeployStatus.DEPLOY_FAILED, - statusMessage: `Deployment failed for ${deploy.uuid}. Check deploy logs in Console > Deploy tab for details.`, - }, - runUUID - ); + await deployService.recordDeployFailure(deploy, runUUID, { + status: DeployStatus.DEPLOY_FAILED, + error, + fallbackMessage: `Deployment failed for ${deploy.uuid}. Check deploy logs in Console > Deploy tab for details.`, + }); throw error; } }); diff --git a/src/server/lib/nativeHelm/helm.ts b/src/server/lib/nativeHelm/helm.ts index dc5aed90..ee45e87d 100644 --- a/src/server/lib/nativeHelm/helm.ts +++ b/src/server/lib/nativeHelm/helm.ts @@ -470,6 +470,10 @@ export async function deployHelm(deploys: Deploy[]): Promise { const startTime = Date.now(); const runUUID = deploy.runUUID ?? nanoid(); const deployService = new DeployService(); + if (deploy.runUUID !== runUUID) { + await deploy.$query().patch({ runUUID }); + deploy.runUUID = runUUID; + } try { const useNative = await shouldUseNativeHelm(deploy); @@ -503,18 +507,14 @@ export async function deployHelm(deploys: Deploy[]): Promise { await trackHelmDeploymentMetrics(deploy, 'success', Date.now() - startTime); } catch (error) { - await trackHelmDeploymentMetrics(deploy, 'failure', Date.now() - startTime, error.message); - - await deployService.patchAndUpdateActivityFeed( - deploy, - { - status: DeployStatus.DEPLOY_FAILED, - statusMessage: error.message.includes('timed out') - ? error.message - : `${error.message}. Check deploy logs in Console > Deploy tab for details.`, - }, - runUUID - ); + const errorMessage = error instanceof Error ? error.message : String(error); + await trackHelmDeploymentMetrics(deploy, 'failure', Date.now() - startTime, errorMessage); + + await deployService.recordDeployFailure(deploy, runUUID, { + status: DeployStatus.DEPLOY_FAILED, + error, + fallbackMessage: `Helm deployment failed for ${deploy.uuid}. Check deploy logs in Console > Deploy tab for details.`, + }); throw error; } diff --git a/src/server/lib/terminalFailure.ts b/src/server/lib/terminalFailure.ts new file mode 100644 index 00000000..bf309112 --- /dev/null +++ b/src/server/lib/terminalFailure.ts @@ -0,0 +1,33 @@ +import { DeployStatus } from 'shared/constants'; + +const MAX_STATUS_MESSAGE_LENGTH = 1000; + +export function compactStatusMessage(message: string): string { + const compact = message.replace(/\s+/g, ' ').trim(); + return compact.length > MAX_STATUS_MESSAGE_LENGTH ? `${compact.slice(0, MAX_STATUS_MESSAGE_LENGTH - 3)}...` : compact; +} + +export function statusMessageFromError(error: unknown, fallbackMessage: string): string { + if (error instanceof Error && error.message) { + return compactStatusMessage(error.message); + } + + if (typeof error === 'string' && error.trim().length > 0) { + return compactStatusMessage(error); + } + + return compactStatusMessage(fallbackMessage); +} + +export function fallbackDeployStatusMessage(status: DeployStatus): string { + switch (status) { + case DeployStatus.BUILD_FAILED: + return 'Build failed. Check build logs for details.'; + case DeployStatus.DEPLOY_FAILED: + return 'Deployment failed. Check deploy logs for details.'; + case DeployStatus.ERROR: + return 'Deploy failed unexpectedly.'; + default: + return ''; + } +} diff --git a/src/server/services/__tests__/build.test.ts b/src/server/services/__tests__/build.test.ts new file mode 100644 index 00000000..c8a92205 --- /dev/null +++ b/src/server/services/__tests__/build.test.ts @@ -0,0 +1,171 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const mockDeployQuery = jest.fn(); +const mockGenerateManifest = jest.fn(); +const mockApplyManifests = jest.fn(); +const mockWaitForPodReady = jest.fn(); +const mockGetAllConfigs = jest.fn(); +const mockQueueAdd = jest.fn(); + +jest.mock('server/lib/dependencies', () => ({ + defaultDb: {}, + defaultRedis: {}, + defaultRedlock: {}, + defaultQueueManager: {}, + redisClient: { + getConnection: jest.fn(), + }, +})); + +jest.mock('server/lib/tracer', () => ({ + Tracer: { + getInstance: jest.fn(() => ({ + initialize: jest.fn(), + })), + }, +})); + +jest.mock('server/lib/logger', () => ({ + getLogger: jest.fn(() => ({ + error: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + debug: jest.fn(), + })), + withLogContext: jest.fn((ctx, fn) => fn()), + extractContextForQueue: jest.fn(() => ({})), + updateLogContext: jest.fn(), + LogStage: {}, +})); + +jest.mock('server/models', () => ({ + Build: class {}, + Deploy: { + query: () => mockDeployQuery(), + }, + Environment: class {}, + Service: class {}, + BuildServiceOverride: class {}, +})); + +jest.mock('server/lib/kubernetes', () => ({ + generateManifest: (...args: any[]) => mockGenerateManifest(...args), + applyManifests: (...args: any[]) => mockApplyManifests(...args), + waitForPodReady: (...args: any[]) => mockWaitForPodReady(...args), + createOrUpdateNamespace: jest.fn(), + createOrUpdateServiceAccount: jest.fn(), +})); + +jest.mock('server/services/globalConfig', () => ({ + __esModule: true, + default: { + getInstance: jest.fn(() => ({ + getAllConfigs: (...args: any[]) => mockGetAllConfigs(...args), + })), + }, +})); + +jest.mock('server/lib/fastly', () => + jest.fn().mockImplementation(() => ({ + getServiceDashboardUrl: jest.fn(), + })) +); + +import BuildService from '../build'; +import { DeployStatus, DeployTypes } from 'shared/constants'; + +describe('BuildService failure boundaries', () => { + let buildService: BuildService; + let recordDeployFailure: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + recordDeployFailure = jest.fn(); + const queueManager = { + registerQueue: jest.fn(() => ({ + add: mockQueueAdd, + process: jest.fn(), + on: jest.fn(), + })), + }; + buildService = new BuildService( + { + services: { + Deploy: { + recordDeployFailure, + }, + }, + } as any, + {} as any, + {} as any, + queueManager as any + ); + (buildService as any).ingressService = { + ingressManifestQueue: { + add: mockQueueAdd, + }, + }; + mockGetAllConfigs.mockResolvedValue({ serviceAccount: { name: 'sample-service-account' } }); + }); + + test('classic manifest failures stay build-scoped when no deploy can be identified', async () => { + const deploys = [ + { + id: 1, + active: true, + uuid: 'sample-api', + status: DeployStatus.BUILT, + service: { type: DeployTypes.GITHUB, name: 'sample-api' }, + }, + { + id: 2, + active: true, + uuid: 'sample-worker', + status: DeployStatus.BUILT, + service: { type: DeployTypes.DOCKER, name: 'sample-worker' }, + }, + ]; + const query = { + where: jest.fn().mockReturnThis(), + withGraphFetched: jest.fn().mockResolvedValue(deploys), + }; + const rolloutError = new Error('Pods for build not ready after 15 minutes'); + mockDeployQuery.mockReturnValue(query); + mockGenerateManifest.mockReturnValue('apiVersion: apps/v1\nkind: Deployment\n'); + mockApplyManifests.mockResolvedValue([]); + mockWaitForPodReady.mockRejectedValue(rolloutError); + + await expect( + buildService.generateAndApplyManifests({ + build: { + id: 123, + uuid: 'sample-build', + runUUID: 'run-1', + namespace: 'sample-namespace', + enableFullYaml: false, + $query: jest.fn(() => ({ + patch: jest.fn().mockResolvedValue(undefined), + })), + } as any, + githubRepositoryId: null, + namespace: 'sample-namespace', + }) + ).rejects.toThrow(rolloutError); + + expect(recordDeployFailure).not.toHaveBeenCalled(); + }); +}); diff --git a/src/server/services/__tests__/deploy.test.ts b/src/server/services/__tests__/deploy.test.ts index 8ffe241b..c505fa4e 100644 --- a/src/server/services/__tests__/deploy.test.ts +++ b/src/server/services/__tests__/deploy.test.ts @@ -16,11 +16,14 @@ import mockRedisClient from 'server/lib/__mocks__/redisClientMock'; import DeployService from '../deploy'; -import { DeployTypes } from 'shared/constants'; +import { DeployStatus, DeployTypes } from 'shared/constants'; import { ChartType } from 'server/lib/nativeHelm'; +import * as github from 'server/lib/github'; mockRedisClient(); +const mockCliDeploy = jest.fn(); + jest.mock('server/lib/logger', () => ({ getLogger: jest.fn(() => ({ error: jest.fn(), @@ -49,6 +52,17 @@ jest.mock('server/lib/nativeHelm', () => ({ determineChartType: (...args: any[]) => mockDetermineChartType(...args), })); +jest.mock('server/lib/github', () => ({ + getSHAForBranch: jest.fn(), + getShaForDeploy: jest.fn(), +})); + +jest.mock('server/lib/cli', () => ({ + cliDeploy: (...args: any[]) => mockCliDeploy(...args), + codefreshDeploy: jest.fn(), + waitForCodefresh: jest.fn(), +})); + describe('DeployService - shouldTriggerGithubDeployment', () => { let deployService: DeployService; let mockDb: any; @@ -76,6 +90,7 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { beforeEach(() => { jest.clearAllMocks(); + mockCliDeploy.mockReset(); mockDetermineChartType.mockResolvedValue(ChartType.PUBLIC); mockDb = { @@ -258,4 +273,128 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { expect(result).toBe(false); }); }); + + describe('failure boundaries', () => { + test('recordDeployFailure writes a terminal status with the original error message', async () => { + const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); + const deploy = { + uuid: 'sample-service-build', + runUUID: 'run-1', + $query: jest.fn(() => ({ + patch: jest.fn().mockResolvedValue(undefined), + })), + }; + + const result = await deployService.recordDeployFailure(deploy as any, 'run-1', { + status: DeployStatus.DEPLOY_FAILED, + error: new Error('Kubernetes apply job failed: pod quota exceeded'), + fallbackMessage: 'Kubernetes deployment failed.', + }); + + expect(result).toBe(false); + expect(patchSpy).toHaveBeenCalledWith( + deploy, + { + status: DeployStatus.DEPLOY_FAILED, + statusMessage: 'Kubernetes apply job failed: pod quota exceeded', + }, + 'run-1' + ); + }); + + test('buildImage boundary records a source resolution failure statusMessage', async () => { + (github.getSHAForBranch as jest.Mock).mockRejectedValue(new Error('Not Found')); + const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); + const deploy = { + uuid: 'sample-service-build', + runUUID: 'run-1', + branchName: 'missing-branch', + env: {}, + tag: 'latest', + $query: jest.fn(() => ({ + patch: jest.fn().mockResolvedValue(undefined), + })), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + deployable: { + name: 'sample-service', + type: DeployTypes.GITHUB, + dockerfilePath: './Dockerfile', + initDockerfilePath: null, + repository: { + fullName: 'example-org/example-repo', + }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }, + build: { + uuid: 'sample-build', + enableFullYaml: true, + commentRuntimeEnv: {}, + enabledFeatures: [], + pullRequest: { + githubLogin: 'sample-user', + }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }, + }; + + const result = await deployService.buildImage(deploy as any, true, 0); + + expect(result).toBe(false); + expect(github.getSHAForBranch).toHaveBeenCalledWith('missing-branch', 'example-org', 'example-repo'); + expect(patchSpy).toHaveBeenLastCalledWith( + deploy, + { + status: DeployStatus.BUILD_FAILED, + statusMessage: + 'Unable to resolve branch "missing-branch" in repository "example-org/example-repo". Verify the branch exists and the repository matches the selected service.', + }, + 'run-1' + ); + }); + + test('deployAurora records failures with the newly assigned runUUID', async () => { + const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); + jest.spyOn(deployService as any, 'findExistingAuroraDatabase').mockResolvedValue(null); + mockCliDeploy.mockRejectedValue(new Error('restore command failed')); + + const patches: any[] = []; + const deploy = { + uuid: 'sample-aurora-restore', + runUUID: 'old-run', + status: DeployStatus.PENDING, + buildLogs: null, + build: { + uuid: 'sample-build', + }, + deployable: { + name: 'sample-database', + type: DeployTypes.AURORA_RESTORE, + }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + $query: jest.fn(() => ({ + patch: jest.fn((params) => { + patches.push(params); + return Promise.resolve(undefined); + }), + })), + }; + + const result = await deployService.deployAurora(deploy as any); + const assignedRunUUID = patches.find((params) => params.status === DeployStatus.BUILDING)?.runUUID; + + expect(result).toBe(false); + expect(assignedRunUUID).toBeDefined(); + expect(assignedRunUUID).not.toBe('old-run'); + expect(deploy.runUUID).toBe(assignedRunUUID); + expect(patchSpy).toHaveBeenLastCalledWith( + deploy, + { + status: DeployStatus.ERROR, + statusMessage: 'restore command failed', + }, + assignedRunUUID + ); + }); + }); }); diff --git a/src/server/services/build.ts b/src/server/services/build.ts index 8b29e4bb..a64bb81d 100644 --- a/src/server/services/build.ts +++ b/src/server/services/build.ts @@ -48,6 +48,7 @@ import AgentPrewarmService from './agentPrewarm'; import { paginate, PaginationMetadata, PaginationParams } from 'server/lib/paginate'; import { getYamlFileContentFromBranch } from 'server/lib/github'; import WebhookService from './webhook'; +import { compactStatusMessage, statusMessageFromError } from 'server/lib/terminalFailure'; const tracer = Tracer.getInstance(); tracer.initialize('build-service'); @@ -734,9 +735,16 @@ export default class BuildService extends BaseService { ); } } catch (error) { - if (error instanceof ParsingError || error instanceof ValidationError) { - await this.updateStatusAndComment(build, BuildStatus.CONFIG_ERROR, runUUID, true, true, error); - } + const isConfigError = error instanceof ParsingError || error instanceof ValidationError; + await this.recordBuildFailure( + build, + isConfigError ? BuildStatus.CONFIG_ERROR : BuildStatus.ERROR, + runUUID, + error, + isConfigError + ? 'Lifecycle configuration failed validation.' + : 'Build setup failed before deploys could be created.' + ); } } else { throw new Error('Missing build or deployment options from environment.'); @@ -827,7 +835,7 @@ export default class BuildService extends BaseService { } } catch (error) { getLogger().error({ error }, 'Build: deploy failed'); - await this.updateStatusAndComment(build, BuildStatus.ERROR, runUUID, true, true, error); + await this.recordBuildFailure(build, BuildStatus.ERROR, runUUID, error, 'Build failed unexpectedly.'); } return build; @@ -994,7 +1002,7 @@ export default class BuildService extends BaseService { runUUID: string, updateMissionControl: boolean, updateStatus: boolean, - error: Error = null + error: Error | null = null ) { return withLogContext({ buildUuid: build.uuid }, async () => { try { @@ -1008,8 +1016,10 @@ export default class BuildService extends BaseService { if (build.runUUID !== runUUID) { return; } else { + const statusMessage = this.resolveBuildStatusMessage(status, deploys || [], error); await build.$query().patch({ status, + statusMessage, }); // add dashboard links to build database @@ -1054,6 +1064,56 @@ export default class BuildService extends BaseService { }); } + private async recordBuildFailure( + build: Build, + status: BuildStatus, + runUUID: string | null | undefined, + error: unknown, + fallbackMessage: string + ): Promise { + const activeRunUUID = runUUID || build.runUUID || nanoid(); + if (build.runUUID !== activeRunUUID) { + await build.$query().patch({ runUUID: activeRunUUID }); + build.runUUID = activeRunUUID; + } + + const statusError = error instanceof Error ? error : new Error(statusMessageFromError(error, fallbackMessage)); + await this.updateStatusAndComment(build, status, activeRunUUID, true, true, statusError); + } + + private resolveBuildStatusMessage(status: BuildStatus, deploys: Deploy[], error: Error | null): string { + if (status === BuildStatus.CONFIG_ERROR) { + return compactStatusMessage(error?.message || 'Lifecycle configuration failed validation.'); + } + + if (status !== BuildStatus.ERROR) { + return ''; + } + + if (error) { + return compactStatusMessage(error.message || 'Build failed unexpectedly.'); + } + + const failedDeployMessages = (deploys || []) + .filter( + (deploy) => + deploy.active !== false && + [DeployStatus.ERROR, DeployStatus.BUILD_FAILED, DeployStatus.DEPLOY_FAILED].includes( + deploy.status as DeployStatus + ) + ) + .map((deploy) => { + const serviceName = deploy.deployable?.name || deploy.service?.name || deploy.uuid || 'unknown service'; + return deploy.statusMessage ? `${serviceName}: ${deploy.statusMessage}` : `${serviceName}: ${deploy.status}`; + }); + + if (failedDeployMessages.length === 0) { + return 'Build failed. Check service status messages for details.'; + } + + return compactStatusMessage(`Build failed because ${failedDeployMessages.slice(0, 3).join('; ')}`); + } + async markConfigurationsAsBuilt(build: Build) { try { await build?.$fetchGraph({ @@ -1112,7 +1172,11 @@ export default class BuildService extends BaseService { return result; } catch (err) { getLogger().error({ error: err }, `CLI: deploy failed uuid=${deploy?.uuid}`); - return false; + return this.db.services.Deploy.recordDeployFailure(deploy, deploy.runUUID || build.runUUID, { + status: DeployStatus.ERROR, + error: err, + fallbackMessage: 'CLI deploy failed.', + }); } }) ) @@ -1128,7 +1192,11 @@ export default class BuildService extends BaseService { } const result = await this.db.services.Deploy.deployCLI(deploy).catch((error) => { getLogger().error({ error }, 'CLI: deploy failed'); - return false; + return this.db.services.Deploy.recordDeployFailure(deploy, deploy.runUUID || build.runUUID, { + status: DeployStatus.ERROR, + error, + fallbackMessage: 'CLI deploy failed.', + }); }); if (!result) getLogger().info(`CLI: deploy failed uuid=${deploy.uuid}`); @@ -1345,6 +1413,7 @@ export default class BuildService extends BaseService { throw e; } } else { + let deploys: Deploy[] = []; try { const buildId = build?.id; if (!buildId) { @@ -1354,7 +1423,7 @@ export default class BuildService extends BaseService { const { serviceAccount } = await GlobalConfigService.getInstance().getAllConfigs(); const serviceAccountName = serviceAccount?.name || 'default'; - const deploys = ( + deploys = ( await Deploy.query() .where({ buildId }) .withGraphFetched({ @@ -1403,7 +1472,7 @@ export default class BuildService extends BaseService { return true; } catch (e) { getLogger().warn({ error: e }, 'K8s: deploy failed'); - return false; + throw e; } } } @@ -1535,10 +1604,25 @@ export default class BuildService extends BaseService { getLogger({ stage: LogStage.BUILD_COMPLETE }).info('Build: completed'); } catch (error) { - if (error instanceof ParsingError || error instanceof ValidationError) { - this.updateStatusAndComment(build, BuildStatus.CONFIG_ERROR, build?.runUUID, true, true, error); + if (!build) { + getLogger({ stage: LogStage.BUILD_FAILED }).fatal({ error }, `Build: queue failed buildId=${buildId}`); + } else if (error instanceof ParsingError || error instanceof ValidationError) { + await this.recordBuildFailure( + build, + BuildStatus.CONFIG_ERROR, + build.runUUID, + error, + 'Lifecycle configuration failed validation.' + ); } else { getLogger({ stage: LogStage.BUILD_FAILED }).fatal({ error }, 'Build: uncaught exception'); + await this.recordBuildFailure( + build, + BuildStatus.ERROR, + build.runUUID, + error, + 'Build queue processing failed.' + ); } } }); diff --git a/src/server/services/deploy.ts b/src/server/services/deploy.ts index e5d55cae..75dd6c09 100644 --- a/src/server/services/deploy.ts +++ b/src/server/services/deploy.ts @@ -39,6 +39,7 @@ import { constructEcrTag } from 'server/lib/codefresh/utils'; import { ChartType, determineChartType } from 'server/lib/nativeHelm'; import { parseSecretRefsFromEnv } from 'server/lib/secretRefs'; import { SecretProcessor } from 'server/services/secretProcessor'; +import { fallbackDeployStatusMessage, statusMessageFromError } from 'server/lib/terminalFailure'; export interface DeployOptions { ownerId?: number; @@ -375,13 +376,15 @@ export default class DeployService extends BaseService { return withLogContext( { deployUuid: deploy.uuid, serviceName: deploy.deployable?.name || deploy.service?.name }, async () => { + let runUUID = deploy.runUUID; try { await deploy.reload(); await deploy.$fetchGraph('[build, deployable]'); + runUUID = deploy.runUUID; if (!deploy.deployable) { getLogger().error('Aurora: deployable missing for=restore'); - return false; + throw new Error('Aurora restore deployable is missing.'); } if ((deploy.status === DeployStatus.BUILT || deploy.status === DeployStatus.READY) && deploy.cname) { @@ -400,11 +403,13 @@ export default class DeployService extends BaseService { } const uuid = nanoid(); + runUUID = nanoid(); await deploy.$query().patch({ status: DeployStatus.BUILDING, buildLogs: uuid, - runUUID: nanoid(), + runUUID, }); + deploy.runUUID = runUUID; getLogger().info('Aurora: restoring'); await cli.cliDeploy(deploy); @@ -425,10 +430,11 @@ export default class DeployService extends BaseService { return true; } catch (e) { getLogger().error({ error: e }, 'Aurora: cluster restore failed'); - await deploy.$query().patch({ + return this.recordDeployFailure(deploy, runUUID || deploy.runUUID, { status: DeployStatus.ERROR, + error: e, + fallbackMessage: 'Aurora restore failed.', }); - return false; } } ); @@ -444,25 +450,16 @@ export default class DeployService extends BaseService { await deploy.$query().patch({ runUUID, }); + deploy.runUUID = runUUID; await deploy.reload(); await deploy.$fetchGraph('[service.[repository], deployable.[repository], build]'); const { build, service, deployable } = deploy; - const { repository } = build.enableFullYaml ? deployable : service; - const repo = repository?.fullName; - const [owner, name] = repo?.split('/') || []; - const fullSha = await github.getSHAForBranch(deploy.branchName, owner, name).catch((error) => { - getLogger().warn( - { error, owner, name, branch: deploy.branchName }, - 'Failed to retrieve commit SHA from github' - ); - }); - - if (!fullSha) { - getLogger().warn({ owner, name, branch: deploy.branchName }, 'Git: SHA missing'); + const source = build.enableFullYaml ? deployable : service; + const repo = source?.repository?.fullName; - result = false; - } else { + try { + const fullSha = await this.resolveSourceSha(repo, deploy.branchName); const shortSha = fullSha.substring(0, 7); const envSha = hash(merge(deploy.env || {}, build.commentRuntimeEnv)); const buildSha = `${shortSha}-${envSha}`; @@ -539,6 +536,12 @@ export default class DeployService extends BaseService { result = false; } } + } catch (error) { + return this.recordDeployFailure(deploy, runUUID, { + status: DeployStatus.BUILD_FAILED, + error, + fallbackMessage: 'CI build failed.', + }); } return result; @@ -570,11 +573,12 @@ export default class DeployService extends BaseService { return withLogContext( { deployUuid: deploy.uuid, serviceName: deploy.deployable?.name || deploy.service?.name }, async () => { + const runUUID = deploy.runUUID ?? nanoid(); try { - const runUUID = deploy.runUUID ?? nanoid(); await deploy.$query().patch({ runUUID, }); + deploy.runUUID = runUUID; await deploy.$fetchGraph('[service, build.[environment], deployable]'); const { service, build, deployable } = deploy; @@ -612,8 +616,7 @@ export default class DeployService extends BaseService { const enabledFeatures = build?.enabledFeatures || []; const repository = service?.repository; const repo = repository?.fullName; - const [owner, name] = repo?.split('/') || []; - const fullSha = await github.getSHAForBranch(deploy.branchName, owner, name); + const fullSha = await this.resolveSourceSha(repo, deploy.branchName); let repositoryName: string = service.repository.fullName; let branchName: string = deploy.branchName; @@ -637,13 +640,6 @@ export default class DeployService extends BaseService { } } - // Verify we actually have a SHA from github before proceeding - if (!fullSha) { - // We were unable to retrieve this branch/repo combo - await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.ERROR }, runUUID); - return false; - } - const shortSha = fullSha.substring(0, 7); getLogger().debug( @@ -732,7 +728,7 @@ export default class DeployService extends BaseService { } else { switch (deployable.type) { case DeployTypes.GITHUB: - return this.buildImageForHelmAndGithub(deploy, runUUID); + return await this.buildImageForHelmAndGithub(deploy, runUUID); case DeployTypes.DOCKER: await this.patchAndUpdateActivityFeed( deploy, @@ -749,7 +745,7 @@ export default class DeployService extends BaseService { const chartType = await determineChartType(deploy); if (chartType !== ChartType.PUBLIC) { - return this.buildImageForHelmAndGithub(deploy, runUUID); + return await this.buildImageForHelmAndGithub(deploy, runUUID); } let fullSha = null; @@ -788,12 +784,73 @@ export default class DeployService extends BaseService { } } catch (e) { getLogger().error({ error: e }, 'Docker: build error'); - return false; + return this.recordDeployFailure(deploy, runUUID, { + status: DeployStatus.BUILD_FAILED, + error: e, + fallbackMessage: 'Build failed unexpectedly. Check build logs for details.', + }); } } ); } + async recordDeployFailure( + deploy: Deploy, + runUUID: string | null | undefined, + { + status, + error, + fallbackMessage, + }: { + status: DeployStatus; + error?: unknown; + fallbackMessage: string; + } + ): Promise { + const activeRunUUID = runUUID || deploy.runUUID || nanoid(); + + if (deploy.runUUID !== activeRunUUID) { + await deploy.$query().patch({ runUUID: activeRunUUID }); + deploy.runUUID = activeRunUUID; + } + + await this.patchAndUpdateActivityFeed( + deploy, + { + status, + statusMessage: statusMessageFromError(error, fallbackMessage), + }, + activeRunUUID + ); + + return false; + } + + private async resolveSourceSha(repo: string | null | undefined, branchName: string | null): Promise { + const [owner, name] = repo?.split('/') || []; + + if (!owner || !name || !branchName) { + throw this.sourceResolutionFailure(repo, branchName); + } + + try { + const fullSha = await github.getSHAForBranch(branchName, owner, name); + if (fullSha) return fullSha; + } catch (error) { + getLogger().warn({ error, repo, branch: branchName }, 'Git: source resolution failed'); + } + + throw this.sourceResolutionFailure(repo, branchName); + } + + private sourceResolutionFailure(repo?: string | null, branchName?: string | null): Error { + const repositoryLabel = repo || 'the selected repository'; + const branchLabel = branchName || 'the selected branch'; + return new Error( + `Unable to resolve branch "${branchLabel}" in repository "${repositoryLabel}". Verify the branch exists and the repository matches the selected service.` + ); + } + public async patchAndUpdateActivityFeed( deploy: Deploy, params: Objection.PartialModelObject, @@ -803,7 +860,13 @@ export default class DeployService extends BaseService { let build: Build; try { const id = deploy?.id; - await this.db.models.Deploy.query().where({ id, runUUID }).patch(params); + const failureStatuses = [DeployStatus.ERROR, DeployStatus.BUILD_FAILED, DeployStatus.DEPLOY_FAILED]; + const status = params.status as DeployStatus; + const fallbackStatusMessage = + !params.statusMessage && failureStatuses.includes(status) ? fallbackDeployStatusMessage(status) : ''; + const patchParams = fallbackStatusMessage ? { ...params, statusMessage: fallbackStatusMessage } : params; + + await this.db.models.Deploy.query().where({ id, runUUID }).patch(patchParams); if (deploy.runUUID !== runUUID) { getLogger().debug(`runUUID mismatch: deployRunUUID=${deploy.runUUID} providedRunUUID=${runUUID}`); return; @@ -822,7 +885,7 @@ export default class DeployService extends BaseService { DeployStatus.DEPLOY_FAILED, DeployStatus.TORN_DOWN, ]; - const isTerminalStatus = terminalStatuses.includes(params.status as DeployStatus); + const isTerminalStatus = terminalStatuses.includes(status); if (isTerminalStatus && build?.githubDeployments && !isSandboxBuild) { await deploy.$fetchGraph('[service, deployable]'); @@ -954,26 +1017,17 @@ export default class DeployService extends BaseService { const repository = deployable?.repository; if (!repository) { - await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.ERROR }, runUUID); - return false; + throw this.sourceResolutionFailure(null, deploy.branchName); } const repo = repository?.fullName; - const [owner, name] = repo?.split('/') || []; - const fullSha = await github.getSHAForBranch(deploy.branchName, owner, name); + const fullSha = await this.resolveSourceSha(repo, deploy.branchName); const repositoryName: string = deployable.repository.fullName; const branchName: string = deploy.branchName; const dockerfilePath: string = deployable.dockerfilePath; const initDockerfilePath: string = deployable.initDockerfilePath; - // Verify we actually have a SHA from github before proceeding - if (!fullSha) { - await this.patchAndUpdateActivityFeed(deploy, { status: DeployStatus.ERROR }, runUUID); - getLogger().error({ owner, name, branch: deploy.branchName }, 'Git: SHA fetch failed'); - return false; - } - const shortSha = fullSha.substring(0, 7); await build?.$fetchGraph('pullRequest.[repository]'); diff --git a/src/shared/openApiSpec.ts b/src/shared/openApiSpec.ts index 8f5c2eb7..4b3e730d 100644 --- a/src/shared/openApiSpec.ts +++ b/src/shared/openApiSpec.ts @@ -1082,6 +1082,13 @@ export const openApiSpecificationForV2Api: OAS3Options = { manifest: { type: 'string', example: 'version: 1.0.0\nservices:\n web:\n image: myapp:web\n' }, uuid: { type: 'string', example: 'white-poetry-596195' }, status: { $ref: '#/components/schemas/BuildStatus' }, + statusMessage: { + type: 'string', + maxLength: 1000, + nullable: true, + example: + 'Build failed because web: Unable to resolve branch "feature/sample" in repository "example-org/example-repo".', + }, kind: { $ref: '#/components/schemas/BuildKind' }, namespace: { type: 'string', example: 'env-white-poetry-596195' }, isStatic: { type: 'boolean', example: false }, @@ -1170,7 +1177,7 @@ export const openApiSpecificationForV2Api: OAS3Options = { id: { type: 'integer' }, uuid: { type: 'string', example: 'deploy-uuid' }, status: { $ref: '#/components/schemas/DeployStatus' }, - statusMessage: { type: 'string', example: 'Deployment in progress' }, + statusMessage: { type: 'string', maxLength: 1000, example: 'Deployment in progress' }, dockerImage: { type: 'string', example: 'myapp:web' }, buildLogs: { type: 'string', example: 'https://g.codefresh.io/build/123...' }, active: { type: 'boolean', example: true },