Skip to content

Commit d46c585

Browse files
NahutabDevelopNahutabDevelopebihimself
authored
fix: Detect deployment rollback after service stabilization (aws-actions#860)
Co-authored-by: NahutabDevelop <baykac@amazon.com> Co-authored-by: ebihimself <ebihimself@users.noreply.github.com>
1 parent fc8fc60 commit d46c585

4 files changed

Lines changed: 286 additions & 24 deletions

File tree

‎dist/index.js‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ async function tasksExitCode(ecs, clusterName, taskArns) {
203203
}
204204

205205
// Deploy to a service that uses the 'ECS' deployment controller
206-
async function updateEcsService(ecs, clusterName, service, taskDefArn, waitForService, waitForMinutes, forceNewDeployment, desiredCount, enableECSManagedTags, propagateTags, waitMaxDelaySeconds) {
206+
async function updateEcsService(ecs, clusterName, service, taskDefArn, waitForService, waitForMinutes, forceNewDeployment, desiredCount, enableECSManagedTags, propagateTags, waitMaxDelaySeconds, beforeDeploymentId) {
207207
core.debug('Updating the service');
208208

209209
const serviceManagedEBSVolumeName = core.getInput('service-managed-ebs-volume-name', { required: false }) || '';
@@ -238,7 +238,11 @@ async function updateEcsService(ecs, clusterName, service, taskDefArn, waitForSe
238238
if (!isNaN(desiredCount) && desiredCount !== undefined) {
239239
params.desiredCount = desiredCount;
240240
}
241-
await ecs.updateService(params);
241+
const updateResponse = await ecs.updateService(params);
242+
243+
// Extract the PRIMARY deployment ID from the update response
244+
const primaryDeployment = (updateResponse.service.deployments || []).find(d => d.status === 'PRIMARY');
245+
const afterDeploymentId = primaryDeployment ? primaryDeployment.id : null;
242246

243247
const region = await ecs.config.region();
244248
const consoleHostname = region.startsWith('cn') ? 'console.amazonaws.cn' : 'console.aws.amazon.com';
@@ -263,6 +267,36 @@ async function updateEcsService(ecs, clusterName, service, taskDefArn, waitForSe
263267
services: [service],
264268
cluster: clusterName
265269
});
270+
271+
// Verify the deployment was successful (not rolled back)
272+
if (afterDeploymentId && afterDeploymentId !== beforeDeploymentId) {
273+
core.debug('Verifying deployment succeeded after service stability...');
274+
275+
const verifyResponse = await ecs.describeServices({
276+
services: [service],
277+
cluster: clusterName
278+
});
279+
280+
const verifiedService = verifyResponse.services[0];
281+
const deployment = (verifiedService.deployments || []).find(d => d.id === afterDeploymentId);
282+
283+
if (!deployment) {
284+
throw new Error(
285+
`Deployment ${afterDeploymentId} not found after stabilization. ` +
286+
`The deployment was likely rolled back by the deployment circuit breaker.`
287+
);
288+
}
289+
290+
if (deployment.rolloutState === 'FAILED') {
291+
throw new Error(
292+
`Deployment ${afterDeploymentId} FAILED: ${deployment.rolloutStateReason || 'unknown reason'}`
293+
);
294+
}
295+
296+
core.info(`Deployment ${afterDeploymentId} verified: rolloutState=${deployment.rolloutState || 'N/A'}`);
297+
} else {
298+
core.debug('No new deployment was created by the update, skipping deployment verification');
299+
}
266300
} else {
267301
core.debug('Not waiting for the service to become stable');
268302
}
@@ -591,8 +625,11 @@ async function run() {
591625

592626
if (!serviceResponse.deploymentController || !serviceResponse.deploymentController.type || serviceResponse.deploymentController.type === 'ECS') {
593627
// Service uses the 'ECS' deployment controller, so we can call UpdateService
628+
const beforePrimaryDeployment = (serviceResponse.deployments || []).find(d => d.status === 'PRIMARY');
629+
const beforeDeploymentId = beforePrimaryDeployment ? beforePrimaryDeployment.id : null;
630+
594631
core.debug('Updating service...');
595-
await updateEcsService(ecs, clusterName, service, taskDefArn, waitForService, waitForMinutes, forceNewDeployment, desiredCount, enableECSManagedTags, propagateTags, waitMaxDelaySeconds);
632+
await updateEcsService(ecs, clusterName, service, taskDefArn, waitForService, waitForMinutes, forceNewDeployment, desiredCount, enableECSManagedTags, propagateTags, waitMaxDelaySeconds, beforeDeploymentId);
596633

597634
} else if (serviceResponse.deploymentController.type === 'CODE_DEPLOY') {
598635
// Service uses CodeDeploy, so we should start a CodeDeploy deployment

‎index.js‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ async function tasksExitCode(ecs, clusterName, taskArns) {
197197
}
198198

199199
// Deploy to a service that uses the 'ECS' deployment controller
200-
async function updateEcsService(ecs, clusterName, service, taskDefArn, waitForService, waitForMinutes, forceNewDeployment, desiredCount, enableECSManagedTags, propagateTags, waitMaxDelaySeconds) {
200+
async function updateEcsService(ecs, clusterName, service, taskDefArn, waitForService, waitForMinutes, forceNewDeployment, desiredCount, enableECSManagedTags, propagateTags, waitMaxDelaySeconds, beforeDeploymentId) {
201201
core.debug('Updating the service');
202202

203203
const serviceManagedEBSVolumeName = core.getInput('service-managed-ebs-volume-name', { required: false }) || '';
@@ -232,7 +232,11 @@ async function updateEcsService(ecs, clusterName, service, taskDefArn, waitForSe
232232
if (!isNaN(desiredCount) && desiredCount !== undefined) {
233233
params.desiredCount = desiredCount;
234234
}
235-
await ecs.updateService(params);
235+
const updateResponse = await ecs.updateService(params);
236+
237+
// Extract the PRIMARY deployment ID from the update response
238+
const primaryDeployment = (updateResponse.service.deployments || []).find(d => d.status === 'PRIMARY');
239+
const afterDeploymentId = primaryDeployment ? primaryDeployment.id : null;
236240

237241
const region = await ecs.config.region();
238242
const consoleHostname = region.startsWith('cn') ? 'console.amazonaws.cn' : 'console.aws.amazon.com';
@@ -257,6 +261,36 @@ async function updateEcsService(ecs, clusterName, service, taskDefArn, waitForSe
257261
services: [service],
258262
cluster: clusterName
259263
});
264+
265+
// Verify the deployment was successful (not rolled back)
266+
if (afterDeploymentId && afterDeploymentId !== beforeDeploymentId) {
267+
core.debug('Verifying deployment succeeded after service stability...');
268+
269+
const verifyResponse = await ecs.describeServices({
270+
services: [service],
271+
cluster: clusterName
272+
});
273+
274+
const verifiedService = verifyResponse.services[0];
275+
const deployment = (verifiedService.deployments || []).find(d => d.id === afterDeploymentId);
276+
277+
if (!deployment) {
278+
throw new Error(
279+
`Deployment ${afterDeploymentId} not found after stabilization. ` +
280+
`The deployment was likely rolled back by the deployment circuit breaker.`
281+
);
282+
}
283+
284+
if (deployment.rolloutState === 'FAILED') {
285+
throw new Error(
286+
`Deployment ${afterDeploymentId} FAILED: ${deployment.rolloutStateReason || 'unknown reason'}`
287+
);
288+
}
289+
290+
core.info(`Deployment ${afterDeploymentId} verified: rolloutState=${deployment.rolloutState || 'N/A'}`);
291+
} else {
292+
core.debug('No new deployment was created by the update, skipping deployment verification');
293+
}
260294
} else {
261295
core.debug('Not waiting for the service to become stable');
262296
}
@@ -585,8 +619,11 @@ async function run() {
585619

586620
if (!serviceResponse.deploymentController || !serviceResponse.deploymentController.type || serviceResponse.deploymentController.type === 'ECS') {
587621
// Service uses the 'ECS' deployment controller, so we can call UpdateService
622+
const beforePrimaryDeployment = (serviceResponse.deployments || []).find(d => d.status === 'PRIMARY');
623+
const beforeDeploymentId = beforePrimaryDeployment ? beforePrimaryDeployment.id : null;
624+
588625
core.debug('Updating service...');
589-
await updateEcsService(ecs, clusterName, service, taskDefArn, waitForService, waitForMinutes, forceNewDeployment, desiredCount, enableECSManagedTags, propagateTags, waitMaxDelaySeconds);
626+
await updateEcsService(ecs, clusterName, service, taskDefArn, waitForService, waitForMinutes, forceNewDeployment, desiredCount, enableECSManagedTags, propagateTags, waitMaxDelaySeconds, beforeDeploymentId);
590627

591628
} else if (serviceResponse.deploymentController.type === 'CODE_DEPLOY') {
592629
// Service uses CodeDeploy, so we should start a CodeDeploy deployment

0 commit comments

Comments
 (0)