fix(3174): Stages - metadata passed to stage teardown is incomplete w…

…hen there is failure in any stage job (#3175)
screwdriver-cd · Aug 26, 2024 · 919680b · 919680b
1 parent 02a8dfe
commit 919680b
Show file tree

Hide file tree

Showing 5 changed files with 335 additions and 224 deletions.
diff --git a/plugins/builds/index.js b/plugins/builds/index.js
@@ -26,7 +26,7 @@ const {
     createJoinObject,
     createEvent,
     parseJobInfo,
-    handleStageFailure,
+    ensureStageTeardownBuildExists,
     getJobId,
     isOrTrigger,
     extractExternalJoinData,
@@ -39,6 +39,7 @@ const {
     isStartFromMiddleOfCurrentStage,
     Status
 } = require('./triggers/helpers');
+const { getFullStageJobName } = require('../helper');
 
 /**
  * Delete a build
@@ -320,6 +321,70 @@ async function triggerNextJobs(config, app) {
     return null;
 }
 
+/**
+ * Create or update stage teardown build
+ * @method createOrUpdateStageTeardownBuild
+ * @param {Object}      config              Configuration object
+ * @param {Pipeline}    config.pipeline     Current pipeline
+ * @param {Job}         config.job          Current job
+ * @param {Build}       config.build        Current build
+ * @param {Build}       config.event        Current event
+ * @param {Build}       config.stage        Current stage
+ * @param {String}      config.username     Username
+ * @param {String}      config.scmContext   SCM context
+ * @param {String}      app                 Server app object
+ * @return {Promise}                        Create a new build or update an existing build
+ */
+async function createOrUpdateStageTeardownBuild(config, app) {
+    const { pipeline, job, build, username, scmContext, event, stage } = config;
+    const { buildFactory, jobFactory, eventFactory } = app;
+    const current = {
+        pipeline,
+        job,
+        build,
+        event,
+        stage
+    };
+
+    const stageTeardownName = getFullStageJobName({ stageName: current.stage.name, jobName: 'teardown' });
+
+    const nextJobsTrigger = [stageTeardownName];
+    const pipelineJoinData = await createJoinObject(nextJobsTrigger, current, eventFactory);
+
+    const resource = `pipeline:${pipeline.id}:groupEvent:${event.groupEventId}`;
+    let lock;
+    let teardownBuild;
+
+    try {
+        lock = await locker.lock(resource);
+        const { parentBuilds } = parseJobInfo({
+            joinObj: pipelineJoinData,
+            currentBuild: build,
+            currentPipeline: pipeline,
+            currentJob: job,
+            nextJobName: stageTeardownName
+        });
+
+        teardownBuild = await ensureStageTeardownBuildExists({
+            jobFactory,
+            buildFactory,
+            current,
+            parentBuilds,
+            stageTeardownName,
+            username,
+            scmContext
+        });
+    } catch (err) {
+        logger.error(
+            `Error in createOrUpdateStageTeardownBuild:${stageTeardownName} from pipeline:${pipeline.id}-event:${event.id} `,
+            err
+        );
+    }
+    await locker.unlock(lock, resource);
+
+    return teardownBuild;
+}
+
 /**
  * Build API Plugin
  * @method register
@@ -338,13 +403,12 @@ const buildsPlugin = {
          * @param {Pipeline}    config.pipeline     Current pipeline
          * @param {Job}         config.job          Current job
          * @param {Build}       config.build        Current build
-         * @param {String}      config.username     Username
          * @param {String}  app                      Server app object
          * @return {Promise}                        Resolves to the removed build or null
          */
         server.expose('removeJoinBuilds', async (config, app) => {
-            const { pipeline, job, build, username, scmContext, event, stage } = config;
-            const { eventFactory, buildFactory, jobFactory } = app;
+            const { pipeline, job, build, event, stage } = config;
+            const { eventFactory, buildFactory } = app;
             const current = {
                 pipeline,
                 job,
@@ -374,20 +438,19 @@ const buildsPlugin = {
                             buildConfig.eventId = hoek.reach(pipelineJoinData[pid], 'event.id');
                         }
 
-                        //   if nextBuild is stage teardown, just return nextBuild
-                        if (current.stage) {
-                            const buildDeletePromises = await handleStageFailure({
-                                nextJobName,
-                                current,
-                                buildConfig,
-                                jobFactory,
-                                buildFactory,
-                                username,
-                                scmContext
-                            });
-
-                            deletePromises.concat(buildDeletePromises);
-                        } else if (buildConfig.eventId) {
+                        if (buildConfig.eventId) {
+                            if (current.stage) {
+                                const stageTeardownName = getFullStageJobName({
+                                    stageName: current.stage.name,
+                                    jobName: 'teardown'
+                                });
+
+                                // Do not remove stage teardown builds as they need to be executed on stage failure as well.
+                                if (nextJobName !== stageTeardownName) {
+                                    deletePromises.push(deleteBuild(buildConfig, buildFactory));
+                                }
+                            }
+
                             deletePromises.push(deleteBuild(buildConfig, buildFactory));
                         }
                     } catch (err) {
@@ -425,6 +488,11 @@ const buildsPlugin = {
          */
         server.expose('triggerNextJobs', triggerNextJobs);
 
+        /**
+         * Create or Update stage teardown build on stage failure
+         */
+        server.expose('createOrUpdateStageTeardownBuild', createOrUpdateStageTeardownBuild);
+
         server.route([
             getRoute(),
             getBuildStatusesRoute(),

diff --git a/plugins/builds/triggers/helpers.js b/plugins/builds/triggers/helpers.js
@@ -834,6 +834,7 @@ async function createJoinObject(nextJobNames, current, eventFactory) {
  * @param {BuildFactory} arg.buildFactory Build factory
  * @param {Object} arg.current Current object
  * @param {Event} arg.current.event Current event
+ * @param {ParentBuilds} arg.parentBuilds Parent builds
  * @param {String} arg.stageTeardownName Stage teardown name
  * @param {String} arg.username Username
  * @param {String} arg.scmContext SCM context
@@ -842,6 +843,7 @@ async function ensureStageTeardownBuildExists({
     jobFactory,
     buildFactory,
     current,
+    parentBuilds,
     stageTeardownName,
     username,
     scmContext
@@ -865,54 +867,21 @@ async function ensureStageTeardownBuildExists({
             jobName: stageTeardownName,
             username,
             scmContext,
+            parentBuilds,
+            parentBuildId: current.build.id,
             event: current.event, // this is the parentBuild for the next build
             baseBranch: current.event.baseBranch || null,
             start: false
         });
+    } else {
+        await updateParentBuilds({
+            joinParentBuilds: parentBuilds,
+            nextBuild: existingStageTeardownBuild,
+            build: current.build
+        });
     }
 }
 
-/**
- * Delete nextBuild, create teardown build if it doesn't exist, and return teardown build or return null
- * @param {Object} arg
- * @param {String} arg.nextJobName Next job name
- * @param {Object} arg.current Object with stage, event, pipeline info
- * @param {Object} arg.buildConfig Build config
- * @param {JobFactory} arg.jobFactory Job factory
- * @param {BuildFactory} arg.buildFactory Build factory
- * @param {String} arg.username Username
- * @param {String} arg.scmContext Scm context
- * @returns {Promise<Array>} Array of promises
- */
-async function handleStageFailure({
-    nextJobName,
-    current,
-    buildConfig,
-    jobFactory,
-    buildFactory,
-    username,
-    scmContext
-}) {
-    const buildDeletePromises = [];
-    const stageTeardownName = getFullStageJobName({ stageName: current.stage.name, jobName: 'teardown' });
-
-    // Remove next build
-    if (buildConfig.eventId && nextJobName !== stageTeardownName) {
-        buildDeletePromises.push(deleteBuild(buildConfig, buildFactory));
-    }
-
-    await ensureStageTeardownBuildExists({
-        jobFactory,
-        buildFactory,
-        current,
-        stageTeardownName,
-        username,
-        scmContext
-    });
-
-    return buildDeletePromises;
-}
-
 /**
  * Get parentBuildId from parentBuilds object
  * @param {Object} arg
@@ -1107,7 +1076,7 @@ module.exports = {
     updateParentBuilds,
     getParentBuildStatus,
     handleNewBuild,
-    handleStageFailure,
+    ensureStageTeardownBuildExists,
     getBuildsForGroupEvent,
     createJoinObject,
     createExternalEvent,

diff --git a/plugins/builds/update.js b/plugins/builds/update.js
@@ -221,7 +221,8 @@ module.exports = () => ({
             const { statusMessage, stats, status: desiredStatus } = request.payload;
             const { username, scmContext, scope } = request.auth.credentials;
             const isBuild = scope.includes('build') || scope.includes('temporal');
-            const { triggerNextJobs, removeJoinBuilds } = request.server.plugins.builds;
+            const { triggerNextJobs, removeJoinBuilds, createOrUpdateStageTeardownBuild } =
+                request.server.plugins.builds;
 
             // Check token permissions
             if (isBuild && username !== id) {
@@ -326,16 +327,20 @@ module.exports = () => ({
                 // Check for failed jobs and remove any child jobs in created state
                 if (newBuild.status === 'FAILURE') {
                     await removeJoinBuilds(
-                        { pipeline, job, build: newBuild, username, scmContext, event: newEvent, stage },
+                        { pipeline, job, build: newBuild, event: newEvent, stage },
                         request.server.app
                     );
+
+                    if (stage && !isStageTeardown) {
+                        await createOrUpdateStageTeardownBuild(
+                            { pipeline, job, build, username, scmContext, event, stage },
+                            request.server.app
+                        );
+                    }
                 }
                 // Do not continue downstream is current job is stage teardown and statusBuild has failure
             } else if (newBuild.status === 'SUCCESS' && isStageTeardown && stageBuildHasFailure) {
-                await removeJoinBuilds(
-                    { pipeline, job, build: newBuild, username, scmContext, event: newEvent, stage },
-                    request.server.app
-                );
+                await removeJoinBuilds({ pipeline, job, build: newBuild, event: newEvent, stage }, request.server.app);
             } else {
                 await triggerNextJobs(
                     { pipeline, job, build: newBuild, username, scmContext, event: newEvent },