fix: defer functions execution change

ci-runner/app/CiCdProcessor.go

@@ -100,14 +100,22 @@ func (impl *CiCdProcessor) ProcessCiCdEvent(ciCdRequest *helper.CiCdTriggerEvent
 	if logLevel == "" || logLevel == "DEBUG" {
 		log.Println(util.DEVTRON, " ci-cd request details -----> ", ciCdRequest)
 	}
+	impl.handleEventBasedOnType(ciCdRequest, &exitCode)
+	impl.HandleCleanup(*ciCdRequest, &exitCode, util.Source_Defer)
+	return
+}
 
-	defer impl.HandleCleanup(*ciCdRequest, &exitCode, util.Source_Defer)
+func (impl *CiCdProcessor) handleEventBasedOnType(ciCdRequest *helper.CiCdTriggerEvent, exitCode *int) {
+	defer func() { //recover in this function allows us to process further cleanup even if the code crashes
+		if r := recover(); r != nil {
+			log.Println("recovered from panic in handleEventBasedOnType:", r)
+		}
+	}()
 	if helper.IsCIOrJobTypeEvent(ciCdRequest.Type) {
-		impl.ciStage.HandleCIEvent(ciCdRequest, &exitCode)
+		impl.ciStage.HandleCIEvent(ciCdRequest, exitCode)
 	} else {
-		impl.cdStage.HandleCDEvent(ciCdRequest, &exitCode)
+		impl.cdStage.HandleCDEvent(ciCdRequest, exitCode)
 	}
-	return
 }
 
 func (impl *CiCdProcessor) CleanUpBuildxK8sDriver(ciCdRequest helper.CiCdTriggerEvent, wg *sync.WaitGroup) {

ci-runner/executor/stage/cdStages.go

@@ -56,9 +56,14 @@ func (impl *CdStage) HandleCDEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod
 	return
 }
 
-func (impl *CdStage) handleCDEvent(ciCdRequest *helper.CiCdTriggerEvent) (*helper.HandleCdEventResponse, error) {
+func (impl *CdStage) handleCDEvent(ciCdRequest *helper.CiCdTriggerEvent) (resp *helper.HandleCdEventResponse, err error) {
+	defer func() { //recover in this function allows us to send event even if the code crashes
+		if r := recover(); r != nil {
+			log.Println("recovered from panic in handleCDEvent:", r)
+			err = fmt.Errorf("panic occurred during CD event handling")
+		}
+	}()
 	var artifactUploaded bool
-	var err error
 	var allPluginArtifacts *helper.PluginArtifacts
 
 	allPluginArtifacts, err = impl.runCDStages(ciCdRequest)

ci-runner/executor/stage/ciStages.go

@@ -74,8 +74,8 @@ func NewCiStage(gitManager helper.GitManager, dockerHelper helper.DockerHelper,
 	}
 }
 
-func deferCIEvent(ciRequest *helper.CommonWorkflowRequest, artifactUploaded bool, err error) (exitCode int) {
-	log.Println(util.DEVTRON, "defer CI stage data.", "err: ", err, "artifactUploaded: ", artifactUploaded)
+func sendCIEventResult(ciRequest *helper.CommonWorkflowRequest, artifactUploaded bool, err error) (exitCode int) {
+	log.Println(util.DEVTRON, "sendCIEventResult CI stage data.", "err: ", err, "artifactUploaded: ", artifactUploaded)
 	if err != nil {
 		var stageError *helper.CiStageError
 		if errors.As(err, &stageError) {
@@ -99,13 +99,23 @@ func deferCIEvent(ciRequest *helper.CommonWorkflowRequest, artifactUploaded bool
 }
 
 func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCode *int) {
-	var artifactUploaded bool
-	var err error
+	artifactUploaded, err := impl.handleCIEvent(ciCdRequest)
+	if err != nil {
+		//log error and send event result
+		log.Println("ci stage error: ", err)
+	}
+	*exitCode = sendCIEventResult(ciCdRequest.CommonWorkflowRequest, artifactUploaded, err)
+}
+
+func (impl *CiStage) handleCIEvent(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, err error) {
+	defer func() { //recover in this function allows us to send event even if the code crashes
+		if r := recover(); r != nil {
+			log.Println("recovered from panic in handleCIEvent:", r)
+			err = fmt.Errorf("panic occurred during CI event handling")
+		}
+	}()
 	ciRequest := ciCdRequest.CommonWorkflowRequest
 	ciContext := cicxt.BuildCiContext(context.Background(), ciRequest.EnableSecretMasking)
-	defer func() {
-		*exitCode = deferCIEvent(ciRequest, artifactUploaded, err)
-	}()
 	artifactUploaded, err = impl.runCIStages(ciContext, ciCdRequest)
 	log.Println(util.DEVTRON, artifactUploaded, err)
 	var artifactUploadErr error
@@ -116,16 +126,16 @@ func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod
 
 	if err != nil {
 		log.Println(util.DEVTRON, err)
-		return
+		return artifactUploaded, err
 	}
 
 	if artifactUploadErr != nil {
 		log.Println(util.DEVTRON, "error in artifact upload: ", artifactUploadErr)
 		if ciCdRequest.CommonWorkflowRequest.IsExtRun {
 			log.Println(util.DEVTRON, "Ignoring artifactUploadErr")
-			return
+			return artifactUploaded, err
 		}
-		return
+		return artifactUploaded, err
 	}
 
 	// sync cache
@@ -148,7 +158,7 @@ func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod
 	if err != nil {
 		log.Println("error in cache push", err)
 	}
-	return
+	return artifactUploaded, err
 }
 
 // TODO: take as tech debt and break this function into parts for better code readability
@@ -672,17 +682,10 @@ func pullCache(metrics *helper.CIMetrics, ciCdRequest *helper.CiCdTriggerEvent)
 	log.Println(util.DEVTRON, " cache-pull")
 	start := time.Now()
 	metrics.CacheDownStartTime = start
-
-	defer func() {
-		log.Println(util.DEVTRON, " /cache-pull")
-		metrics.CacheDownDuration = time.Since(start).Seconds()
-	}()
-
 	err := helper.GetCache(ciCdRequest.CommonWorkflowRequest)
-	if err != nil {
-		return err
-	}
-	return nil
+	log.Println(util.DEVTRON, " /cache-pull")
+	metrics.CacheDownDuration = time.Since(start).Seconds()
+	return err
 }
 
 func (impl *CiStage) prepareStep(ciCdRequest *helper.CiCdTriggerEvent, metrics *helper.CIMetrics, skipCheckout bool) error {

ci-runner/helper/DockerHelper.go

@@ -548,14 +548,14 @@ func (impl *DockerHelperImpl) getBuildxExportCacheFunc(ciContext cicxt.CiContext
 			wg.Add(len(exportCacheCmds))
 			for platform, exportCacheCmd := range exportCacheCmds {
 				go func(platform, exportCacheCmd string) {
-					defer wg.Done()
 					log.Println("exporting build cache, platform : ", platform)
 					log.Println(exportCacheCmd)
 					err := impl.executeCmd(ciContext, exportCacheCmd)
 					if err != nil {
-						log.Println("error in exporting ", "err : ", err)
-						return
+						log.Println("error in exporting", "err:", err)
+						//not returning as need to mark waitGroup done
 					}
+					wg.Done()
 				}(platform, exportCacheCmd)
 			}
 			wg.Wait()
@@ -1006,27 +1006,31 @@ func (impl *DockerHelperImpl) CleanBuildxK8sDriver(ciContext cicxt.CiContext, no
 
 func (impl *DockerHelperImpl) leaveNodesFromBuildxK8sDriver(ciContext cicxt.CiContext, nodeNames []string) error {
 	var err error
-	defer func() {
-		removeCmd := fmt.Sprintf("docker buildx rm %s", BUILDX_K8S_DRIVER_NAME)
-		fmt.Println(util.DEVTRON, " cmd : ", removeCmd)
-		execRemoveCmd := impl.GetCommandToExecute(removeCmd)
-		_ = impl.cmdExecutor.RunCommand(ciContext, execRemoveCmd)
-
-	}()
-
 	for _, node := range nodeNames {
 		createCmd := fmt.Sprintf("docker buildx create --name=%s --node=%s --leave", BUILDX_K8S_DRIVER_NAME, node)
 		fmt.Println(util.DEVTRON, " cmd : ", createCmd)
 		execCreateCmd := impl.GetCommandToExecute(createCmd)
 		err = impl.cmdExecutor.RunCommand(ciContext, execCreateCmd)
 		if err != nil {
 			log.Println(util.DEVTRON, "error in leaving node : ", err)
-			return err
+			break
 		}
 	}
+	impl.removeBuildxDriver(ciContext) //driver cleanup
 	return err
 }
 
+func (impl *DockerHelperImpl) removeBuildxDriver(ciContext cicxt.CiContext) {
+	removeCmd := fmt.Sprintf("docker buildx rm %s", BUILDX_K8S_DRIVER_NAME)
+	fmt.Println(util.DEVTRON, " cmd : ", removeCmd)
+	execRemoveCmd := impl.GetCommandToExecute(removeCmd)
+	err := impl.cmdExecutor.RunCommand(ciContext, execRemoveCmd)
+	if err != nil {
+		log.Println("error in executing docker buildx remove command", "err", err)
+		//not returning error here as this is just a cleanup job, not making it blocking
+	}
+}
+
 // this function is deprecated, use cmdExecutor.RunCommand instead
 func (impl *DockerHelperImpl) runCmd(cmd string) (error, *bytes.Buffer) {
 	fmt.Println(util.DEVTRON, " cmd : ", cmd)

ci-runner/helper/GitCliManager.go

@@ -53,35 +53,30 @@ const GIT_AKS_PASS = "/git-ask-pass.sh"
 const DefaultRemoteName = "origin"
 
 func (impl *GitCliManagerImpl) Fetch(gitContext GitContext, rootDir string) (response, errMsg string, err error) {
-
 	log.Println(util.DEVTRON, "git fetch ", "location", rootDir)
 	cmd := exec.Command("git", "-C", rootDir, "fetch", "origin", "--tags", "--force")
-
-	tlsPathInfo, err := git_manager.CreateFilesForTlsData(git_manager.BuildTlsData(gitContext.TLSKey, gitContext.TLSCertificate, gitContext.CACert, gitContext.TLSVerificationEnabled), git_manager.TLS_FILES_DIR)
-	if err != nil {
-		//making it non-blocking
-		log.Println("error encountered in createFilesForTlsData", "err", err)
-	}
-	defer git_manager.DeleteTlsFiles(tlsPathInfo)
-
-	output, errMsg, err := impl.RunCommandWithCred(cmd, gitContext.Auth.Username, gitContext.Auth.Password, tlsPathInfo)
-	log.Println(util.DEVTRON, "fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
-	return output, errMsg, err
+	return impl.handleTLSDataAndExecuteCommand(cmd, gitContext)
 }
 
 func (impl *GitCliManagerImpl) Checkout(gitContext GitContext, rootDir string, checkout string) (response, errMsg string, err error) {
 	log.Println(util.DEVTRON, "git checkout ", "location", rootDir)
 	cmd := exec.Command("git", "-C", rootDir, "checkout", checkout, "--force")
+	return impl.handleTLSDataAndExecuteCommand(cmd, gitContext)
+}
 
+func (impl *GitCliManagerImpl) handleTLSDataAndExecuteCommand(cmd *exec.Cmd, gitContext GitContext) (response, errMsg string, err error) {
 	tlsPathInfo, err := git_manager.CreateFilesForTlsData(git_manager.BuildTlsData(gitContext.TLSKey, gitContext.TLSCertificate, gitContext.CACert, gitContext.TLSVerificationEnabled), git_manager.TLS_FILES_DIR)
 	if err != nil {
 		//making it non-blocking
 		log.Println("error encountered in createFilesForTlsData", "err", err)
 	}
-	defer git_manager.DeleteTlsFiles(tlsPathInfo)
 
 	output, errMsg, err := impl.RunCommandWithCred(cmd, gitContext.Auth.Username, gitContext.Auth.Password, tlsPathInfo)
-	log.Println(util.DEVTRON, "checkout output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
+	log.Println(util.DEVTRON, "checkout output", "opt", output, "errMsg", errMsg, "error", err)
+
+	//deleting tls files
+	git_manager.DeleteTlsFiles(tlsPathInfo)
+
 	return output, errMsg, err
 }
 

ci-runner/util/Util.go

@@ -235,15 +235,13 @@ func ExecuteWithStageInfoLog(stageName string, stageExecutor func() error) (err
 func ExecuteWithStageInfoLogWithMetadata(stageName string, metadata interface{}, stageExecutor func() error) (err error) {
 	startDockerStageInfo := newStageInfo(stageName).withCurrentStartTime().withMetadata(metadata)
 	startDockerStageInfo.log()
-	defer func() {
-		status := success
-		if err != nil {
-			status = failure
-		}
-		startDockerStageInfo.withStatus(status).withCurrentEndTime().log()
-	}()
-
-	return stageExecutor()
+	err = stageExecutor()
+	status := success
+	if err != nil {
+		status = failure
+	}
+	startDockerStageInfo.withStatus(status).withCurrentEndTime().log()
+	return
 }
 
 func GenerateBuildkitdContent(host string) string {