From 657b353d5cba794029378c569b0855564b184577 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 15 May 2025 01:10:24 +0530 Subject: [PATCH 1/3] wip --- ci-runner/app/CiCdProcessor.go | 16 +++++++--- ci-runner/executor/stage/cdStages.go | 9 ++++-- ci-runner/executor/stage/ciStages.go | 45 +++++++++++++++------------- ci-runner/helper/DockerHelper.go | 24 +++++++-------- ci-runner/helper/GitCliManager.go | 23 ++++++-------- ci-runner/util/Util.go | 16 +++++----- 6 files changed, 71 insertions(+), 62 deletions(-) diff --git a/ci-runner/app/CiCdProcessor.go b/ci-runner/app/CiCdProcessor.go index fb57db0b7..f496a6122 100644 --- a/ci-runner/app/CiCdProcessor.go +++ b/ci-runner/app/CiCdProcessor.go @@ -100,14 +100,22 @@ func (impl *CiCdProcessor) ProcessCiCdEvent(ciCdRequest *helper.CiCdTriggerEvent if logLevel == "" || logLevel == "DEBUG" { log.Println(util.DEVTRON, " ci-cd request details -----> ", ciCdRequest) } + impl.handleEventBasedOnType(ciCdRequest, &exitCode) + impl.HandleCleanup(*ciCdRequest, &exitCode, util.Source_Defer) + return +} - defer impl.HandleCleanup(*ciCdRequest, &exitCode, util.Source_Defer) +func (impl *CiCdProcessor) handleEventBasedOnType(ciCdRequest *helper.CiCdTriggerEvent, exitCode *int) { + defer func() { //recover in this function allows us to process further cleanup even if the code crashes + if r := recover(); r != nil { + log.Println("recovered from panic in handleEventBasedOnType:", r) + } + }() if helper.IsCIOrJobTypeEvent(ciCdRequest.Type) { - impl.ciStage.HandleCIEvent(ciCdRequest, &exitCode) + impl.ciStage.HandleCIEvent(ciCdRequest, exitCode) } else { - impl.cdStage.HandleCDEvent(ciCdRequest, &exitCode) + impl.cdStage.HandleCDEvent(ciCdRequest, exitCode) } - return } func (impl *CiCdProcessor) CleanUpBuildxK8sDriver(ciCdRequest helper.CiCdTriggerEvent, wg *sync.WaitGroup) { diff --git a/ci-runner/executor/stage/cdStages.go b/ci-runner/executor/stage/cdStages.go index f7073ac4a..d290297b7 100644 --- a/ci-runner/executor/stage/cdStages.go +++ b/ci-runner/executor/stage/cdStages.go @@ -56,9 +56,14 @@ func (impl *CdStage) HandleCDEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod return } -func (impl *CdStage) handleCDEvent(ciCdRequest *helper.CiCdTriggerEvent) (*helper.HandleCdEventResponse, error) { +func (impl *CdStage) handleCDEvent(ciCdRequest *helper.CiCdTriggerEvent) (resp *helper.HandleCdEventResponse, err error) { + defer func() { //recover in this function allows us to send event even if the code crashes + if r := recover(); r != nil { + log.Println("recovered from panic in handleCDEvent:", r) + err = fmt.Errorf("panic occurred during CD event handling") + } + }() var artifactUploaded bool - var err error var allPluginArtifacts *helper.PluginArtifacts allPluginArtifacts, err = impl.runCDStages(ciCdRequest) diff --git a/ci-runner/executor/stage/ciStages.go b/ci-runner/executor/stage/ciStages.go index 055e33fe1..220cdc397 100644 --- a/ci-runner/executor/stage/ciStages.go +++ b/ci-runner/executor/stage/ciStages.go @@ -74,8 +74,8 @@ func NewCiStage(gitManager helper.GitManager, dockerHelper helper.DockerHelper, } } -func deferCIEvent(ciRequest *helper.CommonWorkflowRequest, artifactUploaded bool, err error) (exitCode int) { - log.Println(util.DEVTRON, "defer CI stage data.", "err: ", err, "artifactUploaded: ", artifactUploaded) +func sendCIEventResult(ciRequest *helper.CommonWorkflowRequest, artifactUploaded bool, err error) (exitCode int) { + log.Println(util.DEVTRON, "sendCIEventResult CI stage data.", "err: ", err, "artifactUploaded: ", artifactUploaded) if err != nil { var stageError *helper.CiStageError if errors.As(err, &stageError) { @@ -99,13 +99,23 @@ func deferCIEvent(ciRequest *helper.CommonWorkflowRequest, artifactUploaded bool } func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCode *int) { - var artifactUploaded bool - var err error + artifactUploaded, err := impl.handleCIEvent(ciCdRequest) + if err != nil { + //log error and send event result + log.Println("ci stage error: ", err) + } + *exitCode = sendCIEventResult(ciCdRequest.CommonWorkflowRequest, artifactUploaded, err) +} + +func (impl *CiStage) handleCIEvent(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, err error) { + defer func() { //recover in this function allows us to send event even if the code crashes + if r := recover(); r != nil { + log.Println("recovered from panic in handleCIEvent:", r) + err = fmt.Errorf("panic occurred during CI event handling") + } + }() ciRequest := ciCdRequest.CommonWorkflowRequest ciContext := cicxt.BuildCiContext(context.Background(), ciRequest.EnableSecretMasking) - defer func() { - *exitCode = deferCIEvent(ciRequest, artifactUploaded, err) - }() artifactUploaded, err = impl.runCIStages(ciContext, ciCdRequest) log.Println(util.DEVTRON, artifactUploaded, err) var artifactUploadErr error @@ -116,16 +126,16 @@ func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod if err != nil { log.Println(util.DEVTRON, err) - return + return artifactUploaded, err } if artifactUploadErr != nil { log.Println(util.DEVTRON, "error in artifact upload: ", artifactUploadErr) if ciCdRequest.CommonWorkflowRequest.IsExtRun { log.Println(util.DEVTRON, "Ignoring artifactUploadErr") - return + return artifactUploaded, err } - return + return artifactUploaded, err } // sync cache @@ -148,7 +158,7 @@ func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod if err != nil { log.Println("error in cache push", err) } - return + return artifactUploaded, err } // TODO: take as tech debt and break this function into parts for better code readability @@ -672,17 +682,10 @@ func pullCache(metrics *helper.CIMetrics, ciCdRequest *helper.CiCdTriggerEvent) log.Println(util.DEVTRON, " cache-pull") start := time.Now() metrics.CacheDownStartTime = start - - defer func() { - log.Println(util.DEVTRON, " /cache-pull") - metrics.CacheDownDuration = time.Since(start).Seconds() - }() - err := helper.GetCache(ciCdRequest.CommonWorkflowRequest) - if err != nil { - return err - } - return nil + log.Println(util.DEVTRON, " /cache-pull") + metrics.CacheDownDuration = time.Since(start).Seconds() + return err } func (impl *CiStage) prepareStep(ciCdRequest *helper.CiCdTriggerEvent, metrics *helper.CIMetrics, skipCheckout bool) error { diff --git a/ci-runner/helper/DockerHelper.go b/ci-runner/helper/DockerHelper.go index 84f573ecb..8ced5cbe7 100644 --- a/ci-runner/helper/DockerHelper.go +++ b/ci-runner/helper/DockerHelper.go @@ -547,14 +547,14 @@ func (impl *DockerHelperImpl) getBuildxExportCacheFunc(ciContext cicxt.CiContext wg.Add(len(exportCacheCmds)) for platform, exportCacheCmd := range exportCacheCmds { go func(platform, exportCacheCmd string) { - defer wg.Done() log.Println("exporting build cache, platform : ", platform) log.Println(exportCacheCmd) err := impl.executeCmd(ciContext, exportCacheCmd) if err != nil { - log.Println("error in exporting ", "err : ", err) - return + log.Println("error in exporting", "err:", err) + //not returning as need to mark waitGroup done } + wg.Done() }(platform, exportCacheCmd) } wg.Wait() @@ -1005,14 +1005,6 @@ func (impl *DockerHelperImpl) CleanBuildxK8sDriver(ciContext cicxt.CiContext, no func (impl *DockerHelperImpl) leaveNodesFromBuildxK8sDriver(ciContext cicxt.CiContext, nodeNames []string) error { var err error - defer func() { - removeCmd := fmt.Sprintf("docker buildx rm %s", BUILDX_K8S_DRIVER_NAME) - fmt.Println(util.DEVTRON, " cmd : ", removeCmd) - execRemoveCmd := impl.GetCommandToExecute(removeCmd) - _ = impl.cmdExecutor.RunCommand(ciContext, execRemoveCmd) - - }() - for _, node := range nodeNames { createCmd := fmt.Sprintf("docker buildx create --name=%s --node=%s --leave", BUILDX_K8S_DRIVER_NAME, node) fmt.Println(util.DEVTRON, " cmd : ", createCmd) @@ -1020,12 +1012,20 @@ func (impl *DockerHelperImpl) leaveNodesFromBuildxK8sDriver(ciContext cicxt.CiCo err = impl.cmdExecutor.RunCommand(ciContext, execCreateCmd) if err != nil { log.Println(util.DEVTRON, "error in leaving node : ", err) - return err + break } } + impl.removeBuildxDriver(ciContext) //driver cleanup return err } +func (impl *DockerHelperImpl) removeBuildxDriver(ciContext cicxt.CiContext) { + removeCmd := fmt.Sprintf("docker buildx rm %s", BUILDX_K8S_DRIVER_NAME) + fmt.Println(util.DEVTRON, " cmd : ", removeCmd) + execRemoveCmd := impl.GetCommandToExecute(removeCmd) + _ = impl.cmdExecutor.RunCommand(ciContext, execRemoveCmd) +} + // this function is deprecated, use cmdExecutor.RunCommand instead func (impl *DockerHelperImpl) runCmd(cmd string) (error, *bytes.Buffer) { fmt.Println(util.DEVTRON, " cmd : ", cmd) diff --git a/ci-runner/helper/GitCliManager.go b/ci-runner/helper/GitCliManager.go index 24c303c1f..3e0407518 100644 --- a/ci-runner/helper/GitCliManager.go +++ b/ci-runner/helper/GitCliManager.go @@ -53,35 +53,30 @@ const GIT_AKS_PASS = "/git-ask-pass.sh" const DefaultRemoteName = "origin" func (impl *GitCliManagerImpl) Fetch(gitContext GitContext, rootDir string) (response, errMsg string, err error) { - log.Println(util.DEVTRON, "git fetch ", "location", rootDir) cmd := exec.Command("git", "-C", rootDir, "fetch", "origin", "--tags", "--force") - - tlsPathInfo, err := git_manager.CreateFilesForTlsData(git_manager.BuildTlsData(gitContext.TLSKey, gitContext.TLSCertificate, gitContext.CACert, gitContext.TLSVerificationEnabled), git_manager.TLS_FILES_DIR) - if err != nil { - //making it non-blocking - log.Println("error encountered in createFilesForTlsData", "err", err) - } - defer git_manager.DeleteTlsFiles(tlsPathInfo) - - output, errMsg, err := impl.RunCommandWithCred(cmd, gitContext.Auth.Username, gitContext.Auth.Password, tlsPathInfo) - log.Println(util.DEVTRON, "fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) - return output, errMsg, err + return impl.handleTLSDataAndExecuteCommand(cmd, gitContext) } func (impl *GitCliManagerImpl) Checkout(gitContext GitContext, rootDir string, checkout string) (response, errMsg string, err error) { log.Println(util.DEVTRON, "git checkout ", "location", rootDir) cmd := exec.Command("git", "-C", rootDir, "checkout", checkout, "--force") + return impl.handleTLSDataAndExecuteCommand(cmd, gitContext) +} +func (impl *GitCliManagerImpl) handleTLSDataAndExecuteCommand(cmd *exec.Cmd, gitContext GitContext) (response, errMsg string, err error) { tlsPathInfo, err := git_manager.CreateFilesForTlsData(git_manager.BuildTlsData(gitContext.TLSKey, gitContext.TLSCertificate, gitContext.CACert, gitContext.TLSVerificationEnabled), git_manager.TLS_FILES_DIR) if err != nil { //making it non-blocking log.Println("error encountered in createFilesForTlsData", "err", err) } - defer git_manager.DeleteTlsFiles(tlsPathInfo) output, errMsg, err := impl.RunCommandWithCred(cmd, gitContext.Auth.Username, gitContext.Auth.Password, tlsPathInfo) - log.Println(util.DEVTRON, "checkout output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) + log.Println(util.DEVTRON, "checkout output", "opt", output, "errMsg", errMsg, "error", err) + + //deleting tls files + git_manager.DeleteTlsFiles(tlsPathInfo) + return output, errMsg, err } diff --git a/ci-runner/util/Util.go b/ci-runner/util/Util.go index 65ed96d14..596dd1680 100644 --- a/ci-runner/util/Util.go +++ b/ci-runner/util/Util.go @@ -235,15 +235,13 @@ func ExecuteWithStageInfoLog(stageName string, stageExecutor func() error) (err func ExecuteWithStageInfoLogWithMetadata(stageName string, metadata interface{}, stageExecutor func() error) (err error) { startDockerStageInfo := newStageInfo(stageName).withCurrentStartTime().withMetadata(metadata) startDockerStageInfo.log() - defer func() { - status := success - if err != nil { - status = failure - } - startDockerStageInfo.withStatus(status).withCurrentEndTime().log() - }() - - return stageExecutor() + err = stageExecutor() + status := success + if err != nil { + status = failure + } + startDockerStageInfo.withStatus(status).withCurrentEndTime().log() + return } func GenerateBuildkitdContent(host string) string { From 45948dbac85b40b82a8421df1204746781320ecd Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 15 May 2025 15:11:19 +0530 Subject: [PATCH 2/3] wip --- ci-runner/helper/DockerHelper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-runner/helper/DockerHelper.go b/ci-runner/helper/DockerHelper.go index 8ced5cbe7..6293e6964 100644 --- a/ci-runner/helper/DockerHelper.go +++ b/ci-runner/helper/DockerHelper.go @@ -1024,6 +1024,7 @@ func (impl *DockerHelperImpl) removeBuildxDriver(ciContext cicxt.CiContext) { fmt.Println(util.DEVTRON, " cmd : ", removeCmd) execRemoveCmd := impl.GetCommandToExecute(removeCmd) _ = impl.cmdExecutor.RunCommand(ciContext, execRemoveCmd) + //not handling error here as this is just a cleanup job, not making it blocking } // this function is deprecated, use cmdExecutor.RunCommand instead From e777028b526d95324bff39b8cbf0d0c609203bee Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 15 May 2025 15:54:27 +0530 Subject: [PATCH 3/3] wip --- ci-runner/helper/DockerHelper.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci-runner/helper/DockerHelper.go b/ci-runner/helper/DockerHelper.go index 6293e6964..518dc50be 100644 --- a/ci-runner/helper/DockerHelper.go +++ b/ci-runner/helper/DockerHelper.go @@ -1023,8 +1023,11 @@ func (impl *DockerHelperImpl) removeBuildxDriver(ciContext cicxt.CiContext) { removeCmd := fmt.Sprintf("docker buildx rm %s", BUILDX_K8S_DRIVER_NAME) fmt.Println(util.DEVTRON, " cmd : ", removeCmd) execRemoveCmd := impl.GetCommandToExecute(removeCmd) - _ = impl.cmdExecutor.RunCommand(ciContext, execRemoveCmd) - //not handling error here as this is just a cleanup job, not making it blocking + err := impl.cmdExecutor.RunCommand(ciContext, execRemoveCmd) + if err != nil { + log.Println("error in executing docker buildx remove command", "err", err) + //not returning error here as this is just a cleanup job, not making it blocking + } } // this function is deprecated, use cmdExecutor.RunCommand instead