feat: add docker scan

ci-runner/env_gen.json

@@ -1 +1 @@
-[{"Category":"DEVTRON","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}]
+[{"Category":"DEVTRON","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DOCKERFILE_SCAN_FAIL_ON_ERROR","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DOCKERFILE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DOCKERFILE_SCAN_RETRY_WAIT_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-service.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]}]

ci-runner/env_gen.md

@@ -22,7 +22,10 @@
  | DEFAULT_CACHE_BUCKET_REGION | string | |  |  | false |
  | DEFAULT_CD_LOGS_BUCKET_REGION | string | |  |  | false |
  | DEFAULT_LOG_TIME_LIMIT | int64 |1 |  |  | false |
- | IMAGE_SCANNER_ENDPOINT | string |http://image-scanner-new-demo-devtroncd-service.devtroncd:80 |  |  | false |
+ | DOCKERFILE_SCAN_FAIL_ON_ERROR | bool |false |  |  | false |
+ | DOCKERFILE_SCAN_MAX_RETRIES | int |3 |  |  | false |
+ | DOCKERFILE_SCAN_RETRY_WAIT_SECONDS | int |5 |  |  | false |
+ | IMAGE_SCANNER_ENDPOINT | string |http://image-scanner-service.devtroncd:80 |  |  | false |
  | LOG_LEVEL | int |0 |  |  | false |
  | NATS_MSG_ACK_WAIT_IN_SECS | int |120 |  |  | false |
  | NATS_MSG_BUFFER_SIZE | int |-1 |  |  | false |

ci-runner/executor/stage/ciStages.go

@@ -348,6 +348,34 @@ func (impl *CiStage) runBuildArtifact(ciCdRequest *helper.CiCdTriggerEvent, metr
 	// build
 	start := time.Now()
 	metrics.BuildStartTime = start
+
+	// Trigger Dockerfile scan right before build (git clone has definitely completed)
+	// Orchestrator has already made the decision (OR logic: userEnabled OR orgForced)
+	// CI-Runner blindly trusts Orchestrator's decision (no decision logic here)
+	if ciCdRequest.CommonWorkflowRequest.DockerfileScanEnabled {
+		log.Println(util.DEVTRON, "dockerfile scan triggered at build start (git clone completed)",
+			"buildId", ciCdRequest.CommonWorkflowRequest.WorkflowId,
+			"pipelineId", ciCdRequest.CommonWorkflowRequest.PipelineId,
+			"appId", ciCdRequest.CommonWorkflowRequest.AppId,
+			"checkoutPath", ciCdRequest.CommonWorkflowRequest.CheckoutPath)
+		// Trigger scan asynchronously (non-blocking, runs parallel to build)
+		go func() {
+			defer func() {
+				if r := recover(); r != nil {
+					log.Println(util.DEVTRON, "recovered from panic in Dockerfile scan goroutine", "panic", r)
+				}
+			}()
+			log.Println(util.DEVTRON, "dockerfile scan started",
+				"appId", ciCdRequest.CommonWorkflowRequest.AppId,
+				"buildId", ciCdRequest.CommonWorkflowRequest.WorkflowId,
+				"pipelineId", ciCdRequest.CommonWorkflowRequest.PipelineId,
+				"checkoutPath", ciCdRequest.CommonWorkflowRequest.CheckoutPath)
+			helper.InitiateDockerfileScan(ciCdRequest.CommonWorkflowRequest)
+			log.Println(util.DEVTRON, "dockerfile scan request sent to image-scanner",
+				"buildId", ciCdRequest.CommonWorkflowRequest.WorkflowId)
+		}()
+	}
+
 	dest, err := impl.dockerHelper.BuildArtifact(ciCdRequest.CommonWorkflowRequest) // TODO make it skipable
 	metrics.BuildDuration = time.Since(start).Seconds()
 	if err != nil {

ci-runner/helper/DockerfileScanHelper.go

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2024. Devtron Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helper
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/caarlos0/env"
+	"github.com/devtron-labs/ci-runner/helper/bean"
+	"github.com/devtron-labs/ci-runner/util"
+	"github.com/go-resty/resty/v2"
+	"github.com/devtron-labs/ci-runner/pubsub"
+)
+
+// MaxDockerfileSize is the maximum allowed Dockerfile size (1MB)
+const MaxDockerfileSize = 1 * 1024 * 1024 // 1MB
+
+// InitiateDockerfileScan initiates a Dockerfile scan using hadolint
+// It reads the Dockerfile from filesystem and sends content to image-scanner service
+func InitiateDockerfileScan(ciRequest *CommonWorkflowRequest) {
+	log.Println(util.DEVTRON, "initiating Dockerfile scan")
+
+	// Validate config exists
+	if ciRequest.CiBuildConfig == nil || ciRequest.CiBuildConfig.DockerBuildConfig == nil {
+		log.Println(util.DEVTRON, "docker build config not found, skipping Dockerfile scan")
+		return
+	}
+
+	// Resolve Dockerfile path
+	var dockerfilePath string
+	if ciRequest.CiBuildConfig.CiBuildType == "managed-dockerfile-build" {
+		dockerfilePath = filepath.Join(util.WORKINGDIR, ciRequest.CheckoutPath, "./Dockerfile")
+	} else {
+		dockerfilePath = ciRequest.CiBuildConfig.DockerBuildConfig.DockerfilePath
+	}
+
+	// Convert to absolute path
+	var err error
+	dockerfilePath, err = filepath.Abs(dockerfilePath)
+	if err != nil {
+		log.Println(util.DEVTRON, "error in resolving absolute path for Dockerfile", "path", dockerfilePath, "err", err)
+		return
+	}
+
+	// Check if Dockerfile exists (removed polling logic)
+	if _, err := os.Stat(dockerfilePath); os.IsNotExist(err) {
+		log.Println(util.DEVTRON, "dockerfile scan: Dockerfile not found at", dockerfilePath)
+		return
+	}
+
+	// Read Dockerfile from filesystem
+	dockerfileContent, err := os.ReadFile(dockerfilePath)
+	if err != nil {
+		log.Println(util.DEVTRON, "error in reading Dockerfile for scanning", "path", dockerfilePath, "err", err)
+		return
+	}
+
+	// Prepare scan request
+	scanRequest := &bean.DockerfileScanRequest{
+		AppId:             ciRequest.AppId,
+		BuildId:           ciRequest.WorkflowId,
+		PipelineId:        ciRequest.PipelineId,
+		DockerfileContent: string(dockerfileContent),
+	}
+
+	jsonBody, err := json.Marshal(scanRequest)
+	if err != nil {
+		log.Println(util.DEVTRON, "error in marshalling Dockerfile scan request", "err", err)
+		return
+	}
+
+	cfg := &bean.ScanConfig{}
+	err = env.Parse(cfg)
+	if err != nil {
+		log.Println(util.DEVTRON, "error in parsing scan config", "err", err)
+		return
+	}
+
+	scannerCfg := &pubsub.PubSubConfig{}
+	err = env.Parse(scannerCfg)
+	if err != nil {
+		log.Println(util.DEVTRON, "error in parsing scanner endpoint config", "err", err)
+		return
+	}
+
+	// Create HTTP client (Using local instantiation but with improved error logging)
+	client := resty.New()
+	client.SetTimeout(2 * time.Minute)
+	client.
+		SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}).
+		SetRetryCount(cfg.MaxRetries).
+		SetRetryWaitTime(time.Duration(cfg.RetryWaitTimeSeconds) * time.Second)
+
+	resp, err := client.R().
+		SetHeader("Content-Type", "application/json").
+		SetBody(jsonBody).
+		Post(fmt.Sprintf("%s/%s", scannerCfg.ImageScannerEndpoint, "scanner/dockerfile/scan"))
+
+	// Record success/failure with actual error logging
+	if err != nil || (resp != nil && (resp.StatusCode() != http.StatusAccepted && resp.StatusCode() != http.StatusOK)) {
+		var status string
+		if resp != nil {
+			status = resp.Status()
+		}
+		log.Println(util.DEVTRON, "circuit breaker recorded FAILURE for Dockerfile scan", "buildId", ciRequest.WorkflowId, "err", err, "status", status)
+	} else {
+		log.Println(util.DEVTRON, "circuit breaker recorded SUCCESS for Dockerfile scan", "buildId", ciRequest.WorkflowId)
+	}
+
+	if err != nil {
+		log.Println(util.DEVTRON, "error in calling image-scanner for Dockerfile scan", "err", err)
+		return
+	}
+
+	// Accept both 202 (Accepted) and 200 (OK)
+	if resp.StatusCode() != http.StatusAccepted && resp.StatusCode() != http.StatusOK {
+		log.Println(util.DEVTRON, "image-scanner returned non-202/200 status for Dockerfile scan",
+			"status", resp.StatusCode(), "body", string(resp.Body()))
+		return
+	}
+
+	log.Println(util.DEVTRON, "successfully initiated Dockerfile scan",
+		"statusCode", resp.StatusCode(), "buildId", ciRequest.WorkflowId)
+}