Merge branch 'community-fork/feat/docker-logging-options' into 'main'

feat: allow passing `env` and `labels` options to `json-file` Docker logging driver

See merge request https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/5638

Merged-by: Axel von Bertoldi <avonbertoldi@gitlab.com>
Approved-by: Roshni Sarangadharan <rsarangadharan@gitlab.com>
Approved-by: Hannes Hörl <2185075-hoegaarden@users.noreply.gitlab.com>
Approved-by: Axel von Bertoldi <avonbertoldi@gitlab.com>
Reviewed-by: Hannes Hörl <2185075-hoegaarden@users.noreply.gitlab.com>
Reviewed-by: GitLab Duo <gitlab-duo@gitlab.com>
Reviewed-by: Roshni Sarangadharan <rsarangadharan@gitlab.com>
Co-authored-by: Patrick Decat <pdecat@gmail.com>

Author: Axel von Bertoldi

common/config.go

@@ -23,6 +23,7 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"github.com/BurntSushi/toml"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/go-units"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/sirupsen/logrus"
@@ -310,6 +311,7 @@ type DockerConfig struct {
 	EnableIPv6                 bool                `toml:"enable_ipv6,omitempty" json:"enable_ipv6" long:"enable-ipv6" description:"Enable IPv6 for automatically created networks. This is only takes affect when the feature flag FF_NETWORK_PER_BUILD is enabled."`
 	Ulimit                     map[string]string   `toml:"ulimit,omitempty" json:"ulimit,omitempty" long:"ulimit" env:"DOCKER_ULIMIT" description:"Ulimit options for container"`
 	NetworkMTU                 int                 `toml:"network_mtu,omitempty" json:"network_mtu" long:"network-mtu" description:"MTU of the Docker network created for the job IFF the FF_NETWORK_PER_BUILD feature-flag was specified."`
+	LogOptions                 map[string]string   `toml:"log_options,omitempty" json:"log_options,omitempty" long:"log-options" env:"DOCKER_LOG_OPTIONS" description:"Log driver options for json-file logging"`
 }
 
 type InstanceConfig struct {
@@ -1684,6 +1686,36 @@ func (c *DockerConfig) GetServicesLimit() int {
 	return *c.ServicesLimit
 }
 
+// GetLogConfig returns the LogConfig for build containers
+func (c *DockerConfig) GetLogConfig() (container.LogConfig, error) {
+	logConfig := container.LogConfig{
+		Type: "json-file",
+	}
+
+	if c == nil || len(c.LogOptions) == 0 {
+		return logConfig, nil
+	}
+
+	var invalidKeys []string
+	var allowedKeys = []string{"env", "labels"}
+
+	for key := range c.LogOptions {
+		if !slices.Contains(allowedKeys, key) {
+			invalidKeys = append(invalidKeys, key)
+		}
+	}
+
+	slices.Sort(invalidKeys) // to get stable error outputs
+
+	if len(invalidKeys) > 0 {
+		return logConfig, fmt.Errorf("invalid log options: only %q are allowed, but found: %q", allowedKeys, invalidKeys)
+	}
+
+	logConfig.Config = c.LogOptions
+
+	return logConfig, nil
+}
+
 func (c *KubernetesConfig) GetPollTimeout() int {
 	if c.PollTimeout <= 0 {
 		c.PollTimeout = KubernetesPollTimeout

common/config_log_options_test.go

@@ -0,0 +1,102 @@
+//go:build !integration
+
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDockerConfig_ValidateLogOptions(t *testing.T) {
+	tests := []struct {
+		name           string
+		logOptions     map[string]string
+		expectedErrMsg string
+	}{
+		{
+			name: "nil config",
+		},
+		{
+			name:       "empty log options",
+			logOptions: map[string]string{},
+		},
+		{
+			name: "valid env option",
+			logOptions: map[string]string{
+				"env": "GITLAB_CI_JOB_ID,GITLAB_CI_JOB_NAME",
+			},
+		},
+		{
+			name: "valid labels option",
+			logOptions: map[string]string{
+				"labels": "com.gitlab.gitlab-runner.type",
+			},
+		},
+		{
+			name: "valid env and labels options",
+			logOptions: map[string]string{
+				"env":    "GITLAB_CI_JOB_ID,GITLAB_CI_JOB_NAME",
+				"labels": "com.gitlab.gitlab-runner.type",
+			},
+		},
+		{
+			name: "invalid single option",
+			logOptions: map[string]string{
+				"max-size": "10m",
+			},
+			expectedErrMsg: `invalid log options: only ["env" "labels"] are allowed, but found: ["max-size"]`,
+		},
+		{
+			name: "invalid multiple options",
+			logOptions: map[string]string{
+				"max-size": "10m",
+				"max-file": "3",
+			},
+			expectedErrMsg: `invalid log options: only ["env" "labels"] are allowed, but found: ["max-file" "max-size"]`,
+		},
+		{
+			name: "mixed valid and invalid options",
+			logOptions: map[string]string{
+				"env":      "CI_JOB_ID",
+				"max-size": "10m",
+				"labels":   "job_name",
+			},
+			expectedErrMsg: `invalid log options: only ["env" "labels"] are allowed, but found: ["max-size"]`,
+		},
+		{
+			name: "unknown option",
+			logOptions: map[string]string{
+				"unknown-option": "value",
+			},
+			expectedErrMsg: `invalid log options: only ["env" "labels"] are allowed, but found: ["unknown-option"]`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dockerConfig := &DockerConfig{
+				LogOptions: tt.logOptions,
+			}
+
+			logConfig, err := dockerConfig.GetLogConfig()
+
+			if tt.expectedErrMsg != "" {
+				assert.Error(t, err)
+				assert.Contains(t, err.Error(), tt.expectedErrMsg)
+			} else {
+				assert.NoError(t, err)
+				assertMapMatches(t, tt.logOptions, logConfig.Config)
+			}
+		})
+	}
+}
+
+func assertMapMatches(t *testing.T, expected, actual map[string]string) {
+	t.Helper()
+	if len(expected) == 0 {
+		assert.Len(t, actual, 0)
+		return
+	}
+	assert.Equal(t, expected, actual)
+}

docs/configuration/advanced-configuration.md

@@ -559,6 +559,7 @@ The following settings define the Docker container parameters. These settings ar
 | `hostname`                         |                                                  | Custom hostname for the Docker container. |
 | `image`                            | `"ruby:3.3"`                                     | The image to run jobs with. |
 | `links`                            | `["mysql_container:mysql"]`                      | Containers that should be linked with container that runs the job. |
+| `log_options`                      | `{"env": "GITLAB_CI_JOB_ID,GITLAB_CI_JOB_NAME", "labels": "com.gitlab.gitlab-runner.type"}` | Log driver options for Docker containers that use the `json-file` log driver. Only `env` and `labels` options are allowed. For more information, see [Docker log options](#docker-log-options). |
 | `memory`                           | `"128m"`                                         | The memory limit. A string. |
 | `memory_swap`                      | `"256m"`                                         | The total memory limit. A string. |
 | `memory_reservation`               | `"64m"`                                          | The memory soft limit. A string. |
@@ -645,6 +646,7 @@ Example:
   links = ["mysql_container:mysql"]
   allowed_images = ["ruby:*", "python:*", "php:*"]
   allowed_services = ["postgres:9", "redis:*", "mysql:*"]
+  log_options = { env = "GITLAB_CI_JOB_ID,GITLAB_CI_JOB_NAME", labels = "com.gitlab.gitlab-runner.type" }
   [runners.docker.ulimit]
     "rtprio" = "99"
   [[runners.docker.services]]
@@ -711,6 +713,50 @@ GitLab Runner 11.11 and later [mount the host directory](https://gitlab.com/gitl
 for the defined [services](https://docs.gitlab.com/ci/services/) as
 well.
 
+### Docker log options
+
+The `log_options` parameter allows you to configure Docker container log options for the `json-file` log driver.
+For security and compatibility reasons, only the `env` and `labels` options are supported.
+
+#### Supported log options
+
+- `env`: Comma-separated list of environment variable names to include in log entries
+- `labels`: Comma-separated list of container label names to include in log entries
+
+#### Configuration examples
+
+The following are some configuration examples:
+
+```toml
+[[runners]]
+  [runners.docker]
+    # Include specific environment variables in logs
+    log_options = { env = "GITLAB_CI_JOB_ID,GITLAB_CI_JOB_NAME,CI_PIPELINE_ID" }
+```
+
+```toml
+[[runners]]
+  [runners.docker]
+    # Include container labels in logs
+    log_options = { labels = "com.gitlab.gitlab-runner.type" }
+```
+
+```toml
+[[runners]]
+  [runners.docker]
+    # Include both environment variables and labels
+    log_options = { env = "GITLAB_CI_JOB_ID,GITLAB_CI_JOB_NAME", labels = "com.gitlab.gitlab-runner.type" }
+```
+
+#### Validation and error handling
+
+GitLab Runner validates log options during executor preparation. If you specify unsupported options
+such as `max-size`, `max-file`, or `compress`, the job fails immediately with a configuration error.
+
+The log options apply to the main job container and any service containers defined in your CI/CD configuration.
+
+For more information about Docker logging, see the [Docker `json-file` log driver documentation](https://docs.docker.com/config/containers/logging/json-file/).
+
 ### Use a private container registry
 
 To use private registries as a source of images for your jobs, configure authorization

executors/docker/docker.go

@@ -115,7 +115,19 @@ type executor struct {
 
 	projectUniqRandomizedName string
 
-	dockerConn *dockerConnection
+	dockerConn      *dockerConnection
+	dockerConnector dockerConnector
+
+	logConfig container.LogConfig
+}
+
+type dockerConnector func(ctx context.Context, options common.ExecutorPrepareOptions, executor *executor) error
+
+func (dc dockerConnector) Connect(ctx context.Context, options common.ExecutorPrepareOptions, executor *executor) error {
+	if dc == nil {
+		dc = connectDocker
+	}
+	return dc(ctx, options, executor)
 }
 
 type dockerTunnel struct {
@@ -559,10 +571,8 @@ func (e *executor) createHostConfigForService(imageIsPrivileged bool, devices []
 		Binds:         e.volumesManager.Binds(),
 		ShmSize:       e.Config.Docker.ShmSize,
 		Tmpfs:         e.Config.Docker.ServicesTmpfs,
-		LogConfig: container.LogConfig{
-			Type: "json-file",
-		},
-		Init: useInit,
+		LogConfig:     e.logConfig,
+		Init:          useInit,
 	}, nil
 }
 
@@ -992,12 +1002,10 @@ func (e *executor) createHostConfig(isBuildContainer, imageIsPrivileged bool) (*
 		Isolation:     isolation,
 		VolumeDriver:  e.Config.Docker.VolumeDriver,
 		VolumesFrom:   e.Config.Docker.VolumesFrom,
-		LogConfig: container.LogConfig{
-			Type: "json-file",
-		},
-		Tmpfs:   e.Config.Docker.Tmpfs,
-		Sysctls: e.Config.Docker.SysCtls,
-		Init:    useInit,
+		LogConfig:     e.logConfig,
+		Tmpfs:         e.Config.Docker.Tmpfs,
+		Sysctls:       e.Config.Docker.SysCtls,
+		Init:          useInit,
 	}, nil
 }
 
@@ -1142,7 +1150,7 @@ func (e *executor) overwriteEntrypoint(image *spec.Image) []string {
 	return nil
 }
 
-func (e *executor) connectDocker(ctx context.Context, options common.ExecutorPrepareOptions) error {
+func connectDocker(ctx context.Context, options common.ExecutorPrepareOptions, e *executor) error {
 	_ = e.dockerConn.Close()
 
 	dockerConnection, err := createDockerConnection(ctx, options, e)
@@ -1341,7 +1349,16 @@ func (e *executor) Prepare(options common.ExecutorPrepareOptions) error {
 
 	e.AbstractExecutor.PrepareConfiguration(options)
 
-	err := e.connectDocker(e.Context, options)
+	var err error
+	e.logConfig, err = options.Config.Docker.GetLogConfig()
+	if err != nil {
+		return &common.BuildError{
+			Inner:         fmt.Errorf("creating docker log configuration: %w", err),
+			FailureReason: common.RunnerSystemFailure,
+		}
+	}
+
+	err = e.dockerConnector.Connect(e.Context, options, e)
 	if err != nil {
 		return err
 	}
@@ -1607,9 +1624,7 @@ func (e *executor) createHostConfigForServiceHealthCheck(service *serviceInfo) *
 		RestartPolicy: neverRestartPolicy,
 		ExtraHosts:    extraHosts,
 		NetworkMode:   e.networkMode,
-		LogConfig: container.LogConfig{
-			Type: "json-file",
-		},
+		LogConfig:     e.logConfig,
 	}
 }