Load jobs from yaml with compose-go pipeline

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>

Author: ndeloof

loader/environment.go

@@ -25,27 +25,32 @@ import (
 // ResolveEnvironment update the environment variables for the format {- VAR} (without interpolation)
 func ResolveEnvironment(dict map[string]any, environment types.Mapping) {
 	resolveServicesEnvironment(dict, environment)
+	resolveContainerEnvironment(dict, "jobs", environment)
 	resolveSecretsEnvironment(dict, environment)
 	resolveConfigsEnvironment(dict, environment)
 }
 
 func resolveServicesEnvironment(dict map[string]any, environment types.Mapping) {
-	services, ok := dict["services"].(map[string]any)
+	resolveContainerEnvironment(dict, "services", environment)
+}
+
+func resolveContainerEnvironment(dict map[string]any, key string, environment types.Mapping) {
+	containers, ok := dict[key].(map[string]any)
 	if !ok {
 		return
 	}
 
-	for service, cfg := range services {
-		serviceConfig, ok := cfg.(map[string]any)
+	for name, cfg := range containers {
+		config, ok := cfg.(map[string]any)
 		if !ok {
 			continue
 		}
-		serviceEnv, ok := serviceConfig["environment"].([]any)
+		envList, ok := config["environment"].([]any)
 		if !ok {
 			continue
 		}
 		envs := []any{}
-		for _, env := range serviceEnv {
+		for _, env := range envList {
 			varEnv, ok := env.(string)
 			if !ok {
 				continue
@@ -57,10 +62,10 @@ func resolveServicesEnvironment(dict map[string]any, environment types.Mapping)
 				envs = append(envs, varEnv)
 			}
 		}
-		serviceConfig["environment"] = envs
-		services[service] = serviceConfig
+		config["environment"] = envs
+		containers[name] = config
 	}
-	dict["services"] = services
+	dict[key] = containers
 }
 
 func resolveSecretsEnvironment(dict map[string]any, environment types.Mapping) {

loader/extends.go

@@ -28,22 +28,24 @@ import (
 )
 
 func ApplyExtends(ctx context.Context, dict map[string]any, opts *Options, tracker *cycleTracker, post PostProcessor) error {
-	a, ok := dict["services"]
-	if !ok {
-		return nil
-	}
-	services, ok := a.(map[string]any)
-	if !ok {
-		return fmt.Errorf("services must be a mapping")
-	}
-	for name := range services {
-		merged, err := applyServiceExtends(ctx, name, services, opts, tracker, post)
-		if err != nil {
-			return err
+	for _, key := range []string{"services", "jobs"} {
+		a, ok := dict[key]
+		if !ok {
+			continue
+		}
+		entries, ok := a.(map[string]any)
+		if !ok {
+			return fmt.Errorf("%s must be a mapping", key)
+		}
+		for name := range entries {
+			merged, err := applyServiceExtends(ctx, name, entries, opts, tracker, post)
+			if err != nil {
+				return err
+			}
+			entries[name] = merged
 		}
-		services[name] = merged
+		dict[key] = entries
 	}
-	dict["services"] = services
 	return nil
 }
 

loader/include.go

@@ -166,23 +166,10 @@ func ApplyInclude(ctx context.Context, workingDir string, environment types.Mapp
 
 // importResources import into model all resources defined by imported, and report error on conflict
 func importResources(source map[string]any, target map[string]any, processor PostProcessor) error {
-	if err := importResource(source, target, "services", processor); err != nil {
-		return err
-	}
-	if err := importResource(source, target, "volumes", processor); err != nil {
-		return err
-	}
-	if err := importResource(source, target, "networks", processor); err != nil {
-		return err
-	}
-	if err := importResource(source, target, "secrets", processor); err != nil {
-		return err
-	}
-	if err := importResource(source, target, "configs", processor); err != nil {
-		return err
-	}
-	if err := importResource(source, target, "models", processor); err != nil {
-		return err
+	for _, key := range []string{"services", "jobs", "volumes", "networks", "secrets", "configs", "models"} {
+		if err := importResource(source, target, key, processor); err != nil {
+			return err
+		}
 	}
 	return nil
 }

loader/loader.go

@@ -787,9 +787,9 @@ func Transform(source interface{}, target interface{}) error {
 	return decoder.Decode(source)
 }
 
-// nameServices create implicit `name` key for convenience accessing service
+// nameServices create implicit `name` key for convenience accessing service or job
 func nameServices(from reflect.Value, to reflect.Value) (interface{}, error) {
-	if to.Type() == reflect.TypeOf(types.Services{}) {
+	if to.Type() == reflect.TypeOf(types.Services{}) || to.Type() == reflect.TypeOf(types.Jobs{}) {
 		nameK := reflect.ValueOf("name")
 		iter := from.MapRange()
 		for iter.Next() {

loader/normalize.go

@@ -29,21 +29,25 @@ import (
 func Normalize(dict map[string]any, env types.Mapping) (map[string]any, error) {
 	normalizeNetworks(dict)
 
-	if d, ok := dict["services"]; ok {
-		services := d.(map[string]any)
-		for name, s := range services {
-			service := s.(map[string]any)
+	for _, key := range []string{"services", "jobs"} {
+		d, ok := dict[key]
+		if !ok {
+			continue
+		}
+		containers := d.(map[string]any)
+		for name, s := range containers {
+			container := s.(map[string]any)
 
-			if service["pull_policy"] == types.PullPolicyIfNotPresent {
-				service["pull_policy"] = types.PullPolicyMissing
+			if container["pull_policy"] == types.PullPolicyIfNotPresent {
+				container["pull_policy"] = types.PullPolicyMissing
 			}
 
 			fn := func(s string) (string, bool) {
 				v, ok := env[s]
 				return v, ok
 			}
 
-			if b, ok := service["build"]; ok {
+			if b, ok := container["build"]; ok {
 				build := b.(map[string]any)
 				if build["context"] == nil {
 					build["context"] = "."
@@ -56,20 +60,20 @@ func Normalize(dict map[string]any, env types.Mapping) (map[string]any, error) {
 					build["args"], _ = resolve(a, fn, false)
 				}
 
-				service["build"] = build
+				container["build"] = build
 			}
 
-			if e, ok := service["environment"]; ok {
-				service["environment"], _ = resolve(e, fn, true)
+			if e, ok := container["environment"]; ok {
+				container["environment"], _ = resolve(e, fn, true)
 			}
 
 			var dependsOn map[string]any
-			if d, ok := service["depends_on"]; ok {
+			if d, ok := container["depends_on"]; ok {
 				dependsOn = d.(map[string]any)
 			} else {
 				dependsOn = map[string]any{}
 			}
-			if l, ok := service["links"]; ok {
+			if l, ok := container["links"]; ok {
 				links := l.([]any)
 				for _, e := range links {
 					link := e.(string)
@@ -88,7 +92,7 @@ func Normalize(dict map[string]any, env types.Mapping) (map[string]any, error) {
 			}
 
 			for _, namespace := range []string{"network_mode", "ipc", "pid", "uts", "cgroup"} {
-				if n, ok := service[namespace]; ok {
+				if n, ok := container[namespace]; ok {
 					ref := n.(string)
 					if strings.HasPrefix(ref, types.ServicePrefix) {
 						shared := ref[len(types.ServicePrefix):]
@@ -103,18 +107,18 @@ func Normalize(dict map[string]any, env types.Mapping) (map[string]any, error) {
 				}
 			}
 
-			if v, ok := service["volumes"]; ok {
+			if v, ok := container["volumes"]; ok {
 				volumes := v.([]any)
 				for i, volume := range volumes {
 					vol := volume.(map[string]any)
 					target := vol["target"].(string)
 					vol["target"] = path.Clean(target)
 					volumes[i] = vol
 				}
-				service["volumes"] = volumes
+				container["volumes"] = volumes
 			}
 
-			if n, ok := service["volumes_from"]; ok {
+			if n, ok := container["volumes_from"]; ok {
 				volumesFrom := n.([]any)
 				for _, v := range volumesFrom {
 					vol := v.(string)
@@ -131,12 +135,12 @@ func Normalize(dict map[string]any, env types.Mapping) (map[string]any, error) {
 				}
 			}
 			if len(dependsOn) > 0 {
-				service["depends_on"] = dependsOn
+				container["depends_on"] = dependsOn
 			}
-			services[name] = service
+			containers[name] = container
 		}
 
-		dict["services"] = services
+		dict[key] = containers
 	}
 	setNameFromKey(dict)
 
@@ -154,33 +158,37 @@ func normalizeNetworks(dict map[string]any) {
 	// implicit `default` network must be introduced only if actually used by some service
 	usesDefaultNetwork := false
 
-	if s, ok := dict["services"]; ok {
-		services := s.(map[string]any)
-		for name, se := range services {
-			service := se.(map[string]any)
-			if _, ok := service["provider"]; ok {
+	for _, key := range []string{"services", "jobs"} {
+		s, ok := dict[key]
+		if !ok {
+			continue
+		}
+		containers := s.(map[string]any)
+		for name, se := range containers {
+			container := se.(map[string]any)
+			if _, ok := container["provider"]; ok {
 				continue
 			}
-			if _, ok := service["network_mode"]; ok {
+			if _, ok := container["network_mode"]; ok {
 				continue
 			}
-			if n, ok := service["networks"]; !ok {
-				// If none explicitly declared, service is connected to default network
-				service["networks"] = map[string]any{"default": nil}
+			if n, ok := container["networks"]; !ok {
+				// If none explicitly declared, container is connected to default network
+				container["networks"] = map[string]any{"default": nil}
 				usesDefaultNetwork = true
 			} else {
 				net := n.(map[string]any)
 				if len(net) == 0 {
 					// networks section declared but empty (corner case)
-					service["networks"] = map[string]any{"default": nil}
+					container["networks"] = map[string]any{"default": nil}
 					usesDefaultNetwork = true
 				} else if _, ok := net["default"]; ok {
 					usesDefaultNetwork = true
 				}
 			}
-			services[name] = service
+			containers[name] = container
 		}
-		dict["services"] = services
+		dict[key] = containers
 	}
 
 	if _, ok := networks["default"]; !ok && usesDefaultNetwork {

loader/omitEmpty.go

@@ -20,6 +20,7 @@ import "github.com/compose-spec/compose-go/v2/tree"
 
 var omitempty = []tree.Path{
 	"services.*.dns",
+	"jobs.*.dns",
 }
 
 // OmitEmpty removes empty attributes which are irrelevant when unset

loader/tests/annotations_test.go

@@ -35,11 +35,22 @@ services:
     image: alpine
     annotations:
       com.example.foo: bar
+jobs:
+  list:
+    image: alpine
+    annotations:
+      - com.example.foo=bar
+  map:
+    image: alpine
+    annotations:
+      com.example.foo: bar
 `)
 	expect := func(p *types.Project) {
 		expected := types.Mapping{"com.example.foo": "bar"}
 		assert.DeepEqual(t, p.Services["list"].Annotations, expected)
 		assert.DeepEqual(t, p.Services["map"].Annotations, expected)
+		assert.DeepEqual(t, p.Jobs["list"].Annotations, expected)
+		assert.DeepEqual(t, p.Jobs["map"].Annotations, expected)
 	}
 	expect(p)
 

loader/tests/attach_test.go

@@ -35,12 +35,24 @@ services:
     attach: false
   default:
     image: alpine
+jobs:
+  attached:
+    image: alpine
+    attach: true
+  detached:
+    image: alpine
+    attach: false
+  default:
+    image: alpine
 `)
 
 	expect := func(p *types.Project) {
 		assert.Equal(t, *p.Services["attached"].Attach, true)
 		assert.Equal(t, *p.Services["detached"].Attach, false)
 		assert.Assert(t, p.Services["default"].Attach == nil)
+		assert.Equal(t, *p.Jobs["attached"].Attach, true)
+		assert.Equal(t, *p.Jobs["detached"].Attach, false)
+		assert.Assert(t, p.Jobs["default"].Attach == nil)
 	}
 	expect(p)
 

loader/tests/blkio_config_test.go

@@ -46,6 +46,26 @@ services:
       device_write_iops:
         - path: /dev/sda
           rate: 200
+jobs:
+  foo:
+    image: busybox
+    blkio_config:
+      weight: 300
+      weight_device:
+        - path: /dev/sda
+          weight: 400
+      device_read_bps:
+        - path: /dev/sda
+          rate: 1024k
+      device_write_bps:
+        - path: /dev/sda
+          rate: 1024
+      device_read_iops:
+        - path: /dev/sda
+          rate: 100
+      device_write_iops:
+        - path: /dev/sda
+          rate: 200
 `)
 	expect := func(p *types.Project) {
 		bc := p.Services["foo"].BlkioConfig
@@ -57,6 +77,15 @@ services:
 		assert.Equal(t, bc.DeviceWriteBps[0].Rate, types.UnitBytes(1024))
 		assert.Equal(t, bc.DeviceReadIOps[0].Rate, types.UnitBytes(100))
 		assert.Equal(t, bc.DeviceWriteIOps[0].Rate, types.UnitBytes(200))
+		jbc := p.Jobs["foo"].BlkioConfig
+		assert.Equal(t, jbc.Weight, uint16(300))
+		assert.Equal(t, jbc.WeightDevice[0].Path, "/dev/sda")
+		assert.Equal(t, jbc.WeightDevice[0].Weight, uint16(400))
+		assert.Equal(t, jbc.DeviceReadBps[0].Path, "/dev/sda")
+		assert.Equal(t, jbc.DeviceReadBps[0].Rate, types.UnitBytes(1024*1024))
+		assert.Equal(t, jbc.DeviceWriteBps[0].Rate, types.UnitBytes(1024))
+		assert.Equal(t, jbc.DeviceReadIOps[0].Rate, types.UnitBytes(100))
+		assert.Equal(t, jbc.DeviceWriteIOps[0].Rate, types.UnitBytes(200))
 	}
 	expect(p)