osquery_manager: generate ECS section mappings from config (#18989)

Move ECS keep fields into osquery-gen config so parent object mappings such as *.sections are regenerated consistently, and refresh generated artifacts for osquery 5.23.0.

Author: marc-gr

packages/osquery_manager/_dev/scripts/osquery-gen/README.md

@@ -2,7 +2,7 @@
 
 Config-driven generator for osquery_manager field/schema generation.
 
-This tool reads `osquery` version and `beats` git selection from `config.yml`,
+This tool reads `osquery` version, `beats` git selection, and ECS keep fields from `config.yml`,
 reads the ECS git ref from `packages/osquery_manager/_dev/build/build.yml`
 (`dependencies.ecs.reference`, e.g. `git@v9.3.0`), resolves the latest matching
 patch for osquery/beats when using semver config, and generates:
@@ -20,13 +20,16 @@ Extension metadata in osquery schema is enforced:
 
 ## Config
 
-`config.yml` — osquery and beats only:
+`config.yml`:
 
 ```yaml
 osquery:
   version: "5.21.0"
 beats:
   version: "9.3"
+ecs:
+  keep_fields:
+    - file.pe.sections
 ```
 
 **Beats** (extension specs under `elastic/beats`): set **either** an explicit git ref **or** a semver-style `version`. Precedence is **tag > branch > version**:
@@ -51,6 +54,10 @@ dependencies:
     reference: git@v9.3.0
 ```
 
+**ECS** `keep_fields` entries force specific ECS field names into the generated
+`data_stream/result/fields/ecs.yml` even when the field is an object/nested node
+or has a type outside the generator's default allow-list.
+
 Notes:
 
 - Output is always written to `packages/osquery_manager` under the detected `integrations` repo root.

packages/osquery_manager/_dev/scripts/osquery-gen/config.yml

@@ -1,10 +1,61 @@
 osquery:
   # Exact patch (5.21.0) or prefix (5.21) to auto-pick latest patch.
-  version: "5.22.1"
+  version: "5.23.0"
 beats:
   # Git ref for elastic/beats osquery extension specs (raw.githubusercontent.com).
   # Precedence: tag > branch > version (semver / prefix resolution).
   # tag: "v9.4.2"
   # branch: "main"
   # Exact patch (9.3.1) or prefix (9.3) to auto-pick latest patch when tag/branch unset:
   version: "9.4.0"
+ecs:
+  # ECS field names to include even when they are objects/nested or otherwise
+  # filtered out by the default type allow-list.
+  keep_fields:
+    - dll.pe.sections
+    - email
+    - email.attachments
+    - email.attachments.file.extension
+    - email.attachments.file.hash.md5
+    - email.attachments.file.hash.sha1
+    - email.attachments.file.hash.sha256
+    - email.attachments.file.hash.sha384
+    - email.attachments.file.hash.sha512
+    - email.attachments.file.hash.ssdeep
+    - email.attachments.file.hash.tlsh
+    - email.attachments.file.mime_type
+    - email.attachments.file.name
+    - email.attachments.file.size
+    - email.bcc.address
+    - email.cc.address
+    - email.content_type
+    - email.delivery_timestamp
+    - email.direction
+    - email.from.address
+    - email.local_id
+    - email.message_id
+    - email.origination_timestamp
+    - email.reply_to.address
+    - email.sender.address
+    - email.subject
+    - email.to.address
+    - email.x_mailer
+    - file.elf.sections
+    - file.macho.sections
+    - file.pe.sections
+    - group.id
+    - network.iana_number
+    - network.type
+    - process.elf.sections
+    - process.macho.sections
+    - process.parent.elf.sections
+    - process.parent.macho.sections
+    - process.parent.pe.sections
+    - process.pe.sections
+    - threat.enrichments
+    - threat.enrichments.indicator.file.elf.sections
+    - threat.enrichments.indicator.file.pe.sections
+    - threat.indicator.file.elf.sections
+    - threat.indicator.file.pe.sections
+    - user.group.id
+    - user.id

packages/osquery_manager/_dev/scripts/osquery-gen/ecs_keep_fields.txt

@@ -6,7 +6,6 @@ package main
 
 import (
 	"bytes"
-	_ "embed"
 	"encoding/csv"
 	"encoding/json"
 	"fmt"
@@ -45,9 +44,6 @@ var (
 	ecsRestrictedSet       = makeSet(ecsRestrictedFields)
 )
 
-//go:embed ecs_keep_fields.txt
-var ecsKeepFieldsContent string
-
 type kibanaOsqueryTable struct {
 	Name        string             `json:"name"`
 	Description string             `json:"description"`
@@ -123,7 +119,7 @@ type columnInfo struct {
 	Column osqueryCol
 }
 
-func generateArtifacts(outputRoot, osqueryVersion, ecsVersion, beatsRef string, runPackageCheck bool) error {
+func generateArtifacts(outputRoot, osqueryVersion, ecsVersion, beatsRef string, ecsKeepFields []string, runPackageCheck bool) error {
 	outFields := filepath.Join(outputRoot, "packages", "osquery_manager", "data_stream", "result", "fields")
 	outSchemas := filepath.Join(outputRoot, "packages", "osquery_manager", "schemas")
 
@@ -165,7 +161,7 @@ func generateArtifacts(outputRoot, osqueryVersion, ecsVersion, beatsRef string,
 	if err != nil {
 		return fmt.Errorf("download ECS fields: %w", err)
 	}
-	ecsOut, err := generateECSFieldsYAML(ecsYAML)
+	ecsOut, err := generateECSFieldsYAML(ecsYAML, ecsKeepFields)
 	if err != nil {
 		return fmt.Errorf("generate ecs.yml: %w", err)
 	}
@@ -500,20 +496,8 @@ func marshalYAMLWithIndent(v any, indent int) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-func loadKeepFields() []string {
-	var out []string
-	for _, line := range strings.Split(ecsKeepFieldsContent, "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		out = append(out, line)
-	}
-	return out
-}
-
-func generateECSFieldsYAML(ecsBeatsYAML []byte) ([]byte, error) {
-	keepFields := makeSet(loadKeepFields())
+func generateECSFieldsYAML(ecsBeatsYAML []byte, ecsKeepFields []string) ([]byte, error) {
+	keepFields := makeSet(ecsKeepFields)
 	var root any
 	if err := yaml.Unmarshal(ecsBeatsYAML, &root); err != nil {
 		return nil, err
@@ -550,12 +534,16 @@ func collectECSFieldNames(root any, parent string, keepFields map[string]struct{
 		if parent == "" && inSet(excludeFromTopLevelSet, name) {
 			continue
 		}
+		fieldName := joinPath(parent, name)
+		kept := inSet(keepFields, fieldName)
+		if kept && name != "@timestamp" {
+			*ecsFields = append(*ecsFields, fieldName)
+		}
 		if childFields, hasFields := m["fields"]; hasFields {
 			collectECSFieldNames(childFields, joinPath(parent, name), keepFields, ecsFields)
 		} else {
-			fieldName := joinPath(parent, name)
 			typ, _ := m["type"].(string)
-			if (inSet(keepFields, fieldName) || isAllowedECSType(typ)) && name != "@timestamp" {
+			if !kept && isAllowedECSType(typ) && name != "@timestamp" {
 				*ecsFields = append(*ecsFields, fieldName)
 			}
 		}

packages/osquery_manager/_dev/scripts/osquery-gen/generator.go

@@ -37,9 +37,14 @@ type beatsConfig struct {
 	Version string `yaml:"version"`
 }
 
+type ecsConfig struct {
+	KeepFields []string `yaml:"keep_fields"`
+}
+
 type config struct {
 	Osquery versionConfig `yaml:"osquery"`
 	Beats   beatsConfig   `yaml:"beats"`
+	ECS     ecsConfig     `yaml:"ecs"`
 }
 
 type packageBuildYAML struct {
@@ -93,7 +98,7 @@ func main() {
 	}
 
 	log.Printf("Resolved versions: osquery=%s beats=%s ecs=%s", osqueryVersion, beatsRef, ecsVersion)
-	if err := generateArtifacts(outputRoot, osqueryVersion, ecsVersion, beatsRef, !*skipPackageCheck); err != nil {
+	if err := generateArtifacts(outputRoot, osqueryVersion, ecsVersion, beatsRef, cfg.ECS.KeepFields, !*skipPackageCheck); err != nil {
 		log.Fatalf("generate artifacts: %v", err)
 	}
 	log.Println("Done.")

packages/osquery_manager/_dev/scripts/osquery-gen/main.go

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.29.0"
+  changes:
+    - description: Generate ECS `*.sections` parent mappings from osquery-gen config
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/16650
+    - description: Upgrade osquery version to 5.23.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/16650
 - version: "1.28.1"
   changes:
     - description: Update ECS mappings for `*.sections` fields in results dataset

packages/osquery_manager/changelog.yml

@@ -77,6 +77,8 @@
   name: dns.answers.ttl
 - external: ecs
   name: dns.resolved_ip
+- external: ecs
+  name: email
 - external: ecs
   name: email.attachments
 - external: ecs