Prepare release 3.6.3

.buildkite/pipeline.trigger.compliance.tests.sh

@@ -28,7 +28,8 @@ steps:
 EOF
 
 # Generate each test we want to do.
-compliance_test 9.4.0-SNAPSHOT 3.6.2
+compliance_test 9.5.0-SNAPSHOT 3.6.3
+compliance_test 9.4.0 3.6.3
 compliance_test 8.19.3 3.4.2
 compliance_test 9.0.6 3.3.5
 compliance_test 8.14.0 3.1.5

.cursor/skills/package-spec-release/references/workflow.md

@@ -60,7 +60,19 @@ Let `VERSION` = the agreed value (e.g. `3.6.1`).
 
 3. Do **not** change other `compliance_test` rows unless the user explicitly asked to refresh those stack/spec pairs.
 
-## 5. Go toolchain
+## 5. Review pending TODOs
+
+Scan for `# Pending on <url>` comments in both places:
+
+- **`spec/changelog.yml`** — comments above changelog entries waiting on external work.
+- **`compliance/features/*.feature`** — `@skip` tags with `# Pending …` comments on blocked scenarios.
+
+For each pending item, check the status of every linked issue/PR (use `gh pr view` / `gh issue view`):
+
+- **All blockers merged/closed?** Remove the `# Pending on …` comment (and the `@skip` tag if in a feature file). If the same blocker appears in both files, clean up both. Commit resolved items together with a short message describing what was unblocked.
+- **Any blocker still open?** Leave the item as-is; do nothing.
+
+## 6. Go toolchain
 
 1. Read `go.mod` `go` directive (repo minor; line may be `1.MM` or `1.MM.P`) and `.go-version` (full `1.MM.P`, e.g. `1.25.8`).
 2. From **official Go releases**, determine the **latest stable** version (minor + patch).
@@ -72,7 +84,7 @@ Let `VERSION` = the agreed value (e.g. `3.6.1`).
 4. **Patch-only path** (no newer stable minor than `go.mod`, **or** user chose patches only in step 3): resolve the **latest patch** for **`go.mod`’s minor**. If that patch is **greater** than `.go-version`, update **only** `.go-version`. Optionally run `go mod tidy` / `make check`.
 5. Do not churn unrelated `require` blocks; scope edits to the Go version lines unless `tidy` demands otherwise.
 
-## 6. `AGENTS.md`
+## 7. `AGENTS.md`
 
 After changelog and CI edits, scan `AGENTS.md` for:
 
@@ -81,17 +93,18 @@ After changelog and CI edits, scan `AGENTS.md` for:
 
 Update only what is **factually wrong** after the release edit (keep the doc accurate; avoid unrelated rewrites).
 
-## 7. Commits
+## 8. Commits
 
 Create **one commit per topic**, short imperative subject, e.g.:
 
 - `Update changelog for ${VERSION}`
 - `Bump compliance CI spec version to ${VERSION}`
+- `Resolve pending TODOs now that blockers are merged` (if applicable)
 - `Bump Go patch version in .go-version` (if applicable)
 - `Bump Go to 1.MM.P` (if `go.mod` / `.go-version` minor bump)
 - `Update AGENTS.md for ${VERSION} release`
 
-## 8. Push and pull request
+## 9. Push and pull request
 
 1. `git push ${FORK_REMOTE}` (same branch as in step 2; add `--set-upstream` on first push if not already set).
 2. Open a PR **from the fork** → **`elastic/package-spec` `main`** (GitHub UI or `gh pr create --repo elastic/package-spec --base main --head <fork-owner>:prepare-release-${VERSION} --title "Prepare release ${VERSION}"`). Derive `<fork-owner>` from the **`FORK_REMOTE`** URL if needed. If `gh` defaults the head correctly after `push`, `--head` may be omitted.

code/go/pkg/validator/validator_test.go

@@ -37,6 +37,7 @@ func TestValidateFile(t *testing.T) {
 		"good_geo_shape":                         {},
 		"good_input":                             {},
 		"good_input_otel":                        {},
+		"good_integration_otel":                  {},
 		"good_input_qualifier":                   {},
 		"good_input_fleet_reserved_vars":         {},
 		"good_integration_fleet_reserved_vars":   {},

compliance/compliance_test.go

@@ -134,33 +134,42 @@ type contextKey string
 
 const agentPolicyIDKey contextKey = "agentPolicyID"
 
-func aPolicyIsCreatedWithPackage(ctx context.Context, packageName string) (context.Context, error) {
-	const version = "1.0.0"
-	return aPolicyIsCreatedWithPackageAndVersion(ctx, packageName, version)
-}
-
-func aPolicyIsCreatedWithPackageAndVersion(ctx context.Context, packageName, packageVersion string) (context.Context, error) {
+// withAgentPolicyID creates a Kibana client, runs fn, and stores the returned agent policy ID
+// in the context for use by subsequent steps.
+func withAgentPolicyID(ctx context.Context, fn func(*Kibana) (string, error)) (context.Context, error) {
 	kibana, err := NewKibanaClient()
 	if err != nil {
 		return ctx, err
 	}
-	agentPolicyID, err := kibana.createPolicyForPackage(packageName, packageVersion)
+	agentPolicyID, err := fn(kibana)
 	if err != nil {
 		return ctx, err
 	}
 	return context.WithValue(ctx, agentPolicyIDKey, agentPolicyID), nil
 }
 
+func aPolicyIsCreatedWithPackage(ctx context.Context, packageName string) (context.Context, error) {
+	const version = "1.0.0"
+	return aPolicyIsCreatedWithPackageAndVersion(ctx, packageName, version)
+}
+
+func aPolicyIsCreatedWithPackageAndVersion(ctx context.Context, packageName, packageVersion string) (context.Context, error) {
+	return withAgentPolicyID(ctx, func(k *Kibana) (string, error) {
+		return k.createPolicyForPackageInputAndDataset(packageName, packageVersion, "", "", "", "", "")
+	})
+}
+
 func aPolicyIsCreatedWithPackageInputAndDataset(ctx context.Context, packageName, packageVersion, templateName, inputName, inputType, dataset string) (context.Context, error) {
-	kibana, err := NewKibanaClient()
-	if err != nil {
-		return ctx, err
-	}
-	agentPolicyID, err := kibana.createPolicyForPackageInputAndDataset(packageName, packageVersion, templateName, inputName, inputType, dataset)
-	if err != nil {
-		return ctx, err
-	}
-	return context.WithValue(ctx, agentPolicyIDKey, agentPolicyID), nil
+	return withAgentPolicyID(ctx, func(k *Kibana) (string, error) {
+		// When no explicit input effective name is given, it equals the input type.
+		return k.createPolicyForPackageInputAndDataset(packageName, packageVersion, templateName, inputName, inputType, inputType, dataset)
+	})
+}
+
+func aPolicyIsCreatedWithPackageInputEffectiveNameAndDataset(ctx context.Context, packageName, packageVersion, templateName, inputEffectiveName, inputName, inputType, dataset string) (context.Context, error) {
+	return withAgentPolicyID(ctx, func(k *Kibana) (string, error) {
+		return k.createPolicyForPackageInputAndDataset(packageName, packageVersion, templateName, inputName, inputType, inputEffectiveName, dataset)
+	})
 }
 
 func thereIsAnIndexTemplateWithPattern(indexTemplateName, pattern string) error {
@@ -432,6 +441,7 @@ func InitializeScenario(ctx *godog.ScenarioContext) {
 	ctx.Step(`^a policy is created with "([^"]*)" package$`, aPolicyIsCreatedWithPackage)
 	ctx.Step(`^a policy is created with "([^"]*)" package and "([^"]*)" version$`, aPolicyIsCreatedWithPackageAndVersion)
 	ctx.Step(`^a policy is created with "([^"]*)" package, "([^"]*)" version, "([^"]*)" template, "([^"]*)" input, "([^"]*)" input type and dataset "([^"]*)"$`, aPolicyIsCreatedWithPackageInputAndDataset)
+	ctx.Step(`^a policy is created with "([^"]*)" package, "([^"]*)" version, "([^"]*)" template, input effective name "([^"]*)", "([^"]*)" stream, "([^"]*)" input type and dataset "([^"]*)"$`, aPolicyIsCreatedWithPackageInputEffectiveNameAndDataset)
 	ctx.Step(`^there is an index template "([^"]*)" with pattern "([^"]*)"$`, thereIsAnIndexTemplateWithPattern)
 	ctx.Step(`^there is a transform "([^"]*)"$`, thereIsATransform)
 	ctx.Step(`^the transform "([^"]*)" has alias "([^"]*)" configured$`, theTransformHasAliasConfigured)

compliance/features/basic.feature

@@ -37,14 +37,14 @@ Feature: Basic package types support
 
   @3.6.0
   @skip
-  # Pending support for qualified input names: https://github.com/elastic/package-spec/pull/1135 and https://github.com/elastic/kibana/pull/262138
-  Scenario: Integration package with OTel input can be installed
-   Given the "good_v3" package is installed
-     And a policy is created with "good_v3" package, "1.1.0" version, "otel" template, "otel_logs" input, "otelcol" input type and dataset ""
-    Then there is an index template "logs-good_v3.otel_logs" with pattern "logs-good_v3.otel_logs.otel-*"
+  # Still not working on 9.4.0, to be investigated.
+  Scenario: Integration package with named OTel inputs can be installed
+   Given the "good_integration_otel" package is installed
+     And a policy is created with "good_integration_otel" package, "0.0.1" version, "otel" template, input effective name "otel_logs", "otel_logs" stream, "otelcol" input type and dataset ""
+    Then there is an index template "logs-good_integration_otel.otel_logs" with pattern "logs-good_integration_otel.otel_logs.otel-*"
 
   @3.6.0
   Scenario: OTel input package with profiles type can be installed
    Given the "good_input_profiles" package is installed
      And a policy is created with "good_input_profiles" package, "0.0.1" version, "profilingreceiver" template, "profilingreceiver" input, "otelcol" input type and dataset "spec.otel_input_test"
-     # We omit assertions here because 'profiles' OTel packages should not produce ES assets which is currently handled by an ES plugin. https://github.com/elastic/kibana/pull/254090
+     # We omit assertions here because 'profiles' OTel packages should not produce ES assets which is currently handled by an ES plugin. https://github.com/elastic/kibana/pull/254090

compliance/kibana.go

@@ -120,8 +120,12 @@ func NewKibanaClient() (*Kibana, error) {
 	}, nil
 }
 
-// createPolicyForPackage creates a new policy for a package.
-func (k *Kibana) createPolicyForPackage(name string, version string) (string, error) {
+// createPolicyForPackageInputAndDataset creates an agent policy and a package policy for the
+// given package. When templateName, inputName, and inputType are all non-empty, the package
+// policy targets a specific input; otherwise a default policy with no custom inputs is created.
+// inputEffectiveName is the input's effective name as defined by Fleet (name field when present,
+// otherwise the input type), used to build the simplified policy inputs key.
+func (k *Kibana) createPolicyForPackageInputAndDataset(name, version, templateName, inputName, inputType, inputEffectiveName, dataset string) (string, error) {
 	err := k.deletePackagePolicyForPackage(name)
 	if err != nil {
 		return "", fmt.Errorf("failed to delete agent policy: %w", err)
@@ -132,27 +136,7 @@ func (k *Kibana) createPolicyForPackage(name string, version string) (string, er
 		return "", fmt.Errorf("failed to create agent policy: %w", err)
 	}
 
-	err = k.createPackagePolicy(agentPolicy.Item.ID, name, version, "", "", "", "")
-	if err != nil {
-		return "", fmt.Errorf("failed to create package policy: %w", err)
-	}
-
-	return agentPolicy.Item.ID, nil
-}
-
-// createPolicyForPackageInputAndDataset creates a policy for a package with a custom dataset.
-func (k *Kibana) createPolicyForPackageInputAndDataset(name, version, templateName, inputName, inputType, dataset string) (string, error) {
-	err := k.deletePackagePolicyForPackage(name)
-	if err != nil {
-		return "", fmt.Errorf("failed to delete agent policy: %w", err)
-	}
-
-	agentPolicy, err := k.createAgentPolicyForPackage(name)
-	if err != nil {
-		return "", fmt.Errorf("failed to create agent policy: %w", err)
-	}
-
-	err = k.createPackagePolicy(agentPolicy.Item.ID, name, version, templateName, inputName, inputType, dataset)
+	err = k.createPackagePolicy(agentPolicy.Item.ID, name, version, templateName, inputName, inputType, inputEffectiveName, dataset)
 	if err != nil {
 		return "", fmt.Errorf("failed to create package policy: %w", err)
 	}
@@ -274,15 +258,15 @@ func (k *Kibana) createAgentPolicyForPackage(name string) (*agentPolicyResponse,
 	return &agentPolicy, nil
 }
 
-func (k *Kibana) createPackagePolicy(agentPolicyID, name, version, templateName, inputName, inputType, dataset string) error {
+func (k *Kibana) createPackagePolicy(agentPolicyID, name, version, templateName, inputName, inputType, inputEffectiveName, dataset string) error {
 	var packagePolicyRequest createPackagePolicyRequest
 	packagePolicyRequest.Name = name + "-test-1"
 	packagePolicyRequest.PolicyID = agentPolicyID
 	packagePolicyRequest.Package.Name = name
 	packagePolicyRequest.Package.Version = version
 
 	if templateName != "" && inputName != "" && inputType != "" {
-		policyInputName := templateName + "-" + inputType
+		policyInputName := templateName + "-" + inputEffectiveName
 		policyStreamName := name + "." + inputName
 		vars := make(map[string]any)
 		if dataset != "" {

spec/changelog.yml

@@ -8,6 +8,8 @@
     - description: Add support for semantic_text field definition.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/807
+- version: 3.6.3
+  changes:
     - description: Add optional `release` field to agentless deployment mode to explicitly declare its release stage.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/1130
@@ -21,7 +23,6 @@
     - description: Add var_groups support to policy template and input levels in integration packages, and to policy template and package root levels in input packages.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/1120
-    # Pending on https://github.com/elastic/kibana/pull/262138
     - description: Add support for named inputs in policy templates to disambiguate multiple inputs of the same type.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/1135

test/packages/good_integration_otel/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial package
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/1168

test/packages/good_integration_otel/data_stream/otel_logs/agent/stream/stream.yml.hbs

@@ -0,0 +1,9 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4318
+service:
+  pipelines:
+    logs:
+      receivers: [otlp]

test/packages/good_integration_otel/data_stream/otel_logs/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good_integration_otel/data_stream/otel_logs/manifest.yml

@@ -0,0 +1,6 @@
+title: OTel logs
+type: logs
+streams:
+  - input: otel_logs
+    title: Collect logs via OTel receiver
+    description: Collecting logs via OpenTelemetry receiver

test/packages/good_integration_otel/data_stream/otel_metrics/agent/stream/stream.yml.hbs

@@ -0,0 +1,9 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4317
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]

test/packages/good_integration_otel/data_stream/otel_metrics/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good_integration_otel/data_stream/otel_metrics/manifest.yml

@@ -0,0 +1,6 @@
+title: OTel metrics
+type: metrics
+streams:
+  - input: otel_metrics
+    title: Collect metrics via OTel receiver
+    description: Collecting metrics via OpenTelemetry receiver

test/packages/good_integration_otel/docs/README.md

@@ -0,0 +1,3 @@
+# Good OTel Integration package
+
+Minimal integration package with an OTel input and an otel_logs data stream, used by basic compliance tests.

test/packages/good_integration_otel/manifest.yml

@@ -0,0 +1,36 @@
+format_version: 3.6.0
+name: good_integration_otel
+title: "Good OTel Integration package"
+version: 0.0.1
+source:
+  license: "Elastic-2.0"
+description: "Minimal integration package with an OTel input and an otel_logs data stream, used by basic compliance tests."
+type: integration
+categories:
+  - custom
+conditions:
+  kibana:
+    version: "^9.4.0"
+  elastic:
+    subscription: "basic"
+icons:
+  - src: /img/sample-logo.svg
+    title: Sample logo
+    size: 32x32
+    type: image/svg+xml
+policy_templates:
+  - name: otel
+    title: OTel logs
+    description: Collect logs and metrics via OTel receiver
+    inputs:
+      - name: otel_logs
+        type: otelcol
+        title: Collect logs via OTel receiver
+        description: Collecting logs via OpenTelemetry receiver
+      - name: otel_metrics
+        type: otelcol
+        title: Collect metrics via OTel receiver
+        description: Collecting metrics via OpenTelemetry receiver
+owner:
+  github: elastic/ecosystem
+  type: elastic

test/packages/good_v3/manifest.yml

@@ -146,6 +146,10 @@ policy_templates:
             show_user: true
             default:
               - http://127.0.0.1
+      # deployment_modes on this input is intentionally kept to test the spec feature.
+      # Note: this causes Kibana's simplified package-policy API to reject policy creation
+      # for other templates in this package (https://github.com/elastic/kibana/issues/268930).
+      # The OTel compliance scenario uses good_integration_otel instead of good_v3 for this reason.
       - type: aws/s3
         title: Collect S3 logs (agentless only)
         description: Collecting logs from AWS S3 in agentless mode