Skip to content

Commit ae86bb5

Browse files
authored
Add observability.otlp.ignore-if-missing to downgrade missing OTLP config to warnings (#32173)
1 parent 1797426 commit ae86bb5

8 files changed

Lines changed: 395 additions & 9 deletions

File tree

actions/setup/js/assign_agent_helpers.cjs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ async function findAgent(owner, repo, agentName, githubClient = github) {
126126
core.error(`Failed to find ${agentName} agent: ${errorMessage}`);
127127

128128
// Re-throw authentication/permission errors so they can be handled by the caller
129-
// This allows ignore-if-missing logic to work properly
129+
// This allows if-missing: ignore logic to work properly
130130
if (
131131
errorMessage.includes("Bad credentials") ||
132132
errorMessage.includes("Not Authenticated") ||

actions/setup/js/start_mcp_gateway.cjs

Lines changed: 106 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,95 @@ function sleep(ms) {
6464
return new Promise(resolve => setTimeout(resolve, ms));
6565
}
6666

67+
/**
68+
* Normalizes GH_AW_OTLP_IF_MISSING to a supported mode.
69+
* @param {string | undefined} value
70+
* @returns {"error" | "warn" | "ignore"}
71+
*/
72+
function getOTLPIfMissingMode(value) {
73+
const normalized = (value || "").trim().toLowerCase();
74+
if (normalized === "warn" || normalized === "ignore") {
75+
return normalized;
76+
}
77+
return "error";
78+
}
79+
80+
/**
81+
* Returns true when GH_AW_OTLP_IF_MISSING is set to "ignore".
82+
* @param {string | undefined} value
83+
* @returns {boolean}
84+
*/
85+
function isOTLPIfMissingIgnore(value) {
86+
return getOTLPIfMissingMode(value) === "ignore";
87+
}
88+
89+
/**
90+
* Returns true when OTLP headers contain at least one non-empty value.
91+
* NOTE: Unexpected primitive values are treated as non-empty so they are
92+
* preserved for downstream validation/error handling.
93+
* @param {unknown} headers
94+
* @returns {boolean}
95+
*/
96+
function hasNonEmptyOTLPHeaders(headers) {
97+
if (headers == null) {
98+
return false;
99+
}
100+
if (typeof headers === "string") {
101+
return headers.trim() !== "";
102+
}
103+
if (Array.isArray(headers)) {
104+
return headers.some(item => hasNonEmptyOTLPHeaders(item));
105+
}
106+
if (typeof headers === "object") {
107+
return Object.values(headers).some(value => hasNonEmptyOTLPHeaders(value));
108+
}
109+
// For unexpected primitive types, keep headers
110+
// intact so downstream validation can fail explicitly rather than silently
111+
// treating them as "missing".
112+
return true;
113+
}
114+
115+
/**
116+
* Apply observability.otlp.if-missing: ignore behavior to gateway OTLP config.
117+
* When enabled, missing endpoint values are downgraded to warnings and OTLP
118+
* gateway configuration is skipped instead of hard-failing the workflow.
119+
*
120+
* @param {Record<string, unknown>} configObj
121+
*/
122+
function applyOTLPIgnoreIfMissing(configObj) {
123+
const mode = getOTLPIfMissingMode(process.env.GH_AW_OTLP_IF_MISSING);
124+
const shouldDowngradeMissing = mode === "warn" || mode === "ignore";
125+
const shouldWarn = mode === "warn";
126+
if (!shouldDowngradeMissing) {
127+
return;
128+
}
129+
const gw = configObj.gateway;
130+
if (!gw || typeof gw !== "object" || Array.isArray(gw)) {
131+
return;
132+
}
133+
const gateway = /** @type {Record<string, unknown>} */ gw;
134+
const otel = gateway["opentelemetry"];
135+
if (!otel || typeof otel !== "object" || Array.isArray(otel)) {
136+
return;
137+
}
138+
const otelConfig = /** @type {Record<string, unknown>} */ otel;
139+
const endpoint = typeof otelConfig["endpoint"] === "string" ? otelConfig["endpoint"].trim() : "";
140+
if (!endpoint) {
141+
delete gateway["opentelemetry"];
142+
if (shouldWarn) {
143+
core.warning("OTLP endpoint is missing/empty and GH_AW_OTLP_IF_MISSING=warn; skipping MCP gateway OTLP configuration.");
144+
}
145+
return;
146+
}
147+
const headers = otelConfig["headers"];
148+
if ("headers" in otelConfig && !hasNonEmptyOTLPHeaders(headers)) {
149+
delete otelConfig["headers"];
150+
if (shouldWarn) {
151+
core.warning("OTLP headers are missing/empty and GH_AW_OTLP_IF_MISSING=warn; continuing without MCP gateway OTLP headers.");
152+
}
153+
}
154+
}
155+
67156
/**
68157
* Check whether a process is alive.
69158
* @param {number} pid
@@ -258,6 +347,8 @@ async function main() {
258347
process.exit(1);
259348
}
260349

350+
applyOTLPIgnoreIfMissing(configObj);
351+
261352
core.info("Configuration validated successfully");
262353
printTiming(configValidationStart, "Configuration validation");
263354
core.info("");
@@ -297,7 +388,7 @@ async function main() {
297388
core.error("ERROR: Gateway process stdin is not available");
298389
process.exit(1);
299390
}
300-
child.stdin.write(mcpConfig);
391+
child.stdin.write(JSON.stringify(configObj));
301392
child.stdin.end();
302393

303394
// Allow the child to run independently
@@ -740,11 +831,18 @@ async function main() {
740831
}
741832
}
742833

743-
main().catch(err => {
744-
const message = err instanceof Error ? err.message : String(err);
745-
const stack = err instanceof Error ? err.stack : undefined;
746-
if (stack) core.error(stack);
747-
core.setFailed(`FATAL: ${message}`);
748-
});
834+
if (require.main === module) {
835+
main().catch(err => {
836+
const message = err instanceof Error ? err.message : String(err);
837+
const stack = err instanceof Error ? err.stack : undefined;
838+
if (stack) core.error(stack);
839+
core.setFailed(`FATAL: ${message}`);
840+
});
841+
}
749842

750-
module.exports = {};
843+
module.exports = {
844+
applyOTLPIgnoreIfMissing,
845+
getOTLPIfMissingMode,
846+
hasNonEmptyOTLPHeaders,
847+
isOTLPIfMissingIgnore,
848+
};
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
2+
import { applyOTLPIgnoreIfMissing, getOTLPIfMissingMode, hasNonEmptyOTLPHeaders } from "./start_mcp_gateway.cjs";
3+
4+
describe("start_mcp_gateway OTLP if-missing helpers", () => {
5+
let originalWarning;
6+
7+
beforeEach(() => {
8+
originalWarning = global.core.warning;
9+
global.core.warning = vi.fn();
10+
});
11+
12+
afterEach(() => {
13+
delete process.env.GH_AW_OTLP_IF_MISSING;
14+
global.core.warning = originalWarning;
15+
});
16+
17+
it("normalizes if-missing mode", () => {
18+
expect(getOTLPIfMissingMode(undefined)).toBe("error");
19+
expect(getOTLPIfMissingMode(" warn ")).toBe("warn");
20+
expect(getOTLPIfMissingMode("ignore")).toBe("ignore");
21+
expect(getOTLPIfMissingMode("invalid")).toBe("error");
22+
});
23+
24+
it("detects non-empty OTLP headers for string/map/array forms", () => {
25+
expect(hasNonEmptyOTLPHeaders("")).toBe(false);
26+
expect(hasNonEmptyOTLPHeaders("Authorization=Bearer token")).toBe(true);
27+
expect(hasNonEmptyOTLPHeaders({ Authorization: "" })).toBe(false);
28+
expect(hasNonEmptyOTLPHeaders({ Authorization: "Bearer token" })).toBe(true);
29+
expect(hasNonEmptyOTLPHeaders(["", " "])).toBe(false);
30+
expect(hasNonEmptyOTLPHeaders(["", "token"])).toBe(true);
31+
});
32+
33+
it("is a no-op when if-missing mode is unset/error", () => {
34+
const config = {
35+
gateway: {
36+
opentelemetry: {
37+
endpoint: " ",
38+
headers: "",
39+
},
40+
},
41+
};
42+
applyOTLPIgnoreIfMissing(config);
43+
expect(config.gateway.opentelemetry).toEqual({
44+
endpoint: " ",
45+
headers: "",
46+
});
47+
});
48+
49+
it("removes opentelemetry when endpoint is empty for warn mode and emits a warning", () => {
50+
const warningSpy = vi.fn();
51+
global.core.warning = warningSpy;
52+
process.env.GH_AW_OTLP_IF_MISSING = "warn";
53+
54+
const config = {
55+
gateway: {
56+
opentelemetry: {
57+
endpoint: " ",
58+
headers: { Authorization: "" },
59+
},
60+
},
61+
};
62+
63+
applyOTLPIgnoreIfMissing(config);
64+
65+
expect(config.gateway.opentelemetry).toBeUndefined();
66+
expect(warningSpy).toHaveBeenCalledOnce();
67+
expect(warningSpy).toHaveBeenCalledWith(expect.stringContaining("OTLP endpoint is missing/empty"));
68+
});
69+
70+
it("removes empty headers object for warn mode and emits a warning", () => {
71+
const warningSpy = vi.fn();
72+
global.core.warning = warningSpy;
73+
process.env.GH_AW_OTLP_IF_MISSING = "warn";
74+
75+
const config = {
76+
gateway: {
77+
opentelemetry: {
78+
endpoint: "https://collector.example/v1/traces",
79+
headers: { Authorization: "", "X-Tenant": " " },
80+
},
81+
},
82+
};
83+
84+
applyOTLPIgnoreIfMissing(config);
85+
86+
expect(config.gateway.opentelemetry.headers).toBeUndefined();
87+
expect(warningSpy).toHaveBeenCalledOnce();
88+
expect(warningSpy).toHaveBeenCalledWith(expect.stringContaining("OTLP headers are missing/empty"));
89+
});
90+
91+
it("removes empty headers object for ignore mode without warning", () => {
92+
const warningSpy = vi.fn();
93+
global.core.warning = warningSpy;
94+
process.env.GH_AW_OTLP_IF_MISSING = "ignore";
95+
96+
const config = {
97+
gateway: {
98+
opentelemetry: {
99+
endpoint: "https://collector.example/v1/traces",
100+
headers: { Authorization: "" },
101+
},
102+
},
103+
};
104+
105+
applyOTLPIgnoreIfMissing(config);
106+
107+
expect(config.gateway.opentelemetry.headers).toBeUndefined();
108+
expect(warningSpy).not.toHaveBeenCalled();
109+
});
110+
});

docs/src/content/docs/reference/frontmatter.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -895,6 +895,7 @@ observability:
895895
|-------|------|-------------|
896896
| `observability.otlp.endpoint` | string, object, or array | OTLP/HTTP collector endpoint URL. Accepts a plain URL string, a single `{url, headers}` object, or an array of `{url, headers}` objects for concurrent fan-out to multiple collectors. When a static URL is provided, its hostname is automatically added to the network firewall allowlist. |
897897
| `observability.otlp.headers` | map or string | HTTP headers sent with every OTLP export request. Only applies when `endpoint` is a plain string; object and array endpoint entries carry their own per-endpoint headers. |
898+
| `observability.otlp.if-missing` | string (`error`, `warn`, `ignore`) | Controls behavior when OTLP endpoint/header values resolve to empty values at runtime (for example because a referenced secret is unset): `error` (default) fails startup, `warn` logs a warning and skips MCP gateway OTLP configuration, `ignore` skips MCP gateway OTLP configuration without warning. This setting affects MCP gateway setup only. |
898899

899900
### `observability.otlp.endpoint`
900901

@@ -972,6 +973,7 @@ When `observability.otlp` is configured, the following environment variables are
972973
| `OTEL_EXPORTER_OTLP_HEADERS` | Comma-separated `key=value` headers for the first endpoint. Set only when headers are configured. |
973974
| `OTEL_SERVICE_NAME` | Always `gh-aw`. |
974975
| `GH_AW_OTLP_ENDPOINTS` | JSON-encoded array of all endpoint entries (`[{"url":"...","headers":"..."}]`). Used by JavaScript action scripts to fan out spans to multiple endpoints. |
976+
| `GH_AW_OTLP_IF_MISSING` | Set to `warn` or `ignore` when `observability.otlp.if-missing` is configured. Used by runtime gateway setup to downgrade missing OTLP endpoint/header values from startup errors to non-fatal behavior for MCP gateway setup only. |
975977
| `COPILOT_OTEL_FILE_EXPORTER_PATH` | Path where Copilot CLI writes its own OTLP spans (`/tmp/gh-aw/copilot-otel.jsonl`). Copilot CLI detects this variable and writes its traces here; gh-aw forwards these traces to configured endpoints at the end of each run. |
976978

977979
> [!NOTE]

pkg/parser/schemas/main_workflow_schema.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9317,6 +9317,12 @@
93179317
"description": "Deprecated: use the map form instead. Comma-separated list of key=value HTTP headers to include with every OTLP export request (e.g. 'Authorization=Bearer <token>'). Supports GitHub Actions expressions such as ${{ secrets.OTLP_HEADERS }}. Injected as the OTEL_EXPORTER_OTLP_HEADERS environment variable."
93189318
}
93199319
]
9320+
},
9321+
"if-missing": {
9322+
"type": "string",
9323+
"enum": ["error", "warn", "ignore"],
9324+
"default": "error",
9325+
"description": "How to handle missing OTLP endpoint/header values at runtime (for example from unset secrets). 'error' fails workflow startup (default), 'warn' logs a warning and skips MCP gateway OTLP configuration, and 'ignore' skips MCP gateway OTLP configuration without warning. This affects MCP gateway setup only; workflow-level OTEL_* environment variables are still injected."
93209326
}
93219327
},
93229328
"additionalProperties": false

pkg/workflow/frontmatter_types.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,15 @@ type OTLPConfig struct {
225225
// - a comma-separated list of key=value pairs (e.g. "Authorization=Bearer <token>")
226226
// Both forms are injected as the standard OTEL_EXPORTER_OTLP_HEADERS environment variable.
227227
Headers any `json:"headers,omitempty"`
228+
229+
// IfMissing controls runtime behavior when OTLP endpoint/header values are
230+
// missing (for example because a referenced secret is unset). Supported values:
231+
// - "error" (default): fail workflow startup
232+
// - "warn": log a warning and skip MCP gateway OTLP config
233+
// - "ignore": skip MCP gateway OTLP config without warning
234+
// This setting affects MCP gateway setup only. Other OTLP-aware steps still
235+
// receive workflow-level OTEL_* environment variables.
236+
IfMissing string `json:"if-missing,omitempty"`
228237
}
229238

230239
// ObservabilityConfig represents workflow observability options.

pkg/workflow/observability_otlp.go

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,57 @@ func getOTLPEndpointEnvValue(config *FrontmatterConfig) string {
9999
return ""
100100
}
101101

102+
// normalizeOTLPIfMissingMode returns a validated if-missing mode.
103+
// Empty string means "unset/default (error)".
104+
func normalizeOTLPIfMissingMode(mode string) string {
105+
switch strings.ToLower(strings.TrimSpace(mode)) {
106+
case "":
107+
return ""
108+
case "error", "warn", "ignore":
109+
return strings.ToLower(strings.TrimSpace(mode))
110+
default:
111+
return ""
112+
}
113+
}
114+
115+
// getOTLPIfMissingMode returns observability.otlp.if-missing mode.
116+
// Returns empty string when unset or invalid.
117+
func getOTLPIfMissingMode(config *FrontmatterConfig, frontmatter map[string]any) string {
118+
if config != nil && config.Observability != nil && config.Observability.OTLP != nil {
119+
if mode := normalizeOTLPIfMissingMode(config.Observability.OTLP.IfMissing); mode != "" {
120+
return mode
121+
}
122+
}
123+
if frontmatter == nil {
124+
return ""
125+
}
126+
obsAny, ok := frontmatter["observability"]
127+
if !ok {
128+
return ""
129+
}
130+
obsMap, ok := obsAny.(map[string]any)
131+
if !ok {
132+
return ""
133+
}
134+
otlpAny, ok := obsMap["otlp"]
135+
if !ok {
136+
return ""
137+
}
138+
otlpMap, ok := otlpAny.(map[string]any)
139+
if !ok {
140+
return ""
141+
}
142+
if v, ok := otlpMap["if-missing"].(string); ok {
143+
if mode := normalizeOTLPIfMissingMode(v); mode != "" {
144+
return mode
145+
}
146+
if strings.TrimSpace(v) != "" {
147+
otlpLog.Printf("Ignoring invalid observability.otlp.if-missing value %q (expected one of: error, warn, ignore)", v)
148+
}
149+
}
150+
return ""
151+
}
152+
102153
// isOTLPHeadersPresent returns true when OTEL_EXPORTER_OTLP_HEADERS or
103154
// GH_AW_OTLP_ALL_HEADERS has been injected into the workflow-level env block.
104155
// This indicates that header masking is needed so that authentication tokens in
@@ -349,6 +400,7 @@ func (c *Compiler) injectOTLPConfig(workflowData *WorkflowData) {
349400

350401
firstEndpoint := entries[0].URL
351402
firstHeaders := entries[0].Headers
403+
ifMissingMode := getOTLPIfMissingMode(workflowData.ParsedFrontmatter, workflowData.RawFrontmatter)
352404

353405
// 2. Inject OTEL env vars into the workflow-level env: block.
354406
// OTEL_EXPORTER_OTLP_ENDPOINT and OTEL_SERVICE_NAME are set to the first
@@ -376,6 +428,10 @@ func (c *Compiler) injectOTLPConfig(workflowData *WorkflowData) {
376428
otlpEnvLines += "\n GH_AW_OTLP_ENDPOINTS: '" + escapedEncoded + "'"
377429
otlpLog.Printf("Injected GH_AW_OTLP_ENDPOINTS env var")
378430
}
431+
if ifMissingMode == "warn" || ifMissingMode == "ignore" {
432+
otlpEnvLines += "\n GH_AW_OTLP_IF_MISSING: " + ifMissingMode
433+
otlpLog.Printf("Injected GH_AW_OTLP_IF_MISSING env var (%s)", ifMissingMode)
434+
}
379435

380436
if workflowData.Env == "" {
381437
workflowData.Env = "env:\n" + otlpEnvLines

0 commit comments

Comments
 (0)