diff --git a/.changeset/fuzzy-lions-federate.md b/.changeset/fuzzy-lions-federate.md new file mode 100644 index 00000000000..1b6d31b13c7 --- /dev/null +++ b/.changeset/fuzzy-lions-federate.md @@ -0,0 +1,16 @@ +--- +'hive': patch +--- + +Add Azure Workload Identity Federation support for OIDC SSO in self-hosted deployments. Enabled +organizations use the projected Azure token as a client assertion instead of an OIDC client secret. + +For example, enable federation for an organization with: + +```env +OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER=azure +AZURE_FEDERATED_TOKEN_FILE=/var/run/secrets/azure/tokens/azure-identity-token +OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS=00000000-0000-4000-8000-000000000000 +``` + +This will effectively override the client secret configured for that organization to use the OIDC provider. diff --git a/.changeset/modern-badgers-fall.md b/.changeset/modern-badgers-fall.md new file mode 100644 index 00000000000..b422c53ee3e --- /dev/null +++ b/.changeset/modern-badgers-fall.md @@ -0,0 +1,5 @@ +--- +'hive': patch +--- + +Track schema composition worker memory usage; add to schema service dashboard diff --git a/.changeset/quiet-routers-heal.md b/.changeset/quiet-routers-heal.md new file mode 100644 index 00000000000..0da548f1608 --- /dev/null +++ b/.changeset/quiet-routers-heal.md @@ -0,0 +1,6 @@ +--- +'hive': patch +--- + +Address vulnerabilities [GHSA-45rx-2jwx-cxfr](https://github.com/advisories/GHSA-45rx-2jwx-cxfr), +and [GHSA-c96f-x56v-gq3h](https://github.com/advisories/GHSA-c96f-x56v-gq3h). diff --git a/.changeset/social-drinks-speak.md b/.changeset/social-drinks-speak.md new file mode 100644 index 00000000000..5c9c4e27a25 --- /dev/null +++ b/.changeset/social-drinks-speak.md @@ -0,0 +1,6 @@ +--- +'hive': patch +--- + +Optimize memory for composition worker stitching schemas. Don't store or build the schemas if there +are stitched errors diff --git a/deployment/grafana-dashboards/Metric-Alerts.json b/deployment/grafana-dashboards/Metric-Alerts.json index c23b27b61c1..6276302222f 100644 --- a/deployment/grafana-dashboards/Metric-Alerts.json +++ b/deployment/grafana-dashboards/Metric-Alerts.json @@ -63,7 +63,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 0 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 0 }, "id": 1, "options": { "legend": { @@ -143,7 +143,7 @@ } ] }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 0 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 0 }, "id": 2, "options": { "legend": { @@ -190,7 +190,7 @@ }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 9 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 10 }, "id": 3, "options": { "colorMode": "value", @@ -248,7 +248,7 @@ }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 9 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 10 }, "id": 4, "options": { "legend": { @@ -303,7 +303,7 @@ } ] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 20 }, "id": 5, "options": { "legend": { @@ -366,7 +366,7 @@ } ] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 20 }, "id": 6, "options": { "legend": { @@ -401,7 +401,482 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, + "id": 10, + "panels": [], + "title": "Diagnostics", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "p99 round-trip latency of the metric-alert evaluator's ClickHouse queries, overlaid against p99 of all API-issued ClickHouse queries (api_clickhouse_elapsed_duration, the metric behind the ClickHouse-Latency dashboard). If both lines rise together the ClickHouse cluster is contended for everyone (ingestion, merges, another heavy consumer) and the slowdown is not specific to alert evaluation. If only the evaluator line rises, the cost is in the alert workload itself.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "dashed" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "orange", "value": 2 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "API-wide p99" }, + "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 31 }, + "id": 11, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(hive_metric_alert_clickhouse_query_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "evaluator p99", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(api_clickhouse_elapsed_duration_bucket[$__rate_interval])))", + "legendFormat": "API-wide p99", + "range": true, + "refId": "B" + } + ], + "title": "ClickHouse latency: evaluator vs API-wide (p99)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "p99 time a evaluateMetricAlertRules job waits in the graphile-worker queue before a worker picks it up. The cron enqueues every minute and groups run at GROUP_CONCURRENCY=5; sustained growth here means the evaluator can't drain a tick before the next is enqueued (workers saturated or ticks running long), so alerts are evaluated late even if each query is fast.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 31 }, + "id": 12, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "hive_workflow_job_queue_time_seconds{task_identifier=\"evaluateMetricAlertRules\",quantile=\"0.99\"}", + "legendFormat": "p99 {{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Task queue time (p99)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Success vs error completion rate of the evaluateMetricAlertRules job. This is the task-level failure signal: the 'ClickHouse error ratio' panel above only counts failed ClickHouse queries, whereas a tick can also fail in PG state transitions or the task body. Any sustained 'error' here means whole ticks are throwing.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "error" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "success" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 41 }, + "id": 13, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "sum(rate(hive_workflow_job_success_total{task_identifier=\"evaluateMetricAlertRules\"}[$__rate_interval]))", + "legendFormat": "success", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "sum(rate(hive_workflow_job_error_total{task_identifier=\"evaluateMetricAlertRules\"}[$__rate_interval]))", + "legendFormat": "error", + "range": true, + "refId": "B" + } + ], + "title": "Task success / error rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, + "id": 20, + "panels": [], + "title": "Runtime (workflow-service)", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Per-pod Node.js event-loop lag p99 for the workflow-service. The evaluator's task-duration is wall-clock, so it inflates whenever the pod's event loop is blocked (heavy sync work, GC pauses, another workflow task saturating the pod) even when ClickHouse is fast. A lag climb that tracks the task-duration climb points at the runtime, not the query. Note: this is pod-level (all workflow tasks share the pod), not attributed to alert evaluation alone.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 52 }, + "id": 21, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "nodejs_eventloop_lag_p99_seconds{service=\"workflow-service\"}", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Event-loop lag (p99) per pod", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Per-pod CPU usage (cores) for the workflow-service, from process_cpu_seconds_total. Read alongside event-loop lag: high CPU with high lag means the pod is compute-bound.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "cores", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 52 }, + "id": 22, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{service=\"workflow-service\"}[$__rate_interval])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU (cores) per pod", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Per-pod memory for the workflow-service: resident set (RSS) and Node.js used heap. A steady climb across a tick storm can indicate GC pressure or a leak that also shows up as rising GC time and event-loop lag.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 62 }, + "id": 23, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "process_resident_memory_bytes{service=\"workflow-service\"}", + "legendFormat": "RSS {{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "nodejs_heap_size_used_bytes{service=\"workflow-service\"}", + "legendFormat": "heap {{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "Memory (RSS + heap) per pod", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Per-pod Node.js GC time spent per second for the workflow-service (rate of nodejs_gc_duration_seconds_sum). Rising GC time steals from the event loop and shows up as event-loop lag and inflated task duration.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 62 }, + "id": 24, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "rate(nodejs_gc_duration_seconds_sum{service=\"workflow-service\"}[$__rate_interval])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "GC time per second per pod", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 72 }, "id": 100, "panels": [], "title": "Traces", @@ -423,7 +898,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 25 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 73 }, "id": 101, "options": { "cellHeight": "sm", @@ -461,7 +936,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 25 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 73 }, "id": 102, "options": { "cellHeight": "sm", @@ -516,7 +991,7 @@ } ] }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 34 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 83 }, "id": 104, "options": { "legend": { @@ -570,7 +1045,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 34 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 83 }, "id": 105, "options": { "legend": { @@ -624,7 +1099,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 43 }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 93 }, "id": 106, "options": { "legend": { @@ -668,7 +1143,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 52 }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 103 }, "id": 103, "options": { "cellHeight": "sm", diff --git a/deployment/grafana-dashboards/Schema-Service.json b/deployment/grafana-dashboards/Schema-Service.json index 5a5c5351ea9..5470d808137 100644 --- a/deployment/grafana-dashboards/Schema-Service.json +++ b/deployment/grafana-dashboards/Schema-Service.json @@ -661,9 +661,288 @@ ], "title": "Timeouts", "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "description": "Current memory usage across workers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisShow": false, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "editorMode": "code", + "expr": "sum(composition_worker_memory_used_bytes) by (instance)", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Memory Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "description": "P95 of composition cache value size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisShow": false, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(composition_cache_value_size_bytes_bucket[$__rate_interval])) by (le))", + "instant": false, + "legendFormat": "P95 Size", + "range": true, + "refId": "A" + } + ], + "title": "Cache Value Size (P95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "description": "P95 of worker duration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisShow": false, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(composition_worker_duration_ms_bucket[$__rate_interval])) by (le))", + "instant": false, + "legendFormat": "P95 Duration", + "range": true, + "refId": "A" + } + ], + "title": "Worker Duration (P95)", + "type": "timeseries" } ], - "refresh": "5s", + "refresh": "30s", "schemaVersion": 38, "tags": [], "templating": { diff --git a/deployment/package.json b/deployment/package.json index af475296b48..0642037333e 100644 --- a/deployment/package.json +++ b/deployment/package.json @@ -17,7 +17,7 @@ "@pulumi/kubernetesx": "0.1.6", "@pulumi/pulumi": "3.214.0", "@pulumi/random": "4.18.2", - "js-yaml": "4.2.0", + "js-yaml": "4.3.0", "pg-connection-string": "2.9.1", "prettier": "3.4.2" }, diff --git a/deployment/services/schema.ts b/deployment/services/schema.ts index 1bf9704c0ff..25369b69d46 100644 --- a/deployment/services/schema.ts +++ b/deployment/services/schema.ts @@ -43,6 +43,7 @@ export function deploySchema({ SCHEMA_CACHE_TTL_MS: '65000' /* 65s */, SCHEMA_CACHE_SUCCESS_TTL_MS: String(hourInMS * 2), COMPOSITION_WORKER_MAX_OLD_GENERATION_SIZE_MB: '716', + COMPOSITION_WORKER_TRACK_MEMORY_USAGE: '1', OPENTELEMETRY_COLLECTOR_ENDPOINT: observability.enabled && observability.tracingEndpoint ? observability.tracingEndpoint diff --git a/package.json b/package.json index 8c8ec6e69a7..7dffc0030c5 100644 --- a/package.json +++ b/package.json @@ -177,12 +177,11 @@ "brace-expansion@<1.1.16": "^1.1.16", "@sigstore/core@<=3.2.0": ">=3.2.1", "js-yaml@<3.15.0": "3.15.0", - "js-yaml@>=4.0.0 <4.3.0": "4.3.0", "sigstore@<=4.1.0": ">=4.1.1", "tar@<=7.5.18": "7.5.21", "fast-uri@2.x.x": "3.1.4", "fast-uri@3.x.x": "3.1.4", - "@opentelemetry/propagator-jaeger@2.x.x": "2.9.0", + "@opentelemetry/propagator-jaeger@<2.9.0": "2.9.0", "sharp@<0.35.0": "0.35.0", "body-parser@1.x.x": "1.20.6" }, diff --git a/packages/services/api/src/modules/schema/providers/artifact-storage-writer.ts b/packages/services/api/src/modules/schema/providers/artifact-storage-writer.ts index bddd9c2fb5f..612b651b710 100644 --- a/packages/services/api/src/modules/schema/providers/artifact-storage-writer.ts +++ b/packages/services/api/src/modules/schema/providers/artifact-storage-writer.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { Inject } from 'graphql-modules'; import { buildArtifactStorageKey } from '@hive/cdn-script/artifact-storage-reader'; import { traceFn } from '@hive/service-common'; @@ -182,4 +183,35 @@ export class ArtifactStorageWriter { args.contractName, ); } + + async writeSDLForDebugPurposes(subgraphs: Array<{ sdl: string; name?: string }>) { + const hashBuilder = createHash('sha256'); + for (const subgraph of subgraphs) { + hashBuilder.update(subgraph.sdl); + if (subgraph.name) { + hashBuilder.update(subgraph.name); + } + } + + const hash = hashBuilder.digest('hex'); + + const firstMirror = this.s3Mirrors[0]; + const url = [firstMirror.endpoint, firstMirror.bucket, 'debug-artifacts', hash + '.json'].join( + '/', + ); + + await firstMirror.client.fetch(url, { + method: 'PUT', + headers: { + 'content-type': 'application/json', + }, + body: JSON.stringify(subgraphs), + aws: { + // This boolean makes Google Cloud Storage & AWS happy. + signQuery: true, + }, + }); + + return { id: hash.substring(0, 10), url }; + } } diff --git a/packages/services/api/src/modules/schema/providers/orchestrator/composition-orchestrator.ts b/packages/services/api/src/modules/schema/providers/orchestrator/composition-orchestrator.ts index bb5da0f0c8b..ebc105d2f65 100644 --- a/packages/services/api/src/modules/schema/providers/orchestrator/composition-orchestrator.ts +++ b/packages/services/api/src/modules/schema/providers/orchestrator/composition-orchestrator.ts @@ -1,7 +1,7 @@ import { CONTEXT, Inject, Injectable, Scope } from 'graphql-modules'; import { abortSignalAny } from '@graphql-hive/signal'; import type { ContractsInputType, SchemaBuilderApi } from '@hive/schema'; -import { traceFn } from '@hive/service-common'; +import { trace, traceFn } from '@hive/service-common'; import { createTRPCProxyClient, httpLink } from '@trpc/client'; import { ComposeAndValidateResult, @@ -10,6 +10,7 @@ import { SchemaObject, } from '../../../../shared/entities'; import { Logger } from '../../../shared/providers/logger'; +import { ArtifactStorageWriter } from '../artifact-storage-writer'; import type { SchemaServiceConfig } from './tokens'; import { SCHEMA_SERVICE_CONFIG } from './tokens'; @@ -44,6 +45,7 @@ export class CompositionOrchestrator { constructor( logger: Logger, + private artifactStorageWriter: ArtifactStorageWriter, @Inject(SCHEMA_SERVICE_CONFIG) serviceConfig: SchemaServiceConfig, @Inject(CONTEXT) context: GraphQLModules.ModuleContext, ) { @@ -159,6 +161,21 @@ export class CompositionOrchestrator { signal: abortSignalAny([this.incomingRequestAbortSignal, timeoutAbortSignal]), }, ); + + // If a composition error happens due to OOM we write the subgraphs to S3 + // This makes it easier to reproduce and debug the issue + if (result.errors.some(error => error.code === 'ERR_WORKER_OUT_OF_MEMORY')) { + const span = trace.getActiveSpan(); + if (span) { + const result = await this.artifactStorageWriter.writeSDLForDebugPurposes( + schemas.map(s => ({ sdl: s.raw, name: s.source })), + ); + trace + .getActiveSpan() + ?.setAttribute(`hive.composition.subgraphs.${result.id}`, result.url); + } + } + return result; } catch (err) { // In case of a timeout error we return something the user can process diff --git a/packages/services/api/src/modules/schema/providers/schema-publisher.ts b/packages/services/api/src/modules/schema/providers/schema-publisher.ts index b69e5a0980d..1c3c1322058 100644 --- a/packages/services/api/src/modules/schema/providers/schema-publisher.ts +++ b/packages/services/api/src/modules/schema/providers/schema-publisher.ts @@ -1754,6 +1754,10 @@ export class SchemaPublisher { }), ]); + trace.getActiveSpan()?.setAttributes({ + 'hive.latestSchemaVersion.id': latestVersion?.version.id ?? 'no prior version', + }); + function increaseSchemaPublishCountMetric(conclusion: 'rejected' | 'accepted' | 'ignored') { schemaPublishCount.inc({ model: 'modern', diff --git a/packages/services/schema/src/api.ts b/packages/services/schema/src/api.ts index ca26e112022..c0982a86e7f 100644 --- a/packages/services/schema/src/api.ts +++ b/packages/services/schema/src/api.ts @@ -171,6 +171,7 @@ export const schemaBuilderApiRouter = t.router({ message: 'Composition exceeded resource limits. Please contact the Hive Console Team.', source: 'composition', + code: 'ERR_WORKER_OUT_OF_MEMORY', }, ], sdl: null, diff --git a/packages/services/schema/src/composition-scheduler.ts b/packages/services/schema/src/composition-scheduler.ts index b2ba433a301..1dd6e42406e 100644 --- a/packages/services/schema/src/composition-scheduler.ts +++ b/packages/services/schema/src/composition-scheduler.ts @@ -3,15 +3,22 @@ import { Worker } from 'node:worker_threads'; import fastq from 'fastq'; import * as Sentry from '@sentry/node'; import { registerWorkerLogging, type Logger } from '../../api/src/modules/shared/providers/logger'; -import type { CompositionEvent, CompositionResultEvent } from './composition-worker'; +import type { + CompositionEvent, + CompositionResultEvent, + ErrorResultEvent, + TrackMetricEvent, +} from './composition-worker'; import { compositionQueueDurationMS, compositionTotalDurationMS, compositionWorkerDurationMS, + compositionWorkerMemoryUsedBytes, } from './metrics'; type WorkerRunArgs = { data: CompositionEvent['data']; + targetId?: string; requestId: string; abortSignal: AbortSignal; }; @@ -126,18 +133,24 @@ export class CompositionScheduler { registerWorkerLogging(this.logger, worker, name); - worker.on( - 'message', - (data: CompositionResultEvent | { event: 'error'; id: string; err: Error }) => { - if (data.event === 'error') { - workerState?.task.reject(data.err); - } + worker.on('message', (data: CompositionResultEvent | ErrorResultEvent | TrackMetricEvent) => { + if (data.event === 'error') { + workerState?.task.reject(data.err); + } + + if (data.event === 'compositionResult') { + workerState?.task.resolve(data); + } - if (data.event === 'compositionResult') { - workerState?.task.resolve(data); + if (data.event === 'metric') { + if (data.type === 'heapUsed') { + compositionWorkerMemoryUsedBytes.set( + { target: workerState?.args.targetId, type: workerState?.args.data.type }, + data.value, + ); } - }, - ); + } + }); const { logger: baseLogger } = this; @@ -181,6 +194,7 @@ export class CompositionScheduler { event: 'composition', id: taskId, data: args.data, + targetId: args.targetId, taskId, requestId: args.requestId, } satisfies CompositionEvent); diff --git a/packages/services/schema/src/composition-worker.ts b/packages/services/schema/src/composition-worker.ts index 503cb5aba74..74fd8241397 100644 --- a/packages/services/schema/src/composition-worker.ts +++ b/packages/services/schema/src/composition-worker.ts @@ -9,6 +9,19 @@ import { composeSingle, type ComposeSingleArgs } from './composition/single'; import { composeStitching, type ComposeStitchingArgs } from './composition/stitching'; import type { env } from './environment'; +export type ErrorResultEvent = { + event: 'error'; + id: string; + err: unknown; +}; + +export type TrackMetricEvent = { + event: 'metric'; +} & { + type: 'heapUsed'; + value: number; +}; + export function createCompositionWorker(args: { baseLogger: Logger; port: MessagePort; @@ -33,6 +46,18 @@ export function createCompositionWorker(args: { event: message.event, }); logger.debug('processing message'); + const postResult = (result: CompositionResultEvent | ErrorResultEvent) => { + args.port.postMessage(result); + + if (args.env.compositionWorker.trackMemoryUsage) { + // send metric back to main thread so it can be reported + args.port.postMessage({ + event: 'metric', + type: 'heapUsed', + value: process.memoryUsage().heapUsed, + } satisfies TrackMetricEvent); + } + }; try { if (message.event === 'composition') { @@ -44,7 +69,7 @@ export function createCompositionWorker(args: { }); const composed = await composeFederation(message.data.args); - args.port.postMessage({ + postResult({ event: 'compositionResult', id: message.id, data: { @@ -73,7 +98,7 @@ export function createCompositionWorker(args: { if (message.data.type === 'single') { const result = await composeSingle(message.data.args); - args.port.postMessage({ + postResult({ event: 'compositionResult', id: message.id, data: { @@ -86,7 +111,7 @@ export function createCompositionWorker(args: { if (message.data.type === 'stitching') { const result = await composeStitching(message.data.args); - args.port.postMessage({ + postResult({ event: 'compositionResult', id: message.id, data: { @@ -106,7 +131,7 @@ export function createCompositionWorker(args: { message.id, ); baseLogger.error(String(err)); - args.port.postMessage({ + postResult({ event: 'error', id: message.id, err, @@ -123,6 +148,7 @@ export type CompositionEvent = { id: string; event: 'composition'; requestId: string; + targetId?: string; taskId: string; data: | { diff --git a/packages/services/schema/src/composition/shared.ts b/packages/services/schema/src/composition/shared.ts index 87e36fe4c82..4864e77f17e 100644 --- a/packages/services/schema/src/composition/shared.ts +++ b/packages/services/schema/src/composition/shared.ts @@ -4,6 +4,7 @@ import type { Metadata } from '../types'; export type CompositionErrorType = { message: string; source: CompositionErrorSource; + code?: string; }; type ContractResult = { diff --git a/packages/services/schema/src/composition/stitching.ts b/packages/services/schema/src/composition/stitching.ts index 923c2565103..98b45117a5e 100644 --- a/packages/services/schema/src/composition/stitching.ts +++ b/packages/services/schema/src/composition/stitching.ts @@ -69,7 +69,7 @@ export type ComposeStitchingArgs = { export async function composeStitching(args: ComposeStitchingArgs) { const errors: Array = []; - const subschemas: GraphQLSchema[] = []; + let subschemas: GraphQLSchema[] = []; for (const schema of args.schemas) { // parse inside a loop instead of using map to avoid holding on to the parsed schema // in memory. This matters for large schemas. @@ -77,7 +77,8 @@ export async function composeStitching(args: ComposeStitchingArgs) { const stitchedErrors = validateStitchedSchema(parsed); if (stitchedErrors.length) { errors.push(...stitchedErrors); - } else { + subschemas = []; // no need to store subschemas if there are errors. It wont be used. + } else if (!errors.length) { subschemas.push( buildASTSchema(trimDescriptions(parsed), { assumeValid: true, diff --git a/packages/services/schema/src/environment.ts b/packages/services/schema/src/environment.ts index 1d6ffb9a1c8..5e2f01c5cf3 100644 --- a/packages/services/schema/src/environment.ts +++ b/packages/services/schema/src/environment.ts @@ -38,6 +38,9 @@ const EnvironmentModel = zod.object({ AWS_REGION: emptyString(zod.string().optional()), COMPOSITION_WORKER_COUNT: zod.number().min(1).default(4), COMPOSITION_WORKER_MAX_OLD_GENERATION_SIZE_MB: NumberFromString(1).optional().default(512), + COMPOSITION_WORKER_TRACK_MEMORY_USAGE: emptyString( + zod.union([zod.literal('1'), zod.literal('0')]), + ).optional(), }); const RequestBrokerModel = zod.union([ @@ -206,5 +209,6 @@ export const env = { compositionWorker: { count: base.COMPOSITION_WORKER_COUNT, maxOldGenerationSizeMb: base.COMPOSITION_WORKER_MAX_OLD_GENERATION_SIZE_MB, + trackMemoryUsage: base.COMPOSITION_WORKER_TRACK_MEMORY_USAGE === '1', }, } as const; diff --git a/packages/services/schema/src/metrics.ts b/packages/services/schema/src/metrics.ts index 565ec269e74..54339a83c11 100644 --- a/packages/services/schema/src/metrics.ts +++ b/packages/services/schema/src/metrics.ts @@ -38,3 +38,9 @@ export const compositionCacheValueSizeBytes = new metrics.Histogram({ help: 'The size of the cache entries.', buckets: [200, 500, 1_000, 2_000, 3_000, 4_000, 5_000, 7_000, 10_000, 15_000, 20_000, 30_000], }); + +export const compositionWorkerMemoryUsedBytes = new metrics.Gauge({ + name: 'composition_worker_memory_used_bytes', + help: 'Amount of memory used by the composition worker', + labelNames: ['target', 'type'], +}); diff --git a/packages/services/server/src/environment.ts b/packages/services/server/src/environment.ts index 400865b4fbf..c7525e625b8 100644 --- a/packages/services/server/src/environment.ts +++ b/packages/services/server/src/environment.ts @@ -1,4 +1,6 @@ +import { readFileSync } from 'node:fs'; import zod from 'zod'; +import { isUUID } from '@hive/api/shared/is-uuid'; import { OpenTelemetryConfigurationModel, parseRedisConfigFromEnvironment, @@ -302,6 +304,50 @@ const LogModel = zod.object({ ), }); +const OidcWorkloadFederationModel = zod.union([ + zod.object({ + OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER: zod.union([ + zod.void(), + zod.literal(''), + zod.literal('0'), + ]), + }), + zod + .object({ + OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER: zod.literal('azure'), + AZURE_FEDERATED_TOKEN_FILE: zod.string().min(1), + OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS: zod.string().min(1), + }) + .superRefine((data, ctx) => { + try { + const content = readFileSync(data.AZURE_FEDERATED_TOKEN_FILE, 'utf-8').trim(); + if (!content) { + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + path: ['AZURE_FEDERATED_TOKEN_FILE'], + message: `Federated token file at path '${data.AZURE_FEDERATED_TOKEN_FILE}' is empty.`, + }); + } + } catch { + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + path: ['AZURE_FEDERATED_TOKEN_FILE'], + message: `Cannot read federated token file at path '${data.AZURE_FEDERATED_TOKEN_FILE}'. Ensure the file exists and is readable.`, + }); + } + + const ids = data.OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS.split(',').map(s => s.trim()); + const invalid = ids.filter(id => !isUUID(id)); + if (invalid.length > 0) { + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + path: ['OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS'], + message: `The following values are not valid UUIDs: ${invalid.join(', ')}`, + }); + } + }), +]); + const processEnv = process.env; const configs = { @@ -328,6 +374,7 @@ const configs = { zendeskSupport: ZendeskSupportModel.safeParse(processEnv), tracing: OpenTelemetryConfigurationModel.safeParse(processEnv), hivePersistedDocuments: HivePersistedDocumentsSchema.safeParse(processEnv), + oidcWorkloadFederation: OidcWorkloadFederationModel.safeParse(processEnv), }; const environmentErrors: Array = []; @@ -427,6 +474,7 @@ const s3AuditLog = extractConfig(configs.s3AuditLog); const zendeskSupport = extractConfig(configs.zendeskSupport); const tracing = extractConfig(configs.tracing); const hivePersistedDocuments = extractConfig(configs.hivePersistedDocuments); +const oidcWorkloadFederation = extractConfig(configs.oidcWorkloadFederation); const testUtils = extractConfig(configs.testUtils); const hiveUsageConfig = @@ -656,4 +704,14 @@ export const env = { /** Whether metric alert rules should be enabled cluster-wide. */ metricAlertRulesEnabled: base.FEATURE_FLAGS_METRIC_ALERT_RULES_ENABLED === '1', }, + oidcWorkloadFederation: + oidcWorkloadFederation.OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER === 'azure' + ? { + provider: 'azure' as const, + tokenFilePath: oidcWorkloadFederation.AZURE_FEDERATED_TOKEN_FILE, + organizationIds: oidcWorkloadFederation.OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS.split( + ',', + ).map(s => s.trim()), + } + : null, } as const; diff --git a/packages/services/server/src/index.ts b/packages/services/server/src/index.ts index dda94e6e083..2132d4a12ba 100644 --- a/packages/services/server/src/index.ts +++ b/packages/services/server/src/index.ts @@ -57,6 +57,7 @@ import { clickHouseElapsedDuration, clickHouseReadDuration } from './metrics'; import { createOtelAuthEndpoint } from './otel-auth-endpoint'; import { createPublicGraphQLHandler } from './public-graphql-handler'; import { registerSupertokensAtHome } from './supertokens-at-home'; +import { WorkloadIdentityFederationProvider } from './workload-identity-federation'; class CorsError extends Error { constructor() { @@ -172,6 +173,17 @@ export async function main() { ); const taskScheduler = new TaskScheduler(storage.pool); + const workloadIdentityFederation = env.oidcWorkloadFederation + ? new WorkloadIdentityFederationProvider( + env.oidcWorkloadFederation.tokenFilePath, + server.log.child({ source: 'WorkloadIdentityFederation' }), + ) + : null; + + if (workloadIdentityFederation) { + await workloadIdentityFederation.start(); + } + const redis = await createRedisClient(env.redis, { logger: server.log.child({ source: 'Redis' }), }); @@ -194,6 +206,7 @@ export async function main() { await server.close(); server.log.info('Stopping Storage handler...'); await storage.destroy(); + workloadIdentityFederation?.stop(); server.log.info('Shutdown complete.'); }, }); @@ -541,6 +554,7 @@ export async function main() { registry.injector.get(OAuthCache), broadcastLog, env.supertokens.secrets, + workloadIdentityFederation, ); if (env.cdn.providers.api !== null) { diff --git a/packages/services/server/src/supertokens-at-home.ts b/packages/services/server/src/supertokens-at-home.ts index 55c847fbe28..7412d09fa4b 100644 --- a/packages/services/server/src/supertokens-at-home.ts +++ b/packages/services/server/src/supertokens-at-home.ts @@ -25,6 +25,7 @@ import { TaskScheduler } from '@hive/workflows/kit'; import { PasswordResetTask } from '@hive/workflows/tasks/password-reset'; import { env } from './environment'; import { createNewSession, validatePassword } from './supertokens-at-home/shared'; +import type { WorkloadIdentityFederationProvider } from './workload-identity-federation'; type BroadcastOIDCIntegrationLog = (oidcOrganizationId: string, message: string) => void; @@ -43,6 +44,7 @@ export async function registerSupertokensAtHome( refreshTokenKey: string; accessTokenKey: string; }, + workloadIdentityFederation: WorkloadIdentityFederationProvider | null, ) { const supertokensStore = new SuperTokensStore(storage.pool, server.log); @@ -792,6 +794,13 @@ export async function registerSupertokensAtHome( const scopes = ['openid', 'email', ...oidcIntegration.additionalScopes]; + const useFederation = + workloadIdentityFederation !== null && + (env.oidcWorkloadFederation?.organizationIds.includes( + oidcIntegration.linkedOrganizationId, + ) ?? + false); + const oidClientConfig = new oidClient.Configuration( { issuer: 'noop', @@ -800,9 +809,11 @@ export async function registerSupertokensAtHome( token_endpoint: oidcIntegration.tokenEndpoint, }, oidcIntegration.clientId, - { - client_secret: crypto.decrypt(oidcIntegration.encryptedClientSecret), - }, + useFederation + ? undefined + : { + client_secret: crypto.decrypt(oidcIntegration.encryptedClientSecret), + }, ); oidClient.allowInsecureRequests(oidClientConfig); @@ -1304,20 +1315,48 @@ export async function registerSupertokensAtHome( access_token: z.string(), }); + const useFederationForExchange = + workloadIdentityFederation !== null && + (env.oidcWorkloadFederation?.organizationIds.includes( + oidcIntegration.linkedOrganizationId, + ) ?? + false); + + const grantParams: Record = { + grant_type: 'authorization_code', + code_verifier: cacheRecord.pkceVerifier, + code: parsedBody.data.redirectURIInfo.redirectURIQueryParams.code ?? '', + redirect_uri: current_url.toString(), + client_id: oidcIntegration.clientId, + }; + + if (useFederationForExchange) { + // Workload Identity Federation: use client_assertion with the cached federated token + const federatedToken = workloadIdentityFederation?.getToken() ?? null; + if (!federatedToken) { + req.log.error( + 'Workload identity federation is not configured or the token is not yet available', + ); + return rep.status(200).send({ + status: 'SIGN_IN_UP_NOT_ALLOWED', + reason: 'Sign in failed. Please contact your organization administrator.', + }); + } + + grantParams['client_assertion_type'] = + 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer'; + grantParams['client_assertion'] = federatedToken; + } else { + grantParams['client_secret'] = crypto.decrypt(oidcIntegration.encryptedClientSecret); + } + const grantResponse = await fetch(oidcIntegration.tokenEndpoint, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded', accept: 'application/json', }, - body: new URLSearchParams({ - grant_type: 'authorization_code', - code_verifier: cacheRecord.pkceVerifier, - code: parsedBody.data.redirectURIInfo.redirectURIQueryParams.code ?? '', - redirect_uri: current_url.toString(), - client_id: oidcIntegration.clientId, - client_secret: crypto.decrypt(oidcIntegration.encryptedClientSecret), - }), + body: new URLSearchParams(grantParams), }); if (grantResponse.status != 200) { diff --git a/packages/services/server/src/workload-identity-federation.ts b/packages/services/server/src/workload-identity-federation.ts new file mode 100644 index 00000000000..40311547d19 --- /dev/null +++ b/packages/services/server/src/workload-identity-federation.ts @@ -0,0 +1,57 @@ +import { readFile } from 'node:fs/promises'; +import type { FastifyBaseLogger } from 'fastify'; + +const DEFAULT_POLL_INTERVAL_MS = 60_000; + +/** + * Polls the workload identity federation token file at a regular interval and + * keeps the current token in memory so it can be used without a per-request + * file read. + * + * Supports Azure Workload Identity today; additional providers can be added in + * the future by extending the `provider` discriminant. + */ +export class WorkloadIdentityFederationProvider { + private currentToken: string | null = null; + private timer: ReturnType | null = null; + + constructor( + private readonly tokenFilePath: string, + private readonly logger: FastifyBaseLogger, + private readonly pollIntervalMs: number = DEFAULT_POLL_INTERVAL_MS, + ) {} + + /** Read the token file once and start the polling interval. */ + async start(): Promise { + await this.refresh(); + this.timer = setInterval(() => { + void this.refresh(); + }, this.pollIntervalMs); + // Prevent the interval from keeping the process alive on its own. + this.timer.unref(); + } + + stop(): void { + if (this.timer !== null) { + clearInterval(this.timer); + this.timer = null; + } + } + + /** Returns the most recently cached token, or null if not yet loaded. */ + getToken(): string | null { + return this.currentToken; + } + + private async refresh(): Promise { + try { + this.currentToken = (await readFile(this.tokenFilePath, 'utf-8')).trim(); + } catch (err) { + this.logger.error( + { err }, + 'Failed to read workload identity federation token file (path=%s)', + this.tokenFilePath, + ); + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ead7f8677a7..14d15cad047 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,12 +53,11 @@ overrides: brace-expansion@<1.1.16: ^1.1.16 '@sigstore/core@<=3.2.0': '>=3.2.1' js-yaml@<3.15.0: 3.15.0 - js-yaml@>=4.0.0 <4.3.0: 4.3.0 sigstore@<=4.1.0: '>=4.1.1' tar@<=7.5.18: 7.5.21 fast-uri@2.x.x: 3.1.4 fast-uri@3.x.x: 3.1.4 - '@opentelemetry/propagator-jaeger@2.x.x': 2.9.0 + '@opentelemetry/propagator-jaeger@<2.9.0': 2.9.0 sharp@<0.35.0: 0.35.0 body-parser@1.x.x: 1.20.6 @@ -7113,12 +7112,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-jaeger@1.30.0': - resolution: {integrity: sha512-0hdP495V6HPRkVpowt54+Swn5NdesMIRof+rlp0mbnuIUOM986uF+eNxnPo9q5MmJegVBRTxgMHXXwvnXRnKRg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-jaeger@2.9.0': resolution: {integrity: sha512-4mYGty27rYvSM0jtp1ZUOqd3LfVRCYg9H5G9OFzSx5HViYToU21MFhWfco7x1HwXr7ER8yGOiCIHZUwjPksc0Q==} engines: {node: ^18.19.0 || >=20.6.0} @@ -13597,8 +13590,8 @@ packages: resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==} engines: {node: '>=8'} - find-my-way@9.5.0: - resolution: {integrity: sha512-VW2RfnmscZO5KgBY5XVyKREMW5nMZcxDy+buTOsL+zIPnBlbKm+00sgzoQzq1EVh4aALZLfKdwv6atBGcjvjrQ==} + find-my-way@9.7.0: + resolution: {integrity: sha512-f2JHn75x2JlwUwLenZypgczR7YWMb/uO9BvUXtus+JMgkbIkLADd38cI4EiV+OQqrGo1Zlq6V8wnqMJ8e62wUQ==} engines: {node: '>=20'} find-root@1.1.0: @@ -22151,7 +22144,7 @@ snapshots: dependencies: '@fastify/error': 4.2.0 fastify-plugin: 5.1.0 - find-my-way: 9.5.0 + find-my-way: 9.7.0 path-to-regexp: 8.4.0 reusify: 1.0.4 @@ -26667,6 +26660,11 @@ snapshots: '@opentelemetry/api': 1.9.1 '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/core@2.9.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/core@2.9.0(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -27732,10 +27730,10 @@ snapshots: '@opentelemetry/api': 1.9.1 '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-jaeger@1.30.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/propagator-jaeger@2.9.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 1.30.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.0) '@opentelemetry/propagator-jaeger@2.9.0(@opentelemetry/api@1.9.1)': dependencies: @@ -28007,7 +28005,7 @@ snapshots: '@opentelemetry/context-async-hooks': 1.30.0(@opentelemetry/api@1.9.0) '@opentelemetry/core': 1.30.0(@opentelemetry/api@1.9.0) '@opentelemetry/propagator-b3': 1.30.0(@opentelemetry/api@1.9.0) - '@opentelemetry/propagator-jaeger': 1.30.0(@opentelemetry/api@1.9.0) + '@opentelemetry/propagator-jaeger': 2.9.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.30.0(@opentelemetry/api@1.9.0) semver: 7.7.2 @@ -35167,7 +35165,7 @@ snapshots: abstract-logging: 2.0.1 avvio: 9.1.0 fast-json-stringify: 6.2.0 - find-my-way: 9.5.0 + find-my-way: 9.7.0 light-my-request: 6.6.0 pino: 10.3.0 process-warning: 5.0.0 @@ -35260,7 +35258,7 @@ snapshots: make-dir: 3.1.0 pkg-dir: 4.2.0 - find-my-way@9.5.0: + find-my-way@9.7.0: dependencies: fast-deep-equal: 3.1.3 fast-querystring: 1.1.2