From d94ee0c9ad3b0ba3c794b02e399a9e2a5b91433e Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Apr 2026 08:17:44 +0300 Subject: [PATCH 01/14] feat: add Azure Federated Workload Identity support for OIDC SSO login (#2) * feat: add Azure Federated Identity (OAuthBearer) support for Kafka/EventHub Add support for the `oauthbearer` SASL mechanism in both the usage and usage-ingestor services, enabling Azure Workload Identity authentication to Event Hub when deployed in Kubernetes. Changes: - Add `@azure/identity` dependency to usage and usage-ingestor packages - Add `oauthbearer` SASL mechanism to environment config (Zod schemas) - Create `oauth-bearer-provider.ts` using DefaultAzureCredential - Update Kafka client initialization to support oauthbearer mechanism - Update Pulumi deployment to conditionally configure oauthbearer Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/b3983d22-e4bf-4a52-90f1-c93895adeff8 Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> * refactor: improve OAuth bearer provider error handling and documentation Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/b3983d22-e4bf-4a52-90f1-c93895adeff8 Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> * revert: remove incorrect Kafka OAuthBearer changes Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/755ee5ba-e92c-4df8-b742-e04e182b5a13 Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> * feat: add Azure Federated Workload Identity support for OIDC SSO login When deployed in Kubernetes with Azure Workload Identity, the OIDC integration can now authenticate to the identity provider using a federated token (client_assertion) instead of a client_secret. Changes: - DB migration: add use_federated_credential column, make client_secret nullable - Entity/model: update OIDCIntegration to include useFederatedCredential - GraphQL: make clientSecret optional in create input, add useFederatedCredential field - OIDC provider: allow creation without client secret when federated - Auth server: use client_assertion with AZURE_FEDERATED_TOKEN_FILE - UI: show federated credential status, allow saving without client secret Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/755ee5ba-e92c-4df8-b742-e04e182b5a13 Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> * fix: use strict equality for null check in OIDC provider Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/755ee5ba-e92c-4df8-b742-e04e182b5a13 Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> --- ...4.09T00-00-00.oidc-federated-credential.ts | 12 +++++ packages/migrations/src/run-pg-migrations.ts | 1 + .../oidc-integrations/module.graphql.ts | 20 ++++++- .../providers/oidc-integrations.provider.ts | 32 +++++++++-- .../Mutation/createOIDCIntegration.ts | 3 +- .../Mutation/updateOIDCIntegration.ts | 1 + .../resolvers/OIDCIntegration.ts | 1 + .../src/modules/shared/providers/storage.ts | 4 +- packages/services/api/src/shared/entities.ts | 3 +- .../server/src/supertokens-at-home.ts | 53 +++++++++++++++---- packages/services/storage/src/index.ts | 13 +++-- .../connect-single-sign-on-provider-sheet.tsx | 6 +-- .../oidc-integration-configuration.tsx | 12 ++++- .../single-sign-on/single-sign-on-subpage.tsx | 2 +- 14 files changed, 135 insertions(+), 28 deletions(-) create mode 100644 packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts diff --git a/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts b/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts new file mode 100644 index 00000000000..81b26af75fe --- /dev/null +++ b/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts @@ -0,0 +1,12 @@ +import { type MigrationExecutor } from '../pg-migrator'; + +export default { + name: '2026.04.09T00-00-00.oidc-federated-credential.sql', + run: ({ psql }) => psql` + ALTER TABLE "oidc_integrations" + ADD COLUMN IF NOT EXISTS "use_federated_credential" BOOLEAN NOT NULL DEFAULT FALSE; + + ALTER TABLE "oidc_integrations" + ALTER COLUMN "client_secret" DROP NOT NULL; + `, +} satisfies MigrationExecutor; diff --git a/packages/migrations/src/run-pg-migrations.ts b/packages/migrations/src/run-pg-migrations.ts index 7e8d35806d5..129fbb10ece 100644 --- a/packages/migrations/src/run-pg-migrations.ts +++ b/packages/migrations/src/run-pg-migrations.ts @@ -184,5 +184,6 @@ export const runPGMigrations = async (args: { slonik: PostgresDatabasePool; runT await import('./actions/2026.02.24T00-00-00.proposal-composition'), await import('./actions/2026.02.25T00-00-00.oidc-integration-domains'), await import('./actions/2026.03.25T00-00-00.access-token-expiration'), + await import('./actions/2026.04.09T00-00-00.oidc-federated-credential'), ], }); diff --git a/packages/services/api/src/modules/oidc-integrations/module.graphql.ts b/packages/services/api/src/modules/oidc-integrations/module.graphql.ts index 4fa9f15e5ee..a50deb26b00 100644 --- a/packages/services/api/src/modules/oidc-integrations/module.graphql.ts +++ b/packages/services/api/src/modules/oidc-integrations/module.graphql.ts @@ -13,7 +13,7 @@ export default gql` type OIDCIntegration { id: ID! clientId: ID! - clientSecretPreview: String! + clientSecretPreview: String tokenEndpoint: String! userinfoEndpoint: String! authorizationEndpoint: String! @@ -24,6 +24,11 @@ export default gql` defaultMemberRole: MemberRole! defaultResourceAssignment: ResourceAssignment """ + Whether this OIDC integration uses a federated credential (e.g. Azure Workload Identity) + instead of a client secret for authenticating to the identity provider. + """ + useFederatedCredential: Boolean! + """ List of domains registered with this OIDC integration. """ registeredDomains: [OIDCIntegrationDomain!]! @@ -188,11 +193,17 @@ export default gql` input CreateOIDCIntegrationInput { organizationId: ID! clientId: ID! - clientSecret: String! + clientSecret: String tokenEndpoint: String! userinfoEndpoint: String! authorizationEndpoint: String! additionalScopes: [String!]! + """ + When true, the OIDC integration will use Azure Workload Identity (federated credential) + instead of a client secret for authenticating to the identity provider during token exchange. + The pod must have Azure Workload Identity configured with the appropriate federated token file. + """ + useFederatedCredential: Boolean } type CreateOIDCIntegrationResult { @@ -227,6 +238,11 @@ export default gql` userinfoEndpoint: String authorizationEndpoint: String additionalScopes: [String!] + """ + When true, the OIDC integration will use Azure Workload Identity (federated credential) + instead of a client secret for authenticating to the identity provider during token exchange. + """ + useFederatedCredential: Boolean } type UpdateOIDCIntegrationResult { diff --git a/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts b/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts index 928a50b6d73..27c02d0467e 100644 --- a/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts +++ b/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts @@ -97,6 +97,9 @@ export class OIDCIntegrationsProvider { } async getClientSecretPreview(integration: OIDCIntegration) { + if (integration.encryptedClientSecret === null) { + return null; + } const decryptedSecret = this.crypto.decrypt(integration.encryptedClientSecret); return decryptedSecret.substring(decryptedSecret.length - 4); } @@ -104,11 +107,12 @@ export class OIDCIntegrationsProvider { async createOIDCIntegrationForOrganization(args: { organizationId: string; clientId: string; - clientSecret: string; + clientSecret: string | null; tokenEndpoint: string; userinfoEndpoint: string; authorizationEndpoint: string; additionalScopes: readonly string[]; + useFederatedCredential: boolean; }) { if (this.isEnabled() === false) { return { @@ -133,8 +137,26 @@ export class OIDCIntegrationsProvider { throw new Error(`Failed to locate organization ${args.organizationId}`); } + // When using federated credentials, client secret is not required + if (!args.useFederatedCredential && !args.clientSecret) { + return { + type: 'error', + reason: null, + fieldErrors: { + clientId: null, + clientSecret: 'Client secret is required when not using federated credentials.', + tokenEndpoint: null, + userinfoEndpoint: null, + authorizationEndpoint: null, + additionalScopes: null, + }, + } as const; + } + const clientIdResult = OIDCIntegrationClientIdModel.safeParse(args.clientId); - const clientSecretResult = OIDCClientSecretModel.safeParse(args.clientSecret); + const clientSecretResult = args.useFederatedCredential + ? { success: true as const, data: null } + : OIDCClientSecretModel.safeParse(args.clientSecret); const tokenEndpointResult = OAuthAPIUrlModel.safeParse(args.tokenEndpoint); const userinfoEndpointResult = OAuthAPIUrlModel.safeParse(args.userinfoEndpoint); const authorizationEndpointResult = OAuthAPIUrlModel.safeParse(args.authorizationEndpoint); @@ -151,11 +173,13 @@ export class OIDCIntegrationsProvider { const creationResult = await this.storage.createOIDCIntegrationForOrganization({ organizationId: args.organizationId, clientId: clientIdResult.data, - encryptedClientSecret: this.crypto.encrypt(clientSecretResult.data), + encryptedClientSecret: + clientSecretResult.data !== null ? this.crypto.encrypt(clientSecretResult.data) : null, tokenEndpoint: tokenEndpointResult.data, userinfoEndpoint: userinfoEndpointResult.data, authorizationEndpoint: authorizationEndpointResult.data, additionalScopes: additionalScopesResult.data, + useFederatedCredential: args.useFederatedCredential, }); if (creationResult.type === 'ok') { @@ -216,6 +240,7 @@ export class OIDCIntegrationsProvider { userinfoEndpoint: string | null; authorizationEndpoint: string | null; additionalScopes: readonly string[] | null; + useFederatedCredential: boolean | null; }) { if (this.isEnabled() === false) { return { @@ -277,6 +302,7 @@ export class OIDCIntegrationsProvider { userinfoEndpoint: userinfoEndpointResult.data, authorizationEndpoint: authorizationEndpointResult.data, additionalScopes: additionalScopesResult.data, + useFederatedCredential: args.useFederatedCredential, }); const redactedClientSecret = maskToken(oidcIntegration.clientId); diff --git a/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/createOIDCIntegration.ts b/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/createOIDCIntegration.ts index a99d30611bb..43530d1da7a 100644 --- a/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/createOIDCIntegration.ts +++ b/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/createOIDCIntegration.ts @@ -9,11 +9,12 @@ export const createOIDCIntegration: NonNullable< const result = await oktaIntegrationsProvider.createOIDCIntegrationForOrganization({ organizationId: input.organizationId, clientId: input.clientId, - clientSecret: input.clientSecret, + clientSecret: input.clientSecret ?? null, tokenEndpoint: input.tokenEndpoint, userinfoEndpoint: input.userinfoEndpoint, authorizationEndpoint: input.authorizationEndpoint, additionalScopes: input.additionalScopes, + useFederatedCredential: input.useFederatedCredential ?? false, }); if (result.type === 'ok') { diff --git a/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/updateOIDCIntegration.ts b/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/updateOIDCIntegration.ts index d89bee398c8..b8dad5eab9e 100644 --- a/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/updateOIDCIntegration.ts +++ b/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/updateOIDCIntegration.ts @@ -13,6 +13,7 @@ export const updateOIDCIntegration: NonNullable< userinfoEndpoint: input.userinfoEndpoint ?? null, authorizationEndpoint: input.authorizationEndpoint ?? null, additionalScopes: input.additionalScopes ?? null, + useFederatedCredential: input.useFederatedCredential ?? null, }); if (result.type === 'ok') { diff --git a/packages/services/api/src/modules/oidc-integrations/resolvers/OIDCIntegration.ts b/packages/services/api/src/modules/oidc-integrations/resolvers/OIDCIntegration.ts index 10909b3002f..ca0312e17bc 100644 --- a/packages/services/api/src/modules/oidc-integrations/resolvers/OIDCIntegration.ts +++ b/packages/services/api/src/modules/oidc-integrations/resolvers/OIDCIntegration.ts @@ -11,6 +11,7 @@ export const OIDCIntegration: OidcIntegrationResolvers = { clientId: oidcIntegration => oidcIntegration.clientId, clientSecretPreview: (oidcIntegration, _, { injector }) => injector.get(OIDCIntegrationsProvider).getClientSecretPreview(oidcIntegration), + useFederatedCredential: oidcIntegration => oidcIntegration.useFederatedCredential, /** * Fallbacks to Viewer if default member role is not set */ diff --git a/packages/services/api/src/modules/shared/providers/storage.ts b/packages/services/api/src/modules/shared/providers/storage.ts index 9382a81e748..5ade74b29ea 100644 --- a/packages/services/api/src/modules/shared/providers/storage.ts +++ b/packages/services/api/src/modules/shared/providers/storage.ts @@ -634,11 +634,12 @@ export interface Storage { createOIDCIntegrationForOrganization(_: { organizationId: string; clientId: string; - encryptedClientSecret: string; + encryptedClientSecret: string | null; tokenEndpoint: string; userinfoEndpoint: string; authorizationEndpoint: string; additionalScopes: readonly string[]; + useFederatedCredential: boolean; }): Promise<{ type: 'ok'; oidcIntegration: OIDCIntegration } | { type: 'error'; reason: string }>; updateOIDCIntegration(_: { @@ -649,6 +650,7 @@ export interface Storage { userinfoEndpoint: string | null; authorizationEndpoint: string | null; additionalScopes: readonly string[] | null; + useFederatedCredential: boolean | null; }): Promise; deleteOIDCIntegration(_: { oidcIntegrationId: string }): Promise; diff --git a/packages/services/api/src/shared/entities.ts b/packages/services/api/src/shared/entities.ts index 3b1de843912..0b8c67a0b36 100644 --- a/packages/services/api/src/shared/entities.ts +++ b/packages/services/api/src/shared/entities.ts @@ -183,7 +183,7 @@ export interface OIDCIntegration { id: string; linkedOrganizationId: string; clientId: string; - encryptedClientSecret: string; + encryptedClientSecret: string | null; tokenEndpoint: string; userinfoEndpoint: string; authorizationEndpoint: string; @@ -193,6 +193,7 @@ export interface OIDCIntegration { requireInvitation: boolean; defaultMemberRoleId: string | null; defaultResourceAssignment: ResourceAssignmentGroup | null; + useFederatedCredential: boolean; } export interface CDNAccessToken { diff --git a/packages/services/server/src/supertokens-at-home.ts b/packages/services/server/src/supertokens-at-home.ts index 1d039a125d2..5778567ea30 100644 --- a/packages/services/server/src/supertokens-at-home.ts +++ b/packages/services/server/src/supertokens-at-home.ts @@ -800,9 +800,11 @@ export async function registerSupertokensAtHome( token_endpoint: oidcIntegration.tokenEndpoint, }, oidcIntegration.clientId, - { - client_secret: crypto.decrypt(oidcIntegration.encryptedClientSecret), - }, + oidcIntegration.useFederatedCredential || !oidcIntegration.encryptedClientSecret + ? undefined + : { + client_secret: crypto.decrypt(oidcIntegration.encryptedClientSecret), + }, ); oidClient.allowInsecureRequests(oidClientConfig); @@ -1304,20 +1306,49 @@ export async function registerSupertokensAtHome( access_token: z.string(), }); + const grantParams: Record = { + grant_type: 'authorization_code', + code_verifier: cacheRecord.pkceVerifier, + code: parsedBody.data.redirectURIInfo.redirectURIQueryParams.code ?? '', + redirect_uri: current_url.toString(), + client_id: oidcIntegration.clientId, + }; + + if ( + oidcIntegration.useFederatedCredential || + !oidcIntegration.encryptedClientSecret + ) { + // Azure Workload Identity: use client_assertion with the federated token file + const tokenFilePath = process.env['AZURE_FEDERATED_TOKEN_FILE']; + if (!tokenFilePath) { + req.log.error('AZURE_FEDERATED_TOKEN_FILE environment variable is not set'); + broadcastLog( + oidcIntegration.id, + 'Federated credential is configured but AZURE_FEDERATED_TOKEN_FILE environment variable is not set. ' + + 'Ensure the pod has Azure Workload Identity configured.', + ); + return rep.status(200).send({ + status: 'SIGN_IN_UP_NOT_ALLOWED', + reason: 'Sign in failed. Please contact your organization administrator.', + }); + } + + const { readFileSync } = await import('node:fs'); + const federatedToken = readFileSync(tokenFilePath, 'utf-8'); + grantParams['client_assertion_type'] = + 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer'; + grantParams['client_assertion'] = federatedToken; + } else { + grantParams['client_secret'] = crypto.decrypt(oidcIntegration.encryptedClientSecret); + } + const grantResponse = await fetch(oidcIntegration.tokenEndpoint, { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded', accept: 'application/json', }, - body: new URLSearchParams({ - grant_type: 'authorization_code', - code_verifier: cacheRecord.pkceVerifier, - code: parsedBody.data.redirectURIInfo.redirectURIQueryParams.code ?? '', - redirect_uri: current_url.toString(), - client_id: oidcIntegration.clientId, - client_secret: crypto.decrypt(oidcIntegration.encryptedClientSecret), - }), + body: new URLSearchParams(grantParams), }); if (grantResponse.status != 200) { diff --git a/packages/services/storage/src/index.ts b/packages/services/storage/src/index.ts index 5659d54180c..f7145d4aba4 100644 --- a/packages/services/storage/src/index.ts +++ b/packages/services/storage/src/index.ts @@ -2963,7 +2963,8 @@ export async function createStorage( "token_endpoint", "userinfo_endpoint", "authorization_endpoint", - "additional_scopes" + "additional_scopes", + "use_federated_credential" ) VALUES ( ${args.organizationId}, @@ -2972,7 +2973,8 @@ export async function createStorage( ${args.tokenEndpoint}, ${args.userinfoEndpoint}, ${args.authorizationEndpoint}, - ${psql.array(args.additionalScopes, 'text')} + ${psql.array(args.additionalScopes, 'text')}, + ${args.useFederatedCredential} ) RETURNING ${oidcIntegrationFields()} @@ -3023,6 +3025,7 @@ export async function createStorage( } , "additional_scopes" = ${args.additionalScopes ? psql.array(args.additionalScopes, 'text') : psql`"additional_scopes"`} , "oauth_api_url" = NULL + , "use_federated_credential" = ${args.useFederatedCredential ?? psql`"use_federated_credential"`} WHERE "id" = ${args.oidcIntegrationId} RETURNING @@ -4659,7 +4662,7 @@ const OIDCIntegrationBaseModel = z.object({ id: z.string(), linkedOrganizationId: z.string(), clientId: z.string(), - clientSecret: z.string(), + clientSecret: z.string().nullable(), additionalScopes: z .array(z.string()) .nullable() @@ -4669,6 +4672,7 @@ const OIDCIntegrationBaseModel = z.object({ defaultRoleId: z.string().nullable(), defaultAssignedResources: z.any().nullable(), requireInvitation: z.boolean(), + useFederatedCredential: z.boolean(), }); const OIDCIntegrationLegacyModel = OIDCIntegrationBaseModel.extend({ @@ -4687,6 +4691,7 @@ const OIDCIntegrationLegacyModel = OIDCIntegrationBaseModel.extend({ requireInvitation: record.requireInvitation, defaultMemberRoleId: record.defaultRoleId, defaultResourceAssignment: record.defaultAssignedResources, + useFederatedCredential: record.useFederatedCredential, })); const LatestOIDCIntegrationModel = OIDCIntegrationBaseModel.extend({ @@ -4708,6 +4713,7 @@ const LatestOIDCIntegrationModel = OIDCIntegrationBaseModel.extend({ requireInvitation: record.requireInvitation, defaultMemberRoleId: record.defaultRoleId, defaultResourceAssignment: record.defaultAssignedResources, + useFederatedCredential: record.useFederatedCredential, })); const OIDCIntegrationModel = z.union([OIDCIntegrationLegacyModel, LatestOIDCIntegrationModel]); @@ -5470,6 +5476,7 @@ const oidcIntegrationFields = (prefix: TaggedTemplateLiteralInvocation = psql``) , ${prefix}"default_role_id" AS "defaultRoleId" , ${prefix}"default_assigned_resources" AS "defaultAssignedResources" , ${prefix}"require_invitation" AS "requireInvitation" + , ${prefix}"use_federated_credential" AS "useFederatedCredential" `; const SchemaLogBase = z.object({ diff --git a/packages/web/app/src/components/organization/settings/single-sign-on/connect-single-sign-on-provider-sheet.tsx b/packages/web/app/src/components/organization/settings/single-sign-on/connect-single-sign-on-provider-sheet.tsx index 621555e48d6..b4b924f59c0 100644 --- a/packages/web/app/src/components/organization/settings/single-sign-on/connect-single-sign-on-provider-sheet.tsx +++ b/packages/web/app/src/components/organization/settings/single-sign-on/connect-single-sign-on-provider-sheet.tsx @@ -25,7 +25,7 @@ type ConnectSingleSignOnProviderSheetProps = { tokenEndpoint: string; userinfoEndpoint: string; clientId: string; - clientSecretPreview: string; + clientSecretPreview: string | null; additionalScopes: string; }; onSave: (args: { @@ -210,8 +210,8 @@ export function ConnectSingleSignOnProviderSheet( Client Secret - •••••••{oidcIntegration.clientSecretPreview} + {oidcIntegration.useFederatedCredential ? ( + Using Federated Credential + ) : oidcIntegration.clientSecretPreview ? ( + <>•••••••{oidcIntegration.clientSecretPreview} + ) : ( + Not Set + )} @@ -413,7 +421,7 @@ export function OIDCIntegrationConfiguration(props: { authorizationEndpoint: oidcIntegration.authorizationEndpoint, tokenEndpoint: oidcIntegration.tokenEndpoint, userinfoEndpoint: oidcIntegration.userinfoEndpoint, - clientSecretPreview: oidcIntegration.clientSecretPreview, + clientSecretPreview: oidcIntegration.clientSecretPreview ?? null, }} onSave={async args => { const result = await updateOIDCIntegrationMutate({ diff --git a/packages/web/app/src/components/organization/settings/single-sign-on/single-sign-on-subpage.tsx b/packages/web/app/src/components/organization/settings/single-sign-on/single-sign-on-subpage.tsx index 27440d1a1a8..92949528c63 100644 --- a/packages/web/app/src/components/organization/settings/single-sign-on/single-sign-on-subpage.tsx +++ b/packages/web/app/src/components/organization/settings/single-sign-on/single-sign-on-subpage.tsx @@ -124,7 +124,7 @@ export function SingleSignOnSubpage(props: SingleSignOnSubPageProps): React.Reac input: { organizationId: organization?.id ?? '', clientId: values.clientId, - clientSecret: values.clientSecret ?? '', + clientSecret: values.clientSecret || null, authorizationEndpoint: values.authorizationEndpoint, tokenEndpoint: values.tokenEndpoint, userinfoEndpoint: values.userinfoEndpoint, From c74478092785fd43cf1cf1589e998881740eef2a Mon Sep 17 00:00:00 2001 From: Cosmin Catalin SANDA Date: Sat, 11 Apr 2026 16:55:52 +0300 Subject: [PATCH 02/14] Update supertokens-at-home.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- packages/services/server/src/supertokens-at-home.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/services/server/src/supertokens-at-home.ts b/packages/services/server/src/supertokens-at-home.ts index 5778567ea30..6353594eeda 100644 --- a/packages/services/server/src/supertokens-at-home.ts +++ b/packages/services/server/src/supertokens-at-home.ts @@ -1333,8 +1333,8 @@ export async function registerSupertokensAtHome( }); } - const { readFileSync } = await import('node:fs'); - const federatedToken = readFileSync(tokenFilePath, 'utf-8'); + const { readFile } = await import("node:fs/promises"); + const federatedToken = await readFile(tokenFilePath, "utf-8"); grantParams['client_assertion_type'] = 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer'; grantParams['client_assertion'] = federatedToken; From 74c80b30e53f2cab354017a39d5ab9b99129bc76 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 14:53:53 +0200 Subject: [PATCH 03/14] fix: define AZURE_FEDERATED_TOKEN_FILE in environment.ts instead of direct process.env access (#3) Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/8b554117-9357-40a5-9879-83f5ac168987 Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> --- packages/services/server/src/environment.ts | 2 ++ packages/services/server/src/supertokens-at-home.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/services/server/src/environment.ts b/packages/services/server/src/environment.ts index 34db280e825..d2a915e9817 100644 --- a/packages/services/server/src/environment.ts +++ b/packages/services/server/src/environment.ts @@ -49,6 +49,7 @@ const EnvironmentModel = zod.object({ FEATURE_FLAGS_OTEL_TRACING_ENABLED: emptyString( zod.union([zod.literal('1'), zod.literal('0')]).optional(), ), + AZURE_FEDERATED_TOKEN_FILE: emptyString(zod.string().optional()), }); const CommerceModel = zod.object({ @@ -576,4 +577,5 @@ export const env = { /** Whether OTEL tracing should be enabled for all organizations. */ otelTracingEnabled: base.FEATURE_FLAGS_OTEL_TRACING_ENABLED === '1', }, + azureFederatedTokenFile: base.AZURE_FEDERATED_TOKEN_FILE ?? null, } as const; diff --git a/packages/services/server/src/supertokens-at-home.ts b/packages/services/server/src/supertokens-at-home.ts index 6353594eeda..76e543da11e 100644 --- a/packages/services/server/src/supertokens-at-home.ts +++ b/packages/services/server/src/supertokens-at-home.ts @@ -1319,7 +1319,7 @@ export async function registerSupertokensAtHome( !oidcIntegration.encryptedClientSecret ) { // Azure Workload Identity: use client_assertion with the federated token file - const tokenFilePath = process.env['AZURE_FEDERATED_TOKEN_FILE']; + const tokenFilePath = env.azureFederatedTokenFile; if (!tokenFilePath) { req.log.error('AZURE_FEDERATED_TOKEN_FILE environment variable is not set'); broadcastLog( From 5f3b49dc4f28a41c8d1b37f754f3dd5c3faa67ff Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 14:55:43 +0200 Subject: [PATCH 04/14] Validate auth method invariant in updateOIDCIntegration (#4) * Add validation in updateOIDCIntegration to ensure valid auth method after update Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/3b791a1d-2b1b-4152-af23-11dcd32bc977 Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> * Improve error message for missing auth method validation Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/3b791a1d-2b1b-4152-af23-11dcd32bc977 Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> --- .../providers/oidc-integrations.provider.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts b/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts index 27c02d0467e..668ed40b17c 100644 --- a/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts +++ b/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts @@ -292,6 +292,28 @@ export class OIDCIntegrationsProvider { authorizationEndpointResult.success && additionalScopesResult.success ) { + const effectiveUseFederatedCredential = + args.useFederatedCredential ?? integration.useFederatedCredential; + const effectiveHasClientSecret = clientSecretResult.data + ? true + : integration.encryptedClientSecret !== null; + + if (!effectiveUseFederatedCredential && !effectiveHasClientSecret) { + return { + type: 'error', + message: "Couldn't update integration.", + fieldErrors: { + clientId: null, + clientSecret: + 'Either a client secret or federated credential must be enabled.', + tokenEndpoint: null, + userinfoEndpoint: null, + authorizationEndpoint: null, + additionalScopes: null, + }, + } as const; + } + const oidcIntegration = await this.storage.updateOIDCIntegration({ oidcIntegrationId: args.oidcIntegrationId, clientId: clientIdResult.data, From bebbbb6b6082e174f3b79096ee6128cf08fff871 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 7 May 2026 08:12:12 +0200 Subject: [PATCH 05/14] refactor(oidc): introduce WorkloadIdentityFederationProvider for federated token management (#5) * refactor: move federated token reading to WorkloadIdentityFederationProvider with polling Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/75f0bc25-7acb-495c-bd74-6767a633f1d3 Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> * fix: trim whitespace from federated token file content Agent-Logs-Url: https://github.com/cosmincatalin/console/sessions/75f0bc25-7acb-495c-bd74-6767a633f1d3 Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: cosmincatalin <525590+cosmincatalin@users.noreply.github.com> --- ...4.09T00-00-00.oidc-federated-credential.ts | 4 +- packages/services/server/src/environment.ts | 38 ++++++++++++- packages/services/server/src/index.ts | 14 +++++ .../server/src/supertokens-at-home.ts | 17 +++--- .../src/workload-identity-federation.ts | 57 +++++++++++++++++++ 5 files changed, 115 insertions(+), 15 deletions(-) create mode 100644 packages/services/server/src/workload-identity-federation.ts diff --git a/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts b/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts index 81b26af75fe..04d04398279 100644 --- a/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts +++ b/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts @@ -4,9 +4,7 @@ export default { name: '2026.04.09T00-00-00.oidc-federated-credential.sql', run: ({ psql }) => psql` ALTER TABLE "oidc_integrations" - ADD COLUMN IF NOT EXISTS "use_federated_credential" BOOLEAN NOT NULL DEFAULT FALSE; - - ALTER TABLE "oidc_integrations" + ADD COLUMN IF NOT EXISTS "use_federated_credential" BOOLEAN NOT NULL DEFAULT FALSE, ALTER COLUMN "client_secret" DROP NOT NULL; `, } satisfies MigrationExecutor; diff --git a/packages/services/server/src/environment.ts b/packages/services/server/src/environment.ts index d2a915e9817..555d4fa2b66 100644 --- a/packages/services/server/src/environment.ts +++ b/packages/services/server/src/environment.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs'; import zod from 'zod'; import { OpenTelemetryConfigurationModel } from '@hive/service-common'; @@ -49,7 +50,6 @@ const EnvironmentModel = zod.object({ FEATURE_FLAGS_OTEL_TRACING_ENABLED: emptyString( zod.union([zod.literal('1'), zod.literal('0')]).optional(), ), - AZURE_FEDERATED_TOKEN_FILE: emptyString(zod.string().optional()), }); const CommerceModel = zod.object({ @@ -285,6 +285,32 @@ const LogModel = zod.object({ ), }); +const OidcWorkloadFederationModel = zod.union([ + zod.object({ + OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER: zod.union([ + zod.void(), + zod.literal(''), + zod.literal('0'), + ]), + }), + zod + .object({ + OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER: zod.literal('azure'), + AZURE_FEDERATED_TOKEN_FILE: zod.string().min(1), + }) + .superRefine((data, ctx) => { + try { + readFileSync(data.AZURE_FEDERATED_TOKEN_FILE, 'utf-8'); + } catch { + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + path: ['AZURE_FEDERATED_TOKEN_FILE'], + message: `Cannot read federated token file at path '${data.AZURE_FEDERATED_TOKEN_FILE}'. Ensure the file exists and is readable.`, + }); + } + }), +]); + const processEnv = process.env; const configs = { @@ -311,6 +337,7 @@ const configs = { zendeskSupport: ZendeskSupportModel.safeParse(processEnv), tracing: OpenTelemetryConfigurationModel.safeParse(processEnv), hivePersistedDocuments: HivePersistedDocumentsSchema.safeParse(processEnv), + oidcWorkloadFederation: OidcWorkloadFederationModel.safeParse(processEnv), }; const environmentErrors: Array = []; @@ -357,6 +384,7 @@ const s3AuditLog = extractConfig(configs.s3AuditLog); const zendeskSupport = extractConfig(configs.zendeskSupport); const tracing = extractConfig(configs.tracing); const hivePersistedDocuments = extractConfig(configs.hivePersistedDocuments); +const oidcWorkloadFederation = extractConfig(configs.oidcWorkloadFederation); const hiveUsageConfig = hive.HIVE_USAGE === '1' @@ -577,5 +605,11 @@ export const env = { /** Whether OTEL tracing should be enabled for all organizations. */ otelTracingEnabled: base.FEATURE_FLAGS_OTEL_TRACING_ENABLED === '1', }, - azureFederatedTokenFile: base.AZURE_FEDERATED_TOKEN_FILE ?? null, + oidcWorkloadFederation: + oidcWorkloadFederation.OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER === 'azure' + ? { + provider: 'azure' as const, + tokenFilePath: oidcWorkloadFederation.AZURE_FEDERATED_TOKEN_FILE, + } + : null, } as const; diff --git a/packages/services/server/src/index.ts b/packages/services/server/src/index.ts index c1d430915a3..57af821f2d6 100644 --- a/packages/services/server/src/index.ts +++ b/packages/services/server/src/index.ts @@ -56,6 +56,7 @@ import { clickHouseElapsedDuration, clickHouseReadDuration } from './metrics'; import { createOtelAuthEndpoint } from './otel-auth-endpoint'; import { createPublicGraphQLHandler } from './public-graphql-handler'; import { registerSupertokensAtHome } from './supertokens-at-home'; +import { WorkloadIdentityFederationProvider } from './workload-identity-federation'; class CorsError extends Error { constructor() { @@ -171,6 +172,17 @@ export async function main() { ); const taskScheduler = new TaskScheduler(storage.pool); + const workloadIdentityFederation = env.oidcWorkloadFederation + ? new WorkloadIdentityFederationProvider( + env.oidcWorkloadFederation.tokenFilePath, + server.log.child({ source: 'WorkloadIdentityFederation' }), + ) + : null; + + if (workloadIdentityFederation) { + await workloadIdentityFederation.start(); + } + const redis = createRedisClient('Redis', env.redis, server.log.child({ source: 'Redis' })); const pubSub = createHivePubSub({ @@ -191,6 +203,7 @@ export async function main() { await server.close(); server.log.info('Stopping Storage handler...'); await storage.destroy(); + workloadIdentityFederation?.stop(); server.log.info('Shutdown complete.'); }, }); @@ -528,6 +541,7 @@ export async function main() { registry.injector.get(OAuthCache), broadcastLog, env.supertokens.secrets, + workloadIdentityFederation, ); if (env.cdn.providers.api !== null) { diff --git a/packages/services/server/src/supertokens-at-home.ts b/packages/services/server/src/supertokens-at-home.ts index 76e543da11e..77a71ed33b2 100644 --- a/packages/services/server/src/supertokens-at-home.ts +++ b/packages/services/server/src/supertokens-at-home.ts @@ -25,6 +25,7 @@ import { TaskScheduler } from '@hive/workflows/kit'; import { PasswordResetTask } from '@hive/workflows/tasks/password-reset'; import { env } from './environment'; import { createNewSession, validatePassword } from './supertokens-at-home/shared'; +import type { WorkloadIdentityFederationProvider } from './workload-identity-federation'; type BroadcastOIDCIntegrationLog = (oidcOrganizationId: string, message: string) => void; @@ -43,6 +44,7 @@ export async function registerSupertokensAtHome( refreshTokenKey: string; accessTokenKey: string; }, + workloadIdentityFederation: WorkloadIdentityFederationProvider | null, ) { const supertokensStore = new SuperTokensStore(storage.pool, server.log); @@ -1318,14 +1320,11 @@ export async function registerSupertokensAtHome( oidcIntegration.useFederatedCredential || !oidcIntegration.encryptedClientSecret ) { - // Azure Workload Identity: use client_assertion with the federated token file - const tokenFilePath = env.azureFederatedTokenFile; - if (!tokenFilePath) { - req.log.error('AZURE_FEDERATED_TOKEN_FILE environment variable is not set'); - broadcastLog( - oidcIntegration.id, - 'Federated credential is configured but AZURE_FEDERATED_TOKEN_FILE environment variable is not set. ' + - 'Ensure the pod has Azure Workload Identity configured.', + // Workload Identity Federation: use client_assertion with the cached federated token + const federatedToken = workloadIdentityFederation?.getToken() ?? null; + if (!federatedToken) { + req.log.error( + 'Workload identity federation is not configured or the token is not yet available', ); return rep.status(200).send({ status: 'SIGN_IN_UP_NOT_ALLOWED', @@ -1333,8 +1332,6 @@ export async function registerSupertokensAtHome( }); } - const { readFile } = await import("node:fs/promises"); - const federatedToken = await readFile(tokenFilePath, "utf-8"); grantParams['client_assertion_type'] = 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer'; grantParams['client_assertion'] = federatedToken; diff --git a/packages/services/server/src/workload-identity-federation.ts b/packages/services/server/src/workload-identity-federation.ts new file mode 100644 index 00000000000..40311547d19 --- /dev/null +++ b/packages/services/server/src/workload-identity-federation.ts @@ -0,0 +1,57 @@ +import { readFile } from 'node:fs/promises'; +import type { FastifyBaseLogger } from 'fastify'; + +const DEFAULT_POLL_INTERVAL_MS = 60_000; + +/** + * Polls the workload identity federation token file at a regular interval and + * keeps the current token in memory so it can be used without a per-request + * file read. + * + * Supports Azure Workload Identity today; additional providers can be added in + * the future by extending the `provider` discriminant. + */ +export class WorkloadIdentityFederationProvider { + private currentToken: string | null = null; + private timer: ReturnType | null = null; + + constructor( + private readonly tokenFilePath: string, + private readonly logger: FastifyBaseLogger, + private readonly pollIntervalMs: number = DEFAULT_POLL_INTERVAL_MS, + ) {} + + /** Read the token file once and start the polling interval. */ + async start(): Promise { + await this.refresh(); + this.timer = setInterval(() => { + void this.refresh(); + }, this.pollIntervalMs); + // Prevent the interval from keeping the process alive on its own. + this.timer.unref(); + } + + stop(): void { + if (this.timer !== null) { + clearInterval(this.timer); + this.timer = null; + } + } + + /** Returns the most recently cached token, or null if not yet loaded. */ + getToken(): string | null { + return this.currentToken; + } + + private async refresh(): Promise { + try { + this.currentToken = (await readFile(this.tokenFilePath, 'utf-8')).trim(); + } catch (err) { + this.logger.error( + { err }, + 'Failed to read workload identity federation token file (path=%s)', + this.tokenFilePath, + ); + } + } +} From c4c732c1d8d9264f9b253249c71000fa3fb45a3d Mon Sep 17 00:00:00 2001 From: Cosmin Catalin SANDA Date: Mon, 20 Jul 2026 21:28:09 +0200 Subject: [PATCH 06/14] Remove OIDC federated credential flag from integrations The `useFederatedCredential` flag is no longer stored on OIDC integration entities or exposed in the API/UI. Instead, the decision to use federated credentials is now externalized and controlled solely by environment variables, specifically `OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS`. This simplifies the OIDC integration model by making client secrets always required unless an organization is explicitly listed for federated authentication via environment configuration. --- ...4.09T00-00-00.oidc-federated-credential.ts | 10 ---- packages/migrations/src/run-pg-migrations.ts | 1 - .../oidc-integrations/module.graphql.ts | 20 +------ .../providers/oidc-integrations.provider.ts | 54 ++----------------- .../Mutation/createOIDCIntegration.ts | 3 +- .../Mutation/updateOIDCIntegration.ts | 1 - .../resolvers/OIDCIntegration.ts | 1 - .../src/modules/shared/providers/storage.ts | 4 +- packages/services/api/src/shared/entities.ts | 3 +- packages/services/server/src/environment.ts | 25 ++++++++- .../server/src/supertokens-at-home.ts | 19 +++++-- packages/services/storage/src/index.ts | 13 ++--- .../connect-single-sign-on-provider-sheet.tsx | 6 +-- .../oidc-integration-configuration.tsx | 12 +---- .../single-sign-on/single-sign-on-subpage.tsx | 2 +- 15 files changed, 55 insertions(+), 119 deletions(-) delete mode 100644 packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts diff --git a/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts b/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts deleted file mode 100644 index 04d04398279..00000000000 --- a/packages/migrations/src/actions/2026.04.09T00-00-00.oidc-federated-credential.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { type MigrationExecutor } from '../pg-migrator'; - -export default { - name: '2026.04.09T00-00-00.oidc-federated-credential.sql', - run: ({ psql }) => psql` - ALTER TABLE "oidc_integrations" - ADD COLUMN IF NOT EXISTS "use_federated_credential" BOOLEAN NOT NULL DEFAULT FALSE, - ALTER COLUMN "client_secret" DROP NOT NULL; - `, -} satisfies MigrationExecutor; diff --git a/packages/migrations/src/run-pg-migrations.ts b/packages/migrations/src/run-pg-migrations.ts index e32c420b32c..5eec1962d92 100644 --- a/packages/migrations/src/run-pg-migrations.ts +++ b/packages/migrations/src/run-pg-migrations.ts @@ -116,7 +116,6 @@ export const runPGMigrations = async (args: { slonik: PostgresDatabasePool; runT import('./actions/2026.02.24T00-00-00.proposal-composition'), import('./actions/2026.02.25T00-00-00.oidc-integration-domains'), import('./actions/2026.03.25T00-00-00.access-token-expiration'), - import('./actions/2026.04.09T00-00-00.oidc-federated-credential'), import('./actions/2026.04.15T00-00-01.metric-alert-rules'), import('./actions/2026.05.25T00-00-00.drop-sdl-store-unique-id'), import('./actions/2026.05.27T00-00-00.access-tokens-description-nullability'), diff --git a/packages/services/api/src/modules/oidc-integrations/module.graphql.ts b/packages/services/api/src/modules/oidc-integrations/module.graphql.ts index a50deb26b00..4fa9f15e5ee 100644 --- a/packages/services/api/src/modules/oidc-integrations/module.graphql.ts +++ b/packages/services/api/src/modules/oidc-integrations/module.graphql.ts @@ -13,7 +13,7 @@ export default gql` type OIDCIntegration { id: ID! clientId: ID! - clientSecretPreview: String + clientSecretPreview: String! tokenEndpoint: String! userinfoEndpoint: String! authorizationEndpoint: String! @@ -24,11 +24,6 @@ export default gql` defaultMemberRole: MemberRole! defaultResourceAssignment: ResourceAssignment """ - Whether this OIDC integration uses a federated credential (e.g. Azure Workload Identity) - instead of a client secret for authenticating to the identity provider. - """ - useFederatedCredential: Boolean! - """ List of domains registered with this OIDC integration. """ registeredDomains: [OIDCIntegrationDomain!]! @@ -193,17 +188,11 @@ export default gql` input CreateOIDCIntegrationInput { organizationId: ID! clientId: ID! - clientSecret: String + clientSecret: String! tokenEndpoint: String! userinfoEndpoint: String! authorizationEndpoint: String! additionalScopes: [String!]! - """ - When true, the OIDC integration will use Azure Workload Identity (federated credential) - instead of a client secret for authenticating to the identity provider during token exchange. - The pod must have Azure Workload Identity configured with the appropriate federated token file. - """ - useFederatedCredential: Boolean } type CreateOIDCIntegrationResult { @@ -238,11 +227,6 @@ export default gql` userinfoEndpoint: String authorizationEndpoint: String additionalScopes: [String!] - """ - When true, the OIDC integration will use Azure Workload Identity (federated credential) - instead of a client secret for authenticating to the identity provider during token exchange. - """ - useFederatedCredential: Boolean } type UpdateOIDCIntegrationResult { diff --git a/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts b/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts index 668ed40b17c..928a50b6d73 100644 --- a/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts +++ b/packages/services/api/src/modules/oidc-integrations/providers/oidc-integrations.provider.ts @@ -97,9 +97,6 @@ export class OIDCIntegrationsProvider { } async getClientSecretPreview(integration: OIDCIntegration) { - if (integration.encryptedClientSecret === null) { - return null; - } const decryptedSecret = this.crypto.decrypt(integration.encryptedClientSecret); return decryptedSecret.substring(decryptedSecret.length - 4); } @@ -107,12 +104,11 @@ export class OIDCIntegrationsProvider { async createOIDCIntegrationForOrganization(args: { organizationId: string; clientId: string; - clientSecret: string | null; + clientSecret: string; tokenEndpoint: string; userinfoEndpoint: string; authorizationEndpoint: string; additionalScopes: readonly string[]; - useFederatedCredential: boolean; }) { if (this.isEnabled() === false) { return { @@ -137,26 +133,8 @@ export class OIDCIntegrationsProvider { throw new Error(`Failed to locate organization ${args.organizationId}`); } - // When using federated credentials, client secret is not required - if (!args.useFederatedCredential && !args.clientSecret) { - return { - type: 'error', - reason: null, - fieldErrors: { - clientId: null, - clientSecret: 'Client secret is required when not using federated credentials.', - tokenEndpoint: null, - userinfoEndpoint: null, - authorizationEndpoint: null, - additionalScopes: null, - }, - } as const; - } - const clientIdResult = OIDCIntegrationClientIdModel.safeParse(args.clientId); - const clientSecretResult = args.useFederatedCredential - ? { success: true as const, data: null } - : OIDCClientSecretModel.safeParse(args.clientSecret); + const clientSecretResult = OIDCClientSecretModel.safeParse(args.clientSecret); const tokenEndpointResult = OAuthAPIUrlModel.safeParse(args.tokenEndpoint); const userinfoEndpointResult = OAuthAPIUrlModel.safeParse(args.userinfoEndpoint); const authorizationEndpointResult = OAuthAPIUrlModel.safeParse(args.authorizationEndpoint); @@ -173,13 +151,11 @@ export class OIDCIntegrationsProvider { const creationResult = await this.storage.createOIDCIntegrationForOrganization({ organizationId: args.organizationId, clientId: clientIdResult.data, - encryptedClientSecret: - clientSecretResult.data !== null ? this.crypto.encrypt(clientSecretResult.data) : null, + encryptedClientSecret: this.crypto.encrypt(clientSecretResult.data), tokenEndpoint: tokenEndpointResult.data, userinfoEndpoint: userinfoEndpointResult.data, authorizationEndpoint: authorizationEndpointResult.data, additionalScopes: additionalScopesResult.data, - useFederatedCredential: args.useFederatedCredential, }); if (creationResult.type === 'ok') { @@ -240,7 +216,6 @@ export class OIDCIntegrationsProvider { userinfoEndpoint: string | null; authorizationEndpoint: string | null; additionalScopes: readonly string[] | null; - useFederatedCredential: boolean | null; }) { if (this.isEnabled() === false) { return { @@ -292,28 +267,6 @@ export class OIDCIntegrationsProvider { authorizationEndpointResult.success && additionalScopesResult.success ) { - const effectiveUseFederatedCredential = - args.useFederatedCredential ?? integration.useFederatedCredential; - const effectiveHasClientSecret = clientSecretResult.data - ? true - : integration.encryptedClientSecret !== null; - - if (!effectiveUseFederatedCredential && !effectiveHasClientSecret) { - return { - type: 'error', - message: "Couldn't update integration.", - fieldErrors: { - clientId: null, - clientSecret: - 'Either a client secret or federated credential must be enabled.', - tokenEndpoint: null, - userinfoEndpoint: null, - authorizationEndpoint: null, - additionalScopes: null, - }, - } as const; - } - const oidcIntegration = await this.storage.updateOIDCIntegration({ oidcIntegrationId: args.oidcIntegrationId, clientId: clientIdResult.data, @@ -324,7 +277,6 @@ export class OIDCIntegrationsProvider { userinfoEndpoint: userinfoEndpointResult.data, authorizationEndpoint: authorizationEndpointResult.data, additionalScopes: additionalScopesResult.data, - useFederatedCredential: args.useFederatedCredential, }); const redactedClientSecret = maskToken(oidcIntegration.clientId); diff --git a/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/createOIDCIntegration.ts b/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/createOIDCIntegration.ts index 43530d1da7a..a99d30611bb 100644 --- a/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/createOIDCIntegration.ts +++ b/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/createOIDCIntegration.ts @@ -9,12 +9,11 @@ export const createOIDCIntegration: NonNullable< const result = await oktaIntegrationsProvider.createOIDCIntegrationForOrganization({ organizationId: input.organizationId, clientId: input.clientId, - clientSecret: input.clientSecret ?? null, + clientSecret: input.clientSecret, tokenEndpoint: input.tokenEndpoint, userinfoEndpoint: input.userinfoEndpoint, authorizationEndpoint: input.authorizationEndpoint, additionalScopes: input.additionalScopes, - useFederatedCredential: input.useFederatedCredential ?? false, }); if (result.type === 'ok') { diff --git a/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/updateOIDCIntegration.ts b/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/updateOIDCIntegration.ts index b8dad5eab9e..d89bee398c8 100644 --- a/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/updateOIDCIntegration.ts +++ b/packages/services/api/src/modules/oidc-integrations/resolvers/Mutation/updateOIDCIntegration.ts @@ -13,7 +13,6 @@ export const updateOIDCIntegration: NonNullable< userinfoEndpoint: input.userinfoEndpoint ?? null, authorizationEndpoint: input.authorizationEndpoint ?? null, additionalScopes: input.additionalScopes ?? null, - useFederatedCredential: input.useFederatedCredential ?? null, }); if (result.type === 'ok') { diff --git a/packages/services/api/src/modules/oidc-integrations/resolvers/OIDCIntegration.ts b/packages/services/api/src/modules/oidc-integrations/resolvers/OIDCIntegration.ts index ca0312e17bc..10909b3002f 100644 --- a/packages/services/api/src/modules/oidc-integrations/resolvers/OIDCIntegration.ts +++ b/packages/services/api/src/modules/oidc-integrations/resolvers/OIDCIntegration.ts @@ -11,7 +11,6 @@ export const OIDCIntegration: OidcIntegrationResolvers = { clientId: oidcIntegration => oidcIntegration.clientId, clientSecretPreview: (oidcIntegration, _, { injector }) => injector.get(OIDCIntegrationsProvider).getClientSecretPreview(oidcIntegration), - useFederatedCredential: oidcIntegration => oidcIntegration.useFederatedCredential, /** * Fallbacks to Viewer if default member role is not set */ diff --git a/packages/services/api/src/modules/shared/providers/storage.ts b/packages/services/api/src/modules/shared/providers/storage.ts index bb730980fba..2a15b64d70b 100644 --- a/packages/services/api/src/modules/shared/providers/storage.ts +++ b/packages/services/api/src/modules/shared/providers/storage.ts @@ -471,12 +471,11 @@ export interface Storage { createOIDCIntegrationForOrganization(_: { organizationId: string; clientId: string; - encryptedClientSecret: string | null; + encryptedClientSecret: string; tokenEndpoint: string; userinfoEndpoint: string; authorizationEndpoint: string; additionalScopes: readonly string[]; - useFederatedCredential: boolean; }): Promise<{ type: 'ok'; oidcIntegration: OIDCIntegration } | { type: 'error'; reason: string }>; updateOIDCIntegration(_: { @@ -487,7 +486,6 @@ export interface Storage { userinfoEndpoint: string | null; authorizationEndpoint: string | null; additionalScopes: readonly string[] | null; - useFederatedCredential: boolean | null; }): Promise; deleteOIDCIntegration(_: { oidcIntegrationId: string }): Promise; diff --git a/packages/services/api/src/shared/entities.ts b/packages/services/api/src/shared/entities.ts index 48eb226a5f1..b79d0817929 100644 --- a/packages/services/api/src/shared/entities.ts +++ b/packages/services/api/src/shared/entities.ts @@ -185,7 +185,7 @@ export interface OIDCIntegration { id: string; linkedOrganizationId: string; clientId: string; - encryptedClientSecret: string | null; + encryptedClientSecret: string; tokenEndpoint: string; userinfoEndpoint: string; authorizationEndpoint: string; @@ -195,7 +195,6 @@ export interface OIDCIntegration { requireInvitation: boolean; defaultMemberRoleId: string | null; defaultResourceAssignment: ResourceAssignmentGroup | null; - useFederatedCredential: boolean; } export interface CDNAccessToken { diff --git a/packages/services/server/src/environment.ts b/packages/services/server/src/environment.ts index fbf220739b5..89581462a46 100644 --- a/packages/services/server/src/environment.ts +++ b/packages/services/server/src/environment.ts @@ -303,6 +303,8 @@ const LogModel = zod.object({ ), }); +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + const OidcWorkloadFederationModel = zod.union([ zod.object({ OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER: zod.union([ @@ -315,10 +317,18 @@ const OidcWorkloadFederationModel = zod.union([ .object({ OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER: zod.literal('azure'), AZURE_FEDERATED_TOKEN_FILE: zod.string().min(1), + OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS: zod.string().min(1), }) .superRefine((data, ctx) => { try { - readFileSync(data.AZURE_FEDERATED_TOKEN_FILE, 'utf-8'); + const content = readFileSync(data.AZURE_FEDERATED_TOKEN_FILE, 'utf-8').trim(); + if (!content) { + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + path: ['AZURE_FEDERATED_TOKEN_FILE'], + message: `Federated token file at path '${data.AZURE_FEDERATED_TOKEN_FILE}' is empty.`, + }); + } } catch { ctx.addIssue({ code: zod.ZodIssueCode.custom, @@ -326,6 +336,16 @@ const OidcWorkloadFederationModel = zod.union([ message: `Cannot read federated token file at path '${data.AZURE_FEDERATED_TOKEN_FILE}'. Ensure the file exists and is readable.`, }); } + + const ids = data.OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS.split(',').map(s => s.trim()); + const invalid = ids.filter(id => !UUID_REGEX.test(id)); + if (invalid.length > 0) { + ctx.addIssue({ + code: zod.ZodIssueCode.custom, + path: ['OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS'], + message: `The following values are not valid UUIDs: ${invalid.join(', ')}`, + }); + } }), ]); @@ -690,6 +710,9 @@ export const env = { ? { provider: 'azure' as const, tokenFilePath: oidcWorkloadFederation.AZURE_FEDERATED_TOKEN_FILE, + organizationIds: oidcWorkloadFederation.OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS.split( + ',', + ).map(s => s.trim()), } : null, } as const; diff --git a/packages/services/server/src/supertokens-at-home.ts b/packages/services/server/src/supertokens-at-home.ts index 04dfb5cdd27..1f044994ef4 100644 --- a/packages/services/server/src/supertokens-at-home.ts +++ b/packages/services/server/src/supertokens-at-home.ts @@ -794,6 +794,12 @@ export async function registerSupertokensAtHome( const scopes = ['openid', 'email', ...oidcIntegration.additionalScopes]; + const useFederation = + workloadIdentityFederation !== null && + (env.oidcWorkloadFederation?.organizationIds.includes( + oidcIntegration.linkedOrganizationId, + ) ?? false); + const oidClientConfig = new oidClient.Configuration( { issuer: 'noop', @@ -802,7 +808,7 @@ export async function registerSupertokensAtHome( token_endpoint: oidcIntegration.tokenEndpoint, }, oidcIntegration.clientId, - oidcIntegration.useFederatedCredential || !oidcIntegration.encryptedClientSecret + useFederation ? undefined : { client_secret: crypto.decrypt(oidcIntegration.encryptedClientSecret), @@ -1308,6 +1314,12 @@ export async function registerSupertokensAtHome( access_token: z.string(), }); + const useFederationForExchange = + workloadIdentityFederation !== null && + (env.oidcWorkloadFederation?.organizationIds.includes( + oidcIntegration.linkedOrganizationId, + ) ?? false); + const grantParams: Record = { grant_type: 'authorization_code', code_verifier: cacheRecord.pkceVerifier, @@ -1316,10 +1328,7 @@ export async function registerSupertokensAtHome( client_id: oidcIntegration.clientId, }; - if ( - oidcIntegration.useFederatedCredential || - !oidcIntegration.encryptedClientSecret - ) { + if (useFederationForExchange) { // Workload Identity Federation: use client_assertion with the cached federated token const federatedToken = workloadIdentityFederation?.getToken() ?? null; if (!federatedToken) { diff --git a/packages/services/storage/src/index.ts b/packages/services/storage/src/index.ts index 3e0bfc27fa1..a1194a60c9c 100644 --- a/packages/services/storage/src/index.ts +++ b/packages/services/storage/src/index.ts @@ -2350,8 +2350,7 @@ export async function createStorage( "token_endpoint", "userinfo_endpoint", "authorization_endpoint", - "additional_scopes", - "use_federated_credential" + "additional_scopes" ) VALUES ( ${args.organizationId}, @@ -2360,8 +2359,7 @@ export async function createStorage( ${args.tokenEndpoint}, ${args.userinfoEndpoint}, ${args.authorizationEndpoint}, - ${psql.array(args.additionalScopes, 'text')}, - ${args.useFederatedCredential} + ${psql.array(args.additionalScopes, 'text')} ) RETURNING ${oidcIntegrationFields()} @@ -2412,7 +2410,6 @@ export async function createStorage( } , "additional_scopes" = ${args.additionalScopes ? psql.array(args.additionalScopes, 'text') : psql`"additional_scopes"`} , "oauth_api_url" = NULL - , "use_federated_credential" = ${args.useFederatedCredential ?? psql`"use_federated_credential"`} WHERE "id" = ${args.oidcIntegrationId} RETURNING @@ -4030,7 +4027,7 @@ const OIDCIntegrationBaseModel = z.object({ id: z.string(), linkedOrganizationId: z.string(), clientId: z.string(), - clientSecret: z.string().nullable(), + clientSecret: z.string(), additionalScopes: z .array(z.string()) .nullable() @@ -4040,7 +4037,6 @@ const OIDCIntegrationBaseModel = z.object({ defaultRoleId: z.string().nullable(), defaultAssignedResources: z.any().nullable(), requireInvitation: z.boolean(), - useFederatedCredential: z.boolean(), }); const OIDCIntegrationLegacyModel = OIDCIntegrationBaseModel.extend({ @@ -4059,7 +4055,6 @@ const OIDCIntegrationLegacyModel = OIDCIntegrationBaseModel.extend({ requireInvitation: record.requireInvitation, defaultMemberRoleId: record.defaultRoleId, defaultResourceAssignment: record.defaultAssignedResources, - useFederatedCredential: record.useFederatedCredential, })); const LatestOIDCIntegrationModel = OIDCIntegrationBaseModel.extend({ @@ -4081,7 +4076,6 @@ const LatestOIDCIntegrationModel = OIDCIntegrationBaseModel.extend({ requireInvitation: record.requireInvitation, defaultMemberRoleId: record.defaultRoleId, defaultResourceAssignment: record.defaultAssignedResources, - useFederatedCredential: record.useFederatedCredential, })); const OIDCIntegrationModel = z.union([OIDCIntegrationLegacyModel, LatestOIDCIntegrationModel]); @@ -4493,7 +4487,6 @@ const oidcIntegrationFields = (prefix: TaggedTemplateLiteralInvocation = psql``) , ${prefix}"default_role_id" AS "defaultRoleId" , ${prefix}"default_assigned_resources" AS "defaultAssignedResources" , ${prefix}"require_invitation" AS "requireInvitation" - , ${prefix}"use_federated_credential" AS "useFederatedCredential" `; const OrganizationModel = z diff --git a/packages/web/app/src/components/organization/settings/single-sign-on/connect-single-sign-on-provider-sheet.tsx b/packages/web/app/src/components/organization/settings/single-sign-on/connect-single-sign-on-provider-sheet.tsx index b4b924f59c0..621555e48d6 100644 --- a/packages/web/app/src/components/organization/settings/single-sign-on/connect-single-sign-on-provider-sheet.tsx +++ b/packages/web/app/src/components/organization/settings/single-sign-on/connect-single-sign-on-provider-sheet.tsx @@ -25,7 +25,7 @@ type ConnectSingleSignOnProviderSheetProps = { tokenEndpoint: string; userinfoEndpoint: string; clientId: string; - clientSecretPreview: string | null; + clientSecretPreview: string; additionalScopes: string; }; onSave: (args: { @@ -210,8 +210,8 @@ export function ConnectSingleSignOnProviderSheet( Client Secret - {oidcIntegration.useFederatedCredential ? ( - Using Federated Credential - ) : oidcIntegration.clientSecretPreview ? ( - <>•••••••{oidcIntegration.clientSecretPreview} - ) : ( - Not Set - )} + •••••••{oidcIntegration.clientSecretPreview} @@ -421,7 +413,7 @@ export function OIDCIntegrationConfiguration(props: { authorizationEndpoint: oidcIntegration.authorizationEndpoint, tokenEndpoint: oidcIntegration.tokenEndpoint, userinfoEndpoint: oidcIntegration.userinfoEndpoint, - clientSecretPreview: oidcIntegration.clientSecretPreview ?? null, + clientSecretPreview: oidcIntegration.clientSecretPreview, }} onSave={async args => { const result = await updateOIDCIntegrationMutate({ diff --git a/packages/web/app/src/components/organization/settings/single-sign-on/single-sign-on-subpage.tsx b/packages/web/app/src/components/organization/settings/single-sign-on/single-sign-on-subpage.tsx index 92949528c63..27440d1a1a8 100644 --- a/packages/web/app/src/components/organization/settings/single-sign-on/single-sign-on-subpage.tsx +++ b/packages/web/app/src/components/organization/settings/single-sign-on/single-sign-on-subpage.tsx @@ -124,7 +124,7 @@ export function SingleSignOnSubpage(props: SingleSignOnSubPageProps): React.Reac input: { organizationId: organization?.id ?? '', clientId: values.clientId, - clientSecret: values.clientSecret || null, + clientSecret: values.clientSecret ?? '', authorizationEndpoint: values.authorizationEndpoint, tokenEndpoint: values.tokenEndpoint, userinfoEndpoint: values.userinfoEndpoint, From a53de653a328395756e041e2d3ec4aeff0a5c6de Mon Sep 17 00:00:00 2001 From: Cosmin Catalin SANDA Date: Mon, 20 Jul 2026 21:34:54 +0200 Subject: [PATCH 07/14] Improve readability of OIDC federation condition --- packages/services/server/src/supertokens-at-home.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/services/server/src/supertokens-at-home.ts b/packages/services/server/src/supertokens-at-home.ts index 1f044994ef4..7412d09fa4b 100644 --- a/packages/services/server/src/supertokens-at-home.ts +++ b/packages/services/server/src/supertokens-at-home.ts @@ -798,7 +798,8 @@ export async function registerSupertokensAtHome( workloadIdentityFederation !== null && (env.oidcWorkloadFederation?.organizationIds.includes( oidcIntegration.linkedOrganizationId, - ) ?? false); + ) ?? + false); const oidClientConfig = new oidClient.Configuration( { @@ -1318,7 +1319,8 @@ export async function registerSupertokensAtHome( workloadIdentityFederation !== null && (env.oidcWorkloadFederation?.organizationIds.includes( oidcIntegration.linkedOrganizationId, - ) ?? false); + ) ?? + false); const grantParams: Record = { grant_type: 'authorization_code', From 7ae1bb5d9ee60333cf0609b56443cd4f14e928ff Mon Sep 17 00:00:00 2001 From: Cosmin Catalin SANDA Date: Mon, 20 Jul 2026 21:52:30 +0200 Subject: [PATCH 08/14] Replace local UUID regex with shared `isUUID` utility --- packages/services/server/src/environment.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/services/server/src/environment.ts b/packages/services/server/src/environment.ts index 89581462a46..c7525e625b8 100644 --- a/packages/services/server/src/environment.ts +++ b/packages/services/server/src/environment.ts @@ -1,5 +1,6 @@ import { readFileSync } from 'node:fs'; import zod from 'zod'; +import { isUUID } from '@hive/api/shared/is-uuid'; import { OpenTelemetryConfigurationModel, parseRedisConfigFromEnvironment, @@ -303,8 +304,6 @@ const LogModel = zod.object({ ), }); -const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; - const OidcWorkloadFederationModel = zod.union([ zod.object({ OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER: zod.union([ @@ -338,7 +337,7 @@ const OidcWorkloadFederationModel = zod.union([ } const ids = data.OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS.split(',').map(s => s.trim()); - const invalid = ids.filter(id => !UUID_REGEX.test(id)); + const invalid = ids.filter(id => !isUUID(id)); if (invalid.length > 0) { ctx.addIssue({ code: zod.ZodIssueCode.custom, From d7359b59f5ab22059fe2d68b999aeee265bf9bdc Mon Sep 17 00:00:00 2001 From: Laurin Date: Fri, 24 Jul 2026 16:46:40 +0200 Subject: [PATCH 09/14] feat: make tracking OOM memory issues in schema worker easier (#8258) --- .../providers/artifact-storage-writer.ts | 32 +++++++++++++++++++ .../orchestrator/composition-orchestrator.ts | 19 ++++++++++- .../schema/providers/schema-publisher.ts | 4 +++ packages/services/schema/src/api.ts | 1 + .../services/schema/src/composition/shared.ts | 1 + 5 files changed, 56 insertions(+), 1 deletion(-) diff --git a/packages/services/api/src/modules/schema/providers/artifact-storage-writer.ts b/packages/services/api/src/modules/schema/providers/artifact-storage-writer.ts index bddd9c2fb5f..612b651b710 100644 --- a/packages/services/api/src/modules/schema/providers/artifact-storage-writer.ts +++ b/packages/services/api/src/modules/schema/providers/artifact-storage-writer.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { Inject } from 'graphql-modules'; import { buildArtifactStorageKey } from '@hive/cdn-script/artifact-storage-reader'; import { traceFn } from '@hive/service-common'; @@ -182,4 +183,35 @@ export class ArtifactStorageWriter { args.contractName, ); } + + async writeSDLForDebugPurposes(subgraphs: Array<{ sdl: string; name?: string }>) { + const hashBuilder = createHash('sha256'); + for (const subgraph of subgraphs) { + hashBuilder.update(subgraph.sdl); + if (subgraph.name) { + hashBuilder.update(subgraph.name); + } + } + + const hash = hashBuilder.digest('hex'); + + const firstMirror = this.s3Mirrors[0]; + const url = [firstMirror.endpoint, firstMirror.bucket, 'debug-artifacts', hash + '.json'].join( + '/', + ); + + await firstMirror.client.fetch(url, { + method: 'PUT', + headers: { + 'content-type': 'application/json', + }, + body: JSON.stringify(subgraphs), + aws: { + // This boolean makes Google Cloud Storage & AWS happy. + signQuery: true, + }, + }); + + return { id: hash.substring(0, 10), url }; + } } diff --git a/packages/services/api/src/modules/schema/providers/orchestrator/composition-orchestrator.ts b/packages/services/api/src/modules/schema/providers/orchestrator/composition-orchestrator.ts index bb5da0f0c8b..ebc105d2f65 100644 --- a/packages/services/api/src/modules/schema/providers/orchestrator/composition-orchestrator.ts +++ b/packages/services/api/src/modules/schema/providers/orchestrator/composition-orchestrator.ts @@ -1,7 +1,7 @@ import { CONTEXT, Inject, Injectable, Scope } from 'graphql-modules'; import { abortSignalAny } from '@graphql-hive/signal'; import type { ContractsInputType, SchemaBuilderApi } from '@hive/schema'; -import { traceFn } from '@hive/service-common'; +import { trace, traceFn } from '@hive/service-common'; import { createTRPCProxyClient, httpLink } from '@trpc/client'; import { ComposeAndValidateResult, @@ -10,6 +10,7 @@ import { SchemaObject, } from '../../../../shared/entities'; import { Logger } from '../../../shared/providers/logger'; +import { ArtifactStorageWriter } from '../artifact-storage-writer'; import type { SchemaServiceConfig } from './tokens'; import { SCHEMA_SERVICE_CONFIG } from './tokens'; @@ -44,6 +45,7 @@ export class CompositionOrchestrator { constructor( logger: Logger, + private artifactStorageWriter: ArtifactStorageWriter, @Inject(SCHEMA_SERVICE_CONFIG) serviceConfig: SchemaServiceConfig, @Inject(CONTEXT) context: GraphQLModules.ModuleContext, ) { @@ -159,6 +161,21 @@ export class CompositionOrchestrator { signal: abortSignalAny([this.incomingRequestAbortSignal, timeoutAbortSignal]), }, ); + + // If a composition error happens due to OOM we write the subgraphs to S3 + // This makes it easier to reproduce and debug the issue + if (result.errors.some(error => error.code === 'ERR_WORKER_OUT_OF_MEMORY')) { + const span = trace.getActiveSpan(); + if (span) { + const result = await this.artifactStorageWriter.writeSDLForDebugPurposes( + schemas.map(s => ({ sdl: s.raw, name: s.source })), + ); + trace + .getActiveSpan() + ?.setAttribute(`hive.composition.subgraphs.${result.id}`, result.url); + } + } + return result; } catch (err) { // In case of a timeout error we return something the user can process diff --git a/packages/services/api/src/modules/schema/providers/schema-publisher.ts b/packages/services/api/src/modules/schema/providers/schema-publisher.ts index b69e5a0980d..1c3c1322058 100644 --- a/packages/services/api/src/modules/schema/providers/schema-publisher.ts +++ b/packages/services/api/src/modules/schema/providers/schema-publisher.ts @@ -1754,6 +1754,10 @@ export class SchemaPublisher { }), ]); + trace.getActiveSpan()?.setAttributes({ + 'hive.latestSchemaVersion.id': latestVersion?.version.id ?? 'no prior version', + }); + function increaseSchemaPublishCountMetric(conclusion: 'rejected' | 'accepted' | 'ignored') { schemaPublishCount.inc({ model: 'modern', diff --git a/packages/services/schema/src/api.ts b/packages/services/schema/src/api.ts index ca26e112022..c0982a86e7f 100644 --- a/packages/services/schema/src/api.ts +++ b/packages/services/schema/src/api.ts @@ -171,6 +171,7 @@ export const schemaBuilderApiRouter = t.router({ message: 'Composition exceeded resource limits. Please contact the Hive Console Team.', source: 'composition', + code: 'ERR_WORKER_OUT_OF_MEMORY', }, ], sdl: null, diff --git a/packages/services/schema/src/composition/shared.ts b/packages/services/schema/src/composition/shared.ts index 87e36fe4c82..4864e77f17e 100644 --- a/packages/services/schema/src/composition/shared.ts +++ b/packages/services/schema/src/composition/shared.ts @@ -4,6 +4,7 @@ import type { Metadata } from '../types'; export type CompositionErrorType = { message: string; source: CompositionErrorSource; + code?: string; }; type ContractResult = { From e3df6375247d3bdfaed946046148f5ff7b644015 Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:31:55 -0700 Subject: [PATCH 10/14] Dont store or build schemas if encountering stitched errors (#8259) --- .changeset/social-drinks-speak.md | 6 ++++++ packages/services/schema/src/composition/stitching.ts | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .changeset/social-drinks-speak.md diff --git a/.changeset/social-drinks-speak.md b/.changeset/social-drinks-speak.md new file mode 100644 index 00000000000..5c9c4e27a25 --- /dev/null +++ b/.changeset/social-drinks-speak.md @@ -0,0 +1,6 @@ +--- +'hive': patch +--- + +Optimize memory for composition worker stitching schemas. Don't store or build the schemas if there +are stitched errors diff --git a/packages/services/schema/src/composition/stitching.ts b/packages/services/schema/src/composition/stitching.ts index 923c2565103..98b45117a5e 100644 --- a/packages/services/schema/src/composition/stitching.ts +++ b/packages/services/schema/src/composition/stitching.ts @@ -69,7 +69,7 @@ export type ComposeStitchingArgs = { export async function composeStitching(args: ComposeStitchingArgs) { const errors: Array = []; - const subschemas: GraphQLSchema[] = []; + let subschemas: GraphQLSchema[] = []; for (const schema of args.schemas) { // parse inside a loop instead of using map to avoid holding on to the parsed schema // in memory. This matters for large schemas. @@ -77,7 +77,8 @@ export async function composeStitching(args: ComposeStitchingArgs) { const stitchedErrors = validateStitchedSchema(parsed); if (stitchedErrors.length) { errors.push(...stitchedErrors); - } else { + subschemas = []; // no need to store subschemas if there are errors. It wont be used. + } else if (!errors.length) { subschemas.push( buildASTSchema(trimDescriptions(parsed), { assumeValid: true, From 372e1f2d99482379d5fae5fb3c6bc4ab1b564203 Mon Sep 17 00:00:00 2001 From: Jonathan Brennan Date: Fri, 24 Jul 2026 16:05:02 -0500 Subject: [PATCH 11/14] Add panels to metric alerts dash for outward correlation (#8254) --- .../grafana-dashboards/Metric-Alerts.json | 501 +++++++++++++++++- 1 file changed, 488 insertions(+), 13 deletions(-) diff --git a/deployment/grafana-dashboards/Metric-Alerts.json b/deployment/grafana-dashboards/Metric-Alerts.json index c23b27b61c1..6276302222f 100644 --- a/deployment/grafana-dashboards/Metric-Alerts.json +++ b/deployment/grafana-dashboards/Metric-Alerts.json @@ -63,7 +63,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 0 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 0 }, "id": 1, "options": { "legend": { @@ -143,7 +143,7 @@ } ] }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 0 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 0 }, "id": 2, "options": { "legend": { @@ -190,7 +190,7 @@ }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 9 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 10 }, "id": 3, "options": { "colorMode": "value", @@ -248,7 +248,7 @@ }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 9 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 10 }, "id": 4, "options": { "legend": { @@ -303,7 +303,7 @@ } ] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 20 }, "id": 5, "options": { "legend": { @@ -366,7 +366,7 @@ } ] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 20 }, "id": 6, "options": { "legend": { @@ -401,7 +401,482 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, + "id": 10, + "panels": [], + "title": "Diagnostics", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "p99 round-trip latency of the metric-alert evaluator's ClickHouse queries, overlaid against p99 of all API-issued ClickHouse queries (api_clickhouse_elapsed_duration, the metric behind the ClickHouse-Latency dashboard). If both lines rise together the ClickHouse cluster is contended for everyone (ingestion, merges, another heavy consumer) and the slowdown is not specific to alert evaluation. If only the evaluator line rises, the cost is in the alert workload itself.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "dashed" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "orange", "value": 2 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "API-wide p99" }, + "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 31 }, + "id": 11, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(hive_metric_alert_clickhouse_query_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "evaluator p99", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(api_clickhouse_elapsed_duration_bucket[$__rate_interval])))", + "legendFormat": "API-wide p99", + "range": true, + "refId": "B" + } + ], + "title": "ClickHouse latency: evaluator vs API-wide (p99)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "p99 time a evaluateMetricAlertRules job waits in the graphile-worker queue before a worker picks it up. The cron enqueues every minute and groups run at GROUP_CONCURRENCY=5; sustained growth here means the evaluator can't drain a tick before the next is enqueued (workers saturated or ticks running long), so alerts are evaluated late even if each query is fast.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 31 }, + "id": 12, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "hive_workflow_job_queue_time_seconds{task_identifier=\"evaluateMetricAlertRules\",quantile=\"0.99\"}", + "legendFormat": "p99 {{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Task queue time (p99)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Success vs error completion rate of the evaluateMetricAlertRules job. This is the task-level failure signal: the 'ClickHouse error ratio' panel above only counts failed ClickHouse queries, whereas a tick can also fail in PG state transitions or the task body. Any sustained 'error' here means whole ticks are throwing.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "error" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "success" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 41 }, + "id": 13, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "sum(rate(hive_workflow_job_success_total{task_identifier=\"evaluateMetricAlertRules\"}[$__rate_interval]))", + "legendFormat": "success", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "sum(rate(hive_workflow_job_error_total{task_identifier=\"evaluateMetricAlertRules\"}[$__rate_interval]))", + "legendFormat": "error", + "range": true, + "refId": "B" + } + ], + "title": "Task success / error rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, + "id": 20, + "panels": [], + "title": "Runtime (workflow-service)", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Per-pod Node.js event-loop lag p99 for the workflow-service. The evaluator's task-duration is wall-clock, so it inflates whenever the pod's event loop is blocked (heavy sync work, GC pauses, another workflow task saturating the pod) even when ClickHouse is fast. A lag climb that tracks the task-duration climb points at the runtime, not the query. Note: this is pod-level (all workflow tasks share the pod), not attributed to alert evaluation alone.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 52 }, + "id": 21, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "nodejs_eventloop_lag_p99_seconds{service=\"workflow-service\"}", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Event-loop lag (p99) per pod", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Per-pod CPU usage (cores) for the workflow-service, from process_cpu_seconds_total. Read alongside event-loop lag: high CPU with high lag means the pod is compute-bound.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "cores", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 52 }, + "id": 22, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{service=\"workflow-service\"}[$__rate_interval])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU (cores) per pod", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Per-pod memory for the workflow-service: resident set (RSS) and Node.js used heap. A steady climb across a tick storm can indicate GC pressure or a leak that also shows up as rising GC time and event-loop lag.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 62 }, + "id": 23, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "process_resident_memory_bytes{service=\"workflow-service\"}", + "legendFormat": "RSS {{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "nodejs_heap_size_used_bytes{service=\"workflow-service\"}", + "legendFormat": "heap {{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "Memory (RSS + heap) per pod", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "description": "Per-pod Node.js GC time spent per second for the workflow-service (rate of nodejs_gc_duration_seconds_sum). Rising GC time steals from the event loop and shows up as event-loop lag and inflated task duration.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 62 }, + "id": 24, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PROM_DATASOURCE_UID" }, + "editorMode": "code", + "expr": "rate(nodejs_gc_duration_seconds_sum{service=\"workflow-service\"}[$__rate_interval])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "GC time per second per pod", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 72 }, "id": 100, "panels": [], "title": "Traces", @@ -423,7 +898,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 25 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 73 }, "id": 101, "options": { "cellHeight": "sm", @@ -461,7 +936,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 25 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 73 }, "id": 102, "options": { "cellHeight": "sm", @@ -516,7 +991,7 @@ } ] }, - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 34 }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 83 }, "id": 104, "options": { "legend": { @@ -570,7 +1045,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 34 }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 83 }, "id": 105, "options": { "legend": { @@ -624,7 +1099,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 43 }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 93 }, "id": 106, "options": { "legend": { @@ -668,7 +1143,7 @@ }, "overrides": [] }, - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 52 }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 103 }, "id": 103, "options": { "cellHeight": "sm", From 8f388784ff57576780ef4262bac62570c383811b Mon Sep 17 00:00:00 2001 From: jdolle <1841898+jdolle@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:36:32 -0700 Subject: [PATCH 12/14] Track composition worker memory (#8256) --- .changeset/modern-badgers-fall.md | 5 + .../grafana-dashboards/Schema-Service.json | 281 +++++++++++++++++- deployment/services/schema.ts | 1 + .../schema/src/composition-scheduler.ts | 36 ++- .../services/schema/src/composition-worker.ts | 34 ++- packages/services/schema/src/environment.ts | 4 + packages/services/schema/src/metrics.ts | 6 + 7 files changed, 351 insertions(+), 16 deletions(-) create mode 100644 .changeset/modern-badgers-fall.md diff --git a/.changeset/modern-badgers-fall.md b/.changeset/modern-badgers-fall.md new file mode 100644 index 00000000000..b422c53ee3e --- /dev/null +++ b/.changeset/modern-badgers-fall.md @@ -0,0 +1,5 @@ +--- +'hive': patch +--- + +Track schema composition worker memory usage; add to schema service dashboard diff --git a/deployment/grafana-dashboards/Schema-Service.json b/deployment/grafana-dashboards/Schema-Service.json index 5a5c5351ea9..5470d808137 100644 --- a/deployment/grafana-dashboards/Schema-Service.json +++ b/deployment/grafana-dashboards/Schema-Service.json @@ -661,9 +661,288 @@ ], "title": "Timeouts", "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "description": "Current memory usage across workers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisShow": false, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "editorMode": "code", + "expr": "sum(composition_worker_memory_used_bytes) by (instance)", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Memory Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "description": "P95 of composition cache value size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisShow": false, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(composition_cache_value_size_bytes_bucket[$__rate_interval])) by (le))", + "instant": false, + "legendFormat": "P95 Size", + "range": true, + "refId": "A" + } + ], + "title": "Cache Value Size (P95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "description": "P95 of worker duration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisShow": false, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PROM_DATASOURCE_UID" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(composition_worker_duration_ms_bucket[$__rate_interval])) by (le))", + "instant": false, + "legendFormat": "P95 Duration", + "range": true, + "refId": "A" + } + ], + "title": "Worker Duration (P95)", + "type": "timeseries" } ], - "refresh": "5s", + "refresh": "30s", "schemaVersion": 38, "tags": [], "templating": { diff --git a/deployment/services/schema.ts b/deployment/services/schema.ts index 1bf9704c0ff..25369b69d46 100644 --- a/deployment/services/schema.ts +++ b/deployment/services/schema.ts @@ -43,6 +43,7 @@ export function deploySchema({ SCHEMA_CACHE_TTL_MS: '65000' /* 65s */, SCHEMA_CACHE_SUCCESS_TTL_MS: String(hourInMS * 2), COMPOSITION_WORKER_MAX_OLD_GENERATION_SIZE_MB: '716', + COMPOSITION_WORKER_TRACK_MEMORY_USAGE: '1', OPENTELEMETRY_COLLECTOR_ENDPOINT: observability.enabled && observability.tracingEndpoint ? observability.tracingEndpoint diff --git a/packages/services/schema/src/composition-scheduler.ts b/packages/services/schema/src/composition-scheduler.ts index b2ba433a301..1dd6e42406e 100644 --- a/packages/services/schema/src/composition-scheduler.ts +++ b/packages/services/schema/src/composition-scheduler.ts @@ -3,15 +3,22 @@ import { Worker } from 'node:worker_threads'; import fastq from 'fastq'; import * as Sentry from '@sentry/node'; import { registerWorkerLogging, type Logger } from '../../api/src/modules/shared/providers/logger'; -import type { CompositionEvent, CompositionResultEvent } from './composition-worker'; +import type { + CompositionEvent, + CompositionResultEvent, + ErrorResultEvent, + TrackMetricEvent, +} from './composition-worker'; import { compositionQueueDurationMS, compositionTotalDurationMS, compositionWorkerDurationMS, + compositionWorkerMemoryUsedBytes, } from './metrics'; type WorkerRunArgs = { data: CompositionEvent['data']; + targetId?: string; requestId: string; abortSignal: AbortSignal; }; @@ -126,18 +133,24 @@ export class CompositionScheduler { registerWorkerLogging(this.logger, worker, name); - worker.on( - 'message', - (data: CompositionResultEvent | { event: 'error'; id: string; err: Error }) => { - if (data.event === 'error') { - workerState?.task.reject(data.err); - } + worker.on('message', (data: CompositionResultEvent | ErrorResultEvent | TrackMetricEvent) => { + if (data.event === 'error') { + workerState?.task.reject(data.err); + } + + if (data.event === 'compositionResult') { + workerState?.task.resolve(data); + } - if (data.event === 'compositionResult') { - workerState?.task.resolve(data); + if (data.event === 'metric') { + if (data.type === 'heapUsed') { + compositionWorkerMemoryUsedBytes.set( + { target: workerState?.args.targetId, type: workerState?.args.data.type }, + data.value, + ); } - }, - ); + } + }); const { logger: baseLogger } = this; @@ -181,6 +194,7 @@ export class CompositionScheduler { event: 'composition', id: taskId, data: args.data, + targetId: args.targetId, taskId, requestId: args.requestId, } satisfies CompositionEvent); diff --git a/packages/services/schema/src/composition-worker.ts b/packages/services/schema/src/composition-worker.ts index 503cb5aba74..74fd8241397 100644 --- a/packages/services/schema/src/composition-worker.ts +++ b/packages/services/schema/src/composition-worker.ts @@ -9,6 +9,19 @@ import { composeSingle, type ComposeSingleArgs } from './composition/single'; import { composeStitching, type ComposeStitchingArgs } from './composition/stitching'; import type { env } from './environment'; +export type ErrorResultEvent = { + event: 'error'; + id: string; + err: unknown; +}; + +export type TrackMetricEvent = { + event: 'metric'; +} & { + type: 'heapUsed'; + value: number; +}; + export function createCompositionWorker(args: { baseLogger: Logger; port: MessagePort; @@ -33,6 +46,18 @@ export function createCompositionWorker(args: { event: message.event, }); logger.debug('processing message'); + const postResult = (result: CompositionResultEvent | ErrorResultEvent) => { + args.port.postMessage(result); + + if (args.env.compositionWorker.trackMemoryUsage) { + // send metric back to main thread so it can be reported + args.port.postMessage({ + event: 'metric', + type: 'heapUsed', + value: process.memoryUsage().heapUsed, + } satisfies TrackMetricEvent); + } + }; try { if (message.event === 'composition') { @@ -44,7 +69,7 @@ export function createCompositionWorker(args: { }); const composed = await composeFederation(message.data.args); - args.port.postMessage({ + postResult({ event: 'compositionResult', id: message.id, data: { @@ -73,7 +98,7 @@ export function createCompositionWorker(args: { if (message.data.type === 'single') { const result = await composeSingle(message.data.args); - args.port.postMessage({ + postResult({ event: 'compositionResult', id: message.id, data: { @@ -86,7 +111,7 @@ export function createCompositionWorker(args: { if (message.data.type === 'stitching') { const result = await composeStitching(message.data.args); - args.port.postMessage({ + postResult({ event: 'compositionResult', id: message.id, data: { @@ -106,7 +131,7 @@ export function createCompositionWorker(args: { message.id, ); baseLogger.error(String(err)); - args.port.postMessage({ + postResult({ event: 'error', id: message.id, err, @@ -123,6 +148,7 @@ export type CompositionEvent = { id: string; event: 'composition'; requestId: string; + targetId?: string; taskId: string; data: | { diff --git a/packages/services/schema/src/environment.ts b/packages/services/schema/src/environment.ts index 1d6ffb9a1c8..5e2f01c5cf3 100644 --- a/packages/services/schema/src/environment.ts +++ b/packages/services/schema/src/environment.ts @@ -38,6 +38,9 @@ const EnvironmentModel = zod.object({ AWS_REGION: emptyString(zod.string().optional()), COMPOSITION_WORKER_COUNT: zod.number().min(1).default(4), COMPOSITION_WORKER_MAX_OLD_GENERATION_SIZE_MB: NumberFromString(1).optional().default(512), + COMPOSITION_WORKER_TRACK_MEMORY_USAGE: emptyString( + zod.union([zod.literal('1'), zod.literal('0')]), + ).optional(), }); const RequestBrokerModel = zod.union([ @@ -206,5 +209,6 @@ export const env = { compositionWorker: { count: base.COMPOSITION_WORKER_COUNT, maxOldGenerationSizeMb: base.COMPOSITION_WORKER_MAX_OLD_GENERATION_SIZE_MB, + trackMemoryUsage: base.COMPOSITION_WORKER_TRACK_MEMORY_USAGE === '1', }, } as const; diff --git a/packages/services/schema/src/metrics.ts b/packages/services/schema/src/metrics.ts index 565ec269e74..54339a83c11 100644 --- a/packages/services/schema/src/metrics.ts +++ b/packages/services/schema/src/metrics.ts @@ -38,3 +38,9 @@ export const compositionCacheValueSizeBytes = new metrics.Histogram({ help: 'The size of the cache entries.', buckets: [200, 500, 1_000, 2_000, 3_000, 4_000, 5_000, 7_000, 10_000, 15_000, 20_000, 30_000], }); + +export const compositionWorkerMemoryUsedBytes = new metrics.Gauge({ + name: 'composition_worker_memory_used_bytes', + help: 'Amount of memory used by the composition worker', + labelNames: ['target', 'type'], +}); From b06b26f73fff5dc34a815a28072adbaa7e56fb96 Mon Sep 17 00:00:00 2001 From: Laurin Date: Sat, 25 Jul 2026 23:51:43 +0200 Subject: [PATCH 13/14] address vulnerabilities 2026-07-25 Episode I (#8262) --- .changeset/quiet-routers-heal.md | 6 ++++++ deployment/package.json | 2 +- package.json | 3 +-- pnpm-lock.yaml | 30 ++++++++++++++---------------- 4 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 .changeset/quiet-routers-heal.md diff --git a/.changeset/quiet-routers-heal.md b/.changeset/quiet-routers-heal.md new file mode 100644 index 00000000000..0da548f1608 --- /dev/null +++ b/.changeset/quiet-routers-heal.md @@ -0,0 +1,6 @@ +--- +'hive': patch +--- + +Address vulnerabilities [GHSA-45rx-2jwx-cxfr](https://github.com/advisories/GHSA-45rx-2jwx-cxfr), +and [GHSA-c96f-x56v-gq3h](https://github.com/advisories/GHSA-c96f-x56v-gq3h). diff --git a/deployment/package.json b/deployment/package.json index af475296b48..0642037333e 100644 --- a/deployment/package.json +++ b/deployment/package.json @@ -17,7 +17,7 @@ "@pulumi/kubernetesx": "0.1.6", "@pulumi/pulumi": "3.214.0", "@pulumi/random": "4.18.2", - "js-yaml": "4.2.0", + "js-yaml": "4.3.0", "pg-connection-string": "2.9.1", "prettier": "3.4.2" }, diff --git a/package.json b/package.json index 8c8ec6e69a7..7dffc0030c5 100644 --- a/package.json +++ b/package.json @@ -177,12 +177,11 @@ "brace-expansion@<1.1.16": "^1.1.16", "@sigstore/core@<=3.2.0": ">=3.2.1", "js-yaml@<3.15.0": "3.15.0", - "js-yaml@>=4.0.0 <4.3.0": "4.3.0", "sigstore@<=4.1.0": ">=4.1.1", "tar@<=7.5.18": "7.5.21", "fast-uri@2.x.x": "3.1.4", "fast-uri@3.x.x": "3.1.4", - "@opentelemetry/propagator-jaeger@2.x.x": "2.9.0", + "@opentelemetry/propagator-jaeger@<2.9.0": "2.9.0", "sharp@<0.35.0": "0.35.0", "body-parser@1.x.x": "1.20.6" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ead7f8677a7..14d15cad047 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,12 +53,11 @@ overrides: brace-expansion@<1.1.16: ^1.1.16 '@sigstore/core@<=3.2.0': '>=3.2.1' js-yaml@<3.15.0: 3.15.0 - js-yaml@>=4.0.0 <4.3.0: 4.3.0 sigstore@<=4.1.0: '>=4.1.1' tar@<=7.5.18: 7.5.21 fast-uri@2.x.x: 3.1.4 fast-uri@3.x.x: 3.1.4 - '@opentelemetry/propagator-jaeger@2.x.x': 2.9.0 + '@opentelemetry/propagator-jaeger@<2.9.0': 2.9.0 sharp@<0.35.0: 0.35.0 body-parser@1.x.x: 1.20.6 @@ -7113,12 +7112,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-jaeger@1.30.0': - resolution: {integrity: sha512-0hdP495V6HPRkVpowt54+Swn5NdesMIRof+rlp0mbnuIUOM986uF+eNxnPo9q5MmJegVBRTxgMHXXwvnXRnKRg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/propagator-jaeger@2.9.0': resolution: {integrity: sha512-4mYGty27rYvSM0jtp1ZUOqd3LfVRCYg9H5G9OFzSx5HViYToU21MFhWfco7x1HwXr7ER8yGOiCIHZUwjPksc0Q==} engines: {node: ^18.19.0 || >=20.6.0} @@ -13597,8 +13590,8 @@ packages: resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==} engines: {node: '>=8'} - find-my-way@9.5.0: - resolution: {integrity: sha512-VW2RfnmscZO5KgBY5XVyKREMW5nMZcxDy+buTOsL+zIPnBlbKm+00sgzoQzq1EVh4aALZLfKdwv6atBGcjvjrQ==} + find-my-way@9.7.0: + resolution: {integrity: sha512-f2JHn75x2JlwUwLenZypgczR7YWMb/uO9BvUXtus+JMgkbIkLADd38cI4EiV+OQqrGo1Zlq6V8wnqMJ8e62wUQ==} engines: {node: '>=20'} find-root@1.1.0: @@ -22151,7 +22144,7 @@ snapshots: dependencies: '@fastify/error': 4.2.0 fastify-plugin: 5.1.0 - find-my-way: 9.5.0 + find-my-way: 9.7.0 path-to-regexp: 8.4.0 reusify: 1.0.4 @@ -26667,6 +26660,11 @@ snapshots: '@opentelemetry/api': 1.9.1 '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/core@2.9.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/core@2.9.0(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -27732,10 +27730,10 @@ snapshots: '@opentelemetry/api': 1.9.1 '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) - '@opentelemetry/propagator-jaeger@1.30.0(@opentelemetry/api@1.9.0)': + '@opentelemetry/propagator-jaeger@2.9.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 1.30.0(@opentelemetry/api@1.9.0) + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.0) '@opentelemetry/propagator-jaeger@2.9.0(@opentelemetry/api@1.9.1)': dependencies: @@ -28007,7 +28005,7 @@ snapshots: '@opentelemetry/context-async-hooks': 1.30.0(@opentelemetry/api@1.9.0) '@opentelemetry/core': 1.30.0(@opentelemetry/api@1.9.0) '@opentelemetry/propagator-b3': 1.30.0(@opentelemetry/api@1.9.0) - '@opentelemetry/propagator-jaeger': 1.30.0(@opentelemetry/api@1.9.0) + '@opentelemetry/propagator-jaeger': 2.9.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.30.0(@opentelemetry/api@1.9.0) semver: 7.7.2 @@ -35167,7 +35165,7 @@ snapshots: abstract-logging: 2.0.1 avvio: 9.1.0 fast-json-stringify: 6.2.0 - find-my-way: 9.5.0 + find-my-way: 9.7.0 light-my-request: 6.6.0 pino: 10.3.0 process-warning: 5.0.0 @@ -35260,7 +35258,7 @@ snapshots: make-dir: 3.1.0 pkg-dir: 4.2.0 - find-my-way@9.5.0: + find-my-way@9.7.0: dependencies: fast-deep-equal: 3.1.3 fast-querystring: 1.1.2 From 5c88922d384e9dc215071fcafe325440d13e2f47 Mon Sep 17 00:00:00 2001 From: Laurin Quast Date: Mon, 27 Jul 2026 10:16:28 +0200 Subject: [PATCH 14/14] changeset --- .changeset/fuzzy-lions-federate.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .changeset/fuzzy-lions-federate.md diff --git a/.changeset/fuzzy-lions-federate.md b/.changeset/fuzzy-lions-federate.md new file mode 100644 index 00000000000..1b6d31b13c7 --- /dev/null +++ b/.changeset/fuzzy-lions-federate.md @@ -0,0 +1,16 @@ +--- +'hive': patch +--- + +Add Azure Workload Identity Federation support for OIDC SSO in self-hosted deployments. Enabled +organizations use the projected Azure token as a client assertion instead of an OIDC client secret. + +For example, enable federation for an organization with: + +```env +OIDC_WORKLOAD_FEDERATION_IDENTITY_PROVIDER=azure +AZURE_FEDERATED_TOKEN_FILE=/var/run/secrets/azure/tokens/azure-identity-token +OIDC_WORKLOAD_FEDERATION_ORGANIZATION_IDS=00000000-0000-4000-8000-000000000000 +``` + +This will effectively override the client secret configured for that organization to use the OIDC provider.