fix(rbac): retry plugin permission metadata reads during conditional policy reconcile

workspaces/rbac/.changeset/mighty-wolves-retry.md

@@ -0,0 +1,5 @@
+---
+'@backstage-community/plugin-rbac-backend': minor
+---
+
+Conditional policy reconciliation from the conditional policies file now retries plugin permission metadata reads with bounded exponential backoff, so transient startup races (permission metadata routes mounting after RBAC startup) no longer abort the reconcile. The retry window is configurable via `permission.rbac.conditionalMetadataRetry` (`maxAttempts`, `baseDelayMs`, `maxDelayMs`) and defaults to roughly 4 minutes; set `maxAttempts: 1` to restore the previous fail-fast behavior. Metadata reads triggered through the REST API keep failing fast.

workspaces/rbac/plugins/rbac-backend/README.md

@@ -324,6 +324,32 @@ All values must be positive integers.
 
 If you already use very large conditional payloads or YAML files, raise these limits in config during upgrade rather than relying on defaults.
 
+### Retrying permission metadata reads during conditional policy reconciliation
+
+When conditional policies are applied from `conditionalPoliciesFile`, the RBAC backend reads the permission metadata of every target plugin (for example `catalog`) to map permission actions to permission names. On a fresh boot the permission metadata route of a target plugin can mount later than the RBAC plugin starts, which previously caused the reconciliation to abort until the next file reload or restart.
+
+The RBAC backend now retries these metadata reads with exponential backoff and jitter. The defaults allow roughly 4 minutes of total retry budget (12 attempts, starting at a 2 second delay and capped at 30 seconds per delay). If the metadata endpoint stays unavailable beyond that window, the reconciliation still aborts safely and preserves the stored conditions; it is applied on a later file reload or restart. Reads triggered through the REST API are not retried and keep failing fast.
+
+The retry behavior can be tuned:
+
+```YAML
+permission:
+  enabled: true
+  rbac:
+    conditionalMetadataRetry:
+      maxAttempts: 12
+      baseDelayMs: 2000
+      maxDelayMs: 30000
+```
+
+Field descriptions:
+
+- `permission.rbac.conditionalMetadataRetry.maxAttempts`: Maximum number of read attempts before the reconciliation aborts. Set to `1` to disable retries.
+- `permission.rbac.conditionalMetadataRetry.baseDelayMs`: Initial backoff delay in milliseconds, doubled on every attempt.
+- `permission.rbac.conditionalMetadataRetry.maxDelayMs`: Upper bound in milliseconds for a single backoff delay.
+
+All values must be positive integers.
+
 ### Configuring Database Storage for policies
 
 The RBAC plugin offers the option to store policies in a database. It supports two database storage options:

workspaces/rbac/plugins/rbac-backend/config.d.ts

@@ -94,6 +94,27 @@ export interface Config {
           action: 'create' | 'read' | 'update' | 'delete' | 'use';
         }>;
       };
+      /**
+       * Optional retry tuning for reading plugin permission metadata while
+       * applying conditional policies from `conditionalPoliciesFile`.
+       * Bounds the wait for permission metadata routes of target plugins
+       * that are not mounted yet during startup.
+       */
+      conditionalMetadataRetry?: {
+        /**
+         * Maximum number of read attempts before conditional policy
+         * reconciliation aborts. Set to 1 to disable retries. Defaults to 12.
+         */
+        maxAttempts?: number;
+        /**
+         * Initial backoff delay in milliseconds. Defaults to 2000.
+         */
+        baseDelayMs?: number;
+        /**
+         * Upper bound in milliseconds for a single backoff delay. Defaults to 30000.
+         */
+        maxDelayMs?: number;
+      };
       /**
        * Optional validation tuning for conditional policies.
        */

workspaces/rbac/plugins/rbac-backend/src/file-permissions/yaml-conditional-file-watcher.test.ts

@@ -34,6 +34,7 @@ import {
   YamlConditionalPoliciesFileWatcher,
 } from './yaml-conditional-file-watcher';
 import { DEFAULT_CONDITION_VALIDATION_LIMITS } from '../validation/condition-validation';
+import type { ConditionalMetadataRetryOptions } from '../helper';
 import { mockAuditorService } from '../../__fixtures__/mock-utils';
 import { expectAuditorLog } from '../../__fixtures__/auditor-test-utils';
 import {
@@ -276,6 +277,26 @@ describe('YamlConditionalFileWatcher', () => {
     );
   }
 
+  function createWatcherWithMetadataRetry(
+    filePath: string | undefined,
+    metadataRetry: Partial<ConditionalMetadataRetryOptions>,
+  ): YamlConditionalPoliciesFileWatcher {
+    return new YamlConditionalPoliciesFileWatcher(
+      filePath,
+      false,
+      mockLoggerService,
+      conditionalStorageMock as DataBaseConditionalStorage,
+      mockAuditorService,
+      mockAuthService,
+      pluginMetadataCollectorMock as PluginPermissionMetadataCollector,
+      roleMetadataStorageMock,
+      roleEventEmitterMock,
+      DEFAULT_CONDITION_VALIDATION_LIMITS,
+      {},
+      metadataRetry,
+    );
+  }
+
   describe('conditional policies file limit validation', () => {
     it('throws InputError when maxBytes is zero', () => {
       expect(() => createWatcherWithLimits(csvFileName, 0, 256)).toThrow(
@@ -767,7 +788,9 @@ describe('YamlConditionalFileWatcher', () => {
       '      - group:default/team-b',
     ].join('\n');
 
-    const watcher = createWatcher(csvFileName);
+    const watcher = createWatcherWithMetadataRetry(csvFileName, {
+      maxAttempts: 1,
+    });
     jest.spyOn(watcher, 'getCurrentContents').mockReturnValue(updatedYaml);
     await watcher.onChange();
 
@@ -777,6 +800,85 @@ describe('YamlConditionalFileWatcher', () => {
     expect(mockLoggerService.error).toHaveBeenCalled();
   });
 
+  describe('conditional metadata retry', () => {
+    let randomSpy: jest.SpyInstance;
+
+    beforeEach(() => {
+      randomSpy = jest.spyOn(Math, 'random').mockReturnValue(0);
+    });
+
+    afterEach(() => {
+      randomSpy.mockRestore();
+    });
+
+    it('throws InputError when maxAttempts is not a positive integer', () => {
+      expect(() =>
+        createWatcherWithMetadataRetry(csvFileName, { maxAttempts: 0 }),
+      ).toThrow(InputError);
+      expect(() =>
+        createWatcherWithMetadataRetry(csvFileName, { baseDelayMs: -1 }),
+      ).toThrow(
+        `'baseDelayMs' must be a positive integer for 'permission.rbac.conditionalMetadataRetry'`,
+      );
+    });
+
+    it('applies conditions after transient metadata failures during startup', async () => {
+      conditionalStorageMock.filterConditions = jest
+        .fn()
+        .mockImplementation(() => []);
+      roleMetadataStorageMock.filterRoleMetadata = jest
+        .fn()
+        .mockImplementation(() => csvFileRoles);
+      pluginMetadataCollectorMock.getMetadataByPluginId = jest
+        .fn()
+        .mockResolvedValueOnce(undefined)
+        .mockResolvedValue(testPluginMetadataResp);
+
+      const watcher = createWatcherWithMetadataRetry(csvFileName, {
+        maxAttempts: 3,
+        baseDelayMs: 1,
+        maxDelayMs: 2,
+      });
+      await watcher.initialize();
+
+      expect(
+        pluginMetadataCollectorMock.getMetadataByPluginId,
+      ).toHaveBeenCalledTimes(3);
+      expect(conditionalStorageMock.createCondition).toHaveBeenCalledTimes(2);
+      expect(loggerWarnSpy).toHaveBeenCalledWith(
+        expect.stringMatching(
+          /Unable to get permission list for plugin catalog, retrying in \d+ms \(attempt 1 of 3\)/,
+        ),
+      );
+    });
+
+    it('preserves stored conditions when metadata stays unavailable after retries', async () => {
+      conditionalStorageMock.filterConditions = jest
+        .fn()
+        .mockImplementation(() => [conditionToRemove]);
+      roleMetadataStorageMock.filterRoleMetadata = jest
+        .fn()
+        .mockImplementation(() => csvFileRoles);
+      pluginMetadataCollectorMock.getMetadataByPluginId = jest
+        .fn()
+        .mockResolvedValue(undefined);
+
+      const watcher = createWatcherWithMetadataRetry(csvFileName, {
+        maxAttempts: 2,
+        baseDelayMs: 1,
+        maxDelayMs: 2,
+      });
+      await watcher.initialize();
+
+      expect(
+        pluginMetadataCollectorMock.getMetadataByPluginId,
+      ).toHaveBeenCalledTimes(2);
+      expect(conditionalStorageMock.deleteCondition).not.toHaveBeenCalled();
+      expect(conditionalStorageMock.createCondition).not.toHaveBeenCalled();
+      expect(mockLoggerService.error).toHaveBeenCalled();
+    });
+  });
+
   test('should merge sibling yaml conditions via update and delete', async () => {
     const storedRead = {
       id: 1,

workspaces/rbac/plugins/rbac-backend/src/file-permissions/yaml-conditional-file-watcher.ts

@@ -39,12 +39,14 @@ import {
 } from '../database/role-metadata';
 import {
   abortConditionalPolicyReconcile,
+  type ConditionalMetadataRetryOptions,
   deepSortEqual,
   diffConditionalPolicies,
   pendingDeleteIdsFromPlan,
   permissionMappingToActions,
   planConditionalReconcile,
   processConditionMapping,
+  resolveConditionalMetadataRetryOptions,
   toError,
 } from '../helper';
 import { RoleEventEmitter, RoleEvents } from '../service/enforcer-delegate';
@@ -122,6 +124,7 @@ export class YamlConditionalPoliciesFileWatcher extends AbstractFileWatcher<
   private readonly maxFileBytes: number;
   private readonly maxFileDocuments: number;
   private readonly conditionValidationLimits: ConditionValidationLimits;
+  private readonly metadataRetryOptions: ConditionalMetadataRetryOptions;
 
   constructor(
     filePath: string | undefined,
@@ -135,13 +138,16 @@ export class YamlConditionalPoliciesFileWatcher extends AbstractFileWatcher<
     private readonly roleEventEmitter: RoleEventEmitter<RoleEvents>,
     conditionValidationLimits: ConditionValidationLimits,
     limits: Partial<ConditionalPoliciesFileLimits> = {},
+    metadataRetry: Partial<ConditionalMetadataRetryOptions> = {},
   ) {
     super(filePath, allowReload, logger);
 
     const resolvedLimits = resolveConditionalPoliciesFileLimits(limits);
     this.maxFileBytes = resolvedLimits.maxBytes;
     this.maxFileDocuments = resolvedLimits.maxDocuments;
     this.conditionValidationLimits = conditionValidationLimits;
+    this.metadataRetryOptions =
+      resolveConditionalMetadataRetryOptions(metadataRetry);
   }
 
   async initialize(): Promise<void> {
@@ -247,6 +253,10 @@ export class YamlConditionalPoliciesFileWatcher extends AbstractFileWatcher<
             condition,
             this.pluginMetadataCollector,
             this.auth,
+            {
+              metadataRetry: this.metadataRetryOptions,
+              logger: this.logger,
+            },
           ),
         );
       }