[FME-9871] Add data loader utils: getRolloutPlan, setRolloutPlan, validateRolloutPlan

src/storages/__tests__/dataLoader.spec.ts

@@ -0,0 +1,133 @@
+import { InMemoryStorageFactory } from '../inMemory/InMemoryStorage';
+import { InMemoryStorageCSFactory } from '../inMemory/InMemoryStorageCS';
+import { fullSettings } from '../../utils/settingsValidation/__tests__/settings.mocks';
+import { loggerMock } from '../../logger/__tests__/sdkLogger.mock';
+import { IRBSegment, ISplit } from '../../dtos/types';
+
+import { validateRolloutPlan, setRolloutPlan } from '../setRolloutPlan';
+import { getRolloutPlan } from '../getRolloutPlan';
+
+const otherKey = 'otherKey';
+const expectedRolloutPlan = {
+  splitChanges: {
+    ff: { d: [{ name: 'split1' }], t: 123, s: -1 },
+    rbs: { d: [{ name: 'rbs1' }], t: 321, s: -1 }
+  },
+  memberships: {
+    [fullSettings.core.key as string]: { ms: { k: [{ n: 'segment1' }] }, ls: { k: [] } },
+    [otherKey]: { ms: { k: [{ n: 'segment1' }] }, ls: { k: [] } }
+  },
+  segmentChanges: [{
+    name: 'segment1',
+    added: [fullSettings.core.key as string, otherKey],
+    removed: [],
+    since: -1,
+    till: 123
+  }]
+};
+
+describe('validateRolloutPlan', () => {
+  afterEach(() => {
+    loggerMock.mockClear();
+  });
+
+  test('valid rollout plan and mode', () => {
+    expect(validateRolloutPlan(loggerMock, { mode: 'standalone', initialRolloutPlan: expectedRolloutPlan } as any)).toEqual(expectedRolloutPlan);
+    expect(loggerMock.error).not.toHaveBeenCalled();
+  });
+
+  test('invalid rollout plan', () => {
+    expect(validateRolloutPlan(loggerMock, { mode: 'standalone', initialRolloutPlan: {} } as any)).toBeUndefined();
+    expect(loggerMock.error).toHaveBeenCalledWith('storage: invalid rollout plan provided');
+  });
+
+  test('invalid mode', () => {
+    expect(validateRolloutPlan(loggerMock, { mode: 'consumer', initialRolloutPlan: expectedRolloutPlan } as any)).toBeUndefined();
+    expect(loggerMock.warn).toHaveBeenCalledWith('storage: initial rollout plan is ignored in consumer mode');
+  });
+});
+
+describe('getRolloutPlan & setRolloutPlan (client-side)', () => {
+  // @ts-expect-error Load server-side storage
+  const serverStorage = InMemoryStorageFactory({ settings: fullSettings });
+  serverStorage.splits.update([{ name: 'split1' } as ISplit], [], 123);
+  serverStorage.rbSegments.update([{ name: 'rbs1' } as IRBSegment], [], 321);
+  serverStorage.segments.update('segment1', [fullSettings.core.key as string, otherKey], [], 123);
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('using preloaded data (no memberships, no segments)', () => {
+    const rolloutPlan = getRolloutPlan(loggerMock, serverStorage);
+
+    // @ts-expect-error Load client-side storage with preloaded data
+    const clientStorage = InMemoryStorageCSFactory({ settings: fullSettings });
+    setRolloutPlan(loggerMock, rolloutPlan, clientStorage, fullSettings.core.key as string);
+
+    // Shared client storage
+    const sharedClientStorage = clientStorage.shared!(otherKey);
+    setRolloutPlan(loggerMock, rolloutPlan, { segments: sharedClientStorage.segments, largeSegments: sharedClientStorage.largeSegments }, otherKey);
+
+    expect(clientStorage.segments.getRegisteredSegments()).toEqual([]);
+    expect(sharedClientStorage.segments.getRegisteredSegments()).toEqual([]);
+
+    // Get preloaded data from client-side storage
+    expect(getRolloutPlan(loggerMock, clientStorage)).toEqual(rolloutPlan);
+    expect(rolloutPlan).toEqual({ ...expectedRolloutPlan, memberships: undefined, segmentChanges: undefined });
+  });
+
+  test('using preloaded data with memberships', () => {
+    const rolloutPlan = getRolloutPlan(loggerMock, serverStorage, { keys: [fullSettings.core.key as string, otherKey] });
+
+    // @ts-expect-error Load client-side storage with preloaded data
+    const clientStorage = InMemoryStorageCSFactory({ settings: fullSettings });
+    setRolloutPlan(loggerMock, rolloutPlan, clientStorage, fullSettings.core.key as string);
+
+    // Shared client storage
+    const sharedClientStorage = clientStorage.shared!(otherKey);
+    setRolloutPlan(loggerMock, rolloutPlan, { segments: sharedClientStorage.segments, largeSegments: sharedClientStorage.largeSegments }, otherKey);
+
+    expect(clientStorage.segments.getRegisteredSegments()).toEqual(['segment1']);
+    expect(sharedClientStorage.segments.getRegisteredSegments()).toEqual(['segment1']);
+
+    // @TODO requires internal storage cache for `shared` storages
+    // // Get preloaded data from client-side storage
+    // expect(getRolloutPlan(loggerMock, clientStorage, { keys: [fullSettings.core.key as string, otherKey] })).toEqual(rolloutPlan);
+    // expect(rolloutPlan).toEqual({ ...expectedRolloutPlan, segmentChanges: undefined });
+  });
+
+  test('using preloaded data with segments', () => {
+    const rolloutPlan = getRolloutPlan(loggerMock, serverStorage, { exposeSegments: true });
+
+    // @ts-expect-error Load client-side storage with preloaded data
+    const clientStorage = InMemoryStorageCSFactory({ settings: fullSettings });
+    setRolloutPlan(loggerMock, rolloutPlan, clientStorage, fullSettings.core.key as string);
+
+    // Shared client storage
+    const sharedClientStorage = clientStorage.shared!(otherKey);
+    setRolloutPlan(loggerMock, rolloutPlan, { segments: sharedClientStorage.segments, largeSegments: sharedClientStorage.largeSegments }, otherKey);
+
+    expect(clientStorage.segments.getRegisteredSegments()).toEqual(['segment1']);
+    expect(sharedClientStorage.segments.getRegisteredSegments()).toEqual(['segment1']);
+
+    expect(rolloutPlan).toEqual({ ...expectedRolloutPlan, memberships: undefined });
+  });
+
+  test('using preloaded data with memberships and segments', () => {
+    const rolloutPlan = getRolloutPlan(loggerMock, serverStorage, { keys: [fullSettings.core.key as string], exposeSegments: true });
+
+    // @ts-expect-error Load client-side storage with preloaded data
+    const clientStorage = InMemoryStorageCSFactory({ settings: fullSettings });
+    setRolloutPlan(loggerMock, rolloutPlan, clientStorage, fullSettings.core.key as string);
+
+    // Shared client storage
+    const sharedClientStorage = clientStorage.shared!(otherKey);
+    setRolloutPlan(loggerMock, rolloutPlan, { segments: sharedClientStorage.segments, largeSegments: sharedClientStorage.largeSegments }, otherKey);
+
+    expect(clientStorage.segments.getRegisteredSegments()).toEqual(['segment1']); // main client membership is set via the rollout plan `memberships` field
+    expect(sharedClientStorage.segments.getRegisteredSegments()).toEqual(['segment1']); // shared client membership is set via the rollout plan `segmentChanges` field
+
+    expect(rolloutPlan).toEqual({ ...expectedRolloutPlan, memberships: { [fullSettings.core.key as string]: expectedRolloutPlan.memberships![fullSettings.core.key as string] } });
+  });
+});

src/storages/getRolloutPlan.ts

@@ -0,0 +1,74 @@
+import SplitIO from '../../types/splitio';
+import { IStorageSync } from './types';
+import { setToArray } from '../utils/lang/sets';
+import { getMatching } from '../utils/key';
+import { ILogger } from '../logger/types';
+import { RolloutPlan } from './types';
+import { IMembershipsResponse, IMySegmentsResponse } from '../dtos/types';
+
+/**
+ * Gets the rollout plan snapshot from the given synchronous storage.
+ * If `keys` are provided, the memberships for those keys is returned, to protect segments data.
+ * Otherwise, the segments data is returned.
+ */
+export function getRolloutPlan(log: ILogger, storage: IStorageSync, options: SplitIO.RolloutPlanOptions = {}): RolloutPlan {
+
+  const { keys, exposeSegments } = options;
+  const { splits, segments, rbSegments } = storage;
+
+  log.debug(`storage: get feature flags${keys ? `, and memberships for keys ${keys}` : ''}${exposeSegments ? ', and segments' : ''}`);
+
+  return {
+    splitChanges: {
+      ff: {
+        t: splits.getChangeNumber(),
+        s: -1,
+        d: splits.getAll(),
+      },
+      rbs: {
+        t: rbSegments.getChangeNumber(),
+        s: -1,
+        d: rbSegments.getAll(),
+      }
+    },
+    segmentChanges: exposeSegments ? // @ts-ignore accessing private prop
+      Object.keys(segments.segmentCache).map(segmentName => ({
+        name: segmentName, // @ts-ignore
+        added: setToArray(segments.segmentCache[segmentName] as Set<string>),
+        removed: [],
+        since: -1,
+        till: segments.getChangeNumber(segmentName)!
+      })) :
+      undefined,
+    memberships: keys ?
+      keys.reduce<Record<string, IMembershipsResponse>>((prev, key) => {
+        const matchingKey = getMatching(key);
+        if (storage.shared) { // Client-side segments
+          const sharedStorage = storage.shared(matchingKey);
+          prev[matchingKey] = {
+            ms: { // @ts-ignore
+              k: Object.keys(sharedStorage.segments.segmentCache).map(segmentName => ({ n: segmentName })),
+            },
+            ls: sharedStorage.largeSegments ? { // @ts-ignore
+              k: Object.keys(sharedStorage.largeSegments.segmentCache).map(segmentName => ({ n: segmentName })),
+            } : undefined
+          };
+        } else { // Server-side segments
+          prev[matchingKey] = {
+            ms: { // @ts-ignore
+              k: Object.keys(storage.segments.segmentCache).reduce<IMySegmentsResponse['k']>((prev, segmentName) => { // @ts-ignore
+                return storage.segments.segmentCache[segmentName].has(matchingKey) ?
+                  prev!.concat({ n: segmentName }) :
+                  prev;
+              }, [])
+            },
+            ls: {
+              k: []
+            }
+          };
+        }
+        return prev;
+      }, {}) :
+      undefined
+  };
+}

src/storages/setRolloutPlan.ts

@@ -0,0 +1,71 @@
+import SplitIO from '../../types/splitio';
+import { IRBSegmentsCacheSync, ISegmentsCacheSync, ISplitsCacheSync } from './types';
+import { ILogger } from '../logger/types';
+import { isObject } from '../utils/lang';
+import { isConsumerMode } from '../utils/settingsValidation/mode';
+import { RolloutPlan } from './types';
+
+/**
+ * Validates if the given rollout plan is valid.
+ */
+export function validateRolloutPlan(log: ILogger, settings: SplitIO.ISettings): RolloutPlan | undefined {
+  const { mode, initialRolloutPlan } = settings;
+
+  if (isConsumerMode(mode)) {
+    log.warn('storage: initial rollout plan is ignored in consumer mode');
+    return;
+  }
+
+  if (isObject(initialRolloutPlan) && isObject((initialRolloutPlan as any).splitChanges)) return initialRolloutPlan as RolloutPlan;
+
+  log.error('storage: invalid rollout plan provided');
+  return;
+}
+
+/**
+ * Sets the given synchronous storage with the provided rollout plan snapshot.
+ * If `matchingKey` is provided, the storage is handled as a client-side storage (segments and largeSegments are instances of MySegmentsCache).
+ * Otherwise, the storage is handled as a server-side storage (segments is an instance of SegmentsCache).
+ */
+export function setRolloutPlan(log: ILogger, rolloutPlan: RolloutPlan, storage: { splits?: ISplitsCacheSync, rbSegments?: IRBSegmentsCacheSync, segments: ISegmentsCacheSync, largeSegments?: ISegmentsCacheSync }, matchingKey?: string) {
+  const { splits, rbSegments, segments, largeSegments } = storage;
+  const { splitChanges: { ff, rbs } } = rolloutPlan;
+
+  log.debug(`storage: set feature flags and segments${matchingKey ? ` for key ${matchingKey}` : ''}`);
+
+  if (splits && ff) {
+    splits.clear();
+    splits.update(ff.d, [], ff.t);
+  }
+
+  if (rbSegments && rbs) {
+    rbSegments.clear();
+    rbSegments.update(rbs.d, [], rbs.t);
+  }
+
+  const segmentChanges = rolloutPlan.segmentChanges;
+  if (matchingKey) { // add memberships data (client-side)
+    let memberships = rolloutPlan.memberships && rolloutPlan.memberships[matchingKey];
+    if (!memberships && segmentChanges) {
+      memberships = {
+        ms: {
+          k: segmentChanges.filter(segment => {
+            return segment.added.indexOf(matchingKey) > -1;
+          }).map(segment => ({ n: segment.name }))
+        }
+      };
+    }
+
+    if (memberships) {
+      if (memberships.ms) segments.resetSegments(memberships.ms!);
+      if (memberships.ls && largeSegments) largeSegments.resetSegments(memberships.ls!);
+    }
+  } else { // add segments data (server-side)
+    if (segmentChanges) {
+      segments.clear();
+      segmentChanges.forEach(segment => {
+        segments.update(segment.name, segment.added, segment.removed, segment.till);
+      });
+    }
+  }
+}

src/storages/types.ts

@@ -1,5 +1,5 @@
 import SplitIO from '../../types/splitio';
-import { MaybeThenable, ISplit, IRBSegment, IMySegmentsResponse } from '../dtos/types';
+import { MaybeThenable, ISplit, IRBSegment, IMySegmentsResponse, IMembershipsResponse, ISegmentChangesResponse, ISplitChangesResponse } from '../dtos/types';
 import { MySegmentsData } from '../sync/polling/types';
 import { EventDataType, HttpErrors, HttpLatencies, ImpressionDataType, LastSync, Method, MethodExceptions, MethodLatencies, MultiMethodExceptions, MultiMethodLatencies, MultiConfigs, OperationType, StoredEventWithMetadata, StoredImpressionWithMetadata, StreamingEvent, UniqueKeysPayloadCs, UniqueKeysPayloadSs, TelemetryUsageStatsPayload, UpdatesFromSSEEnum } from '../sync/submitters/types';
 import { ISettings } from '../types';
@@ -522,3 +522,21 @@ export type IStorageAsyncFactory = SplitIO.StorageAsyncFactory & {
   readonly type: SplitIO.StorageType,
   (params: IStorageFactoryParams): IStorageAsync
 }
+
+export type RolloutPlan = {
+  /**
+   * Feature flags and rule-based segments.
+   */
+  splitChanges: ISplitChangesResponse;
+  /**
+   * Optional map of matching keys to their memberships.
+   */
+  memberships?: {
+    [matchingKey: string]: IMembershipsResponse;
+  };
+  /**
+   * Optional list of standard segments.
+   * This property is ignored if `memberships` is provided.
+   */
+  segmentChanges?: ISegmentChangesResponse[];
+};