Merge pull request #609 from objectstack-ai/copilot/optimize-protocol-report-another-one

Author: hotlong

PROTOCOL_OPTIMIZATION_REPORT.md

@@ -662,3 +662,156 @@ describe('ExternalLookupSchema', () => {
     expect(() => ExternalLookupSchema.parse(salesforceLookup)).not.toThrow();
   });
 });
+
+describe('External Lookup Retry Configuration', () => {
+  const baseLookup = {
+    fieldName: 'external_data',
+    dataSource: {
+      id: 'test-api',
+      name: 'Test API',
+      type: 'rest-api' as const,
+      endpoint: 'https://api.example.com',
+      authentication: { type: 'api-key' as const, config: { key: 'test' } },
+    },
+    query: { endpoint: '/data' },
+    fieldMappings: [{ source: 'name', target: 'name' }],
+  };
+
+  it('should accept lookup with retry configuration', () => {
+    const lookup = {
+      ...baseLookup,
+      retry: {
+        maxRetries: 3,
+        initialDelayMs: 1000,
+        maxDelayMs: 30000,
+        backoffMultiplier: 2,
+        retryableStatusCodes: [429, 500, 502, 503, 504],
+      },
+    };
+
+    expect(() => ExternalLookupSchema.parse(lookup)).not.toThrow();
+  });
+
+  it('should apply retry defaults', () => {
+    const result = ExternalLookupSchema.parse({
+      ...baseLookup,
+      retry: {},
+    });
+
+    expect(result.retry?.maxRetries).toBe(3);
+    expect(result.retry?.initialDelayMs).toBe(1000);
+    expect(result.retry?.maxDelayMs).toBe(30000);
+    expect(result.retry?.backoffMultiplier).toBe(2);
+    expect(result.retry?.retryableStatusCodes).toEqual([429, 500, 502, 503, 504]);
+  });
+
+  it('should accept custom retryable status codes', () => {
+    const result = ExternalLookupSchema.parse({
+      ...baseLookup,
+      retry: { retryableStatusCodes: [408, 429, 503] },
+    });
+
+    expect(result.retry?.retryableStatusCodes).toEqual([408, 429, 503]);
+  });
+
+  it('should reject negative maxRetries', () => {
+    expect(() => ExternalLookupSchema.parse({
+      ...baseLookup,
+      retry: { maxRetries: -1 },
+    })).toThrow();
+  });
+});
+
+describe('External Lookup Transform Configuration', () => {
+  const baseLookup = {
+    fieldName: 'external_data',
+    dataSource: {
+      id: 'test-api',
+      name: 'Test API',
+      type: 'rest-api' as const,
+      endpoint: 'https://api.example.com',
+      authentication: { type: 'api-key' as const, config: { key: 'test' } },
+    },
+    query: { endpoint: '/data' },
+    fieldMappings: [{ source: 'name', target: 'name' }],
+  };
+
+  it('should accept lookup with transform pipeline', () => {
+    const lookup = {
+      ...baseLookup,
+      transform: {
+        request: {
+          headers: { 'X-Custom-Header': 'value' },
+          queryParams: { format: 'json' },
+        },
+        response: {
+          dataPath: '$.data.results',
+          totalPath: '$.meta.total',
+        },
+      },
+    };
+
+    expect(() => ExternalLookupSchema.parse(lookup)).not.toThrow();
+  });
+
+  it('should accept partial transform config', () => {
+    const result = ExternalLookupSchema.parse({
+      ...baseLookup,
+      transform: {
+        response: { dataPath: '$.items' },
+      },
+    });
+
+    expect(result.transform?.response?.dataPath).toBe('$.items');
+    expect(result.transform?.request).toBeUndefined();
+  });
+});
+
+describe('External Lookup Pagination Configuration', () => {
+  const baseLookup = {
+    fieldName: 'external_data',
+    dataSource: {
+      id: 'test-api',
+      name: 'Test API',
+      type: 'rest-api' as const,
+      endpoint: 'https://api.example.com',
+      authentication: { type: 'api-key' as const, config: { key: 'test' } },
+    },
+    query: { endpoint: '/data' },
+    fieldMappings: [{ source: 'name', target: 'name' }],
+  };
+
+  it('should accept lookup with pagination', () => {
+    const result = ExternalLookupSchema.parse({
+      ...baseLookup,
+      pagination: {
+        type: 'cursor',
+        pageSize: 50,
+        maxPages: 10,
+      },
+    });
+
+    expect(result.pagination?.type).toBe('cursor');
+    expect(result.pagination?.pageSize).toBe(50);
+  });
+
+  it('should apply pagination defaults', () => {
+    const result = ExternalLookupSchema.parse({
+      ...baseLookup,
+      pagination: {},
+    });
+
+    expect(result.pagination?.type).toBe('offset');
+    expect(result.pagination?.pageSize).toBe(100);
+  });
+
+  it('should accept all pagination types', () => {
+    const types = ['offset', 'cursor', 'page'] as const;
+    types.forEach(type => {
+      expect(() => ExternalLookupSchema.parse({
+        ...baseLookup,
+        pagination: { type },
+      })).not.toThrow();
+    });
+  });
+});

packages/spec/src/data/external-lookup.test.ts

@@ -241,6 +241,67 @@ export const ExternalLookupSchema = z.object({
      */
     burstSize: z.number().optional().describe('Burst size'),
   }).optional().describe('Rate limiting'),
+
+  /**
+   * Retry configuration with exponential backoff
+   *
+   * @example
+   * ```json
+   * {
+   *   "maxRetries": 3,
+   *   "initialDelayMs": 1000,
+   *   "maxDelayMs": 30000,
+   *   "backoffMultiplier": 2,
+   *   "retryableStatusCodes": [429, 500, 502, 503, 504]
+   * }
+   * ```
+   */
+  retry: z.object({
+    /** Maximum number of retry attempts */
+    maxRetries: z.number().min(0).default(3).describe('Maximum retry attempts'),
+    /** Initial delay before first retry (ms) */
+    initialDelayMs: z.number().default(1000).describe('Initial retry delay in milliseconds'),
+    /** Maximum delay between retries (ms) */
+    maxDelayMs: z.number().default(30000).describe('Maximum retry delay in milliseconds'),
+    /** Backoff multiplier for exponential backoff */
+    backoffMultiplier: z.number().default(2).describe('Exponential backoff multiplier'),
+    /** HTTP status codes that trigger a retry */
+    retryableStatusCodes: z.array(z.number()).default([429, 500, 502, 503, 504])
+      .describe('HTTP status codes that are retryable'),
+  }).optional().describe('Retry configuration with exponential backoff'),
+
+  /**
+   * Request/response transformation pipeline
+   *
+   * Allows transforming request parameters and response data
+   * before they are processed by the external lookup system.
+   */
+  transform: z.object({
+    /** Transform request parameters before sending */
+    request: z.object({
+      /** Header transformations (key-value additions) */
+      headers: z.record(z.string(), z.string()).optional().describe('Additional request headers'),
+      /** Query parameter transformations */
+      queryParams: z.record(z.string(), z.string()).optional().describe('Additional query parameters'),
+    }).optional().describe('Request transformation'),
+    /** Transform response data after receiving */
+    response: z.object({
+      /** JSONPath expression to extract data from response */
+      dataPath: z.string().optional().describe('JSONPath to extract data (e.g., "$.data.results")'),
+      /** JSONPath expression to extract total count for pagination */
+      totalPath: z.string().optional().describe('JSONPath to extract total count (e.g., "$.meta.total")'),
+    }).optional().describe('Response transformation'),
+  }).optional().describe('Request/response transformation pipeline'),
+
+  /** Pagination support for external data sources */
+  pagination: z.object({
+    /** Pagination type */
+    type: z.enum(['offset', 'cursor', 'page']).default('offset').describe('Pagination type'),
+    /** Page size */
+    pageSize: z.number().default(100).describe('Items per page'),
+    /** Maximum pages to fetch */
+    maxPages: z.number().optional().describe('Maximum number of pages to fetch'),
+  }).optional().describe('Pagination configuration for external data'),
 });
 
 // Type exports

packages/spec/src/data/external-lookup.zod.ts

@@ -4,6 +4,10 @@ import {
   CacheTierSchema,
   CacheInvalidationSchema,
   CacheConfigSchema,
+  CacheConsistencySchema,
+  CacheAvalanchePreventionSchema,
+  CacheWarmupSchema,
+  DistributedCacheConfigSchema,
 } from './cache.zod';
 
 describe('CacheStrategySchema', () => {
@@ -158,3 +162,137 @@ describe('CacheConfigSchema', () => {
     expect(() => CacheConfigSchema.parse({ tiers: [] })).toThrow();
   });
 });
+
+describe('CacheConsistencySchema', () => {
+  it('should accept all consistency strategies', () => {
+    const strategies = ['write_through', 'write_behind', 'write_around', 'refresh_ahead'] as const;
+    strategies.forEach(strategy => {
+      expect(() => CacheConsistencySchema.parse(strategy)).not.toThrow();
+    });
+  });
+
+  it('should reject invalid strategy', () => {
+    expect(() => CacheConsistencySchema.parse('read_through')).toThrow();
+  });
+});
+
+describe('CacheAvalanchePreventionSchema', () => {
+  it('should accept empty config', () => {
+    expect(() => CacheAvalanchePreventionSchema.parse({})).not.toThrow();
+  });
+
+  it('should accept jitter TTL config', () => {
+    const result = CacheAvalanchePreventionSchema.parse({
+      jitterTtl: { enabled: true, maxJitterSeconds: 30 },
+    });
+    expect(result.jitterTtl?.enabled).toBe(true);
+    expect(result.jitterTtl?.maxJitterSeconds).toBe(30);
+  });
+
+  it('should accept circuit breaker config', () => {
+    const result = CacheAvalanchePreventionSchema.parse({
+      circuitBreaker: { enabled: true, failureThreshold: 10, resetTimeout: 60 },
+    });
+    expect(result.circuitBreaker?.failureThreshold).toBe(10);
+  });
+
+  it('should accept lockout config', () => {
+    const result = CacheAvalanchePreventionSchema.parse({
+      lockout: { enabled: true, lockTimeoutMs: 3000 },
+    });
+    expect(result.lockout?.lockTimeoutMs).toBe(3000);
+  });
+
+  it('should accept full prevention config', () => {
+    expect(() => CacheAvalanchePreventionSchema.parse({
+      jitterTtl: { enabled: true },
+      circuitBreaker: { enabled: true },
+      lockout: { enabled: true },
+    })).not.toThrow();
+  });
+});
+
+describe('CacheWarmupSchema', () => {
+  it('should accept minimal warmup config', () => {
+    const result = CacheWarmupSchema.parse({});
+    expect(result.enabled).toBe(false);
+    expect(result.strategy).toBe('lazy');
+  });
+
+  it('should accept eager warmup with patterns', () => {
+    const result = CacheWarmupSchema.parse({
+      enabled: true,
+      strategy: 'eager',
+      patterns: ['config:*', 'user:*'],
+      concurrency: 20,
+    });
+    expect(result.strategy).toBe('eager');
+    expect(result.patterns).toHaveLength(2);
+    expect(result.concurrency).toBe(20);
+  });
+
+  it('should accept scheduled warmup', () => {
+    const result = CacheWarmupSchema.parse({
+      enabled: true,
+      strategy: 'scheduled',
+      schedule: '0 0 * * *',
+    });
+    expect(result.schedule).toBe('0 0 * * *');
+  });
+});
+
+describe('DistributedCacheConfigSchema', () => {
+  it('should accept basic distributed cache', () => {
+    const config = DistributedCacheConfigSchema.parse({
+      enabled: true,
+      tiers: [
+        { name: 'l1', type: 'memory', maxSize: 100 },
+        { name: 'l2', type: 'redis', maxSize: 1000 },
+      ],
+      invalidation: [{ trigger: 'update', scope: 'key' }],
+      consistency: 'write_through',
+    });
+
+    expect(config.consistency).toBe('write_through');
+  });
+
+  it('should accept full distributed cache config', () => {
+    const config = DistributedCacheConfigSchema.parse({
+      enabled: true,
+      tiers: [
+        { name: 'l1', type: 'memory', maxSize: 100, ttl: 60, strategy: 'lru' },
+        { name: 'l2', type: 'redis', maxSize: 1000, ttl: 300, strategy: 'lru' },
+      ],
+      invalidation: [{ trigger: 'update', scope: 'key' }],
+      consistency: 'write_behind',
+      avalanchePrevention: {
+        jitterTtl: { enabled: true, maxJitterSeconds: 30 },
+        circuitBreaker: { enabled: true, failureThreshold: 5 },
+        lockout: { enabled: true },
+      },
+      warmup: {
+        enabled: true,
+        strategy: 'eager',
+        patterns: ['config:*'],
+      },
+    });
+
+    expect(config.consistency).toBe('write_behind');
+    expect(config.avalanchePrevention?.jitterTtl?.enabled).toBe(true);
+    expect(config.warmup?.strategy).toBe('eager');
+  });
+
+  it('should extend CacheConfigSchema fields', () => {
+    const config = DistributedCacheConfigSchema.parse({
+      enabled: true,
+      tiers: [{ name: 'test', type: 'memory' }],
+      invalidation: [{ trigger: 'update', scope: 'key' }],
+      prefetch: true,
+      compression: true,
+      encryption: true,
+    });
+
+    expect(config.prefetch).toBe(true);
+    expect(config.compression).toBe(true);
+  });
+});