diff --git a/packages/datasource-cosmos/src/collection.ts b/packages/datasource-cosmos/src/collection.ts index a73dece..e9ca970 100644 --- a/packages/datasource-cosmos/src/collection.ts +++ b/packages/datasource-cosmos/src/collection.ts @@ -35,6 +35,7 @@ export default class CosmosCollection extends BaseCollection { model: ModelCosmos, logger: Logger, nativeDriver: CosmosClient, + options?: { maxConditions?: number }, ) { if (!model) throw new Error('Invalid (null) model instance.'); @@ -53,6 +54,7 @@ export default class CosmosCollection extends BaseCollection { validationOptions: { // Allow unknown fields since schema might not include all nested paths allowUnknownFields: true, + maxConditions: options?.maxConditions, }, }); diff --git a/packages/datasource-cosmos/src/datasource.ts b/packages/datasource-cosmos/src/datasource.ts index 9022d16..debc093 100644 --- a/packages/datasource-cosmos/src/datasource.ts +++ b/packages/datasource-cosmos/src/datasource.ts @@ -19,7 +19,7 @@ export default class CosmosDataSource extends BaseDataSource { cosmosClient: CosmosClient, collectionModels: ModelCosmos[], logger: Logger, - options?: { liveQueryConnections?: string; liveQueryDatabase?: string }, + options?: { liveQueryConnections?: string; liveQueryDatabase?: string; maxConditions?: number }, ) { super(); @@ -27,7 +27,7 @@ export default class CosmosDataSource extends BaseDataSource { this.cosmosClient = cosmosClient; // Creating collections - this.createCollections(collectionModels, logger); + this.createCollections(collectionModels, logger, options?.maxConditions); if (options?.liveQueryConnections && options?.liveQueryDatabase) { this.addNativeQueryConnection(options.liveQueryConnections, { @@ -39,12 +39,18 @@ export default class CosmosDataSource extends BaseDataSource { logger?.('Info', 'CosmosDataSource - Built'); } - protected async createCollections(collectionModels: ModelCosmos[], logger: Logger) { + protected async createCollections( + collectionModels: ModelCosmos[], + logger: Logger, + maxConditions?: number, + ) { collectionModels // avoid schema reordering .sort((modelA, modelB) => (modelA.name > modelB.name ? 1 : -1)) .forEach(model => { - const collection = new CosmosCollection(this, model, logger, this.cosmosClient); + const collection = new CosmosCollection(this, model, logger, this.cosmosClient, { + maxConditions, + }); this.addCollection(collection); }); } diff --git a/packages/datasource-cosmos/src/index.ts b/packages/datasource-cosmos/src/index.ts index 620052b..e356667 100644 --- a/packages/datasource-cosmos/src/index.ts +++ b/packages/datasource-cosmos/src/index.ts @@ -184,6 +184,12 @@ export function createCosmosDataSource( * Default: false */ logQueries?: boolean; + /** + * Maximum number of conditions allowed in a single query condition tree + * Increase this if you use charts with many time buckets or complex filters + * Default: 1000 + */ + maxConditions?: number; }, ): DataSourceFactory { return async (logger: Logger) => { @@ -203,6 +209,7 @@ export function createCosmosDataSource( schema, logRuConsumption, logQueries, + maxConditions, } = options || {}; // Configure RU logging if enabled @@ -263,6 +270,7 @@ export function createCosmosDataSource( const datasource = new CosmosDataSource(client, collectionModels, logger, { liveQueryConnections, liveQueryDatabase, + maxConditions, }); // Add virtual array collections if specified diff --git a/packages/datasource-cosmos/src/utils/query-validation-error.ts b/packages/datasource-cosmos/src/utils/query-validation-error.ts index e9dcd71..b1a37a9 100644 --- a/packages/datasource-cosmos/src/utils/query-validation-error.ts +++ b/packages/datasource-cosmos/src/utils/query-validation-error.ts @@ -1,3 +1,5 @@ +import { ValidationError } from '@forestadmin/datasource-toolkit'; + /** * Error codes for query validation failures */ @@ -12,15 +14,15 @@ export enum QueryValidationErrorCode { } /** - * Security and validation errors for query operations + * Security and validation errors for query operations. + * Extends ValidationError so the agent returns 400 (Bad Request) instead of 500. */ -export default class QueryValidationError extends Error { +export default class QueryValidationError extends ValidationError { constructor( message: string, public readonly code: QueryValidationErrorCode, public readonly field?: string, ) { - super(message); - this.name = 'QueryValidationError'; + super(message, undefined, 'QueryValidationError'); } } diff --git a/packages/datasource-cosmos/src/utils/query-validator.ts b/packages/datasource-cosmos/src/utils/query-validator.ts index 86f3623..3fb7351 100644 --- a/packages/datasource-cosmos/src/utils/query-validator.ts +++ b/packages/datasource-cosmos/src/utils/query-validator.ts @@ -36,7 +36,7 @@ export interface QueryValidatorOptions { /** * Maximum number of conditions in a condition tree - * Default: 100 + * Default: 1000 */ maxConditions?: number; } @@ -78,7 +78,7 @@ export default class QueryValidator { validateAgainstSchema: options.validateAgainstSchema ?? schemaFields !== undefined, allowUnknownFields: options.allowUnknownFields ?? false, maxFieldNameLength: options.maxFieldNameLength ?? 256, - maxConditions: options.maxConditions ?? 100, + maxConditions: options.maxConditions ?? 1000, }; if (schemaFields) { diff --git a/packages/datasource-cosmos/test/max-conditions.test.ts b/packages/datasource-cosmos/test/max-conditions.test.ts new file mode 100644 index 0000000..d625abc --- /dev/null +++ b/packages/datasource-cosmos/test/max-conditions.test.ts @@ -0,0 +1,333 @@ +import { CosmosClient } from '@azure/cosmos'; +import { + BusinessError, + ConditionTreeBranch, + ConditionTreeLeaf, + ValidationError, +} from '@forestadmin/datasource-toolkit'; + +import CosmosCollection from '../src/collection'; +import CosmosDataSource from '../src/datasource'; +import QueryConverter from '../src/utils/query-converter'; +import QueryValidator, { QueryValidationError } from '../src/utils/query-validator'; + +/** + * Helper: build a flat AND condition tree with N leaf conditions + */ +function buildConditionTree(count: number) { + const conditions = Array.from( + { length: count }, + (_, i) => new ConditionTreeLeaf(`field${i}`, 'Equal', `value${i}`), + ); + + return new ConditionTreeBranch('And', conditions); +} + +describe('maxConditions', () => { + // ─────────────────────────────────────────────────────────── + // QueryValidator + // ─────────────────────────────────────────────────────────── + describe('QueryValidator', () => { + it('should default maxConditions to 1000', () => { + const validator = new QueryValidator(); + + // 999 leaf conditions + their parent branch = 1000 nodes → should pass + const tree = buildConditionTree(999); + expect(() => validator.validateConditionTree(tree)).not.toThrow(); + }); + + it('should reject condition trees exceeding the default 1000 limit', () => { + const validator = new QueryValidator(); + + // 1001 leaf conditions + their parent branch = 1002 nodes → should fail + const tree = buildConditionTree(1001); + expect(() => validator.validateConditionTree(tree)).toThrow(QueryValidationError); + expect(() => validator.validateConditionTree(tree)).toThrow(/maximum of 1000/); + }); + + it('should accept exactly 1000 conditions at default', () => { + const validator = new QueryValidator(); + + // Build a tree that totals exactly 1000 nodes: + // 1 branch + 999 leaves = 1000 + const tree = buildConditionTree(999); + expect(() => validator.validateConditionTree(tree)).not.toThrow(); + }); + + it('should reject at 1001 conditions at default', () => { + const validator = new QueryValidator(); + + // 1 branch + 1000 leaves = 1001 → exceeds 1000 + const tree = buildConditionTree(1000); + expect(() => validator.validateConditionTree(tree)).toThrow(QueryValidationError); + }); + + it('should respect a custom maxConditions override', () => { + const validator = new QueryValidator(undefined, { maxConditions: 2000 }); + + // 1 branch + 1500 leaves = 1501 → under 2000 + const tree = buildConditionTree(1500); + expect(() => validator.validateConditionTree(tree)).not.toThrow(); + }); + + it('should reject when custom maxConditions is exceeded', () => { + const validator = new QueryValidator(undefined, { maxConditions: 50 }); + + const tree = buildConditionTree(60); + expect(() => validator.validateConditionTree(tree)).toThrow(/maximum of 50/); + }); + + it('should include the correct error code when maxConditions is exceeded', () => { + const validator = new QueryValidator(undefined, { maxConditions: 5 }); + + let thrownError: QueryValidationError | null = null; + + try { + validator.validateConditionTree(buildConditionTree(10)); + } catch (error) { + thrownError = error as QueryValidationError; + } + + expect(thrownError).toBeInstanceOf(QueryValidationError); + expect(thrownError?.message).toContain('maximum of 5'); + }); + + it('should throw a ValidationError (maps to 400 Bad Request in the agent)', () => { + const validator = new QueryValidator(undefined, { maxConditions: 5 }); + + let thrownError: Error | null = null; + + try { + validator.validateConditionTree(buildConditionTree(10)); + } catch (error) { + thrownError = error as Error; + } + + // QueryValidationError extends ValidationError from datasource-toolkit + expect(thrownError).toBeInstanceOf(ValidationError); + expect(thrownError).toBeInstanceOf(BusinessError); + // The agent uses BusinessError.isOfType as a fallback for cross-package version mismatches + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + expect(BusinessError.isOfType(thrownError!, ValidationError)).toBe(true); + // Preserves the custom name for debugging + expect(thrownError?.name).toBe('QueryValidationError'); + }); + + it('should reset condition count between validations', () => { + const validator = new QueryValidator(undefined, { maxConditions: 20 }); + + // First validation: 15 leaves + 1 branch = 16 → passes + expect(() => validator.validateConditionTree(buildConditionTree(15))).not.toThrow(); + + // Second validation should also pass (not accumulate from first) + expect(() => validator.validateConditionTree(buildConditionTree(15))).not.toThrow(); + }); + }); + + // ─────────────────────────────────────────────────────────── + // QueryConverter — maxConditions passthrough + // ─────────────────────────────────────────────────────────── + describe('QueryConverter', () => { + it('should use default maxConditions (1000) when no options provided', () => { + const converter = new QueryConverter(); + + // 999 leaves + 1 branch = 1000 → should pass + const tree = buildConditionTree(999); + expect(() => converter.getSqlQuerySpec(tree)).not.toThrow(); + }); + + it('should reject when default maxConditions (1000) is exceeded', () => { + const converter = new QueryConverter(); + + // 1000 leaves + 1 branch = 1001 → should fail + const tree = buildConditionTree(1000); + expect(() => converter.getSqlQuerySpec(tree)).toThrow(/maximum of 1000/); + }); + + it('should pass custom maxConditions through to validator', () => { + const converter = new QueryConverter({ + validationOptions: { maxConditions: 2000 }, + }); + + // 1500 leaves + 1 branch = 1501 → under 2000 + const tree = buildConditionTree(1500); + expect(() => converter.getSqlQuerySpec(tree)).not.toThrow(); + }); + + it('should reject when custom maxConditions is exceeded via validationOptions', () => { + const converter = new QueryConverter({ + validationOptions: { maxConditions: 50 }, + }); + + const tree = buildConditionTree(60); + expect(() => converter.getSqlQuerySpec(tree)).toThrow(/maximum of 50/); + }); + + it('should not validate conditions when skipValidation is true', () => { + const converter = new QueryConverter({ skipValidation: true }); + + // Even 2000 conditions should work when validation is skipped + const tree = buildConditionTree(2000); + expect(() => converter.getSqlQuerySpec(tree)).not.toThrow(); + }); + + it('should also apply maxConditions in getWhereClause', () => { + const converter = new QueryConverter({ + validationOptions: { maxConditions: 10 }, + }); + + const tree = buildConditionTree(20); + expect(() => converter.getWhereClause(tree)).toThrow(/maximum of 10/); + }); + }); + + // ─────────────────────────────────────────────────────────── + // CosmosCollection — maxConditions passthrough via constructor + // ─────────────────────────────────────────────────────────── + describe('CosmosCollection', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let mockDatasource: any; + let mockLogger: jest.Mock; + let mockClient: jest.Mocked; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let mockModel: any; + + beforeEach(() => { + mockDatasource = { + getCollection: jest.fn(), + }; + mockLogger = jest.fn(); + mockClient = {} as jest.Mocked; + mockModel = { + name: 'TestCollection', + containerName: 'TestContainer', + databaseName: 'TestDB', + enableCount: true, + partitionKeyPath: '/id', + getPartitionKeyPath: jest.fn().mockReturnValue('/id'), + getAttributes: jest.fn().mockReturnValue({ + id: { type: 'String' }, + status: { type: 'String' }, + operationDate: { type: 'Date' }, + }), + overrideTypeConverter: undefined, + }; + }); + + it('should create collection without maxConditions option (uses default 1000)', () => { + expect( + () => new CosmosCollection(mockDatasource, mockModel, mockLogger, mockClient), + ).not.toThrow(); + }); + + it('should create collection with custom maxConditions option', () => { + expect( + () => + new CosmosCollection(mockDatasource, mockModel, mockLogger, mockClient, { + maxConditions: 5000, + }), + ).not.toThrow(); + }); + }); + + // ─────────────────────────────────────────────────────────── + // CosmosDataSource — maxConditions passthrough via options + // ─────────────────────────────────────────────────────────── + describe('CosmosDataSource', () => { + let mockClient: jest.Mocked; + let mockLogger: jest.Mock; + + beforeEach(() => { + mockClient = { + database: jest.fn().mockReturnValue({ + container: jest.fn().mockReturnValue({ + items: { + query: jest.fn().mockReturnValue({ + fetchAll: jest.fn().mockResolvedValue({ resources: [] }), + }), + }, + }), + }), + } as unknown as jest.Mocked; + + mockLogger = jest.fn(); + }); + + it('should accept maxConditions in options', () => { + expect( + () => + new CosmosDataSource(mockClient, [], mockLogger, { + maxConditions: 2000, + }), + ).not.toThrow(); + }); + + it('should work without maxConditions in options', () => { + expect( + () => + new CosmosDataSource(mockClient, [], mockLogger, { + liveQueryConnections: 'cosmos', + liveQueryDatabase: 'mydb', + }), + ).not.toThrow(); + }); + + it('should work with no options at all', () => { + expect(() => new CosmosDataSource(mockClient, [], mockLogger)).not.toThrow(); + }); + }); + + // ─────────────────────────────────────────────────────────── + // Realistic chart scenario: daily time buckets over a year + // ─────────────────────────────────────────────────────────── + describe('realistic chart scenarios', () => { + it('should handle a daily chart over 1 year (365 conditions)', () => { + const converter = new QueryConverter(); + + // Simulate what Forest Admin generates for a daily chart: + // one condition per day bucket over a year + const conditions = Array.from({ length: 365 }, (_, i) => { + const date = new Date(2025, 0, 1 + i).toISOString(); + + return new ConditionTreeLeaf('operationDate', 'Equal', date); + }); + const tree = new ConditionTreeBranch('Or', conditions); + + expect(() => converter.getSqlQuerySpec(tree)).not.toThrow(); + }); + + it('should handle multiple filters combined with daily buckets (3 filters x 365 days)', () => { + // Total nodes: 1 (top AND) + 3 (OR branches) + 3*365 (leaves) = 1099 + // This exceeds the default 1000, so use a higher limit. + const customConverter = new QueryConverter({ + validationOptions: { maxConditions: 1500 }, + }); + + const dateBuckets = Array.from( + { length: 365 }, + (_, i) => + new ConditionTreeLeaf('operationDate', 'Equal', new Date(2025, 0, 1 + i).toISOString()), + ); + + const tree = new ConditionTreeBranch('And', [ + new ConditionTreeBranch('Or', dateBuckets), + new ConditionTreeLeaf('operationType', 'Equal', 'PAYIN'), + new ConditionTreeLeaf('operationStatus', 'Equal', 'Completed'), + ]); + + expect(() => customConverter.getSqlQuerySpec(tree)).not.toThrow(); + }); + + it('should handle a weekly chart over 2 years (104 conditions)', () => { + const converter = new QueryConverter(); + + const conditions = Array.from( + { length: 104 }, + (_, i) => new ConditionTreeLeaf('operationDate', 'Equal', `week-${i}`), + ); + const tree = new ConditionTreeBranch('Or', conditions); + + expect(() => converter.getSqlQuerySpec(tree)).not.toThrow(); + }); + }); +});