Skip to content

Commit 75d08cd

Browse files
hotlonghotlong
authored
Merge pull request #867 from objectstack-ai/copilot/fix-filter-operators-issue
2 parents 85735f1 + f661b46 commit 75d08cd

5 files changed

Lines changed: 286 additions & 6 deletions

File tree

‎packages/plugins/driver-memory/src/memory-driver.test.ts‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -589,4 +589,132 @@ describe('InMemoryDriver', () => {
589589
expect(results).toHaveLength(3);
590590
});
591591
});
592+
593+
describe('FilterCondition Object Format (from parseFilterAST)', () => {
594+
beforeEach(async () => {
595+
await driver.create(testTable, { id: '1', name: 'Alice Johnson', age: 30, email: 'alice@example.com', bio: null });
596+
await driver.create(testTable, { id: '2', name: 'Bob Smith', age: 25, email: 'bob@test.com', bio: 'Developer' });
597+
await driver.create(testTable, { id: '3', name: 'Charlie Brown', age: 35, email: 'charlie@example.com', bio: '' });
598+
await driver.create(testTable, { id: '4', name: 'Evan Davis', age: 28, email: 'evan@test.com', bio: 'Designer' });
599+
});
600+
601+
it('should filter with $contains operator', async () => {
602+
const results = await driver.find(testTable, {
603+
object: testTable,
604+
where: { name: { $contains: 'Evan' } },
605+
});
606+
expect(results).toHaveLength(1);
607+
expect(results[0].name).toBe('Evan Davis');
608+
});
609+
610+
it('should filter with $contains case-insensitively', async () => {
611+
const results = await driver.find(testTable, {
612+
object: testTable,
613+
where: { name: { $contains: 'alice' } },
614+
});
615+
expect(results).toHaveLength(1);
616+
expect(results[0].name).toBe('Alice Johnson');
617+
});
618+
619+
it('should filter with $notContains operator', async () => {
620+
const results = await driver.find(testTable, {
621+
object: testTable,
622+
where: { email: { $notContains: 'example' } },
623+
});
624+
expect(results).toHaveLength(2);
625+
expect(results.map((r: any) => r.name).sort()).toEqual(['Bob Smith', 'Evan Davis']);
626+
});
627+
628+
it('should filter with $startsWith operator', async () => {
629+
const results = await driver.find(testTable, {
630+
object: testTable,
631+
where: { name: { $startsWith: 'Ch' } },
632+
});
633+
expect(results).toHaveLength(1);
634+
expect(results[0].name).toBe('Charlie Brown');
635+
});
636+
637+
it('should filter with $endsWith operator', async () => {
638+
const results = await driver.find(testTable, {
639+
object: testTable,
640+
where: { email: { $endsWith: '.com' } },
641+
});
642+
expect(results).toHaveLength(4);
643+
});
644+
645+
it('should filter with $between operator', async () => {
646+
const results = await driver.find(testTable, {
647+
object: testTable,
648+
where: { age: { $between: [26, 32] } },
649+
});
650+
expect(results).toHaveLength(2);
651+
expect(results.map((r: any) => r.name).sort()).toEqual(['Alice Johnson', 'Evan Davis']);
652+
});
653+
654+
it('should filter with $null: true', async () => {
655+
const results = await driver.find(testTable, {
656+
object: testTable,
657+
where: { bio: { $null: true } },
658+
});
659+
expect(results).toHaveLength(1);
660+
expect(results[0].name).toBe('Alice Johnson');
661+
});
662+
663+
it('should filter with $null: false', async () => {
664+
const results = await driver.find(testTable, {
665+
object: testTable,
666+
where: { bio: { $null: false } },
667+
});
668+
expect(results).toHaveLength(3);
669+
});
670+
671+
it('should handle $contains inside $and', async () => {
672+
const results = await driver.find(testTable, {
673+
object: testTable,
674+
where: { $and: [{ name: { $contains: 'a' } }, { age: { $gte: 30 } }] },
675+
});
676+
expect(results).toHaveLength(2);
677+
expect(results.map((r: any) => r.name).sort()).toEqual(['Alice Johnson', 'Charlie Brown']);
678+
});
679+
680+
it('should handle $startsWith inside $or', async () => {
681+
const results = await driver.find(testTable, {
682+
object: testTable,
683+
where: { $or: [{ name: { $startsWith: 'Al' } }, { name: { $startsWith: 'Ev' } }] },
684+
});
685+
expect(results).toHaveLength(2);
686+
expect(results.map((r: any) => r.name).sort()).toEqual(['Alice Johnson', 'Evan Davis']);
687+
});
688+
689+
it('should handle AST-converted notcontains via convertConditionToMongo', async () => {
690+
const results = await driver.find(testTable, {
691+
object: testTable,
692+
where: { type: 'comparison', field: 'name', operator: 'notcontains', value: 'Bob' },
693+
});
694+
expect(results).toHaveLength(3);
695+
});
696+
697+
it('should handle combined $startsWith + $endsWith on same field', async () => {
698+
const results = await driver.find(testTable, {
699+
object: testTable,
700+
where: { name: { $startsWith: 'A', $endsWith: 'son' } },
701+
});
702+
expect(results).toHaveLength(1);
703+
expect(results[0].name).toBe('Alice Johnson');
704+
});
705+
706+
it('should filter with $null: true on missing fields', async () => {
707+
// bio is null for Alice Johnson — should match
708+
// Records without bio field at all should also match $null: true
709+
await driver.create(testTable, { id: '5', name: 'Frank', age: 40, email: 'frank@test.com' });
710+
const results = await driver.find(testTable, {
711+
object: testTable,
712+
where: { bio: { $null: true } },
713+
});
714+
// Alice (bio: null), Frank (bio: missing)
715+
expect(results.length).toBeGreaterThanOrEqual(2);
716+
expect(results.map((r: any) => r.name)).toContain('Alice Johnson');
717+
expect(results.map((r: any) => r.name)).toContain('Frank');
718+
});
719+
});
592720
});

‎packages/plugins/driver-memory/src/memory-driver.ts‎

Lines changed: 127 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -629,8 +629,9 @@ export class InMemoryDriver implements DriverInterface {
629629
const op = filters.operator === 'or' ? '$or' : '$and';
630630
return { [op]: conditions };
631631
}
632-
// MongoDB format passthrough: { field: value } or { field: { $eq: value } }
633-
return filters;
632+
// MongoDB/FilterCondition format: { field: value } or { field: { $op: value } }
633+
// Translate non-standard operators ($contains, $notContains, etc.) to Mingo-compatible format
634+
return this.normalizeFilterCondition(filters);
634635
}
635636

636637
// Legacy array format
@@ -694,6 +695,8 @@ export class InMemoryDriver implements DriverInterface {
694695
return { [field]: { $nin: value } };
695696
case 'contains': case 'like':
696697
return { [field]: { $regex: new RegExp(this.escapeRegex(value), 'i') } };
698+
case 'notcontains': case 'not_contains':
699+
return { [field]: { $not: { $regex: new RegExp(this.escapeRegex(value), 'i') } } };
697700
case 'startswith': case 'starts_with':
698701
return { [field]: { $regex: new RegExp(`^${this.escapeRegex(value)}`, 'i') } };
699702
case 'endswith': case 'ends_with':
@@ -708,6 +711,128 @@ export class InMemoryDriver implements DriverInterface {
708711
}
709712
}
710713

714+
/**
715+
* Normalize a FilterCondition object by converting non-standard $-prefixed
716+
* operators ($contains, $notContains, $startsWith, $endsWith, $between, $null)
717+
* to Mingo-compatible equivalents ($regex, $gte/$lte, null checks).
718+
*/
719+
private normalizeFilterCondition(filter: Record<string, any>): Record<string, any> {
720+
const result: Record<string, any> = {};
721+
const extraAndConditions: Record<string, any>[] = [];
722+
723+
for (const key of Object.keys(filter)) {
724+
const value = filter[key];
725+
// Recurse into logical operators
726+
if (key === '$and' || key === '$or') {
727+
result[key] = Array.isArray(value)
728+
? value.map((child: any) => this.normalizeFilterCondition(child))
729+
: value;
730+
continue;
731+
}
732+
if (key === '$not') {
733+
result[key] = value && typeof value === 'object'
734+
? this.normalizeFilterCondition(value)
735+
: value;
736+
continue;
737+
}
738+
// Skip $-prefixed keys that aren't field names (already handled or unknown)
739+
if (key.startsWith('$')) {
740+
result[key] = value;
741+
continue;
742+
}
743+
// Field-level: value may be primitive (implicit eq) or operator object
744+
if (value && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date) && !(value instanceof RegExp)) {
745+
const normalized = this.normalizeFieldOperators(value);
746+
// Handle multiple regex conditions on the same field (e.g. $startsWith + $endsWith)
747+
if (normalized._multiRegex) {
748+
const regexConditions: Record<string, any>[] = normalized._multiRegex;
749+
delete normalized._multiRegex;
750+
// Each regex becomes its own { field: { $regex: ... } } inside $and
751+
for (const rc of regexConditions) {
752+
extraAndConditions.push({ [key]: { ...normalized, ...rc } });
753+
}
754+
} else {
755+
result[key] = normalized;
756+
}
757+
} else {
758+
result[key] = value;
759+
}
760+
}
761+
762+
// Merge extra $and conditions from multi-regex fields
763+
if (extraAndConditions.length > 0) {
764+
const existing = result.$and;
765+
const andArray = Array.isArray(existing) ? existing : [];
766+
// Include the rest of result as a condition too
767+
if (Object.keys(result).filter(k => k !== '$and').length > 0) {
768+
const rest = { ...result };
769+
delete rest.$and;
770+
andArray.push(rest);
771+
}
772+
andArray.push(...extraAndConditions);
773+
return { $and: andArray };
774+
}
775+
776+
return result;
777+
}
778+
779+
/**
780+
* Convert non-standard field operators to Mingo-compatible format.
781+
* When multiple regex-producing operators appear on the same field
782+
* (e.g. $startsWith + $endsWith), they are combined via $and.
783+
*/
784+
private normalizeFieldOperators(ops: Record<string, any>): Record<string, any> {
785+
const result: Record<string, any> = {};
786+
const regexConditions: Record<string, any>[] = [];
787+
788+
for (const op of Object.keys(ops)) {
789+
const val = ops[op];
790+
switch (op) {
791+
case '$contains':
792+
regexConditions.push({ $regex: new RegExp(this.escapeRegex(val), 'i') });
793+
break;
794+
case '$notContains':
795+
result.$not = { $regex: new RegExp(this.escapeRegex(val), 'i') };
796+
break;
797+
case '$startsWith':
798+
regexConditions.push({ $regex: new RegExp(`^${this.escapeRegex(val)}`, 'i') });
799+
break;
800+
case '$endsWith':
801+
regexConditions.push({ $regex: new RegExp(`${this.escapeRegex(val)}$`, 'i') });
802+
break;
803+
case '$between':
804+
if (Array.isArray(val) && val.length === 2) {
805+
result.$gte = val[0];
806+
result.$lte = val[1];
807+
}
808+
break;
809+
case '$null':
810+
// $null: true → field is null, $null: false → field is not null
811+
// Use $eq/$ne null for Mingo compatibility
812+
if (val === true) {
813+
result.$eq = null;
814+
} else {
815+
result.$ne = null;
816+
}
817+
break;
818+
default:
819+
result[op] = val;
820+
break;
821+
}
822+
}
823+
824+
// Merge regex conditions: single → inline, multiple → wrap with $and
825+
if (regexConditions.length === 1) {
826+
Object.assign(result, regexConditions[0]);
827+
} else if (regexConditions.length > 1) {
828+
// Cannot have multiple $regex on one object; promote to top-level $and.
829+
// _multiRegex is an internal sentinel consumed by normalizeFilterCondition().
830+
result._multiRegex = regexConditions;
831+
}
832+
833+
return result;
834+
}
835+
711836
/**
712837
* Escape special regex characters for safe literal matching.
713838
*/

‎packages/plugins/driver-memory/src/memory-matcher.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ function checkCondition(value: any, condition: any): boolean {
103103
const target = condition[op];
104104

105105
// Handle undefined values
106-
if (value === undefined && op !== '$exists' && op !== '$ne') {
106+
if (value === undefined && op !== '$exists' && op !== '$ne' && op !== '$null') {
107107
return false;
108108
}
109109

@@ -151,12 +151,20 @@ function checkCondition(value: any, condition: any): boolean {
151151
case '$contains':
152152
if (typeof value !== 'string' || !value.includes(target)) return false;
153153
break;
154+
case '$notContains':
155+
if (typeof value !== 'string' || value.includes(target)) return false;
156+
break;
154157
case '$startsWith':
155158
if (typeof value !== 'string' || !value.startsWith(target)) return false;
156159
break;
157160
case '$endsWith':
158161
if (typeof value !== 'string' || !value.endsWith(target)) return false;
159162
break;
163+
case '$null':
164+
// $null: true → value must be null/undefined; $null: false → value must not be null/undefined
165+
if (target === true && value != null) return false;
166+
if (target === false && value == null) return false;
167+
break;
160168
case '$regex':
161169
try {
162170
const re = new RegExp(target, condition.$options || '');

‎packages/spec/src/data/filter.test.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,11 @@ describe('StringOperatorSchema', () => {
111111
expect(() => StringOperatorSchema.parse(filter)).not.toThrow();
112112
});
113113

114+
it('should accept $notContains operator', () => {
115+
const filter = { $notContains: 'spam' };
116+
expect(() => StringOperatorSchema.parse(filter)).not.toThrow();
117+
});
118+
114119
it('should accept $endsWith operator', () => {
115120
const filter = { $endsWith: '.pdf' };
116121
expect(() => StringOperatorSchema.parse(filter)).not.toThrow();
@@ -826,6 +831,11 @@ describe('parseFilterAST', () => {
826831
expect(parseFilterAST(['name', 'like', 'John'])).toEqual({ name: { $contains: 'John' } });
827832
});
828833

834+
it('should convert notcontains/not_contains operator', () => {
835+
expect(parseFilterAST(['name', 'notcontains', 'spam'])).toEqual({ name: { $notContains: 'spam' } });
836+
expect(parseFilterAST(['name', 'not_contains', 'spam'])).toEqual({ name: { $notContains: 'spam' } });
837+
});
838+
829839
it('should convert startswith operator', () => {
830840
expect(parseFilterAST(['name', 'startswith', 'A'])).toEqual({ name: { $startsWith: 'A' } });
831841
expect(parseFilterAST(['name', 'starts_with', 'A'])).toEqual({ name: { $startsWith: 'A' } });
@@ -948,6 +958,8 @@ describe('isFilterAST', () => {
948958
expect(isFilterAST(['age', '>=', 18])).toBe(true);
949959
expect(isFilterAST(['role', 'in', ['admin', 'editor']])).toBe(true);
950960
expect(isFilterAST(['name', 'contains', 'John'])).toBe(true);
961+
expect(isFilterAST(['name', 'notcontains', 'spam'])).toBe(true);
962+
expect(isFilterAST(['name', 'not_contains', 'spam'])).toBe(true);
951963
expect(isFilterAST(['name', 'like', 'John'])).toBe(true);
952964
expect(isFilterAST(['created_at', 'between', ['2024-01-01', '2024-12-31']])).toBe(true);
953965
expect(isFilterAST(['deleted_at', 'is_null', null])).toBe(true);
@@ -1009,7 +1021,7 @@ describe('isFilterAST', () => {
10091021
describe('VALID_AST_OPERATORS', () => {
10101022
it('should contain all standard comparison operators', () => {
10111023
const expected = ['=', '==', '!=', '<>', '>', '>=', '<', '<=', 'in', 'nin', 'not_in',
1012-
'contains', 'like', 'startswith', 'starts_with', 'endswith', 'ends_with',
1024+
'contains', 'notcontains', 'not_contains', 'like', 'startswith', 'starts_with', 'endswith', 'ends_with',
10131025
'between', 'is_null', 'is_not_null'];
10141026
for (const op of expected) {
10151027
expect(VALID_AST_OPERATORS.has(op)).toBe(true);

0 commit comments

Comments
 (0)