feat: Implement multiple document filtering with various operators in the Data Explorer.

ChingEnLin · ChingEnLin · commit a7dd5dd4217d · 2026-03-12T10:07:34.000+01:00
diff --git a/backend/models/data_documents.py b/backend/models/data_documents.py
@@ -4,7 +4,8 @@
 
 class DataDocumentsFilter(BaseModel):
     key: str
-    value: Any  # Can be str, int, etc. depending on the field type
+    value: Any = ""  # Can be str, int, etc. depending on the field type
+    operator: Optional[str] = "equals" # 'equals', 'exists', 'not_exists'
 
 
 class DataDocumentsRequest(BaseModel):
@@ -14,6 +15,7 @@ class DataDocumentsRequest(BaseModel):
     page: int = 1
     limit: int = 20
     filter: Optional[DataDocumentsFilter] = None
+    filters: Optional[List[DataDocumentsFilter]] = None
 
 
 class DataDocumentsResponse(BaseModel):
diff --git a/backend/routes/data_documents.py b/backend/routes/data_documents.py
@@ -46,6 +46,7 @@ def get_documents(
         page=body.page,
         limit=body.limit,
         filter=body.filter.model_dump() if body.filter else None,
+        filters=[f.model_dump() for f in body.filters] if body.filters else None,
     )
 
 
diff --git a/backend/services/data_documents_service.py b/backend/services/data_documents_service.py
@@ -89,40 +89,85 @@ def fetch_documents(
     page: int,
     limit: int,
     filter: dict = None,
+    filters: list = None,
 ) -> DataDocumentsResponse:
     client = MongoClient(connection_string)
     db = client[database_name]
     collection = db[collection_name]
 
     query = {}
-    if filter and ("key" in filter) and ("value" in filter):
-        key = filter["key"]
-        value = filter["value"]
+    and_clauses = []
+
+    def get_query_for_filter(f: dict):
+        key = f.get("key")
+        value = f.get("value")
+        operator = f.get("operator", "equals")
+        
+        if not key:
+            return {}
+            
+        if operator == "exists":
+            return {key: {"$exists": True}}
+        if operator == "not_exists":
+            return {key: {"$exists": False}}
+            
+        if value is None:
+            return {}
+            
         if key == "all":
             sample_doc = collection.find_one()
             if sample_doc:
                 or_clauses = []
                 for k, v in sample_doc.items():
                     if isinstance(v, str):
                         or_clauses.append(
-                            {k: {"$regex": re.escape(value), "$options": "i"}}
+                            {k: {"$regex": re.escape(str(value)), "$options": "i"}}
                         )
                 if or_clauses:
-                    query = {"$or": or_clauses}
+                    return {"$or": or_clauses}
+            return {}
         else:
-            # If key is '_id', treat value as ObjectId
             if key == "_id":
                 try:
-                    query = {"_id": ObjectId(value)}
+                    query_val = ObjectId(value)
                 except Exception:
-                    # fallback to string match if not a valid ObjectId
-                    query = {key: value}
+                    query_val = value
+            else:
+                query_val = value
+                
+            if operator == "not_equals":
+                return {key: {"$ne": query_val}}
+            if operator == "greater_than":
+                return {key: {"$gt": query_val}}
+            if operator == "less_than":
+                return {key: {"$lt": query_val}}
+            if operator == "contains":
+                return {key: {"$regex": re.escape(str(value)), "$options": "i"}}
+                
+            # default equals
+            if isinstance(query_val, str) and key != "_id":
+                return {key: {"$regex": re.escape(query_val), "$options": "i"}}
             else:
-                # Support dot notation for nested fields
-                if isinstance(value, str):
-                    query = {key: {"$regex": re.escape(value), "$options": "i"}}
-                else:
-                    query = {key: value}
+                return {key: query_val}
+
+    if filter and ("key" in filter) and ("value" in filter):
+        q = get_query_for_filter(filter)
+        if q:
+            and_clauses.append(q)
+
+    if filters:
+        for f in filters:
+            if ("key" in f) and ("value" in f):
+                q = get_query_for_filter(f)
+                if q:
+                    and_clauses.append(q)
+
+    if and_clauses:
+        if len(and_clauses) == 1:
+            query = and_clauses[0]
+        else:
+            query = {"$and": and_clauses}
+
     total_documents = collection.count_documents(query)
     total_pages = max(1, (total_documents + limit - 1) // limit)
     skip = (page - 1) * limit
diff --git a/frontend/pages/DataExplorerPage.tsx b/frontend/pages/DataExplorerPage.tsx
@@ -148,7 +148,6 @@ const DeleteDocumentDialog: React.FC<{
 const DataExplorerPage: React.FC<DataExplorerPageProps> = ({
   resource,
   dbInfo,
-  accountName,
   availableDbs,
   availableAccounts,
   initialDocumentId,
@@ -182,13 +181,31 @@ const DataExplorerPage: React.FC<DataExplorerPageProps> = ({
   const [pageInput, setPageInput] = useState(String(currentPage));
 
   // --- Filtering State ---
-  const [filterKey, setFilterKey] = useState('all');
-  const [filterValue, setFilterValue] = useState('');
-  const [debouncedFilterValue, setDebouncedFilterValue] = useState(filterValue);
+  interface FilterState {
+    id: string;
+    key: string;
+    value: string;
+    isCustom: boolean;
+    operator?: string;
+  }
+  const [filters, setFilters] = useState<FilterState[]>([{ id: 'default', key: 'all', value: '', isCustom: false, operator: 'equals' }]);
+  const [debouncedFilters, setDebouncedFilters] = useState<FilterState[]>(filters);
   const [schemaTree, setSchemaTree] = useState<SchemaKeyNode[]>([]);
   const [isFetchingSchema, setIsFetchingSchema] = useState(false);
   const [currentCollectionInfo, setCurrentCollectionInfo] = useState<CollectionInfo | null>(null);
 
+  const addFilter = () => {
+    setFilters(prev => [...prev, { id: Math.random().toString(36).substring(7), key: 'all', value: '', isCustom: false, operator: 'equals' }]);
+  };
+
+  const removeFilter = (id: string) => {
+    setFilters(prev => prev.filter(f => f.id !== id));
+  };
+
+  const updateFilter = (id: string, updates: Partial<FilterState>) => {
+    setFilters(prev => prev.map(f => f.id === id ? { ...f, ...updates } : f));
+  };
+
   // --- Editor State ---
   const [selectedDocument, setSelectedDocument] = useState<Record<string, any> | null>(null);
   const [editMode, setEditMode] = useState(false);
@@ -354,25 +371,24 @@ const DataExplorerPage: React.FC<DataExplorerPageProps> = ({
     setTotalPages(1);
     setTotalDocuments(0);
     setPageInput('1');
-    setFilterKey('all');
-    setFilterValue('');
-    setDebouncedFilterValue('');
+    setFilters([{ id: 'default', key: 'all', value: '', isCustom: false, operator: 'equals' }]);
+    setDebouncedFilters([{ id: 'default', key: 'all', value: '', isCustom: false, operator: 'equals' }]);
     setSchemaTree([]);
     setIsFetchingSchema(false);
     setSelectedDocument(null);
     setBreadcrumbs([]);
     // Do not reset pinned documents here, as they should persist across DB/collection changes.
   }, []);
 
-  // Debounce search input
+  // Debounce filters
   useEffect(() => {
     const handler = setTimeout(() => {
-      setDebouncedFilterValue(filterValue);
+      setDebouncedFilters(filters);
       setCurrentPage(1); // Reset to page 1 on new search
       setSelectedDocument(null); // Clear selection on new search
     }, 300);
     return () => clearTimeout(handler);
-  }, [filterValue]);
+  }, [filters]);
 
   // Sync page input with current page
   useEffect(() => setPageInput(String(currentPage)), [currentPage]);
@@ -432,8 +448,12 @@ const DataExplorerPage: React.FC<DataExplorerPageProps> = ({
     setIsLoading(true);
     setError(null);
     try {
-      const processedValue = getCoercedFilterValue(debouncedFilterValue);
-      const response = await getDocuments(selectedCollection, currentResource, currentPage, 20, { key: filterKey, value: processedValue });
+      const activeFilters = debouncedFilters.map(f => ({
+        key: f.key,
+        value: getCoercedFilterValue(f.value),
+        operator: f.operator || 'equals'
+      }));
+      const response = await getDocuments(selectedCollection, currentResource, currentPage, 20, undefined, activeFilters);
       setDocuments(response.documents);
       setTotalPages(response.totalPages);
       setTotalDocuments(response.totalDocuments);
@@ -446,7 +466,7 @@ const DataExplorerPage: React.FC<DataExplorerPageProps> = ({
     } finally {
       setIsLoading(false);
     }
-  }, [selectedCollection, currentResource, currentPage, filterKey, debouncedFilterValue]);
+  }, [selectedCollection, currentResource, currentPage, debouncedFilters]);
 
   useEffect(() => {
     if (breadcrumbs.length > 0 && selectedDocument) return;
@@ -569,8 +589,8 @@ const DataExplorerPage: React.FC<DataExplorerPageProps> = ({
     if (selectedCollection === collectionName) return;
     setSelectedCollection(collectionName);
     setCurrentPage(1);
-    setFilterValue('');
-    setFilterKey('all');
+    setFilters([{ id: 'default', key: 'all', value: '', isCustom: false, operator: 'equals' }]);
+    setDebouncedFilters([{ id: 'default', key: 'all', value: '', isCustom: false, operator: 'equals' }]);
     setSelectedDocument(null);
     setBreadcrumbs([]);
     await fetchSchemaForCollection(collectionName);
@@ -715,7 +735,7 @@ const DataExplorerPage: React.FC<DataExplorerPageProps> = ({
       return (
         <div className="text-center text-slate-500 dark:text-slate-400 py-10">
           <p>No documents found.</p>
-          {debouncedFilterValue && <p className="text-xs">Try a different filter or value.</p>}
+          {debouncedFilters.some(f => f.value || f.operator === 'exists' || f.operator === 'not_exists') && <p className="text-xs">Try a different filter or value.</p>}
         </div>
       );
     }
@@ -1237,30 +1257,100 @@ const DataExplorerPage: React.FC<DataExplorerPageProps> = ({
                     </div>
                   </div>
                   <div className="space-y-2">
-                    <div className="relative">
-                      <select
-                        value={filterKey}
-                        onChange={(e) => setFilterKey(e.target.value)}
-                        disabled={!selectedCollection || isFetchingSchema}
-                        className="w-full text-sm appearance-none cursor-pointer p-2 pr-8 bg-slate-100 dark:bg-slate-700/50 border border-slate-300 dark:border-slate-600 rounded-lg text-slate-900 dark:text-slate-100 focus:ring-2 focus:ring-blue-500 focus:border-blue-500 transition-colors disabled:opacity-50"
-                        title="Select a field to filter by"
-                      >
-                        <option value="all">All Fields</option>
-                        <RenderOptions nodes={schemaTree} level={0} />
-                      </select>
-                      <ChevronDownIcon className="absolute right-2 top-1/2 -translate-y-1/2 w-4 h-4 text-slate-400 dark:text-slate-500 pointer-events-none" />
-                    </div>
-                    <div className="relative">
-                      <input
-                        type="text"
-                        placeholder={isFetchingSchema ? 'Loading schema...' : 'Filter value...'}
-                        value={filterValue}
-                        onChange={(e) => setFilterValue(e.target.value)}
-                        disabled={!selectedCollection || isFetchingSchema}
-                        className="w-full pl-9 pr-4 py-2 bg-slate-100 dark:bg-slate-700/50 border border-slate-300 dark:border-slate-600 rounded-lg text-sm focus:ring-2 focus:ring-blue-500 focus:border-blue-500 transition-colors disabled:opacity-50"
-                      />
-                      <SearchIcon className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-slate-400 dark:text-slate-500" />
-                    </div>
+                    {filters.map((f, i) => (
+                      <div key={f.id} className="flex flex-col gap-2 p-3 bg-slate-50 dark:bg-slate-800/80 border border-slate-200 dark:border-slate-700 rounded-lg relative">
+                        {filters.length > 1 && (
+                          <button onClick={() => removeFilter(f.id)} className="absolute -top-2 -right-2 p-1 bg-white dark:bg-slate-700 border border-slate-300 dark:border-slate-600 rounded-full text-slate-400 hover:text-red-500 hover:border-red-500 shadow-sm z-10 transition-colors">
+                            <XIcon className="w-3 h-3" />
+                          </button>
+                        )}
+                        <div className="flex gap-2">
+                          <div className="flex-[2_2_0%] relative">
+                            {!f.isCustom ? (
+                              <>
+                                <select
+                                  value={f.key}
+                                  onChange={(e) => {
+                                    if (e.target.value === '__custom__') {
+                                      updateFilter(f.id, { isCustom: true, key: '' });
+                                    } else {
+                                      updateFilter(f.id, { key: e.target.value });
+                                    }
+                                  }}
+                                  disabled={!selectedCollection || isFetchingSchema}
+                                  className="w-full text-sm appearance-none cursor-pointer p-2 pr-8 bg-white dark:bg-slate-700/50 border border-slate-300 dark:border-slate-600 rounded-lg text-slate-900 dark:text-slate-100 focus:ring-2 focus:ring-blue-500 focus:border-blue-500 transition-colors disabled:opacity-50"
+                                  title="Select a field to filter by"
+                                >
+                                  <option value="all">All Fields</option>
+                                  <RenderOptions nodes={schemaTree} level={0} />
+                                  <option disabled>──────────</option>
+                                  <option value="__custom__">Type custom field...</option>
+                                </select>
+                                <ChevronDownIcon className="absolute right-2 top-1/2 -translate-y-1/2 w-4 h-4 text-slate-400 dark:text-slate-500 pointer-events-none" />
+                              </>
+                            ) : (
+                              <div className="flex items-center relative">
+                                <input
+                                  type="text"
+                                  value={f.key}
+                                  onChange={(e) => updateFilter(f.id, { key: e.target.value })}
+                                  disabled={!selectedCollection || isFetchingSchema}
+                                  placeholder="e.g. internal_info.internal_id"
+                                  className="w-full text-sm p-2 pr-8 bg-white dark:bg-slate-700/50 border border-slate-300 dark:border-slate-600 rounded-lg text-slate-900 dark:text-slate-100 focus:ring-2 focus:ring-blue-500 focus:border-blue-500 transition-colors disabled:opacity-50"
+                                  autoFocus
+                                />
+                                <button
+                                  onClick={() => updateFilter(f.id, { isCustom: false, key: 'all' })}
+                                  className="absolute right-2 top-1/2 -translate-y-1/2 p-0.5 rounded-full text-slate-400 hover:text-slate-600 dark:hover:text-slate-300 transition-colors bg-slate-200 dark:bg-slate-600"
+                                  title="Back to dropdown"
+                                >
+                                  <XIcon className="w-3 h-3" />
+                                </button>
+                              </div>
+                            )}
+                          </div>
+                          {f.key !== 'all' && (
+                            <div className="flex-1 relative">
+                              <select
+                                value={f.operator || 'equals'}
+                                onChange={(e) => updateFilter(f.id, { operator: e.target.value })}
+                                disabled={!selectedCollection || isFetchingSchema}
+                                className="w-full text-sm appearance-none cursor-pointer p-2 pr-8 bg-white dark:bg-slate-700/50 border border-slate-300 dark:border-slate-600 rounded-lg text-slate-900 dark:text-slate-100 focus:ring-2 focus:ring-blue-500 focus:border-blue-500 transition-colors disabled:opacity-50"
+                              >
+                                <option value="equals">Equals</option>
+                                <option value="not_equals">Does Not Equal</option>
+                                <option value="contains">Contains</option>
+                                <option value="greater_than">Greater Than (&gt;)</option>
+                                <option value="less_than">Less Than (&lt;)</option>
+                                <option value="exists">Exists</option>
+                                <option value="not_exists">Does Not Exist</option>
+                              </select>
+                              <ChevronDownIcon className="absolute right-2 top-1/2 -translate-y-1/2 w-4 h-4 text-slate-400 dark:text-slate-500 pointer-events-none" />
+                            </div>
+                          )}
+                        </div>
+                        {(!f.operator || (f.operator !== 'exists' && f.operator !== 'not_exists')) && (
+                          <div className="relative">
+                            <input
+                              type="text"
+                              placeholder={isFetchingSchema ? 'Loading schema...' : 'Filter value...'}
+                              value={f.value}
+                              onChange={(e) => updateFilter(f.id, { value: e.target.value })}
+                              disabled={!selectedCollection || isFetchingSchema}
+                              className="w-full pl-9 pr-4 py-2 bg-white dark:bg-slate-700/50 border border-slate-300 dark:border-slate-600 rounded-lg text-sm focus:ring-2 focus:ring-blue-500 focus:border-blue-500 transition-colors disabled:opacity-50"
+                            />
+                            <SearchIcon className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-slate-400 dark:text-slate-500" />
+                          </div>
+                        )}
+                      </div>
+                    ))}
+                    <button
+                      onClick={addFilter}
+                      disabled={!selectedCollection || isFetchingSchema}
+                      className="w-full flex justify-center items-center gap-1 p-2 text-sm font-medium text-slate-600 dark:text-slate-300 bg-slate-100 dark:bg-slate-700/50 hover:bg-slate-200 dark:hover:bg-slate-600 border border-slate-300 dark:border-slate-600 rounded-lg transition-colors disabled:opacity-50 disabled:cursor-not-allowed"
+                    >
+                      <span>+ Add Filter</span>
+                    </button>
                   </div>
                 </div>
                 <div className="flex-grow overflow-hidden">
diff --git a/frontend/services/dbService.ts b/frontend/services/dbService.ts

Original file line number	Diff line number	Diff line change
`@@ -46,6 +46,7 @@ def get_documents(`
`46`	`46`	`page=body.page,`
`47`	`47`	`limit=body.limit,`
`48`	`48`	`filter=body.filter.model_dump() if body.filter else None,`
	`49`	`+ filters=[f.model_dump() for f in body.filters] if body.filters else None,`
`49`	`50`	`)`
`50`	`51`
`51`	`52`