Merge remote-tracking branch 'origin/develop' into feature/error-analyzer

Author: webarch-ai

CHANGELOG.md

@@ -7,8 +7,14 @@ SPDX-License-Identifier: MIT-0
 
 ### Added
 
+- **Wildcard pattern support for delete-documents** — `idp-cli delete-documents` and `client.batch.delete_documents()` now accept a `--pattern` / `pattern` parameter for fnmatch-style wildcard matching (e.g. `"batch-123/*.pdf"`, `"*invoice*"`). Combines with `--status-filter` to delete e.g. all failed invoices across batches.
+
 - **Chandra OCR Lambda Hook Sample** — New `GENAIIDP-chandra-ocr-hook` sample in `samples/lambda-hook-inference/` that integrates [Datalab Chandra OCR 2](https://github.com/datalab-to/chandra) with the LambdaHook feature for high-quality OCR. Supports 90+ languages, math, tables, forms, and handwriting. Uses the Datalab hosted async API (`/api/v1/convert`) with configurable output format (markdown/json/html) and conversion mode (fast/balanced/accurate). Includes standalone SAM template, local test script, and deployment instructions. See `docs/lambda-hook-inference.md` — Chandra OCR Integration section.
 
+### Fixed
+
+- **`delete-documents` fails with DynamoDB errors** — Fixed two bugs in `get_documents_by_batch()`: (1) passing empty `ExpressionAttributeNames={}` when no status filter caused `ValidationException`, and (2) using low-level DynamoDB client type descriptors (`{"S": "..."}`) with the high-level Table resource caused `begins_with` operand type mismatch. Rewrote to use the high-level `Table.scan()` API with `boto3.dynamodb.conditions.Attr`.
+
 ## [0.5.4]
 
 ### Added

docs/idp-cli.md

@@ -932,10 +932,11 @@ idp-cli delete-documents [OPTIONS]
 **Document Selection (choose ONE):**
 - `--document-ids`: Comma-separated list of document IDs (S3 object keys) to delete
 - `--batch-id`: Delete all documents in this batch
+- `--pattern`: Wildcard pattern to match document keys (e.g. `"batch-123/*.pdf"`, `"*invoice*"`)
 
 **Options:**
 - `--stack-name` (required): CloudFormation stack name
-- `--status-filter`: Only delete documents with this status (use with --batch-id)
+- `--status-filter`: Only delete documents with this status (use with --batch-id or --pattern)
   - Options: `FAILED`, `COMPLETED`, `PROCESSING`, `QUEUED`
 - `--dry-run`: Show what would be deleted without actually deleting
 - `--force`, `-y`: Skip confirmation prompt
@@ -972,6 +973,23 @@ idp-cli delete-documents \
     --batch-id cli-batch-20250123 \
     --dry-run
 
+# Delete documents matching a wildcard pattern
+idp-cli delete-documents \
+    --stack-name my-stack \
+    --pattern "batch-123/*.pdf"
+
+# Delete all failed invoice documents across batches
+idp-cli delete-documents \
+    --stack-name my-stack \
+    --pattern "*invoice*" \
+    --status-filter FAILED
+
+# Dry run with pattern to preview matches
+idp-cli delete-documents \
+    --stack-name my-stack \
+    --pattern "*2024*" \
+    --dry-run
+
 # Force delete without confirmation
 idp-cli delete-documents \
     --stack-name my-stack \

docs/idp-sdk.md

@@ -547,15 +547,18 @@ print(f"Downloaded {result.files_downloaded} source files")
 
 ### batch.delete_documents()
 
-Permanently delete all documents in a batch and their associated data from InputBucket, OutputBucket, and DynamoDB.
+Permanently delete documents and their associated data from InputBucket, OutputBucket, and DynamoDB. Select documents by batch ID or wildcard pattern.
 
 **Parameters:**
-- `batch_id` (str, required): Batch identifier
+- `batch_id` (str, optional): Batch identifier (selects all docs containing this string)
+- `pattern` (str, optional): Wildcard pattern to match document keys (e.g., `"batch-123/*.pdf"`, `"*invoice*"`)
 - `status_filter` (str, optional): Filter by document status (e.g., "FAILED", "COMPLETED")
 - `stack_name` (str, optional): Stack name override
 - `dry_run` (bool, optional): If True, simulate deletion without actually deleting (default: False)
 - `continue_on_error` (bool, optional): Continue deleting if one document fails (default: True)
 
+**Note:** Must specify either `batch_id` or `pattern` (not both).
+
 **Returns:** `BatchDeletionResult` with `success`, `deleted_count`, `failed_count`, `total_count`, `dry_run`, and `results` (list of DocumentDeletionResult)
 
 ```python
@@ -568,6 +571,17 @@ result = client.batch.delete_documents(
     status_filter="FAILED"
 )
 
+# Delete by wildcard pattern
+result = client.batch.delete_documents(
+    pattern="batch-123/*.pdf"
+)
+
+# Delete all failed invoices across batches
+result = client.batch.delete_documents(
+    pattern="*invoice*",
+    status_filter="FAILED"
+)
+
 # Dry run
 result = client.batch.delete_documents(
     batch_id="batch-123",

lib/idp_cli_pkg/idp_cli/cli.py

@@ -1092,10 +1092,14 @@ def delete(
     "--batch-id",
     help="Delete all documents in this batch (alternative to --document-ids)",
 )
+@click.option(
+    "--pattern",
+    help='Wildcard pattern to match document keys (e.g. "batch-123/*.pdf", "*invoice*")',
+)
 @click.option(
     "--status-filter",
     type=click.Choice(["FAILED", "COMPLETED", "PROCESSING", "QUEUED"]),
-    help="Only delete documents with this status (use with --batch-id)",
+    help="Only delete documents with this status (use with --batch-id or --pattern)",
 )
 @click.option(
     "--dry-run",
@@ -1113,6 +1117,7 @@ def delete_documents_cmd(
     stack_name: str,
     document_ids: Optional[str],
     batch_id: Optional[str],
+    pattern: Optional[str],
     status_filter: Optional[str],
     dry_run: bool,
     force: bool,
@@ -1141,6 +1146,12 @@ def delete_documents_cmd(
       # Delete only failed documents in a batch
       idp-cli delete-documents --stack-name my-stack --batch-id cli-batch-20250123 --status-filter FAILED
 
+      # Delete documents matching a wildcard pattern
+      idp-cli delete-documents --stack-name my-stack --pattern "batch-123/*.pdf"
+
+      # Delete all failed invoice documents
+      idp-cli delete-documents --stack-name my-stack --pattern "*invoice*" --status-filter FAILED
+
       # Dry run to see what would be deleted
       idp-cli delete-documents --stack-name my-stack --batch-id cli-batch-20250123 --dry-run
 
@@ -1149,19 +1160,24 @@ def delete_documents_cmd(
     """
     try:
         import boto3
-        from idp_common.delete_documents import delete_documents, get_documents_by_batch
+        from idp_common.delete_documents import (
+            delete_documents,
+            get_documents_by_batch,
+            get_documents_by_pattern,
+        )
         from idp_sdk import IDPClient
 
-        # Validate input
-        if not document_ids and not batch_id:
+        # Validate input - exactly one of document_ids, batch_id, or pattern required
+        selector_count = sum(1 for x in [document_ids, batch_id, pattern] if x)
+        if selector_count == 0:
             console.print(
-                "[red]✗ Error: Must specify either --document-ids or --batch-id[/red]"
+                "[red]✗ Error: Must specify one of --document-ids, --batch-id, or --pattern[/red]"
             )
             sys.exit(1)
 
-        if document_ids and batch_id:
+        if selector_count > 1:
             console.print(
-                "[red]✗ Error: Cannot specify both --document-ids and --batch-id[/red]"
+                "[red]✗ Error: Cannot specify more than one of --document-ids, --batch-id, --pattern[/red]"
             )
             sys.exit(1)
 
@@ -1190,6 +1206,27 @@ def delete_documents_cmd(
         if document_ids:
             doc_list = [d.strip() for d in document_ids.split(",")]
             console.print(f"Selected {len(doc_list)} document(s) for deletion")
+        elif pattern:
+            console.print(
+                f"[bold blue]Finding documents matching pattern: {pattern}[/bold blue]"
+            )
+            doc_list = get_documents_by_pattern(
+                tracking_table=tracking_table,
+                pattern=pattern,
+                status_filter=status_filter,
+            )
+            if not doc_list:
+                console.print(
+                    f"[yellow]No documents found matching pattern: {pattern}[/yellow]"
+                )
+                if status_filter:
+                    console.print(
+                        f"[yellow]  (with status filter: {status_filter})[/yellow]"
+                    )
+                sys.exit(0)
+            console.print(f"Found {len(doc_list)} document(s) matching pattern")
+            if status_filter:
+                console.print(f"  (filtered by status: {status_filter})")
         else:
             console.print(
                 f"[bold blue]Getting documents for batch: {batch_id}[/bold blue]"

lib/idp_common_pkg/idp_common/delete_documents.py

@@ -10,6 +10,7 @@
 - List entry cleanup with timestamp-aware shard handling
 """
 
+import fnmatch
 import logging
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -464,6 +465,38 @@ def delete_documents(
     }
 
 
+def _scan_all_document_keys(
+    tracking_table, status_filter: Optional[str] = None
+) -> List[Dict[str, Any]]:
+    """
+    Scan all document items from the tracking table.
+
+    Args:
+        tracking_table: DynamoDB table resource
+        status_filter: Optional status filter ('COMPLETED', 'FAILED', 'PROCESSING', etc.)
+
+    Returns:
+        List of DynamoDB items with PK starting with 'doc#'
+    """
+    from boto3.dynamodb.conditions import Attr
+
+    items: List[Dict[str, Any]] = []
+    filter_expr = Attr("PK").begins_with("doc#")
+    if status_filter:
+        filter_expr = filter_expr & Attr("Status").eq(status_filter)
+
+    scan_kwargs: Dict[str, Any] = {"FilterExpression": filter_expr}
+
+    while True:
+        response = tracking_table.scan(**scan_kwargs)
+        items.extend(response.get("Items", []))
+        if "LastEvaluatedKey" not in response:
+            break
+        scan_kwargs["ExclusiveStartKey"] = response["LastEvaluatedKey"]
+
+    return items
+
+
 def get_documents_by_batch(
     tracking_table, batch_id: str, status_filter: Optional[str] = None
 ) -> List[str]:
@@ -481,29 +514,47 @@ def get_documents_by_batch(
     object_keys = []
 
     try:
-        # Query the GSI for batch documents (if available) or scan with filter
-        # For now, we'll use a prefix-based query on the tracking table
-        paginator = tracking_table.meta.client.get_paginator("scan")
-
-        filter_expression = "begins_with(PK, :pk_prefix)"
-        expression_values = {":pk_prefix": {"S": "doc#"}}
-
-        if status_filter:
-            filter_expression += " AND #status = :status"
-            expression_values[":status"] = {"S": status_filter}
-
-        for page in paginator.paginate(
-            TableName=tracking_table.table_name,
-            FilterExpression=filter_expression,
-            ExpressionAttributeValues=expression_values,
-            ExpressionAttributeNames={"#status": "Status"} if status_filter else {},
-        ):
-            for item in page.get("Items", []):
-                object_key = item.get("ObjectKey", {}).get("S", "")
-                if batch_id in object_key:
-                    object_keys.append(object_key)
-
+        items = _scan_all_document_keys(tracking_table, status_filter)
+        for item in items:
+            object_key = item.get("ObjectKey", "")
+            if batch_id in object_key:
+                object_keys.append(object_key)
     except Exception as e:
         logger.error(f"Error getting documents for batch {batch_id}: {str(e)}")
 
     return object_keys
+
+
+def get_documents_by_pattern(
+    tracking_table, pattern: str, status_filter: Optional[str] = None
+) -> List[str]:
+    """
+    Get all document object keys matching a wildcard pattern.
+
+    Uses fnmatch-style patterns (*, ?, [seq], [!seq]).
+
+    Examples:
+        - ``"batch-123/*"`` — all docs in batch-123
+        - ``"*/invoice*.pdf"`` — any invoice PDF in any batch
+        - ``"*2025*"`` — any doc with 2025 in the key
+
+    Args:
+        tracking_table: DynamoDB table resource
+        pattern: Wildcard pattern to match against object keys
+        status_filter: Optional status filter ('COMPLETED', 'FAILED', 'PROCESSING', etc.)
+
+    Returns:
+        List of matching object keys
+    """
+    object_keys = []
+
+    try:
+        items = _scan_all_document_keys(tracking_table, status_filter)
+        for item in items:
+            object_key = item.get("ObjectKey", "")
+            if fnmatch.fnmatch(object_key, pattern):
+                object_keys.append(object_key)
+    except Exception as e:
+        logger.error(f"Error getting documents for pattern {pattern}: {str(e)}")
+
+    return object_keys