UN-2813 Reverted unwanted changes

Author: Deepak-Kesavan

.github/workflows/ci-test.yaml

@@ -48,13 +48,22 @@ jobs:
         run: |
           tox
 
-      - name: Render the report to the PR
-        uses: marocchino/sticky-pull-request-comment@v2
+      - name: Render the Runner report to the PR
+        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
+        if: always() && hashFiles('runner-report.md') != ''
         with:
           header: runner-test-report
           recreate: true
           path: runner-report.md
 
+      - name: Render the SDK1 report to the PR
+        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
+        if: always() && hashFiles('sdk1-report.md') != ''
+        with:
+          header: sdk1-test-report
+          recreate: true
+          path: sdk1-report.md
+
       - name: Output reports to the job summary when tests fail
         shell: bash
         run: |
@@ -65,3 +74,10 @@ jobs:
             echo "" >> $GITHUB_STEP_SUMMARY
             echo "</details>" >> $GITHUB_STEP_SUMMARY
           fi
+          if [ -f "sdk1-report.md" ]; then
+            echo "<details><summary>SDK1 Test Report</summary>" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            cat "sdk1-report.md" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "</details>" >> $GITHUB_STEP_SUMMARY
+          fi

backend/backend/settings/base.py

@@ -160,6 +160,10 @@ def get_required_setting(setting_key: str, default: str | None = None) -> str |
     os.environ.get("FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND", 60 * 10)
 )  # 10 minutes
 
+FILE_ACTIVE_CACHE_REDIS_DB = int(
+    os.environ.get("FILE_ACTIVE_CACHE_REDIS_DB", 0)
+)  # Redis DB for active file cache tracking
+
 INSTANT_WF_POLLING_TIMEOUT = int(
     os.environ.get("INSTANT_WF_POLLING_TIMEOUT", "300")
 )  # 5 minutes
@@ -175,7 +179,7 @@ def get_required_setting(setting_key: str, default: str | None = None) -> str |
     os.environ.get("MAX_PARALLEL_FILE_BATCHES_MAX_VALUE", 100)
 )
 
-CELERY_RESULT_CHORD_RETRY_INTERVAL = int(
+CELERY_RESULT_CHORD_RETRY_INTERVAL = float(
     os.environ.get("CELERY_RESULT_CHORD_RETRY_INTERVAL", "3")
 )
 

backend/pyproject.toml

@@ -35,11 +35,7 @@ dependencies = [
     "python-socketio==5.9.0", # For log_events
     "social-auth-app-django==5.3.0", # For OAuth
     "social-auth-core==4.4.2", # For OAuth
-    # TODO: Temporarily removing the extra dependencies of aws and gcs from unstract-sdk
-    # to resolve lock file. Will have to be re-looked into
-    "unstract-sdk[azure]~=0.77.3",
-    "gcsfs==2024.10.0",
-    "s3fs==2024.10.0",
+    "unstract-sdk[aws,gcs,azure]~=0.78.0",
     "azure-identity==1.16.0",
     "azure-mgmt-apimanagement==3.0.0",
     "croniter>=3.0.3",
@@ -50,7 +46,7 @@ dependencies = [
     "unstract-core",
     "unstract-filesystem",
     "unstract-flags",
-    "unstract-sdk1",
+    "unstract-sdk1[aws,gcs,azure]",
     "unstract-tool-registry",
     "unstract-tool-sandbox",
     "unstract-workflow-execution",

backend/sample.env

@@ -78,9 +78,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
 
 # Structure Tool Image (Runs prompt studio exported tools)
 # https://hub.docker.com/r/unstract/tool-structure
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.88"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.89"
 STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.88"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.89"
 
 # Feature Flags
 EVALUATION_SERVER_IP=unstract-flipt
@@ -200,3 +200,6 @@ RUNNER_POLLING_INTERVAL_SECONDS=2
 # Default: 1800 seconds (30 minutes)
 # Examples: 900 (15 min), 1800 (30 min), 3600 (60 min)
 MIN_SCHEDULE_INTERVAL_SECONDS=1800
+
+# File active cache redis db
+FILE_ACTIVE_CACHE_REDIS_DB=0

backend/utils/cache_service.py

@@ -43,12 +43,18 @@ def clear_cache(key_pattern: str) -> Any:
         cache.delete_pattern(key_pattern)
 
     @staticmethod
-    def clear_cache_optimized(key_pattern: str) -> Any:
+    def clear_cache_optimized(key_pattern: str, db: int | None = None) -> Any:
         """Delete keys in bulk using optimized SCAN approach for large datasets.
 
         Uses Redis SCAN instead of KEYS to avoid blocking Redis during deletion.
         Safe for production with large key sets. Use this for heavy operations
         like workflow history clearing.
+
+        Args:
+            key_pattern: Pattern to match keys for deletion (e.g., "file_active:*")
+            db: Optional Redis database number. If provided, clears from that specific DB.
+                If None, uses the default Django cache connection (typically DB 0).
+                Use db=1 for worker cache entries (file_active:*, etc.)
         """
         TIMEOUT_SECONDS = 90  # Generous but bounded timeout
         BATCH_SIZE = 1000
@@ -58,6 +64,20 @@ def clear_cache_optimized(key_pattern: str) -> Any:
         cursor = 0
         completed_naturally = False
 
+        # Use specified DB or default connection
+        if db is not None:
+            import redis
+
+            redis_connection = redis.Redis(
+                host=settings.REDIS_HOST,
+                port=settings.REDIS_PORT,
+                db=db,
+                username=getattr(settings, "REDIS_USER", None),
+                password=getattr(settings, "REDIS_PASSWORD", None),
+            )
+        else:
+            redis_connection = redis_cache
+
         try:
             while True:
                 # Check timeout first
@@ -69,13 +89,13 @@ def clear_cache_optimized(key_pattern: str) -> Any:
                     break
 
                 # SCAN returns (next_cursor, keys_list)
-                cursor, keys = redis_cache.scan(
+                cursor, keys = redis_connection.scan(
                     cursor=cursor, match=key_pattern, count=BATCH_SIZE
                 )
 
                 if keys:
                     # Delete keys in pipeline for efficiency
-                    pipe = redis_cache.pipeline()
+                    pipe = redis_connection.pipeline()
                     for key in keys:
                         pipe.delete(key)
                     pipe.execute()