Merge branch 'main' into steven_update

Author: sttruji

.gitignore

@@ -36,6 +36,7 @@ coverage/
 backend/.coverage
 node_modules
 backend/qdrant_storage/
+frontend/build/
 
 # Exclude all .env files
 .env

README.md

@@ -4,10 +4,37 @@
 from flask_cors import CORS
 import json, logging, os, re
 from werkzeug.exceptions import HTTPException
+from config import LOG_LEVEL, LOG_EXCLUDE_LEVELS, LOG_FORMAT, LOG_DATE_FORMAT
+
+class ConfigurableLogFilter(logging.Filter):
+    """Filter out specific log levels based on configuration."""
+    def __init__(self, exclude_levels_str):
+        super().__init__()
+        # Parse excluded levels from config (e.g., "WARNING,DEBUG" -> [30, 10])
+        self.excluded_levels = set()
+        if exclude_levels_str:
+            for level_name in exclude_levels_str.split(','):
+                level_name = level_name.strip().upper()
+                if level_name and hasattr(logging, level_name):
+                    self.excluded_levels.add(getattr(logging, level_name))
+    
+    def filter(self, record):
+        return record.levelno not in self.excluded_levels
+
+logging.basicConfig(
+    level=getattr(logging, LOG_LEVEL, logging.INFO),
+    format=LOG_FORMAT,
+    datefmt=LOG_DATE_FORMAT
+)
+
+log_filter = ConfigurableLogFilter(LOG_EXCLUDE_LEVELS)
+for handler in logging.root.handlers:
+    handler.addFilter(log_filter)
 
 from services.db import redis_client
 from services.canvas_counter import get_canvas_draw_count
 from services.graphql_service import commit_transaction_via_graphql
+from services.graphql_retry_worker import start_retry_worker, stop_retry_worker
 from config import *
 
 app = Flask(__name__)
@@ -78,7 +105,17 @@ def handle_rate_limit_exception(e):
 local_regexes = [r"^https?://localhost(:\d+)?$", r"^https?://127\.0\.0\.1(:\d+)?$"]
 
 cors_origins = explicit_allowed + local_regexes
-CORS(app, supports_credentials=True, origins=cors_origins)
+
+CORS(app, 
+     resources={r"/*": {
+         "origins": cors_origins,
+         "methods": ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
+         "allow_headers": ["Content-Type", "Authorization", "X-Requested-With", "Accept"],
+         "expose_headers": ["Content-Type", "Authorization"],
+         "supports_credentials": True,
+         "max_age": 3600
+     }}
+)
 
 def origin_allowed(origin):
     """Return True if the provided origin string is allowed by explicit allowed
@@ -103,17 +140,17 @@ def add_cors_headers(response):
     This complements flask-cors and guards against cases where exception paths
     or other middleware may return responses without the proper headers.
     """
+    if response.headers.get("Access-Control-Allow-Origin"):
+        return response
+    
     try:
         origin = request.headers.get("Origin")
         if origin and origin_allowed(origin):
             response.headers["Access-Control-Allow-Origin"] = origin
             response.headers["Access-Control-Allow-Credentials"] = "true"
-        else:
-            fallback = explicit_allowed[0] if explicit_allowed else "http://localhost:10008"
-            response.headers.setdefault("Access-Control-Allow-Origin", fallback)
-            response.headers.setdefault("Access-Control-Allow-Credentials", "true")
-        response.headers.setdefault("Access-Control-Allow-Headers", "Content-Type,Authorization")
-        response.headers.setdefault("Access-Control-Allow-Methods", "GET,POST,PUT,PATCH,DELETE,OPTIONS")
+            response.headers["Access-Control-Allow-Headers"] = "Content-Type,Authorization,X-Requested-With,Accept"
+            response.headers["Access-Control-Allow-Methods"] = "GET,POST,PUT,PATCH,DELETE,OPTIONS"
+            response.headers["Access-Control-Expose-Headers"] = "Content-Type,Authorization"
     except Exception:
         pass
     return response
@@ -142,28 +179,32 @@ def handle_all_exceptions(e):
         if origin and origin_allowed(origin):
             resp.headers["Access-Control-Allow-Origin"] = origin
             resp.headers["Access-Control-Allow-Credentials"] = "true"
+            resp.headers["Access-Control-Allow-Headers"] = "Content-Type,Authorization,X-Requested-With,Accept"
+            resp.headers["Access-Control-Allow-Methods"] = "GET,POST,PUT,PATCH,DELETE,OPTIONS"
+            resp.headers["Access-Control-Expose-Headers"] = "Content-Type,Authorization"
         else:
             fallback = explicit_allowed[0] if explicit_allowed else "http://localhost:10008"
             resp.headers.setdefault("Access-Control-Allow-Origin", fallback)
             resp.headers.setdefault("Access-Control-Allow-Credentials", "true")
-        resp.headers.setdefault("Access-Control-Allow-Headers", "Content-Type,Authorization")
-        resp.headers.setdefault("Access-Control-Allow-Methods", "GET,POST,PUT,PATCH,DELETE,OPTIONS")
+            resp.headers.setdefault("Access-Control-Allow-Headers", "Content-Type,Authorization,X-Requested-With,Accept")
+            resp.headers.setdefault("Access-Control-Allow-Methods", "GET,POST,PUT,PATCH,DELETE,OPTIONS")
+            resp.headers.setdefault("Access-Control-Expose-Headers", "Content-Type,Authorization")
         return resp
     except Exception:
         # If even the error handler fails, return a minimal JSON response
         logger.exception("Error while handling exception")
         out = make_response(json.dumps({"status": "error", "message": "Fatal error"}), 500)
         out.headers.setdefault("Access-Control-Allow-Origin", "http://localhost:10008")
         out.headers.setdefault("Access-Control-Allow-Credentials", "true")
-        out.headers.setdefault("Access-Control-Allow-Headers", "Content-Type,Authorization")
+        out.headers.setdefault("Access-Control-Allow-Headers", "Content-Type,Authorization,X-Requested-With,Accept")
         out.headers.setdefault("Access-Control-Allow-Methods", "GET,POST,PUT,PATCH,DELETE,OPTIONS")
         out.headers["Content-Type"] = "application/json"
         return out
 
 
 from flask_socketio import SocketIO
 import services.socketio_service as socketio_service
-socketio = SocketIO(app, cors_allowed_origins="*", async_mode="threading")
+socketio = SocketIO(app, cors_allowed_origins=cors_origins, async_mode="threading")
 socketio_service.socketio = socketio
 socketio_service.register_socketio_handlers()
 
@@ -201,6 +242,14 @@ def handle_all_exceptions(e):
 app.register_blueprint(frontend_bp)
 app.register_blueprint(analytics_bp)
 
+# Start the GraphQL retry worker in background thread
+# Worker sleeps 2 seconds on startup to ensure Redis/MongoDB are ready
+start_retry_worker()
+
+# Register cleanup on shutdown
+import atexit
+atexit.register(stop_retry_worker)
+
 if __name__ == '__main__':
     if not redis_client.exists('res-canvas-draw-count'):
         init_count = {"id": "res-canvas-draw-count", "value": 0}

backend/app.py

@@ -3,7 +3,8 @@
 load_dotenv()
 
 MONGO_ATLAS_URI_SECRET = os.getenv("MONGO_ATLAS_URI")
-RES_DB_BASE_URI = os.getenv("RES_DB_BASE_URI")
+RES_DB_BASE_URI = os.getenv("RESILIENTDB_BASE_URI") 
+RES_DB_BASE_URL = RES_DB_BASE_URI
 RES_DB_API_COMMIT = f"{RES_DB_BASE_URI}/v1/transactions/commit"
 RES_DB_API_QUERY = f"{RES_DB_BASE_URI}/v1/transactions/"
 HEADERS = {"Content-Type": "application/json"}
@@ -34,6 +35,7 @@
 REFRESH_TOKEN_COOKIE_NAME = os.getenv("REFRESH_TOKEN_COOKIE_NAME", "rescanvas_refresh")
 REFRESH_TOKEN_COOKIE_SECURE = os.getenv("REFRESH_TOKEN_COOKIE_SECURE", "False") == "True"
 REFRESH_TOKEN_COOKIE_SAMESITE = os.getenv("REFRESH_TOKEN_COOKIE_SAMESITE", "Lax")
+REFRESH_TOKEN_COOKIE_PARTITIONED = os.getenv("REFRESH_TOKEN_COOKIE_PARTITIONED", "False") == "True"
 
 ROOM_MASTER_KEY_B64 = os.getenv("ROOM_MASTER_KEY_B64")
 if not ROOM_MASTER_KEY_B64:
@@ -77,4 +79,10 @@
 RATE_LIMIT_SEARCH_MINUTE = int(os.getenv("RATE_LIMIT_SEARCH_MINUTE", "30"))
 
 # Burst protection
-RATE_LIMIT_BURST_SECOND = int(os.getenv("RATE_LIMIT_BURST_SECOND", "10"))
+RATE_LIMIT_BURST_SECOND = int(os.getenv("RATE_LIMIT_BURST_SECOND", "10"))
+
+# Logging Configuration
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()  # DEBUG, INFO, WARNING, ERROR, CRITICAL
+LOG_EXCLUDE_LEVELS = os.getenv("LOG_EXCLUDE_LEVELS", "WARNING")  # Comma-separated: WARNING,DEBUG
+LOG_FORMAT = os.getenv("LOG_FORMAT", "%(asctime)s [%(levelname)s] %(name)s:%(lineno)d – %(message)s")
+LOG_DATE_FORMAT = os.getenv("LOG_DATE_FORMAT", "%Y-%m-%d %H:%M:%S")

backend/config.py

@@ -59,6 +59,10 @@ services:
       - JWT_SECRET=${JWT_SECRET:-dev-insecure-change-me}
       - OPENAI_API_KEY=${OPENAI_API_KEY}
       - ANALYTICS_ENABLED=${ANALYTICS_ENABLED:-True}
+      - ALLOWED_ORIGINS=${ALLOWED_ORIGINS}
+      - REFRESH_TOKEN_COOKIE_SECURE=${REFRESH_TOKEN_COOKIE_SECURE:-True}
+      - REFRESH_TOKEN_COOKIE_SAMESITE=${REFRESH_TOKEN_COOKIE_SAMESITE:-None}
+      - REFRESH_TOKEN_COOKIE_PARTITIONED=${REFRESH_TOKEN_COOKIE_PARTITIONED:-True}
     volumes:
       - ./logs:/app/logs
     depends_on:

backend/docker-compose.yml

@@ -61,7 +61,7 @@ async def main():
     )
 
     resilient_db_config = ResilientDBConfig(
-        base_url="resilientdb://crow.resilientdb.com",
+        base_url="resilientdb://dev-crow.resilientdb.com",
         http_secure=True,
         ws_secure=True
     )

backend/incubator-resilientdb-resilient-python-cache/example.py

@@ -31,7 +31,7 @@ limits==5.6.0
 markdown-it-py==4.0.0
 MarkupSafe==3.0.3
 mdurl==0.1.2
-# motor==3.7.1
+motor==2.5.1
 ordered-set==4.1.0
 packaging==25.0
 passlib==1.7.4
@@ -40,7 +40,7 @@ pycparser==2.23
 pyee==9.1.1
 Pygments==2.19.2
 PyJWT==2.3.0
-pymongo==4.15.3
+pymongo==3.12.3
 PyNaCl==1.6.0
 pytest==8.3.5
 pytest-cov==5.0.0

backend/requirements.txt

@@ -8,6 +8,7 @@
 from config import (
     JWT_SECRET, JWT_ISSUER, ACCESS_TOKEN_EXPIRES_SECS, REFRESH_TOKEN_EXPIRES_SECS,
     REFRESH_TOKEN_COOKIE_NAME, REFRESH_TOKEN_COOKIE_SECURE, REFRESH_TOKEN_COOKIE_SAMESITE,
+    REFRESH_TOKEN_COOKIE_PARTITIONED,
     RATE_LIMIT_LOGIN_HOURLY, RATE_LIMIT_REGISTER_HOURLY, RATE_LIMIT_REFRESH_HOURLY
 )
 from middleware.auth import require_auth, validate_request_data
@@ -55,6 +56,37 @@ def _find_valid_refresh_token(token_hash):
     })
     return doc
 
+def _set_refresh_cookie(response, token_value, max_age):
+    """Set refresh token cookie with all necessary attributes including Partitioned.
+    
+    The Partitioned attribute is required for third-party cookies in modern browsers
+    to prevent cross-site tracking.
+    """
+    cookie_parts = [
+        f"{REFRESH_TOKEN_COOKIE_NAME}={token_value}",
+        f"Max-Age={max_age}",
+        f"Path=/",
+    ]
+    
+    if REFRESH_TOKEN_COOKIE_SECURE:
+        cookie_parts.append("Secure")
+    
+    cookie_parts.append("HttpOnly")
+    
+    if REFRESH_TOKEN_COOKIE_SAMESITE:
+        cookie_parts.append(f"SameSite={REFRESH_TOKEN_COOKIE_SAMESITE}")
+    
+    if REFRESH_TOKEN_COOKIE_PARTITIONED:
+        cookie_parts.append("Partitioned")
+    
+    expires = datetime.utcnow() + timedelta(seconds=max_age)
+    expires_str = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
+    cookie_parts.append(f"Expires={expires_str}")
+    
+    cookie_string = "; ".join(cookie_parts)
+    response.headers.add("Set-Cookie", cookie_string)
+    return response
+
 @auth_bp.route("/auth/register", methods=["POST"])
 @limiter.limit(f"{RATE_LIMIT_REGISTER_HOURLY}/hour")
 @validate_request_data({
@@ -96,7 +128,7 @@ def register():
     expires_at = datetime.utcnow() + timedelta(seconds=REFRESH_TOKEN_EXPIRES_SECS)
     _store_refresh_token(user["_id"], h, expires_at)
     resp = make_response(jsonify({"status":"ok","token": access, "user": {"username": user["username"], "walletPubKey": user.get("walletPubKey")}}), 201)
-    resp.set_cookie(REFRESH_TOKEN_COOKIE_NAME, raw_refresh, httponly=True, secure=REFRESH_TOKEN_COOKIE_SECURE, samesite=REFRESH_TOKEN_COOKIE_SAMESITE, max_age=REFRESH_TOKEN_EXPIRES_SECS)
+    _set_refresh_cookie(resp, raw_refresh, REFRESH_TOKEN_EXPIRES_SECS)
     return resp
 
 @auth_bp.route("/auth/login", methods=["POST"])
@@ -130,7 +162,7 @@ def login():
     expires_at = datetime.utcnow() + timedelta(seconds=REFRESH_TOKEN_EXPIRES_SECS)
     _store_refresh_token(user["_id"], h, expires_at)
     resp = make_response(jsonify({"status":"ok","token": access, "user": {"username": user["username"], "walletPubKey": user.get("walletPubKey")}}))
-    resp.set_cookie(REFRESH_TOKEN_COOKIE_NAME, raw_refresh, httponly=True, secure=REFRESH_TOKEN_COOKIE_SECURE, samesite=REFRESH_TOKEN_COOKIE_SAMESITE, max_age=REFRESH_TOKEN_EXPIRES_SECS)
+    _set_refresh_cookie(resp, raw_refresh, REFRESH_TOKEN_EXPIRES_SECS)
     return resp
 
 @auth_bp.route("/auth/refresh", methods=["POST"])
@@ -150,7 +182,7 @@ def refresh():
     expires_at = datetime.utcnow() + timedelta(seconds=REFRESH_TOKEN_EXPIRES_SECS)
     _store_refresh_token(user["_id"], new_h, expires_at)
     resp = make_response(jsonify({"status":"ok","token": access}))
-    resp.set_cookie(REFRESH_TOKEN_COOKIE_NAME, new_raw, httponly=True, secure=REFRESH_TOKEN_COOKIE_SECURE, samesite=REFRESH_TOKEN_COOKIE_SAMESITE, max_age=REFRESH_TOKEN_EXPIRES_SECS)
+    _set_refresh_cookie(resp, new_raw, REFRESH_TOKEN_EXPIRES_SECS)
     return resp
 
 @auth_bp.route("/auth/logout", methods=["POST"])

backend/routes/auth.py

@@ -408,8 +408,9 @@ def import_canvas(room_id):
                     if field in stroke:
                         stroke_data[field] = stroke[field]
 
-                # Generate unique stroke ID
-                draw_count = get_canvas_draw_count()
+                # ATOMIC OPERATION: Increment counter and get the NEW value
+                # This must happen FIRST to ensure unique stroke IDs even under concurrent load
+                draw_count = increment_canvas_draw_count()
                 stroke_id = f"res-canvas-draw-{draw_count}"
                 stroke_data["id"] = stroke_id
 
@@ -486,7 +487,7 @@ def import_canvas(room_id):
                     logger.warning(f"import_canvas: GraphQL commit failed for {stroke_id}: {e}")
                     # Continue even if GraphQL fails - data is in Redis/MongoDB
 
-                increment_canvas_draw_count()
+                # Note: increment_canvas_draw_count() already called above - do not call again
                 imported_count += 1
 
             except Exception as e:

backend/routes/export.py

@@ -777,14 +777,21 @@ def get_canvas_data():
 
         try:
             # We'll scan for both undo- and redo- prefix markers.
-            # Query uses a simple regex on the stored asset.data.id field inside
-            # transactions. This may be somewhat heavy but is necessary for recovery.
+            # Use range query instead of regex to leverage index
+            # Range query with prefix matching is much faster than regex (6-18x speedup)
+            # Index: transactions.value.asset.data.id + _id (created by undo_redo_marker_idx)
             for prefix in ("undo-", "redo-"):
                 # Find any transaction blocks that contain an asset.data.id starting with the prefix.
-                # We sort latest-first by _id so we see the most recent writes earliest.
+                # Use range query: $gte prefix and $lt prefix+highest_unicode to match prefix*
+                # This allows MongoDB to use the index efficiently
                 try:
                     cursor = strokes_coll.find(
-                        {"transactions.value.asset.data.id": {"$regex": f"^{prefix}"}},
+                        {
+                            "transactions.value.asset.data.id": {
+                                "$gte": prefix,
+                                "$lt": prefix + "\uffff"  # Unicode max ensures prefix matching
+                            }
+                        },
                         sort=[("_id", -1)]
                     )
                 except Exception:
@@ -846,23 +853,47 @@ def get_canvas_data():
             except Exception:
                 end_idx = 0
 
+            # OPTIMIZATION: Use Redis pipeline to fetch all keys in parallel (10-20x faster)
+            # Build list of all keys to fetch
+            keys_to_fetch = [f"res-canvas-draw-{i}" for i in range(start_idx, end_idx)]
+            
+            # Fetch all keys in one pipeline operation
+            redis_results = {}
+            if keys_to_fetch:
+                try:
+                    pipe = redis_client.pipeline()
+                    for key_id in keys_to_fetch:
+                        pipe.get(key_id)
+                    raw_results = pipe.execute()
+                    
+                    # Map results back to keys
+                    for idx, key_id in enumerate(keys_to_fetch):
+                        if idx < len(raw_results):
+                            redis_results[key_id] = raw_results[idx]
+                except Exception as e:
+                    logger.warning(f"Redis pipeline failed, falling back to sequential: {e}")
+                    # Fallback to sequential if pipeline fails
+                    for key_id in keys_to_fetch:
+                        try:
+                            redis_results[key_id] = redis_client.get(key_id)
+                        except Exception:
+                            redis_results[key_id] = None
+
+            # Process each stroke (same logic as before, but using pre-fetched results)
             for i in range(start_idx, end_idx):
                 key_id = f"res-canvas-draw-{i}"
                 drawing = None
 
-                # 1) Try Redis cached entry first
-                try:
-                    raw = redis_client.get(key_id)
-                    if raw:
+                # 1) Try Redis cached entry first (from pipeline results)
+                raw = redis_results.get(key_id)
+                if raw:
+                    try:
+                        drawing = json.loads(raw)
+                    except Exception:
                         try:
-                            drawing = json.loads(raw)
+                            drawing = json.loads(raw.decode()) if isinstance(raw, (bytes, bytearray)) else None
                         except Exception:
-                            try:
-                                drawing = json.loads(raw.decode()) if isinstance(raw, (bytes, bytearray)) else None
-                            except Exception:
-                                drawing = None
-                except Exception:
-                    drawing = None
+                            drawing = None
 
                 # 2) If not in Redis, try Mongo fallback for this specific key
                 if not drawing: