feat: Add orjson for faster JSON processing and update schema caching logic; enhance tests for schema functionality

Author: Sub0X

pyproject.toml

@@ -35,6 +35,7 @@ requires-python = ">=3.8"
 dependencies = [
     "aiohttp>=3.8.0,<5.0.0",
     "dacite>=1.6.0,<2.0.0",
+    "orjson>=3.0.0",
 ]
 
 [project.urls]
@@ -55,3 +56,9 @@ veedb = ["py.typed", "VERSION"]
 
 [tool.setuptools.dynamic]
 version = {file = "src/veedb/VERSION"}
+
+[tool.poetry.group.dev.dependencies]
+pytest-asyncio = "^0.21.0"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

src/veedb/apitypes/common.py

@@ -141,7 +141,7 @@
 class ImageCommon:
     id: VNDBID
     url: str
-    dims: Tuple[int, int]  # [width, height]
+    dims: List[int]  # [width, height]
     sexual: float  # 0.0-2.0 (average flagging vote)
     violence: float  # 0.0-2.0 (average flagging vote)
     votecount: int

src/veedb/schema_validator.py

@@ -1,5 +1,5 @@
 import asyncio
-import json
+import orjson # Added for faster JSON processing
 import time
 import os
 from pathlib import Path
@@ -8,6 +8,7 @@
 import aiohttp
 
 from .exceptions import InvalidRequestError, VNDBAPIError
+from .methods.fetch import _fetch_api # Ensure this import is present
 
 # Forward declaration for type hinting
 if "VNDB" not in globals():
@@ -19,31 +20,89 @@ class SchemaCache:
     """
     Manages the download, caching, and retrieval of the VNDB API schema.
     """
+    
     def __init__(self, cache_dir: str = ".veedb_cache", cache_filename: str = "schema.json", ttl_hours: float = 24.0, local_schema_path: Optional[str] = None):
-        self.cache_dir = Path(cache_dir)
-        self.cache_file = self.cache_dir / cache_filename
+        # Use string paths initially to avoid any Path recursion issues
+        self._cache_dir_str = str(cache_dir) if cache_dir else ".veedb_cache"
+        self._cache_filename_str = str(cache_filename) if cache_filename else "schema.json"
+        self._local_schema_path_str = str(local_schema_path) if local_schema_path else None
+        
+        # Create Path objects only when needed and with error handling
+        self._cache_dir = None
+        self._cache_file = None
+        self._local_schema_path = None
+        
         self.ttl_seconds = ttl_hours * 3600
         self._schema_data: Optional[Dict[str, Any]] = None
-        self.local_schema_path = Path(local_schema_path) if local_schema_path else None
+
+    @property
+    def cache_dir(self) -> Path:
+        """Safely get the cache directory Path object."""
+        if self._cache_dir is None:
+            try:
+                self._cache_dir = Path(self._cache_dir_str)
+            except Exception:
+                self._cache_dir = Path(".veedb_cache")
+        return self._cache_dir
+    
+    @property
+    def cache_file(self) -> Path:
+        """Safely get the cache file Path object."""
+        if self._cache_file is None:
+            try:
+                self._cache_file = self.cache_dir / self._cache_filename_str
+            except Exception:
+                self._cache_file = Path(".veedb_cache") / "schema.json"
+        return self._cache_file
+    
+    @property
+    def local_schema_path(self) -> Optional[Path]:
+        """Safely get the local schema path Path object."""
+        if self._local_schema_path is None and self._local_schema_path_str:
+            try:
+                self._local_schema_path = Path(self._local_schema_path_str)
+            except Exception:
+                self._local_schema_path = None
+        return self._local_schema_path
 
     def is_cached(self) -> bool:
         """Check if the schema file exists in the cache or if a local path is provided."""
-        if self.local_schema_path and self.local_schema_path.is_file():
-            return True
-        return self.cache_file.is_file()
+        if self._local_schema_path_str:
+            try:
+                # Use os.path.isfile instead of Path.is_file() to avoid recursion
+                return os.path.isfile(self._local_schema_path_str)
+            except (RecursionError, OSError, Exception):
+                pass
+        
+        try:
+            # Use os.path.exists instead of Path.exists() to avoid recursion
+            cache_path = os.path.join(self._cache_dir_str, self._cache_filename_str)
+            return os.path.exists(cache_path)
+        except (RecursionError, OSError, Exception):
+            return False
 
     def get_cache_age(self) -> float:
         """Get the age of the cache file in seconds. Returns 0 if using local_schema_path."""
-        if self.local_schema_path and self.local_schema_path.is_file():
-            # Treat local schema as always up-to-date unless explicitly updated
-            return 0.0
-        if not self.cache_file.is_file(): # Changed from self.is_cached() to self.cache_file.is_file()
+        if self._local_schema_path_str:
+            try:
+                # Use os.path.isfile instead of Path methods to avoid recursion
+                if os.path.isfile(self._local_schema_path_str):
+                    # Treat local schema as always up-to-date unless explicitly updated
+                    return 0.0
+            except (RecursionError, OSError, Exception):
+                pass
+        
+        try:
+            cache_path = os.path.join(self._cache_dir_str, self._cache_filename_str)
+            if not os.path.exists(cache_path):
+                return float('inf')
+            return time.time() - os.path.getmtime(cache_path)
+        except (RecursionError, OSError, Exception):
             return float('inf')
-        return time.time() - self.cache_file.stat().st_mtime
 
     def is_cache_expired(self) -> bool:
         """Check if the cached schema has expired. Local schema path is never considered expired by this check."""
-        if self.local_schema_path and self.local_schema_path.is_file():
+        if self._local_schema_path_str and os.path.isfile(self._local_schema_path_str):
             return False # Local schema is not subject to TTL expiration, only manual updates
         return self.get_cache_age() > self.ttl_seconds
 
@@ -57,24 +116,35 @@ def save_schema(self, schema_data: Dict[str, Any], to_local_path: bool = False):
         target_dir = target_path.parent
         target_dir.mkdir(parents=True, exist_ok=True)
 
-        with open(target_path, 'w', encoding='utf-8') as f:
-            json.dump(schema_data, f, indent=2)
+        # Use orjson for writing
+        with open(target_path, 'wb') as f: # Open in binary mode for orjson
+            f.write(orjson.dumps(schema_data, option=orjson.OPT_INDENT_2))
         self._schema_data = schema_data # Update in-memory cache as well
-
+        
     def load_schema(self) -> Optional[Dict[str, Any]]:
         """Load the schema data from the local_schema_path (if provided) or the cache file."""
-        if self.local_schema_path and self.local_schema_path.is_file():
+        if self._local_schema_path_str and os.path.isfile(self._local_schema_path_str):
             try:
-                with open(self.local_schema_path, 'r', encoding='utf-8') as f:
-                    return json.load(f)
+                # Use orjson for reading
+                with open(self._local_schema_path_str, 'rb') as f: # Open in binary mode for orjson
+                    return orjson.loads(f.read())
             except Exception:
                 # If local schema fails to load, fall back to cache or download
                 pass 
 
-        if self.cache_file.is_file(): # Changed from self.is_cached()
+        # Use os.path instead of Path methods to avoid recursion issues
+        try:
+            cache_path_str = os.path.join(self._cache_dir_str, self._cache_filename_str)
+            cache_file_exists = os.path.isfile(cache_path_str) # Changed from os.path.exists to os.path.isfile
+        except (RecursionError, OSError, Exception):
+            # If there's any issue with the cache_file path, return None
+            return None
+            
+        if cache_file_exists:
             try:
-                with open(self.cache_file, 'r', encoding='utf-8') as f:
-                    return json.load(f)
+                # Use orjson for reading
+                with open(cache_path_str, 'rb') as f: # Open in binary mode for orjson
+                    return orjson.loads(f.read())
             except Exception:
                 pass
         return None
@@ -83,8 +153,9 @@ def invalidate_cache(self):
         """Remove the cache file. Does not remove user-provided local_schema_path."""
         self._schema_data = None
         try:
-            if self.cache_file.exists():
-                os.remove(self.cache_file)
+            cache_path = os.path.join(self._cache_dir_str, self._cache_filename_str)
+            if os.path.exists(cache_path):
+                os.remove(cache_path)
         except FileNotFoundError:
             pass
 
@@ -124,16 +195,61 @@ async def update_local_schema_from_api(self, client: 'VNDB') -> Dict[str, Any]:
         return await self.get_schema(client, force_download=True)
 
     async def _download_schema(self, client: 'VNDB') -> Dict[str, Any]:
-        """Fetch the schema from the VNDB API."""
+        """Fetch the schema from the VNDB API directly."""
         try:
-            return await client.get_schema()
-        except VNDBAPIError as e:
-            # If download fails but a stale cache exists, use it as a fallback
-            if self.is_cached():
-                schema = self.load_schema()
-                if schema:
-                    return schema
-            raise e # Re-raise if there's no cache at all
+            # Call the API directly to avoid recursion - do NOT call client.get_schema()
+            url = f"{client.base_url}/schema"
+            session = client._get_session()
+            
+            # Use the imported _fetch_api
+            # The schema endpoint typically does not require a token.
+            response_data = await _fetch_api(
+                session=session,
+                method="GET",
+                url=url,
+                token=None # Explicitly None for public schema endpoint
+            )
+            if not isinstance(response_data, dict):
+                raise VNDBAPIError(f"Schema download did not return a valid JSON object. Received type: {type(response_data)}")
+            return response_data
+        except aiohttp.ClientError as e:
+            raise VNDBAPIError(f"Failed to download schema due to network/HTTP error: {e}") from e
+        except Exception as e:
+            # Catch other potential errors during the fetch or processing
+            raise VNDBAPIError(f"An unexpected error occurred while downloading schema: {e}") from e
+
+    async def update_local_schema_from_api(self, client: 'VNDB') -> Dict[str, Any]:
+        """Forces a download of the schema and saves it to local_schema_path if configured, else to cache."""
+        if not self.local_schema_path:
+            # If no specific local path, update the default cache file.
+            # Or, one might choose to raise an error if this method is called without a local_schema_path configured.
+            # For now, let's assume it updates the primary schema location (local if set, else cache).
+            pass # Fall through to get_schema with force_download
+        return await self.get_schema(client, force_download=True)
+
+    async def _download_schema(self, client: 'VNDB') -> Dict[str, Any]:
+        """Fetch the schema from the VNDB API directly."""
+        try:
+            # Call the API directly to avoid recursion - do NOT call client.get_schema()
+            url = f"{client.base_url}/schema"
+            session = client._get_session()
+            
+            # Use the imported _fetch_api
+            # The schema endpoint typically does not require a token.
+            response_data = await _fetch_api(
+                session=session,
+                method="GET",
+                url=url,
+                token=None # Explicitly None for public schema endpoint
+            )
+            if not isinstance(response_data, dict):
+                raise VNDBAPIError(f"Schema download did not return a valid JSON object. Received type: {type(response_data)}")
+            return response_data
+        except aiohttp.ClientError as e:
+            raise VNDBAPIError(f"Failed to download schema due to network/HTTP error: {e}") from e
+        except Exception as e:
+            # Catch other potential errors during the fetch or processing
+            raise VNDBAPIError(f"An unexpected error occurred while downloading schema: {e}") from e
 
 class FilterValidator:
     """

test_schema_fix.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+Test script to verify the recursion fix for schema caching.
+This script tests all the schema-related functionality to ensure no recursion occurs.
+"""
+
+import asyncio
+import sys
+import traceback
+from veedb import VNDB, QueryRequest
+
+async def test_comprehensive_schema():
+    """Test all schema-related functionality to ensure no recursion."""
+    print("🧪 Testing comprehensive schema functionality...")
+    
+    try:
+        async with VNDB(use_sandbox=False) as vndb:
+            print("\n1. Testing get_schema()...")
+            schema = await vndb.get_schema()
+            print(f"   ✅ Schema retrieved with keys: {list(schema.keys())}")
+            
+            print("\n2. Testing get_enums()...")
+            enums = await vndb.get_enums()
+            print(f"   ✅ Found {len(enums)} enum types: {list(enums.keys())}")
+            
+            print("\n3. Testing filter validation (uses schema)...")
+            validation = await vndb.validate_filters('/vn', ['title', '=', 'test'])
+            print(f"   ✅ Filter validation successful: {validation.get('valid', False)}")
+            
+            print("\n4. Testing get_available_fields (uses schema)...")
+            fields = await vndb.get_available_fields('/vn')
+            print(f"   ✅ Found {len(fields)} available fields for /vn")
+            
+            print("\n5. Testing VN query (autocomplete scenario)...")
+            search_req = QueryRequest(
+                filters=['search', '=', 'fate'],
+                results=3,
+                fields='id, title, released, olang'
+            )
+            results = await vndb.vn.query(search_req)
+            print(f"   ✅ VN search successful: {len(results.results)} results")
+            
+            print("\n6. Testing language enum (Discord autocomplete scenario)...")
+            languages = enums.get('language', [])
+            filtered_langs = [
+                lang for lang in languages 
+                if 'en' in lang['label'].lower() or lang['id'].startswith('en')
+            ][:5]
+            print(f"   ✅ Language filtering successful: {len(filtered_langs)} matching languages")
+            
+            print("\n🎉 All tests passed! No recursion detected.")
+            return True
+            
+    except RecursionError as e:
+        print(f"\n❌ RECURSION ERROR DETECTED!")
+        print(f"Error: {e}")
+        traceback.print_exc()
+        return False
+    except Exception as e:
+        print(f"\n❌ OTHER ERROR: {e}")
+        traceback.print_exc()
+        return False
+
+async def main():
+    """Main test function."""
+    print("VeeDB Schema Recursion Fix Test")
+    print("=" * 40)
+    
+    success = await test_comprehensive_schema()
+    
+    if success:
+        print("\n✅ ALL TESTS PASSED - Schema caching works correctly!")
+        sys.exit(0)
+    else:
+        print("\n❌ TESTS FAILED - Check the error messages above")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    asyncio.run(main())