From 9a2ed39bfe508b909df7ea30bcee299707e55043 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Wed, 16 Jul 2025 10:17:51 +0300 Subject: [PATCH 1/7] Add entry size limit per cache core --- README.rst | 17 ++++++++++++++++ src/cachier/config.py | 23 +++++++++++++++++++++ src/cachier/core.py | 37 +++++++++++++++++++++++++--------- src/cachier/cores/base.py | 22 ++++++++++++++++++-- src/cachier/cores/memory.py | 8 ++++++-- src/cachier/cores/mongo.py | 10 +++++++-- src/cachier/cores/pickle.py | 16 +++++++++------ src/cachier/cores/redis.py | 11 ++++++++-- src/cachier/cores/sql.py | 18 +++++++++++------ tests/test_entry_size_limit.py | 37 ++++++++++++++++++++++++++++++++++ 10 files changed, 170 insertions(+), 29 deletions(-) create mode 100644 tests/test_entry_size_limit.py diff --git a/README.rst b/README.rst index 5ec99e37..947a9622 100644 --- a/README.rst +++ b/README.rst @@ -138,6 +138,7 @@ The following parameters will only be applied to decorators defined after `set_d * `cache_dir` * `pickle_reload` * `separate_files` +* `entry_size_limit` These parameters can be changed at any time and they will apply to all decorators: @@ -269,6 +270,22 @@ You can specify a maximum allowed age for a cached value on a per-call basis usi - If the cached value is older than this threshold, a new calculation is triggered and the cache is updated. - If not, the cached value is returned as usual. +Entry Size Limit +~~~~~~~~~~~~~~~~ +You can prevent very large return values from being cached by specifying +``entry_size_limit`` on the decorator. Values larger than this limit are +returned but not stored. The limit accepts an integer number of bytes or a +human readable string like ``"200MB"``. + +.. code-block:: python + + @cachier(entry_size_limit="10KB") + def load_data(): + ... + +When ``cachier__verbose=True`` is passed to a call that returns a value +exceeding the limit, an informative message is printed. + Ignore Cache ~~~~~~~~~~~~ diff --git a/src/cachier/config.py b/src/cachier/config.py index 53dfbe82..3ac0cb33 100644 --- a/src/cachier/config.py +++ b/src/cachier/config.py @@ -1,6 +1,7 @@ import hashlib import os import pickle +import re import threading from dataclasses import dataclass, field, replace from datetime import datetime, timedelta @@ -32,6 +33,27 @@ def _default_cache_dir(): return os.path.expanduser("~/.cachier/") +def parse_bytes(size: Union[int, str, None]) -> Optional[int]: + """Convert a human friendly size string to bytes.""" + if size is None: + return None + if isinstance(size, int): + return size + match = re.fullmatch(r"(?i)\s*(\d+(?:\.\d+)?)\s*([kmgt]?b)?\s*", str(size)) + if not match: + raise ValueError(f"Invalid size value: {size}") + number = float(match.group(1)) + unit = (match.group(2) or "b").upper() + factor = { + "B": 1, + "KB": 1024, + "MB": 1024**2, + "GB": 1024**3, + "TB": 1024**4, + }[unit] + return int(number * factor) + + class LazyCacheDir: """Lazily resolve the default cache directory using $XDG_CACHE_HOME.""" @@ -65,6 +87,7 @@ class Params: allow_none: bool = False cleanup_stale: bool = False cleanup_interval: timedelta = timedelta(days=1) + entry_size_limit: Optional[int] = None _global_params = Params() diff --git a/src/cachier/core.py b/src/cachier/core.py index 4db5e329..8aa33f0d 100644 --- a/src/cachier/core.py +++ b/src/cachier/core.py @@ -24,6 +24,7 @@ HashFunc, Mongetter, _update_with_defaults, + parse_bytes, ) from .cores.base import RecalculationNeeded, _BaseCore from .cores.memory import _MemoryCore @@ -60,11 +61,15 @@ def _function_thread(core, key, func, args, kwds): print(f"Function call failed with the following exception:\n{exc}") -def _calc_entry(core, key, func, args, kwds) -> Optional[Any]: +def _calc_entry( + core, key, func, args, kwds, printer=lambda *_: None +) -> Optional[Any]: core.mark_entry_being_calculated(key) try: func_res = func(*args, **kwds) - core.set_entry(key, func_res) + stored = core.set_entry(key, func_res) + if not stored: + printer("Result exceeds entry_size_limit; not cached") return func_res finally: core.mark_entry_not_calculated(key) @@ -123,6 +128,7 @@ def cachier( allow_none: Optional[bool] = None, cleanup_stale: Optional[bool] = None, cleanup_interval: Optional[timedelta] = None, + entry_size_limit: Optional[Union[int, str]] = None, ): """Wrap as a persistent, stale-free memoization decorator. @@ -191,6 +197,10 @@ def cachier( thread. Defaults to False. cleanup_interval: datetime.timedelta, optional Minimum time between automatic cleanup runs. Defaults to one day. + entry_size_limit: int or str, optional + Maximum serialized size of a cached value. Values exceeding the limit + are returned but not cached. Human readable strings like ``"10MB"`` are + allowed. """ # Check for deprecated parameters @@ -204,6 +214,9 @@ def cachier( # Update parameters with defaults if input is None backend = _update_with_defaults(backend, "backend") mongetter = _update_with_defaults(mongetter, "mongetter") + size_limit_bytes = parse_bytes( + _update_with_defaults(entry_size_limit, "entry_size_limit") + ) # Override the backend parameter if a mongetter is provided. if callable(mongetter): backend = "mongo" @@ -215,28 +228,34 @@ def cachier( cache_dir=cache_dir, separate_files=separate_files, wait_for_calc_timeout=wait_for_calc_timeout, + entry_size_limit=size_limit_bytes, ) elif backend == "mongo": core = _MongoCore( hash_func=hash_func, mongetter=mongetter, wait_for_calc_timeout=wait_for_calc_timeout, + entry_size_limit=size_limit_bytes, ) elif backend == "memory": core = _MemoryCore( - hash_func=hash_func, wait_for_calc_timeout=wait_for_calc_timeout + hash_func=hash_func, + wait_for_calc_timeout=wait_for_calc_timeout, + entry_size_limit=size_limit_bytes, ) elif backend == "sql": core = _SQLCore( hash_func=hash_func, sql_engine=sql_engine, wait_for_calc_timeout=wait_for_calc_timeout, + entry_size_limit=size_limit_bytes, ) elif backend == "redis": core = _RedisCore( hash_func=hash_func, redis_client=redis_client, wait_for_calc_timeout=wait_for_calc_timeout, + entry_size_limit=size_limit_bytes, ) else: raise ValueError("specified an invalid core: %s" % backend) @@ -324,12 +343,12 @@ def _call(*args, max_age: Optional[timedelta] = None, **kwds): ) key, entry = core.get_entry((), kwargs) if overwrite_cache: - return _calc_entry(core, key, func, args, kwds) + return _calc_entry(core, key, func, args, kwds, _print) if entry is None or ( not entry._completed and not entry._processing ): _print("No entry found. No current calc. Calling like a boss.") - return _calc_entry(core, key, func, args, kwds) + return _calc_entry(core, key, func, args, kwds, _print) _print("Entry found.") if _allow_none or entry.value is not None: _print("Cached result found.") @@ -362,7 +381,7 @@ def _call(*args, max_age: Optional[timedelta] = None, **kwds): try: return core.wait_on_entry_calc(key) except RecalculationNeeded: - return _calc_entry(core, key, func, args, kwds) + return _calc_entry(core, key, func, args, kwds, _print) if _next_time: _print("Async calc and return stale") core.mark_entry_being_calculated(key) @@ -374,15 +393,15 @@ def _call(*args, max_age: Optional[timedelta] = None, **kwds): core.mark_entry_not_calculated(key) return entry.value _print("Calling decorated function and waiting") - return _calc_entry(core, key, func, args, kwds) + return _calc_entry(core, key, func, args, kwds, _print) if entry._processing: _print("No value but being calculated. Waiting.") try: return core.wait_on_entry_calc(key) except RecalculationNeeded: - return _calc_entry(core, key, func, args, kwds) + return _calc_entry(core, key, func, args, kwds, _print) _print("No entry found. No current calc. Calling like a boss.") - return _calc_entry(core, key, func, args, kwds) + return _calc_entry(core, key, func, args, kwds, _print) # MAINTAINER NOTE: The main function wrapper is now a standard function # that passes *args and **kwargs to _call. This ensures that user diff --git a/src/cachier/cores/base.py b/src/cachier/cores/base.py index edb8e7ed..14e5836b 100644 --- a/src/cachier/cores/base.py +++ b/src/cachier/cores/base.py @@ -9,9 +9,11 @@ import abc # for the _BaseCore abstract base class import inspect +import pickle +import sys import threading from datetime import timedelta -from typing import Callable, Optional, Tuple +from typing import Any, Callable, Optional, Tuple from .._types import HashFunc from ..config import CacheEntry, _update_with_defaults @@ -34,10 +36,12 @@ def __init__( self, hash_func: Optional[HashFunc], wait_for_calc_timeout: Optional[int], + entry_size_limit: Optional[int] = None, ): self.hash_func = _update_with_defaults(hash_func, "hash_func") self.wait_for_calc_timeout = wait_for_calc_timeout self.lock = threading.RLock() + self.entry_size_limit = entry_size_limit def set_func(self, func): """Set the function this core will use. @@ -90,8 +94,22 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: """ + def _estimate_size(self, value: Any) -> int: + try: + return len(pickle.dumps(value)) + except Exception: + return sys.getsizeof(value) + + def _should_store(self, value: Any) -> bool: + if self.entry_size_limit is None: + return True + try: + return self._estimate_size(value) <= self.entry_size_limit + except Exception: + return True + @abc.abstractmethod - def set_entry(self, key: str, func_res): + def set_entry(self, key: str, func_res: Any) -> bool: """Map the given result to the given key in this core's cache.""" @abc.abstractmethod diff --git a/src/cachier/cores/memory.py b/src/cachier/cores/memory.py index ddd0acdb..21386b4b 100644 --- a/src/cachier/cores/memory.py +++ b/src/cachier/cores/memory.py @@ -16,8 +16,9 @@ def __init__( self, hash_func: Optional[HashFunc], wait_for_calc_timeout: Optional[int], + entry_size_limit: Optional[int] = None, ): - super().__init__(hash_func, wait_for_calc_timeout) + super().__init__(hash_func, wait_for_calc_timeout, entry_size_limit) self.cache: Dict[str, CacheEntry] = {} def _hash_func_key(self, key: str) -> str: @@ -29,7 +30,9 @@ def get_entry_by_key( with self.lock: return key, self.cache.get(self._hash_func_key(key), None) - def set_entry(self, key: str, func_res: Any) -> None: + def set_entry(self, key: str, func_res: Any) -> bool: + if not self._should_store(func_res): + return False hash_key = self._hash_func_key(key) with self.lock: try: @@ -47,6 +50,7 @@ def set_entry(self, key: str, func_res: Any) -> None: _condition=cond, _completed=True, ) + return True def mark_entry_being_calculated(self, key: str) -> None: with self.lock: diff --git a/src/cachier/cores/mongo.py b/src/cachier/cores/mongo.py index fbc93711..9a28dd1c 100644 --- a/src/cachier/cores/mongo.py +++ b/src/cachier/cores/mongo.py @@ -40,6 +40,7 @@ def __init__( hash_func: Optional[HashFunc], mongetter: Optional[Mongetter], wait_for_calc_timeout: Optional[int], + entry_size_limit: Optional[int] = None, ): if "pymongo" not in sys.modules: warnings.warn( @@ -49,7 +50,9 @@ def __init__( ) # pragma: no cover super().__init__( - hash_func=hash_func, wait_for_calc_timeout=wait_for_calc_timeout + hash_func=hash_func, + wait_for_calc_timeout=wait_for_calc_timeout, + entry_size_limit=entry_size_limit, ) if mongetter is None: raise MissingMongetter( @@ -87,7 +90,9 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: ) return key, entry - def set_entry(self, key: str, func_res: Any) -> None: + def set_entry(self, key: str, func_res: Any) -> bool: + if not self._should_store(func_res): + return False thebytes = pickle.dumps(func_res) self.mongo_collection.update_one( filter={"func": self._func_str, "key": key}, @@ -104,6 +109,7 @@ def set_entry(self, key: str, func_res: Any) -> None: }, upsert=True, ) + return True def mark_entry_being_calculated(self, key: str) -> None: self.mongo_collection.update_one( diff --git a/src/cachier/cores/pickle.py b/src/cachier/cores/pickle.py index 344fcba7..78209cab 100644 --- a/src/cachier/cores/pickle.py +++ b/src/cachier/cores/pickle.py @@ -78,8 +78,9 @@ def __init__( cache_dir: Optional[Union[str, os.PathLike]], separate_files: Optional[bool], wait_for_calc_timeout: Optional[int], + entry_size_limit: Optional[int] = None, ): - super().__init__(hash_func, wait_for_calc_timeout) + super().__init__(hash_func, wait_for_calc_timeout, entry_size_limit) self._cache_dict: Dict[str, CacheEntry] = {} self.reload = _update_with_defaults(pickle_reload, "pickle_reload") self.cache_dir = os.path.expanduser( @@ -119,7 +120,7 @@ def _convert_legacy_cache_entry( def _load_cache_dict(self) -> Dict[str, CacheEntry]: try: with portalocker.Lock(self.cache_fpath, mode="rb") as cf: - cache = pickle.load(cf) + cache = pickle.load(cf) # type: ignore[arg-type] self._cache_used_fpath = str(self.cache_fpath) except (FileNotFoundError, EOFError): cache = {} @@ -146,7 +147,7 @@ def _load_cache_by_key( fpath += f"_{hash_str or key}" try: with portalocker.Lock(fpath, mode="rb") as cache_file: - entry = pickle.load(cache_file) + entry = pickle.load(cache_file) # type: ignore[arg-type] return _PickleCore._convert_legacy_cache_entry(entry) except (FileNotFoundError, EOFError): return None @@ -185,7 +186,7 @@ def _save_cache( fpath += f"_{hash_str}" with self.lock: with portalocker.Lock(fpath, mode="wb") as cf: - pickle.dump(cache, cf, protocol=4) + pickle.dump(cache, cf, protocol=4) # type: ignore[arg-type] # the same as check for separate_file, but changed for typing if isinstance(cache, dict): self._cache_dict = cache @@ -198,7 +199,9 @@ def get_entry_by_key( return key, self._load_cache_by_key(key) return key, self.get_cache_dict(reload).get(key) - def set_entry(self, key: str, func_res: Any) -> None: + def set_entry(self, key: str, func_res: Any) -> bool: + if not self._should_store(func_res): + return False key_data = CacheEntry( value=func_res, time=datetime.now(), @@ -208,12 +211,13 @@ def set_entry(self, key: str, func_res: Any) -> None: ) if self.separate_files: self._save_cache(key_data, key) - return # pragma: no cover + return True # pragma: no cover with self.lock: cache = self.get_cache_dict() cache[key] = key_data self._save_cache(cache) + return True def mark_entry_being_calculated_separate_files(self, key: str) -> None: self._save_cache( diff --git a/src/cachier/cores/redis.py b/src/cachier/cores/redis.py index ccd0ffe0..ff4d8fd0 100644 --- a/src/cachier/cores/redis.py +++ b/src/cachier/cores/redis.py @@ -35,6 +35,7 @@ def __init__( ], wait_for_calc_timeout: Optional[int] = None, key_prefix: str = "cachier", + entry_size_limit: Optional[int] = None, ): if not REDIS_AVAILABLE: warnings.warn( @@ -45,7 +46,9 @@ def __init__( ) super().__init__( - hash_func=hash_func, wait_for_calc_timeout=wait_for_calc_timeout + hash_func=hash_func, + wait_for_calc_timeout=wait_for_calc_timeout, + entry_size_limit=entry_size_limit, ) if redis_client is None: raise MissingRedisClient( @@ -122,7 +125,9 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: warnings.warn(f"Redis get_entry_by_key failed: {e}", stacklevel=2) return key, None - def set_entry(self, key: str, func_res: Any) -> None: + def set_entry(self, key: str, func_res: Any) -> bool: + if not self._should_store(func_res): + return False """Map the given result to the given key in Redis.""" redis_client = self._resolve_redis_client() redis_key = self._get_redis_key(key) @@ -143,8 +148,10 @@ def set_entry(self, key: str, func_res: Any) -> None: "completed": "true", }, ) + return True except Exception as e: warnings.warn(f"Redis set_entry failed: {e}", stacklevel=2) + return False def mark_entry_being_calculated(self, key: str) -> None: """Mark the entry mapped by the given key as being calculated.""" diff --git a/src/cachier/cores/sql.py b/src/cachier/cores/sql.py index 543531ef..94159528 100644 --- a/src/cachier/cores/sql.py +++ b/src/cachier/cores/sql.py @@ -63,6 +63,7 @@ def __init__( hash_func: Optional[HashFunc], sql_engine: Optional[Union[str, "Engine", Callable[[], "Engine"]]], wait_for_calc_timeout: Optional[int] = None, + entry_size_limit: Optional[int] = None, ): if not SQLALCHEMY_AVAILABLE: raise ImportError( @@ -70,7 +71,9 @@ def __init__( "Install with `pip install SQLAlchemy`." ) super().__init__( - hash_func=hash_func, wait_for_calc_timeout=wait_for_calc_timeout + hash_func=hash_func, + wait_for_calc_timeout=wait_for_calc_timeout, + entry_size_limit=entry_size_limit, ) self._engine = self._resolve_engine(sql_engine) self._Session = sessionmaker(bind=self._engine) @@ -109,14 +112,16 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: value = pickle.loads(row.value) if row.value is not None else None entry = CacheEntry( value=value, - time=row.timestamp, - stale=row.stale, - _processing=row.processing, - _completed=row.completed, + time=row.timestamp, # type: ignore[arg-type] + stale=row.stale, # type: ignore[arg-type] + _processing=row.processing, # type: ignore[arg-type] + _completed=row.completed, # type: ignore[arg-type] ) return key, entry - def set_entry(self, key: str, func_res: Any) -> None: + def set_entry(self, key: str, func_res: Any) -> bool: + if not self._should_store(func_res): + return False with self._lock, self._Session() as session: thebytes = pickle.dumps(func_res) now = datetime.now() @@ -187,6 +192,7 @@ def set_entry(self, key: str, func_res: Any) -> None: ) ) session.commit() + return True def mark_entry_being_calculated(self, key: str) -> None: with self._lock, self._Session() as session: diff --git a/tests/test_entry_size_limit.py b/tests/test_entry_size_limit.py new file mode 100644 index 00000000..f2784967 --- /dev/null +++ b/tests/test_entry_size_limit.py @@ -0,0 +1,37 @@ +import pytest + +import cachier + + +@pytest.mark.memory +def test_entry_size_limit_not_cached(): + call_count = 0 + + @cachier.cachier(backend="memory", entry_size_limit="10B") + def func(x): + nonlocal call_count + call_count += 1 + return "a" * 50 + + func.clear_cache() + val1 = func(1) + val2 = func(1) + assert val1 == val2 + assert call_count == 2 + + +@pytest.mark.memory +def test_entry_size_limit_cached(): + call_count = 0 + + @cachier.cachier(backend="memory", entry_size_limit="1KB") + def func(x): + nonlocal call_count + call_count += 1 + return "small" + + func.clear_cache() + val1 = func(1) + val2 = func(1) + assert val1 == val2 + assert call_count == 1 From e763d07a986c11414ca409561aecfcea75d14ce0 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Wed, 16 Jul 2025 12:12:46 +0300 Subject: [PATCH 2/7] Fix default params test and export parse_bytes --- src/cachier/__init__.py | 2 ++ tests/test_core_lookup.py | 1 + 2 files changed, 3 insertions(+) diff --git a/src/cachier/__init__.py b/src/cachier/__init__.py index cfaeaea3..d53b3535 100644 --- a/src/cachier/__init__.py +++ b/src/cachier/__init__.py @@ -4,6 +4,7 @@ enable_caching, get_default_params, get_global_params, + parse_bytes, set_default_params, set_global_params, ) @@ -15,6 +16,7 @@ "get_default_params", "set_global_params", "get_global_params", + "parse_bytes", "enable_caching", "disable_caching", "__version__", diff --git a/tests/test_core_lookup.py b/tests/test_core_lookup.py index 2b2a9191..c39b653d 100644 --- a/tests/test_core_lookup.py +++ b/tests/test_core_lookup.py @@ -14,6 +14,7 @@ def test_get_default_params(): "caching_enabled", "cleanup_interval", "cleanup_stale", + "entry_size_limit", "hash_func", "mongetter", "next_time", From 3102e08586a6a8827ea828ce0ec168d761c0c9a8 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Wed, 16 Jul 2025 12:12:59 +0300 Subject: [PATCH 3/7] Remove mypy usage and add size util --- .github/copilot-instructions.md | 2 +- .pre-commit-config.yaml | 6 ------ AGENTS.md | 8 +------- CLAUDE.md | 6 ------ Makefile | 6 +----- pyproject.toml | 3 ++- src/cachier/__init__.py | 2 +- src/cachier/config.py | 22 ---------------------- src/cachier/core.py | 9 ++------- src/cachier/cores/base.py | 5 +++-- src/cachier/cores/pickle.py | 6 +++--- src/cachier/cores/sql.py | 8 ++++---- src/cachier/util.py | 25 +++++++++++++++++++++++++ tests/requirements.txt | 1 + 14 files changed, 44 insertions(+), 65 deletions(-) create mode 100644 src/cachier/util.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1da246da..ced25991 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -26,7 +26,7 @@ Welcome to the Cachier codebase! Please follow these guidelines to ensure code s ## 4. Coverage, Linting, and Typing -- Code must pass `mypy`, `ruff`, and `pytest`. +- Code must pass `ruff` and `pytest`. - Use per-file or per-line ignores for known, justified issues (e.g., SQLAlchemy model base class typing, intentional use of `pickle`). - All new code must include full type annotations and docstrings matching the style of the existing codebase. - All docstrings should follow numpy docstring conventions. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8046aaa9..3920f669 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,12 +65,6 @@ repos: name: Ruff check args: ["--fix"] - # it needs to be after formatting hooks because the lines might be changed - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.16.1 - hooks: - - id: mypy - files: "src/*" - repo: https://github.com/tox-dev/pyproject-fmt rev: v2.6.0 diff --git a/AGENTS.md b/AGENTS.md index 4174b470..675eb234 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,6 @@ - **Key Dependencies:** `portalocker`, `watchdog` (optional: `pymongo`, `sqlalchemy`, `redis`) - **Test Framework:** `pytest` with backend-specific markers - **Linting:** `ruff` (replaces black/flake8) -- **Type Checking:** `mypy` - **CI:** GitHub Actions (matrix for backends/OS with Dockerized services) - **Issue Tracking:** GitHub Issues - **Additional Docs:** `.github/copilot-instructions.md` for contributor guidelines @@ -65,11 +64,10 @@ ______________________________________________________________________ pytest -m "not (mongo or sql)" # Exclude external service backends ``` -3. **Lint and type-check:** +3. **Lint:** ```bash ruff check . - mypy src/cachier/ ``` 4. **Try an example:** @@ -102,7 +100,6 @@ ______________________________________________________________________ - **Type annotations** required for all new code. - **Docstrings:** Use numpy style, multi-line, no single-line docstrings. - **Lint:** Run `ruff` before PRs. Use per-line/file ignores only for justified cases. -- **Type check:** Run `mypy` before PRs. - **Testing:** All public methods must have at least one test. Use `pytest.mark.` for backend-specific tests. - **No warnings/errors for missing optional dependencies at import time.** Only raise when backend is used. @@ -415,7 +412,6 @@ ______________________________________________________________________ - **Run multiple backends:** `pytest -m "redis or sql"` - **Exclude backends:** `pytest -m "not mongo"` - **Lint:** `ruff check .` -- **Type check:** `mypy src/cachier/` - **Format:** `ruff format .` - **Pre-commit:** `pre-commit run --all-files` - **Build package:** `python -m build` @@ -505,7 +501,6 @@ ______________________________________________________________________ - **If adding new dependencies, use context7 MCP to get latest versions.** - **Always check GitHub Issues before starting new features/PRs.** - **Create a relevant issue for every new PR.** -- **Use per-file or per-line ignores for mypy/ruff only when justified.** - **All new code must have full type annotations and numpy-style docstrings.** ______________________________________________________________________ @@ -557,7 +552,6 @@ ______________________________________________________________________ | Test multiple backends | `pytest -m "redis or sql"` | | Exclude backends | `pytest -m "not mongo"` | | Lint | `ruff check .` | -| Type check | `mypy src/cachier/` | | Format code | `ruff format .` | | Build package | `python -m build` | | Check docs | `python setup.py checkdocs` | diff --git a/CLAUDE.md b/CLAUDE.md index ca0143f9..9faa4b15 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,6 @@ - **Key Dependencies:** `portalocker`, `watchdog` (optional: `pymongo`, `sqlalchemy`, `redis`) - **Test Framework:** `pytest` with backend-specific markers - **Linting:** `ruff` (replaces black/flake8) -- **Type Checking:** `mypy` - **CI:** GitHub Actions (matrix for backends/OS with Dockerized services) - **Issue Tracking:** GitHub Issues - **Additional Docs:** `.github/copilot-instructions.md` for contributor guidelines @@ -69,7 +68,6 @@ ______________________________________________________________________ ```bash ruff check . - mypy src/cachier/ ``` 4. **Try an example:** @@ -102,7 +100,6 @@ ______________________________________________________________________ - **Type annotations** required for all new code. - **Docstrings:** Use numpy style, multi-line, no single-line docstrings. - **Lint:** Run `ruff` before PRs. Use per-line/file ignores only for justified cases. -- **Type check:** Run `mypy` before PRs. - **Testing:** All public methods must have at least one test. Use `pytest.mark.` for backend-specific tests. - **No warnings/errors for missing optional dependencies at import time.** Only raise when backend is used. @@ -415,7 +412,6 @@ ______________________________________________________________________ - **Run multiple backends:** `pytest -m "redis or sql"` - **Exclude backends:** `pytest -m "not mongo"` - **Lint:** `ruff check .` -- **Type check:** `mypy src/cachier/` - **Format:** `ruff format .` - **Pre-commit:** `pre-commit run --all-files` - **Build package:** `python -m build` @@ -505,7 +501,6 @@ ______________________________________________________________________ - **If adding new dependencies, use context7 MCP to get latest versions.** - **Always check GitHub Issues before starting new features/PRs.** - **Create a relevant issue for every new PR.** -- **Use per-file or per-line ignores for mypy/ruff only when justified.** - **All new code must have full type annotations and numpy-style docstrings.** ______________________________________________________________________ @@ -557,7 +552,6 @@ ______________________________________________________________________ | Test multiple backends | `pytest -m "redis or sql"` | | Exclude backends | `pytest -m "not mongo"` | | Lint | `ruff check .` | -| Type check | `mypy src/cachier/` | | Format code | `ruff format .` | | Build package | `python -m build` | | Check docs | `python setup.py checkdocs` | diff --git a/Makefile b/Makefile index 22918c71..95cbf8c1 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test-mongo-local test-mongo-docker test-mongo-inmemory test-mongo-also-local \ test-redis-local test-sql-local \ services-start services-stop services-logs \ - mongo-start mongo-stop mongo-logs lint type-check format clean \ +mongo-start mongo-stop mongo-logs lint format clean \ install install-dev install-all # Default target @@ -30,7 +30,6 @@ help: @echo "" @echo "Code Quality:" @echo " make lint - Run ruff linter" - @echo " make type-check - Run mypy type checker" @echo " make format - Format code with ruff" @echo "" @echo "Installation:" @@ -131,8 +130,6 @@ mongo-logs: lint: ruff check . -type-check: - mypy src/cachier/ format: ruff format . @@ -145,7 +142,6 @@ clean: rm -rf .coverage rm -rf htmlcov/ rm -rf .pytest_cache/ - rm -rf .mypy_cache/ rm -rf .ruff_cache/ find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true find . -type f -name "*.pyc" -delete diff --git a/pyproject.toml b/pyproject.toml index 85bb1135..bb203fa1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dynamic = [ dependencies = [ "portalocker>=2.3.2", "watchdog>=2.3.1", + "pympler>=1.0", ] urls.Source = "https://github.com/python-cachier/cachier" # --- setuptools --- @@ -197,5 +198,5 @@ show_missing = true exclude_lines = [ "pragma: no cover", # Have to re-enable the standard pragma "raise NotImplementedError", # Don't complain if tests don't hit defensive assertion code: - "if TYPE_CHECKING:", # Is only true when running mypy, not tests + "if TYPE_CHECKING:", ] diff --git a/src/cachier/__init__.py b/src/cachier/__init__.py index d53b3535..922ab021 100644 --- a/src/cachier/__init__.py +++ b/src/cachier/__init__.py @@ -4,11 +4,11 @@ enable_caching, get_default_params, get_global_params, - parse_bytes, set_default_params, set_global_params, ) from .core import cachier +from .util import parse_bytes __all__ = [ "cachier", diff --git a/src/cachier/config.py b/src/cachier/config.py index 3ac0cb33..4c7bb1d7 100644 --- a/src/cachier/config.py +++ b/src/cachier/config.py @@ -1,7 +1,6 @@ import hashlib import os import pickle -import re import threading from dataclasses import dataclass, field, replace from datetime import datetime, timedelta @@ -33,27 +32,6 @@ def _default_cache_dir(): return os.path.expanduser("~/.cachier/") -def parse_bytes(size: Union[int, str, None]) -> Optional[int]: - """Convert a human friendly size string to bytes.""" - if size is None: - return None - if isinstance(size, int): - return size - match = re.fullmatch(r"(?i)\s*(\d+(?:\.\d+)?)\s*([kmgt]?b)?\s*", str(size)) - if not match: - raise ValueError(f"Invalid size value: {size}") - number = float(match.group(1)) - unit = (match.group(2) or "b").upper() - factor = { - "B": 1, - "KB": 1024, - "MB": 1024**2, - "GB": 1024**3, - "TB": 1024**4, - }[unit] - return int(number * factor) - - class LazyCacheDir: """Lazily resolve the default cache directory using $XDG_CACHE_HOME.""" diff --git a/src/cachier/core.py b/src/cachier/core.py index 8aa33f0d..8c56d960 100644 --- a/src/cachier/core.py +++ b/src/cachier/core.py @@ -19,19 +19,14 @@ from warnings import warn from ._types import RedisClient -from .config import ( - Backend, - HashFunc, - Mongetter, - _update_with_defaults, - parse_bytes, -) +from .config import Backend, HashFunc, Mongetter, _update_with_defaults from .cores.base import RecalculationNeeded, _BaseCore from .cores.memory import _MemoryCore from .cores.mongo import _MongoCore from .cores.pickle import _PickleCore from .cores.redis import _RedisCore from .cores.sql import _SQLCore +from .util import parse_bytes MAX_WORKERS_ENVAR_NAME = "CACHIER_MAX_WORKERS" DEFAULT_MAX_WORKERS = 8 diff --git a/src/cachier/cores/base.py b/src/cachier/cores/base.py index 14e5836b..99cf998e 100644 --- a/src/cachier/cores/base.py +++ b/src/cachier/cores/base.py @@ -9,12 +9,13 @@ import abc # for the _BaseCore abstract base class import inspect -import pickle import sys import threading from datetime import timedelta from typing import Any, Callable, Optional, Tuple +from pympler import asizeof + from .._types import HashFunc from ..config import CacheEntry, _update_with_defaults @@ -96,7 +97,7 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: def _estimate_size(self, value: Any) -> int: try: - return len(pickle.dumps(value)) + return asizeof.asizeof(value) except Exception: return sys.getsizeof(value) diff --git a/src/cachier/cores/pickle.py b/src/cachier/cores/pickle.py index 78209cab..903dcc1f 100644 --- a/src/cachier/cores/pickle.py +++ b/src/cachier/cores/pickle.py @@ -120,7 +120,7 @@ def _convert_legacy_cache_entry( def _load_cache_dict(self) -> Dict[str, CacheEntry]: try: with portalocker.Lock(self.cache_fpath, mode="rb") as cf: - cache = pickle.load(cf) # type: ignore[arg-type] + cache = pickle.load(cf) self._cache_used_fpath = str(self.cache_fpath) except (FileNotFoundError, EOFError): cache = {} @@ -147,7 +147,7 @@ def _load_cache_by_key( fpath += f"_{hash_str or key}" try: with portalocker.Lock(fpath, mode="rb") as cache_file: - entry = pickle.load(cache_file) # type: ignore[arg-type] + entry = pickle.load(cache_file) return _PickleCore._convert_legacy_cache_entry(entry) except (FileNotFoundError, EOFError): return None @@ -186,7 +186,7 @@ def _save_cache( fpath += f"_{hash_str}" with self.lock: with portalocker.Lock(fpath, mode="wb") as cf: - pickle.dump(cache, cf, protocol=4) # type: ignore[arg-type] + pickle.dump(cache, cf, protocol=4) # the same as check for separate_file, but changed for typing if isinstance(cache, dict): self._cache_dict = cache diff --git a/src/cachier/cores/sql.py b/src/cachier/cores/sql.py index 94159528..2fa199d5 100644 --- a/src/cachier/cores/sql.py +++ b/src/cachier/cores/sql.py @@ -112,10 +112,10 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: value = pickle.loads(row.value) if row.value is not None else None entry = CacheEntry( value=value, - time=row.timestamp, # type: ignore[arg-type] - stale=row.stale, # type: ignore[arg-type] - _processing=row.processing, # type: ignore[arg-type] - _completed=row.completed, # type: ignore[arg-type] + time=row.timestamp, + stale=row.stale, + _processing=row.processing, + _completed=row.completed, ) return key, entry diff --git a/src/cachier/util.py b/src/cachier/util.py new file mode 100644 index 00000000..dda540bf --- /dev/null +++ b/src/cachier/util.py @@ -0,0 +1,25 @@ +"""Utility helpers for Cachier.""" + +import re +from typing import Optional, Union + + +def parse_bytes(size: Union[int, str, None]) -> Optional[int]: + """Convert a human friendly size string to bytes.""" + if size is None: + return None + if isinstance(size, int): + return size + match = re.fullmatch(r"(?i)\s*(\d+(?:\.\d+)?)\s*([kmgt]?b)?\s*", str(size)) + if not match: + raise ValueError(f"Invalid size value: {size}") + number = float(match.group(1)) + unit = (match.group(2) or "b").upper() + factor = { + "B": 1, + "KB": 1024, + "MB": 1024**2, + "GB": 1024**3, + "TB": 1024**4, + }[unit] + return int(number * factor) diff --git a/tests/requirements.txt b/tests/requirements.txt index 23c73edc..5aa12f1a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,3 +9,4 @@ collective.checkdocs pygments # the memory core tests dataframe caching pandas +pympler From 5bd9ac5c13c51c03ff65eae9b4a7be053d9cce49 Mon Sep 17 00:00:00 2001 From: Shay Palachy-Affek Date: Wed, 16 Jul 2025 12:13:11 +0300 Subject: [PATCH 4/7] Restore mypy usage --- .github/copilot-instructions.md | 2 +- .pre-commit-config.yaml | 7 +++++++ AGENTS.md | 7 ++++++- CLAUDE.md | 5 +++++ Makefile | 6 +++++- pyproject.toml | 2 +- src/cachier/cores/base.py | 2 +- src/cachier/cores/pickle.py | 8 ++++---- src/cachier/cores/sql.py | 10 +++++----- 9 files changed, 35 insertions(+), 14 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index ced25991..1da246da 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -26,7 +26,7 @@ Welcome to the Cachier codebase! Please follow these guidelines to ensure code s ## 4. Coverage, Linting, and Typing -- Code must pass `ruff` and `pytest`. +- Code must pass `mypy`, `ruff`, and `pytest`. - Use per-file or per-line ignores for known, justified issues (e.g., SQLAlchemy model base class typing, intentional use of `pickle`). - All new code must include full type annotations and docstrings matching the style of the existing codebase. - All docstrings should follow numpy docstring conventions. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3920f669..4ade40bf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,6 +65,13 @@ repos: name: Ruff check args: ["--fix"] + # it needs to be after formatting hooks because the lines might be changed + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.16.1 + hooks: + - id: mypy + files: "src/*" + - repo: https://github.com/tox-dev/pyproject-fmt rev: v2.6.0 diff --git a/AGENTS.md b/AGENTS.md index 675eb234..74b8211d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,7 @@ - **Key Dependencies:** `portalocker`, `watchdog` (optional: `pymongo`, `sqlalchemy`, `redis`) - **Test Framework:** `pytest` with backend-specific markers - **Linting:** `ruff` (replaces black/flake8) +- **Type Checking:** `mypy` - **CI:** GitHub Actions (matrix for backends/OS with Dockerized services) - **Issue Tracking:** GitHub Issues - **Additional Docs:** `.github/copilot-instructions.md` for contributor guidelines @@ -64,10 +65,11 @@ ______________________________________________________________________ pytest -m "not (mongo or sql)" # Exclude external service backends ``` -3. **Lint:** +3. **Lint and type-check:** ```bash ruff check . + mypy src/cachier/ ``` 4. **Try an example:** @@ -100,6 +102,7 @@ ______________________________________________________________________ - **Type annotations** required for all new code. - **Docstrings:** Use numpy style, multi-line, no single-line docstrings. - **Lint:** Run `ruff` before PRs. Use per-line/file ignores only for justified cases. +- **Type check:** Run `mypy` before PRs. - **Testing:** All public methods must have at least one test. Use `pytest.mark.` for backend-specific tests. - **No warnings/errors for missing optional dependencies at import time.** Only raise when backend is used. @@ -501,6 +504,7 @@ ______________________________________________________________________ - **If adding new dependencies, use context7 MCP to get latest versions.** - **Always check GitHub Issues before starting new features/PRs.** - **Create a relevant issue for every new PR.** +- **Use per-file or per-line ignores for mypy/ruff only when justified.** - **All new code must have full type annotations and numpy-style docstrings.** ______________________________________________________________________ @@ -552,6 +556,7 @@ ______________________________________________________________________ | Test multiple backends | `pytest -m "redis or sql"` | | Exclude backends | `pytest -m "not mongo"` | | Lint | `ruff check .` | +| Type check | `mypy src/cachier/` | | Format code | `ruff format .` | | Build package | `python -m build` | | Check docs | `python setup.py checkdocs` | diff --git a/CLAUDE.md b/CLAUDE.md index 9faa4b15..3d52b391 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,6 +9,7 @@ - **Key Dependencies:** `portalocker`, `watchdog` (optional: `pymongo`, `sqlalchemy`, `redis`) - **Test Framework:** `pytest` with backend-specific markers - **Linting:** `ruff` (replaces black/flake8) +- **Type Checking:** `mypy` - **CI:** GitHub Actions (matrix for backends/OS with Dockerized services) - **Issue Tracking:** GitHub Issues - **Additional Docs:** `.github/copilot-instructions.md` for contributor guidelines @@ -68,6 +69,7 @@ ______________________________________________________________________ ```bash ruff check . + mypy src/cachier/ ``` 4. **Try an example:** @@ -100,6 +102,7 @@ ______________________________________________________________________ - **Type annotations** required for all new code. - **Docstrings:** Use numpy style, multi-line, no single-line docstrings. - **Lint:** Run `ruff` before PRs. Use per-line/file ignores only for justified cases. +- **Type check:** Run `mypy` before PRs. - **Testing:** All public methods must have at least one test. Use `pytest.mark.` for backend-specific tests. - **No warnings/errors for missing optional dependencies at import time.** Only raise when backend is used. @@ -501,6 +504,7 @@ ______________________________________________________________________ - **If adding new dependencies, use context7 MCP to get latest versions.** - **Always check GitHub Issues before starting new features/PRs.** - **Create a relevant issue for every new PR.** +- **Use per-file or per-line ignores for mypy/ruff only when justified.** - **All new code must have full type annotations and numpy-style docstrings.** ______________________________________________________________________ @@ -552,6 +556,7 @@ ______________________________________________________________________ | Test multiple backends | `pytest -m "redis or sql"` | | Exclude backends | `pytest -m "not mongo"` | | Lint | `ruff check .` | +| Type check | `mypy src/cachier/` | | Format code | `ruff format .` | | Build package | `python -m build` | | Check docs | `python setup.py checkdocs` | diff --git a/Makefile b/Makefile index 95cbf8c1..22918c71 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test-mongo-local test-mongo-docker test-mongo-inmemory test-mongo-also-local \ test-redis-local test-sql-local \ services-start services-stop services-logs \ -mongo-start mongo-stop mongo-logs lint format clean \ + mongo-start mongo-stop mongo-logs lint type-check format clean \ install install-dev install-all # Default target @@ -30,6 +30,7 @@ help: @echo "" @echo "Code Quality:" @echo " make lint - Run ruff linter" + @echo " make type-check - Run mypy type checker" @echo " make format - Format code with ruff" @echo "" @echo "Installation:" @@ -130,6 +131,8 @@ mongo-logs: lint: ruff check . +type-check: + mypy src/cachier/ format: ruff format . @@ -142,6 +145,7 @@ clean: rm -rf .coverage rm -rf htmlcov/ rm -rf .pytest_cache/ + rm -rf .mypy_cache/ rm -rf .ruff_cache/ find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true find . -type f -name "*.pyc" -delete diff --git a/pyproject.toml b/pyproject.toml index bb203fa1..17c4454a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -198,5 +198,5 @@ show_missing = true exclude_lines = [ "pragma: no cover", # Have to re-enable the standard pragma "raise NotImplementedError", # Don't complain if tests don't hit defensive assertion code: - "if TYPE_CHECKING:", + "if TYPE_CHECKING:", # Is only true when running mypy, not tests ] diff --git a/src/cachier/cores/base.py b/src/cachier/cores/base.py index 99cf998e..ef631850 100644 --- a/src/cachier/cores/base.py +++ b/src/cachier/cores/base.py @@ -14,7 +14,7 @@ from datetime import timedelta from typing import Any, Callable, Optional, Tuple -from pympler import asizeof +from pympler import asizeof # type: ignore from .._types import HashFunc from ..config import CacheEntry, _update_with_defaults diff --git a/src/cachier/cores/pickle.py b/src/cachier/cores/pickle.py index 903dcc1f..1e446b9a 100644 --- a/src/cachier/cores/pickle.py +++ b/src/cachier/cores/pickle.py @@ -12,7 +12,7 @@ import time from contextlib import suppress from datetime import datetime, timedelta -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Optional, Tuple, Union, IO, cast import portalocker # to lock on pickle cache IO from watchdog.events import PatternMatchingEventHandler @@ -120,7 +120,7 @@ def _convert_legacy_cache_entry( def _load_cache_dict(self) -> Dict[str, CacheEntry]: try: with portalocker.Lock(self.cache_fpath, mode="rb") as cf: - cache = pickle.load(cf) + cache = pickle.load(cast(IO[bytes], cf)) self._cache_used_fpath = str(self.cache_fpath) except (FileNotFoundError, EOFError): cache = {} @@ -147,7 +147,7 @@ def _load_cache_by_key( fpath += f"_{hash_str or key}" try: with portalocker.Lock(fpath, mode="rb") as cache_file: - entry = pickle.load(cache_file) + entry = pickle.load(cast(IO[bytes], cache_file)) return _PickleCore._convert_legacy_cache_entry(entry) except (FileNotFoundError, EOFError): return None @@ -186,7 +186,7 @@ def _save_cache( fpath += f"_{hash_str}" with self.lock: with portalocker.Lock(fpath, mode="wb") as cf: - pickle.dump(cache, cf, protocol=4) + pickle.dump(cache, cast(IO[bytes], cf), protocol=4) # the same as check for separate_file, but changed for typing if isinstance(cache, dict): self._cache_dict = cache diff --git a/src/cachier/cores/sql.py b/src/cachier/cores/sql.py index 2fa199d5..16de020f 100644 --- a/src/cachier/cores/sql.py +++ b/src/cachier/cores/sql.py @@ -3,7 +3,7 @@ import pickle import threading from datetime import datetime, timedelta -from typing import Any, Callable, Optional, Tuple, Union +from typing import Any, Callable, Optional, Tuple, Union, cast try: from sqlalchemy import ( @@ -112,10 +112,10 @@ def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]: value = pickle.loads(row.value) if row.value is not None else None entry = CacheEntry( value=value, - time=row.timestamp, - stale=row.stale, - _processing=row.processing, - _completed=row.completed, + time=cast(datetime, row.timestamp), + stale=cast(bool, row.stale), + _processing=cast(bool, row.processing), + _completed=cast(bool, row.completed), ) return key, entry From a7d82b67b0c4f8fa89dc208eba640a12e4518c58 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Jul 2025 09:14:41 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .pre-commit-config.yaml | 1 - pyproject.toml | 2 +- src/cachier/cores/pickle.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4ade40bf..8046aaa9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,7 +72,6 @@ repos: - id: mypy files: "src/*" - - repo: https://github.com/tox-dev/pyproject-fmt rev: v2.6.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 17c4454a..e04af960 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,8 +45,8 @@ dynamic = [ ] dependencies = [ "portalocker>=2.3.2", + "pympler>=1", "watchdog>=2.3.1", - "pympler>=1.0", ] urls.Source = "https://github.com/python-cachier/cachier" # --- setuptools --- diff --git a/src/cachier/cores/pickle.py b/src/cachier/cores/pickle.py index 1e446b9a..6a49cb2e 100644 --- a/src/cachier/cores/pickle.py +++ b/src/cachier/cores/pickle.py @@ -12,7 +12,7 @@ import time from contextlib import suppress from datetime import datetime, timedelta -from typing import Any, Dict, Optional, Tuple, Union, IO, cast +from typing import IO, Any, Dict, Optional, Tuple, Union, cast import portalocker # to lock on pickle cache IO from watchdog.events import PatternMatchingEventHandler From 5371986e32a8157ca4fa5a2c2b5ea4f51c59a706 Mon Sep 17 00:00:00 2001 From: Shay Palachy Date: Wed, 16 Jul 2025 12:18:10 +0300 Subject: [PATCH 6/7] restore mentions of mypy --- AGENTS.md | 1 + CLAUDE.md | 1 + uv.lock | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 74b8211d..4174b470 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -415,6 +415,7 @@ ______________________________________________________________________ - **Run multiple backends:** `pytest -m "redis or sql"` - **Exclude backends:** `pytest -m "not mongo"` - **Lint:** `ruff check .` +- **Type check:** `mypy src/cachier/` - **Format:** `ruff format .` - **Pre-commit:** `pre-commit run --all-files` - **Build package:** `python -m build` diff --git a/CLAUDE.md b/CLAUDE.md index 3d52b391..ca0143f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -415,6 +415,7 @@ ______________________________________________________________________ - **Run multiple backends:** `pytest -m "redis or sql"` - **Exclude backends:** `pytest -m "not mongo"` - **Lint:** `ruff check .` +- **Type check:** `mypy src/cachier/` - **Format:** `ruff format .` - **Pre-commit:** `pre-commit run --all-files` - **Build package:** `python -m build` diff --git a/uv.lock b/uv.lock index eaef5993..9ac89397 100644 --- a/uv.lock +++ b/uv.lock @@ -7,12 +7,14 @@ name = "cachier" source = { editable = "." } dependencies = [ { name = "portalocker" }, + { name = "pympler" }, { name = "watchdog" }, ] [package.metadata] requires-dist = [ { name = "portalocker", specifier = ">=2.3.2" }, + { name = "pympler", specifier = ">=1.0" }, { name = "watchdog", specifier = ">=2.3.1" }, ] @@ -28,6 +30,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424, upload-time = "2025-06-14T13:20:38.083Z" }, ] +[[package]] +name = "pympler" +version = "1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/37/c384631908029676d8e7213dd956bb686af303a80db7afbc9be36bc49495/pympler-1.1.tar.gz", hash = "sha256:1eaa867cb8992c218430f1708fdaccda53df064144d1c5656b1e6f1ee6000424", size = 179954, upload-time = "2024-06-28T19:56:06.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/4f/a6a2e2b202d7fd97eadfe90979845b8706676b41cbd3b42ba75adf329d1f/Pympler-1.1-py3-none-any.whl", hash = "sha256:5b223d6027d0619584116a0cbc28e8d2e378f7a79c1e5e024f9ff3b673c58506", size = 165766, upload-time = "2024-06-28T19:56:05.087Z" }, +] + [[package]] name = "pywin32" version = "310" From 4ef660f7f9a069f3c326efa504c4bcc5c41eb0f2 Mon Sep 17 00:00:00 2001 From: Shay Palachy Date: Thu, 17 Jul 2025 23:21:13 +0300 Subject: [PATCH 7/7] Add missing docker error message to the local testing script --- scripts/test-local.sh | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/scripts/test-local.sh b/scripts/test-local.sh index 9c9b9b8e..f5d784a0 100755 --- a/scripts/test-local.sh +++ b/scripts/test-local.sh @@ -168,16 +168,40 @@ validate_cores "$SELECTED_CORES" # Function to check if Docker is available check_docker() { if ! command -v docker &> /dev/null; then - print_message $RED "Error: Docker is required but not installed." + print_message $RED "═══════════════════════════════════════════════════════════════" + print_message $RED "ERROR: Docker is not installed!" + print_message $RED "═══════════════════════════════════════════════════════════════" + echo "" + echo "This script requires Docker to run external backend tests (MongoDB, Redis, PostgreSQL)." echo "Please install Docker from: https://www.docker.com/products/docker-desktop" + echo "" exit 1 fi - if ! docker ps &> /dev/null; then - print_message $RED "Error: Docker daemon is not running." - echo "Please start Docker and try again." + # Try to run docker ps and capture the actual error + if ! docker ps > /dev/null 2>&1; then + print_message $RED "═══════════════════════════════════════════════════════════════" + print_message $RED "ERROR: Docker daemon is not running!" + print_message $RED "═══════════════════════════════════════════════════════════════" + echo "" + echo "Docker is installed but the Docker daemon is not running." + echo "" + echo "To fix this:" + echo " • On macOS: Start Docker Desktop from Applications" + echo " • On Linux: Run 'sudo systemctl start docker' or 'sudo service docker start'" + echo " • On Windows: Start Docker Desktop from the Start Menu" + echo "" + echo "After starting Docker, wait a few seconds and try running this script again." + echo "" + + # Show the actual docker error for debugging + echo "Technical details:" + docker ps 2>&1 | sed 's/^/ /' + echo "" exit 1 fi + + print_message $GREEN "✓ Docker is installed and running" } # Function to check and install dependencies