Use diskcache

Author: layday

.gitignore

@@ -1,2 +1,2 @@
-/http-cache.db*
 __pycache__
+/_http_v1

_http_cache.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import contextlib
+import os
+from collections.abc import Callable, Set
+from functools import partial
+from typing import Never
+
+import aiohttp_client_cache
+import diskcache
+
+_http_cache_name = "_http_v1"
+
+
+@contextlib.asynccontextmanager
+async def make_cache():
+    with concurrent.futures.ThreadPoolExecutor(1, "_http_cache") as executor:
+        loop = asyncio.get_running_loop()
+
+        def run_in_thread2[**P, T](fn: Callable[P, T]):
+            async def wrapper(*args: P.args, **kwargs: P.kwargs):
+                return await loop.run_in_executor(executor, partial(fn, *args, **kwargs))
+
+            return wrapper
+
+        # diskcache will only close the sqlite connection if it was initialised
+        # in the same thread.
+        cache = await run_in_thread2(diskcache.Cache)(
+            os.path.join(os.getcwd(), _http_cache_name), size_limit=2**30 * 10
+        )
+
+        def make_cache_wrapper(prefix: str):
+            class Cache(aiohttp_client_cache.BaseCache):
+                @run_in_thread2
+                def bulk_delete(self, keys: Set[str]):
+                    for key in keys:
+                        cache.delete((prefix, key))
+
+                @run_in_thread2
+                def contains(self, key: str):
+                    return (prefix, key) in cache
+
+                @run_in_thread2
+                def delete(self, key: str):
+                    cache.delete((prefix, key))
+
+                @run_in_thread2
+                def read(self, key: str):
+                    return cache.get((prefix, key))
+
+                @run_in_thread2
+                def write(self, key: str, item: aiohttp_client_cache.ResponseOrKey):
+                    cache[prefix, key] = item
+
+                async def clear(self) -> Never:
+                    raise NotImplementedError
+
+                async def keys(self):
+                    if False:  # pragma: no cover
+                        yield
+                    raise NotImplementedError
+
+                async def values(self):
+                    if False:  # pragma: no cover
+                        yield
+                    raise NotImplementedError
+
+                async def size(self) -> Never:
+                    raise NotImplementedError
+
+            return Cache()
+
+        try:
+            yield make_cache_wrapper("responses"), make_cache_wrapper("redirects")
+        finally:
+            await run_in_thread2(cache.close)()

cataloguer.py

@@ -23,9 +23,7 @@
 import aiohttp
 import click
 import structlog
-from aiohttp_client_cache import SQLiteBackend
 from aiohttp_client_cache.cache_control import ExpirationPatterns
-from aiohttp_client_cache.session import CachedSession
 from attrs import field, fields, frozen
 from cattrs import BaseValidationError, Converter
 from yarl import URL
@@ -415,14 +413,23 @@ async def parse_repo(get: Get, repo: Mapping[str, Any]):
 
 @asynccontextmanager
 async def _make_http_client(token: str):
-    async with AsyncExitStack() as stack:
-        cache_backend = SQLiteBackend(
+    from aiohttp_client_cache import CacheBackend
+    from aiohttp_client_cache.session import CachedSession
+
+    from _http_cache import make_cache
+
+    async with AsyncExitStack() as exit_stack:
+        cache_backend = CacheBackend(
             autoclose=False,
             cache_name="http-cache.db",
             expire_after=CACHE_INDEFINITELY,
             urls_expire_after=EXPIRE_URLS,
+            timeout=20,
+        )
+        cache_backend.responses, cache_backend.redirects = await exit_stack.enter_async_context(
+            make_cache()
         )
-        client = await stack.enter_async_context(
+        client = await exit_stack.enter_async_context(
             CachedSession(
                 cache=cache_backend,
                 connector=aiohttp.TCPConnector(limit_per_host=8),
@@ -434,7 +441,6 @@ async def _make_http_client(token: str):
                 timeout=aiohttp.ClientTimeout(sock_connect=10, sock_read=10),
             )
         )
-        stack.push_async_callback(cache_backend.close)
 
         @asynccontextmanager
         async def get(url: str | URL, **kwargs: Any):

pyproject.toml

@@ -9,11 +9,12 @@ description = "Script which collects WoW add-on metadata from GitHub"
 requires-python = ">= 3.11"
 license.file = "LICENSE"
 dependencies = [
-  "aiohttp-client-cache[sqlite]",
   "aiohttp",
+  "aiohttp-client-cache",
   "attrs",
   "cattrs",
   "click",
+  "diskcache",
   "structlog",
   "yarl",
 ]