diff --git a/docs/.pages b/docs/.pages index 1b153383..a77446c5 100644 --- a/docs/.pages +++ b/docs/.pages @@ -5,3 +5,5 @@ nav: - conformance.md - Reference: - reference + - Project: + - project diff --git a/docs/conformance.md b/docs/conformance.md index b30e1f96..03522889 100644 --- a/docs/conformance.md +++ b/docs/conformance.md @@ -13,6 +13,8 @@ PyLD aims to conform with: - [Universal RDF Graph Canonicalization Algorithm 2012 (URGNA2012)](https://www.w3.org/TR/rdf-canon/#URDNA2012) - The JSON-LD Working Group [test suite](https://github.com/w3c/json-ld-api/tree/master/tests) +PyLD aims to follow [JSON-LD Best Practices](https://w3c.github.io/json-ld-bp/). + The test runner is updated over time to note or skip newer tests that are not yet supported. diff --git a/docs/examples/document_loaders/sqlite_cache_basic.py b/docs/examples/document_loaders/sqlite_cache_basic.py new file mode 100644 index 00000000..3409ae71 --- /dev/null +++ b/docs/examples/document_loaders/sqlite_cache_basic.py @@ -0,0 +1,15 @@ +import json +from pathlib import Path + +from pyld import SqliteCacheRequestsDocumentLoader, jsonld + +doc = { + "@context": {"name": "http://schema.org/name"}, + "name": "Earth", +} + +loader = SqliteCacheRequestsDocumentLoader( + sqlite_file_path=Path("/tmp/pyld_example_http_cache.sqlite"), +) +result = jsonld.expand(doc, options={"documentLoader": loader}) +print(json.dumps(result, indent=2)) diff --git a/docs/installation.md b/docs/installation.md index 7659708f..1e0d2a48 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -13,6 +13,7 @@ matching extra: ```bash pip install "PyLD[requests]" pip install "PyLD[aiohttp]" +pip install "PyLD[requests-cache]" ``` You can also depend on `requests` or `aiohttp` directly if your project already diff --git a/docs/javascripts/mermaid.mjs b/docs/javascripts/mermaid.mjs new file mode 100644 index 00000000..de171579 --- /dev/null +++ b/docs/javascripts/mermaid.mjs @@ -0,0 +1,8 @@ +import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs'; + +mermaid.initialize({ + startOnLoad: false, + securityLevel: 'loose', +}); + +window.mermaid = mermaid; diff --git a/docs/project/decisions/use-requests-cache-for-sync-http-caching-in-document-loaders.md b/docs/project/decisions/use-requests-cache-for-sync-http-caching-in-document-loaders.md new file mode 100644 index 00000000..40f54e48 --- /dev/null +++ b/docs/project/decisions/use-requests-cache-for-sync-http-caching-in-document-loaders.md @@ -0,0 +1,183 @@ +--- +title: Use requests-cache for persistent HTTP caching in synchronous Python code +status: decided +date: 2026-06-29 +author: Anatoly Scherbakov +tags: [decision] +hide: [toc] +--- + +# Use `requests-cache` for persistent HTTP caching in synchronous Python code + +{{ adr_metadata(date, status) }} + +## :material-text-box-outline: Context + +[:material-file-download-outline: PyLD document loaders](/pyld/reference/document-loaders/) fetch remote JSON-LD contexts and other documents over HTTP. [JSON-LD Best Practices](https://w3c.github.io/json-ld-bp/) recommends: + +> **Best Practice 14:** Cache JSON-LD Contexts +> +> Services providing a JSON-LD Context *SHOULD* set HTTP cache-control headers to allow liberal caching of such contexts, and clients *SHOULD* attempt to use a locally cached version of these documents. +> +> — [§ 8.1 Cache JSON-LD Contexts](https://w3c.github.io/json-ld-bp/#cache-json-ld-contexts) + +### Status Quo + +PyLD already caches remote contexts in process memory only. [`ContextResolver`](https://github.com/digitalbazaar/pyld/blob/master/lib/pyld/context_resolver.py) shares a module-level [`LRUCache`](https://github.com/digitalbazaar/pyld/blob/master/lib/pyld/jsonld.py#L160-L162) of resolved contexts, and [`ResolvedContext`](https://github.com/digitalbazaar/pyld/blob/master/lib/pyld/resolved_context.py) keeps a per-document [`LRUCache`](https://github.com/digitalbazaar/pyld/blob/master/lib/pyld/resolved_context.py#L28) for processed contexts relative to an active context. + +[`FrozenDocumentLoader`](/pyld/reference/document-loaders/frozen/) serves files from disk based on its configuration options set in code. + +None of this is HTTP-aware or persistent across processes: [`RequestsDocumentLoader`](/pyld/reference/document-loaders/requests/) and [`AioHttpDocumentLoader`](/pyld/reference/document-loaders/aiohttp/) issue a fresh HTTP request on every load with no `Cache-Control` handling. + +### HTTP Caching Standards + +Client-side caching semantics are governed by the following standards: + +```mermaid +flowchart LR + rfc2616("RFC 2616
HTTP/1.1") + rfc7234("RFC 7234
HTTP/1.1: Caching") + rfc9111("RFC 9111
HTTP Caching") + rfc9110("RFC 9110
HTTP Semantics") + rfc5861("RFC 5861
Cache-Control Extensions for Stale Content") + rfc8246("RFC 8246
HTTP Immutable Responses") + + rfc2616 -->|is superseded by| rfc7234 + rfc7234 -->|is superseded by| rfc9111 + rfc9110 -->|complements| rfc9111 + rfc5861 -->|complements| rfc9111 + rfc8246 -->|complements| rfc9111 + + click rfc2616 "https://www.rfc-editor.org/rfc/rfc2616.html" "RFC 2616" + click rfc7234 "https://www.rfc-editor.org/rfc/rfc7234.html" "RFC 7234" + click rfc9111 "https://www.rfc-editor.org/rfc/rfc9111.html" "RFC 9111" + click rfc9110 "https://www.rfc-editor.org/rfc/rfc9110.html" "RFC 9110" + click rfc5861 "https://www.rfc-editor.org/rfc/rfc5861.html" "RFC 5861" + click rfc8246 "https://www.rfc-editor.org/rfc/rfc8246.html" "RFC 8246" +``` + +This ADR compares Python HTTP caching libraries that could extend document loaders with standards-aware, persistent caching. + +### Alternatives Rejected + +| Library | HTTP-aware? | Persistent? | +|---------|:----------:|:----------:| +| [:fontawesome-brands-github: `tkem/cachetools`](https://github.com/tkem/cachetools) | :x: | :x: | +| [:fontawesome-brands-github: `grantjenks/python-diskcache`](https://github.com/grantjenks/python-diskcache) | :x: | :white_check_mark: | + +Neither library implements HTTP cache semantics — `cachetools` is a generic in-memory structure and `diskcache` is persistent but not HTTP-aware — so they do not fit document-loader HTTP caching. + +## :material-arrow-decision-outline: Decision + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
[:fontawesome-brands-github: `requests-cache/requests-cache`](https://github.com/requests-cache/requests-cache)[:fontawesome-brands-github: `psf/cachecontrol`](https://github.com/psf/cachecontrol)[:fontawesome-brands-github: `requests-cache/aiohttp-client-cache`](https://github.com/requests-cache/aiohttp-client-cache)[:fontawesome-brands-github: `karpetrosyan/hishel`](https://github.com/karpetrosyan/hishel)
Backend
[:fontawesome-brands-github: `psf/requests`](https://github.com/psf/requests)[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/#quickstart)[:white_check_mark:](https://cachecontrol.readthedocs.io/en/latest/usage.html#wrapper)[:x:](https://aiohttp-client-cache.readthedocs.io/en/stable/#aiohttp-client-cache)[:white_check_mark:](https://hishel.com/requests.html)
[:fontawesome-brands-github: `encode/httpx`](https://github.com/encode/httpx)[:x:](https://requests-cache.readthedocs.io/en/stable/#summary)[:x:](https://cachecontrol.readthedocs.io/en/latest/#welcome-to-cachecontrol-s-documentation)[:x:](https://aiohttp-client-cache.readthedocs.io/en/stable/#aiohttp-client-cache)[:white_check_mark:](https://hishel.com/quickstart.html#with-httpx)
[:fontawesome-brands-github: `aio-libs/aiohttp`](https://github.com/aio-libs/aiohttp)[:x:](https://requests-cache.readthedocs.io/en/stable/project_info/related_projects.html#client-side-http-caching)[:x:](https://cachecontrol.readthedocs.io/en/latest/#welcome-to-cachecontrol-s-documentation)[:white_check_mark:](https://aiohttp-client-cache.readthedocs.io/en/stable/#quickstart)[:x:](https://github.com/karpetrosyan/hishel#-features)
HTTP Headers Support
`Cache-Control` request[Broad directive support.](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[`max-age`, `no-cache`, `no-store`, `min-fresh`; other directives parsed but not necessarily honored.](https://cachecontrol.readthedocs.io/en/latest/usage.html)[Simplified: `max-age`, `no-cache`, `no-store`.](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-request-directives)
`Cache-Control` response[Broad directive support.](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[`max-age`, `no-store`, validator-driven caching.](https://cachecontrol.readthedocs.io/en/latest/etags.html)[Simplified: `max-age`, `no-store`.](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-response-directives)
`Expires`[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[:white_check_mark:](https://cachecontrol.readthedocs.io/en/latest/etags.html)[:white_check_mark:](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-expires)
`ETag` / `If-None-Match`[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[:white_check_mark:](https://cachecontrol.readthedocs.io/en/latest/etags.html)[:warning:](https://aiohttp-client-cache.readthedocs.io/en/stable/modules/aiohttp_client_cache.cache_control.html#aiohttp_client_cache.cache_control.compose_refresh_headers)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-validation)
`Last-Modified` / `If-Modified-Since`[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[:white_check_mark:](https://cachecontrol.readthedocs.io/en/latest/etags.html)[:warning:](https://aiohttp-client-cache.readthedocs.io/en/stable/modules/aiohttp_client_cache.cache_control.html#aiohttp_client_cache.cache_control.compose_refresh_headers)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-validation)
`Vary`[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[:white_check_mark:](https://github.com/psf/cachecontrol/blob/master/cachecontrol/serialize.py#L56-L66)[:x:](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-calculating-cache-keys-with)
Meta *(last updated: 28 June 2026)*
Last release[1.3.2](https://pypi.org/project/requests-cache/1.3.2/)
2026-05-11
[0.14.4](https://pypi.org/project/CacheControl/0.14.4/)
2025-11-14
[0.14.3](https://pypi.org/project/aiohttp-client-cache/0.14.3/)
2026-01-07
[1.3.0](https://pypi.org/project/hishel/1.3.0/)
2026-06-11
GitHub Stars:star: [1,495](https://github.com/requests-cache/requests-cache/stargazers):star: [499](https://github.com/psf/cachecontrol/stargazers):star: [152](https://github.com/requests-cache/aiohttp-client-cache/stargazers):star: [395](https://github.com/karpetrosyan/hishel/stargazers)
Decision:white_check_mark: Adopt as optional sync HTTP cache support for [`RequestsDocumentLoader`](/pyld/reference/document-loaders/requests/).:x: Exclude — shares the `requests` backend with `requests-cache`, but [defaults to an in-memory store](https://cachecontrol.readthedocs.io/en/latest/#quick-start) with no first-class persistent backend comparable to [`requests-cache` storage options](https://requests-cache.readthedocs.io/en/stable/user_guide/backends.html), offers narrower [`Cache-Control` directive support](https://cachecontrol.readthedocs.io/en/latest/usage.html), and trails on maintenance signals in this comparison.:question: Defer for async support to limit dependency and maintenance footprint; candidate if [`AioHttpDocumentLoader`](/pyld/reference/document-loaders/aiohttp/) gains HTTP caching.:question: Defer — strong HTTP cache policy design, but adoption would target [`httpx`](https://github.com/karpetrosyan/hishel#-features) rather than PyLD's current [`AioHttpDocumentLoader`](/pyld/reference/document-loaders/aiohttp/).
+ +:warning: marks indirect or limited support. `aiohttp-client-cache` has [conditional refresh support for validators](https://aiohttp-client-cache.readthedocs.io/en/stable/modules/aiohttp_client_cache.cache_control.html#aiohttp_client_cache.cache_control.compose_refresh_headers), but its own documentation describes cache-header handling as a [simplified subset](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control) rather than full automatic validator-driven HTTP cache revalidation. For `hishel`, header rows link to the governing [RFC 9111](https://www.rfc-editor.org/rfc/rfc9111.html) sections because its default [`SpecificationPolicy`](https://hishel.com/policies) implements the full specification rather than documenting per-header support separately. + +## :material-arrow-right-bold-outline: Consequences + +- PyLD will add optional, HTTP-aware sync caching via `requests-cache` without making caching a required dependency. +- Opt-in sync caching will now be provided by [`SqliteCacheRequestsDocumentLoader`](/pyld/reference/document-loaders/sqlite-cache-requests/), composing `RequestsDocumentLoader` with a persistent SQLite `CachedSession`. +- A custom in-process HTTP cache implementation is rejected; with a fitting library available, PyLD will integrate `requests-cache` rather than expand the codebase to implement RFC 9111 caching itself. +- CacheControl is excluded for the sync `requests` path in favor of `requests-cache`. +- Async HTTP caching remains out of scope for this decision; `aiohttp-client-cache` and `hishel` stay as future candidates. diff --git a/docs/project/index.md b/docs/project/index.md new file mode 100644 index 00000000..09e8009b --- /dev/null +++ b/docs/project/index.md @@ -0,0 +1,9 @@ +# :material-hard-hat: Project + +Project documentation for PyLD — architecture decisions, development notes, and other material that sits alongside the API reference. + +## :material-gavel: Architecture Decision Records + +Architecture Decision Records (ADRs) document the technical choices taken during development of PyLD. + +{{ adr_index() }} diff --git a/docs/reference/document-loaders/index.md b/docs/reference/document-loaders/index.md index c784d2bc..b8b8d567 100644 --- a/docs/reference/document-loaders/index.md +++ b/docs/reference/document-loaders/index.md @@ -12,6 +12,12 @@ class-based loaders for common cases and supports custom subclasses of Synchronous remote document loading with `requests`. +- [:material-database:{ .lg .middle } `SqliteCacheRequestsDocumentLoader`](sqlite-cache-requests.md) + + --- + + Persistent SQLite HTTP caching for JSON-LD contexts with `requests-cache`. + - [:material-sync:{ .lg .middle } `AioHttpDocumentLoader`](aiohttp.md) --- diff --git a/docs/reference/document-loaders/sqlite-cache-requests.md b/docs/reference/document-loaders/sqlite-cache-requests.md new file mode 100644 index 00000000..21b563e9 --- /dev/null +++ b/docs/reference/document-loaders/sqlite-cache-requests.md @@ -0,0 +1,43 @@ +# :material-database: `SqliteCacheRequestsDocumentLoader` + +`SqliteCacheRequestsDocumentLoader` retrieves JSON-LD documents with +`requests-cache`, storing responses in a persistent SQLite file. HTTP cache +headers (`Cache-Control`, `Expires`, validators) are always honored. + +Caching is opt-in: pass the loader via `options["documentLoader"]`. It composes +`RequestsDocumentLoader` internally. + +Specify an absolute path to the `.sqlite` cache file: + +{{ example('document_loaders/sqlite_cache_basic.py', output_syntax='json') }} + +When `sqlite_file_path` is omitted, the cache defaults to the platform user +cache directory: + +| OS | Default path | +|----|--------------| +| Linux | `~/.cache/pyld/http_cache.sqlite` | +| macOS | `~/Library/Caches/pyld/http_cache.sqlite` | +| Windows | `%LOCALAPPDATA%\pyld\http_cache.sqlite` | + +Inspect the resolved path at runtime with `loader.session.cache.db_path`. + +For other `CachedSession` options (memory backend, custom TTL, and so on), use +`RequestsDocumentLoader` with a pre-built session: + +```python +from requests_cache import CachedSession + +from pyld import RequestsDocumentLoader + +loader = RequestsDocumentLoader( + session=CachedSession("cache", backend="memory", cache_control=True), + timeout=10, +) +``` + +Install the optional dependency with: + +```bash +pip install "PyLD[requests-cache]" +``` diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 097493ff..e4d04d23 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -64,3 +64,23 @@ margin-left: auto; font-weight: normal; } + +/* ADR comparison column tints — match mkdocs-material admonition hues at 10%. */ +:root, +[data-md-color-scheme="slate"] { + --adr-col-included-bg: color-mix(in srgb, #00c853 10%, transparent); + --adr-col-undecided-bg: color-mix(in srgb, #ff9100 10%, transparent); + --adr-col-rejected-bg: color-mix(in srgb, #ff1744 10%, transparent); +} + +.md-typeset :is(th, td).adr-col-included { + background-color: var(--adr-col-included-bg); +} + +.md-typeset :is(th, td).adr-col-undecided { + background-color: var(--adr-col-undecided-bg); +} + +.md-typeset :is(th, td).adr-col-rejected { + background-color: var(--adr-col-rejected-bg); +} diff --git a/docs_macros.py b/docs_macros.py index b49ccae6..c5612db7 100644 --- a/docs_macros.py +++ b/docs_macros.py @@ -3,10 +3,16 @@ import re import subprocess import sys +from datetime import date from pathlib import Path +import yaml +from mkdocs.utils.meta import YAML_RE +from yaml import SafeLoader + ROOT_DIR = Path(__file__).resolve().parent EXAMPLES_DIR = ROOT_DIR / 'docs' / 'examples' +DECISIONS_DIR = ROOT_DIR / 'docs' / 'project' / 'decisions' sys.path.insert(0, str(ROOT_DIR / 'lib')) sys.path.insert(0, str(ROOT_DIR / 'tests')) @@ -130,7 +136,132 @@ def _example_github_url(name, repo_url): return f'{repo_url.rstrip("/")}/blob/{branch}/{rel_path.as_posix()}' +def _human_date(value): + if not value: + return '' + parsed = date.fromisoformat(value) if isinstance(value, str) else value + return f'{parsed.day} {parsed.strftime("%B %Y")}' + + +_ADR_STATUS = { + 'draft': (':material-pencil-outline:', 'Draft'), + 'undecided': (':question:', 'Undecided'), + 'decided': (':white_check_mark:', 'Decided'), +} + + +_ADR_STATUS_ADMONITION = { + 'draft': 'note', + 'undecided': 'warning', + 'decided': 'success', +} + + +def _adr_status_label(value): + if not value: + return '' + key = str(value).lower() + return _ADR_STATUS.get(key, (None, str(value).replace('_', ' ').title()))[1] + + +def _adr_metadata_date(value): + if not value: + return '' + return f':material-calendar-clock: {_human_date(value)}' + + +def _adr_metadata(date, status): + kind = _ADR_STATUS_ADMONITION.get(str(status).lower(), 'note') + parts = [] + label = _adr_status_label(status) + if label: + parts.append(label) + date_part = _adr_metadata_date(date) + if date_part: + parts.append(date_part) + title = ' · '.join(parts) + return f'!!! {kind} "{title}"\n' + + +def _adr_status(value): + if not value: + return '' + key = str(value).lower() + icon, label = _ADR_STATUS.get( + key, + (':material-information-outline:', str(value).replace('_', ' ').title()), + ) + return f'{icon} {label}' + + +def _adr_status_icon(value): + if not value: + return ':material-information-outline:' + key = str(value).lower() + return _ADR_STATUS.get( + key, + (':material-information-outline:', ''), + )[0] + + +def _parse_frontmatter(path): + text = path.read_text(encoding='utf-8-sig') + match = YAML_RE.match(text) + if not match: + return {} + return yaml.load(match.group(1), SafeLoader) or {} + + +def _parse_adr_date(value): + if not value: + return None + if isinstance(value, date): + return value + if isinstance(value, str): + return date.fromisoformat(value) + if hasattr(value, 'date'): + return value.date() + return None + + +def _adr_index(): + decisions = [] + for path in sorted(DECISIONS_DIR.glob('*.md')): + meta = _parse_frontmatter(path) + title = meta.get('title', path.stem.replace('-', ' ').title()) + parsed_date = _parse_adr_date(meta.get('date')) + status = meta.get('status', '') + url = f'decisions/{path.stem}/' + decisions.append((parsed_date or date.min, title, status, url)) + + decisions.sort(key=lambda item: item[0], reverse=True) + + lines = [] + for parsed_date, title, status, url in decisions: + icon = _adr_status_icon(status) + date_part = _human_date(parsed_date) if parsed_date != date.min else '' + suffix = f' · {date_part}' if date_part else '' + lines.append(f'- {icon} [{title}]({url}){suffix}') + return '\n'.join(lines) + + def define_env(env): + @env.filter + def human_date(value): + return _human_date(value) + + @env.filter + def adr_status(value): + return _adr_status(value) + + @env.macro + def adr_metadata(date, status): + return _adr_metadata(date, status) + + @env.macro + def adr_index(): + return _adr_index() + @env.macro def bundled_contexts_table(): from pyld import BUNDLED_CONTEXTS diff --git a/lib/pyld/__init__.py b/lib/pyld/__init__.py index 42de12b5..ae05cb95 100644 --- a/lib/pyld/__init__.py +++ b/lib/pyld/__init__.py @@ -6,6 +6,7 @@ from .documentloader.base import DocumentLoader, RemoteDocument from .documentloader.frozen import BUNDLED_CONTEXTS, FrozenDocumentLoader from .documentloader.requests import RequestsDocumentLoader +from .documentloader.requests_sqlite_cache import SqliteCacheRequestsDocumentLoader __all__ = [ 'AioHttpDocumentLoader', @@ -15,5 +16,6 @@ 'FrozenDocumentLoader', 'RequestsDocumentLoader', 'RemoteDocument', + 'SqliteCacheRequestsDocumentLoader', 'jsonld', ] diff --git a/lib/pyld/documentloader/requests.py b/lib/pyld/documentloader/requests.py index 9ef9caf6..4a6e6e3b 100644 --- a/lib/pyld/documentloader/requests.py +++ b/lib/pyld/documentloader/requests.py @@ -15,16 +15,18 @@ from pyld import iri_resolver from pyld.documentloader.base import DocumentLoader, RemoteDocument -from pyld.jsonld import JsonLdError, parse_link_header, LINK_HEADER_REL +from pyld.jsonld import LINK_HEADER_REL, JsonLdError, parse_link_header class RequestsDocumentLoader(DocumentLoader): """Remote document loader using Requests.""" - def __init__(self, secure=False, **kwargs): - import requests + def __init__(self, secure=False, session=None, **kwargs): + if session is None: + import requests - self.requests = requests + session = requests.Session() + self.session = session self.secure = secure self.kwargs = kwargs @@ -61,7 +63,7 @@ def __call__(self, url, options=None) -> RemoteDocument: headers = { 'Accept': 'application/ld+json, application/json' } - response = self.requests.get(url, headers=headers, **self.kwargs) + response = self.session.get(url, headers=headers, **self.kwargs) content_type = response.headers.get('content-type') if not content_type: diff --git a/lib/pyld/documentloader/requests_sqlite_cache.py b/lib/pyld/documentloader/requests_sqlite_cache.py new file mode 100644 index 00000000..09f5a756 --- /dev/null +++ b/lib/pyld/documentloader/requests_sqlite_cache.py @@ -0,0 +1,58 @@ +""" +SQLite-backed HTTP cache document loader using requests-cache. + +.. module:: jsonld.documentloader.requests_sqlite_cache + :synopsis: Persistent SQLite HTTP caching for Requests document loader +""" +from pathlib import Path + +from pyld.documentloader.base import DocumentLoader, RemoteDocument +from pyld.documentloader.requests import RequestsDocumentLoader + + +def _resolve_sqlite_file_path(sqlite_file_path: Path | None) -> Path: + """Return absolute path to the SQLite cache file.""" + from platformdirs import user_cache_dir + + if sqlite_file_path is None: + return Path(user_cache_dir('pyld')) / 'http_cache.sqlite' + path = sqlite_file_path.expanduser() + if not path.is_absolute(): + raise ValueError( + 'sqlite_file_path must be an absolute path to the .sqlite file') + return path + + +class SqliteCacheRequestsDocumentLoader(DocumentLoader): + """Remote document loader with persistent SQLite HTTP caching. + + Composes :class:`RequestsDocumentLoader` with a + :class:`requests_cache.CachedSession` backed by SQLite. HTTP cache headers + are always honored. For other ``CachedSession`` options, use + ``RequestsDocumentLoader(session=CachedSession(...))`` directly. + + :param secure: require all requests to use HTTPS (default: False). + :param sqlite_file_path: absolute path to the ``.sqlite`` cache file; when + omitted, defaults to the platform user cache directory under ``pyld/``. + """ + + def __init__( + self, + secure=False, + *, + sqlite_file_path: Path | None = None, + ): + from requests_cache import CachedSession + + path = _resolve_sqlite_file_path(sqlite_file_path) + self.session = CachedSession( + cache_name=str(path), + backend='sqlite', + cache_control=True, + expire_after=-1, + ) + self._loader = RequestsDocumentLoader( + secure=secure, session=self.session) + + def __call__(self, url, options=None) -> RemoteDocument: + return self._loader(url, options=options) diff --git a/mkdocs.yml b/mkdocs.yml index 4fad1cb3..17d52419 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,6 +7,9 @@ repo_name: digitalbazaar/pyld extra_css: - stylesheets/extra.css +extra_javascript: + - javascripts/mermaid.mjs + theme: name: material features: @@ -29,12 +32,18 @@ markdown_extensions: emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: anchor_linenums: true - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.tabbed: alternate_style: true slugify: !!python/object/apply:pymdownx.slugs.slugify kwds: case: lower + - pymdownx.tasklist: + custom_checkbox: true - toc: permalink: true diff --git a/requirements-test.txt b/requirements-test.txt index 28a66eae..8584b40a 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -2,3 +2,4 @@ ruff pytest pytest-cov typing_extensions +requests-cache>=1.3 diff --git a/setup.py b/setup.py index 62f533b1..0fc11db4 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ extras_require={ 'requests': ['requests'], 'aiohttp': ['aiohttp'], + 'requests-cache': ['requests-cache>=1.3'], 'cachetools': ['cachetools'], 'frozendict': ['frozendict'], } diff --git a/tests/test_sqlite_cache_requests_document_loader.py b/tests/test_sqlite_cache_requests_document_loader.py new file mode 100644 index 00000000..aa4e8c98 --- /dev/null +++ b/tests/test_sqlite_cache_requests_document_loader.py @@ -0,0 +1,117 @@ +"""Tests for SqliteCacheRequestsDocumentLoader and HTTP cache behavior.""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest +from requests_cache import CachedSession + +pytest.importorskip('requests_cache') + +from pyld import ( + DocumentLoader, + RequestsDocumentLoader, + SqliteCacheRequestsDocumentLoader, +) + + +class _ContextHandler(BaseHTTPRequestHandler): + request_count = 0 + + def do_GET(self): + type(self).request_count += 1 + body = json.dumps({ + '@context': {'name': 'http://example.org/name'}, + }).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/ld+json') + self.send_header('Cache-Control', 'max-age=3600') + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + pass + + +@pytest.fixture +def context_url(): + """Local HTTP server returning JSON-LD with Cache-Control: max-age=3600.""" + _ContextHandler.request_count = 0 + server = HTTPServer(('127.0.0.1', 0), _ContextHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + url = f'http://127.0.0.1:{port}/context.jsonld' + yield url + server.shutdown() + + +def test_requests_document_loader_accepts_custom_session(): + """RequestsDocumentLoader accepts a CachedSession via session=.""" + loader = RequestsDocumentLoader( + session=CachedSession(backend='memory', cache_control=True)) + assert isinstance(loader, DocumentLoader) + assert callable(loader) + loader.session.close() + + +def test_sqlite_cache_requests_document_loader_is_document_loader(): + """Sqlite loader is a DocumentLoader composing RequestsDocumentLoader.""" + loader = SqliteCacheRequestsDocumentLoader() + assert isinstance(loader, DocumentLoader) + assert isinstance(loader._loader, RequestsDocumentLoader) + assert callable(loader) + loader.session.close() + + +def test_sqlite_cache_requests_document_loader_rejects_relative_sqlite_file_path(): + """Relative sqlite_file_path is rejected.""" + with pytest.raises(ValueError, match='absolute path'): + SqliteCacheRequestsDocumentLoader( + sqlite_file_path=Path('relative.sqlite')) + + +def test_http_cache_headers_serve_from_cache_with_cache_control(context_url): + """With cache_control=True, Cache-Control max-age avoids a second HTTP hit.""" + loader = RequestsDocumentLoader( + session=CachedSession( + 'test_memory_cache_control', + backend='memory', + cache_control=True, + )) + loader(context_url) + loader(context_url) + assert _ContextHandler.request_count == 1 + loader.session.close() + + +def test_http_cache_headers_without_cache_control_hits_server_twice(context_url): + """With cache_control=False, response Cache-Control headers are ignored.""" + loader = RequestsDocumentLoader( + session=CachedSession( + 'test_memory_no_cache_control', + backend='memory', + cache_control=False, + expire_after=0, + )) + loader(context_url) + loader(context_url) + assert _ContextHandler.request_count == 2 + loader.session.close() + + +def test_sqlite_cache_requests_document_loader_persists(context_url, tmp_path): + """Second loader instance reuses the on-disk SQLite cache.""" + cache_path = tmp_path / 'contexts.sqlite' + loader = SqliteCacheRequestsDocumentLoader(sqlite_file_path=cache_path) + loader(context_url) + assert _ContextHandler.request_count == 1 + loader.session.close() + + _ContextHandler.request_count = 0 + loader = SqliteCacheRequestsDocumentLoader(sqlite_file_path=cache_path) + loader(context_url) + assert _ContextHandler.request_count == 0 + loader.session.close()