From f75c051af5ce9cb8635ac3150c97fec6add23d3a Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 20:52:58 -0400 Subject: [PATCH 01/10] fixes --- .idea/secrets-cache.iml | 1 + pyproject.toml | 4 +- src/secrets_cache/_aws.py | 76 +++++++++-------------------- src/secrets_cache/_cache.py | 86 +++++++++++++++++++++++++++++++-- src/secrets_cache/_constants.py | 5 ++ 5 files changed, 112 insertions(+), 60 deletions(-) create mode 100644 src/secrets_cache/_constants.py diff --git a/.idea/secrets-cache.iml b/.idea/secrets-cache.iml index 5586b56..ee41d03 100644 --- a/.idea/secrets-cache.iml +++ b/.idea/secrets-cache.iml @@ -2,6 +2,7 @@ + diff --git a/pyproject.toml b/pyproject.toml index 111987a..a9915cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,9 +38,9 @@ cli = [ 'typer', # CLI / optional but handy ] local = [ - 'boto3>=1.26', # AWS SDK for fetching secrets + 'boto3>=1.26', # AWS SDK for fetching secrets + 'tomli-w>=1.0', # TOML writer for Python <3.11 'tomli>=2.0; python_version<"3.11"', # TOML reader for Python <3.11 - 'tomli-w>=1.0; python_version<"3.11"', # TOML writer for Python <3.11 ] lambda = [] # Lambda-friendly, skips boto3 test = [ diff --git a/src/secrets_cache/_aws.py b/src/secrets_cache/_aws.py index ffa42c0..a5e9717 100644 --- a/src/secrets_cache/_aws.py +++ b/src/secrets_cache/_aws.py @@ -1,35 +1,15 @@ """Main module.""" - -from pathlib import Path +from json import loads import boto3 from ._cache import cached_fetch - - -# Optional: import TOML only if available -try: - import tomllib # Python 3.11+ -except ImportError: - try: - import tomli as tomllib # Python 3.10 - except ImportError: - tomllib = None - -try: - import tomli_w -except ImportError: - tomli_w = None +from ._constants import DEFAULT_CACHE_TTL # Module-level caches -_secret_cache = {} -_param_cache = {} _boto_clients = {} -# TOML cache file (optional) -_CACHE_FILE = Path.home() / '.secrets_cache.toml' - def get_boto_client(service: str, region: str = 'us-east-1'): """Cache boto3 clients per service/region.""" @@ -40,49 +20,39 @@ def get_boto_client(service: str, region: str = 'us-east-1'): return _client -def _read_toml_cache(): - if not tomllib or not _CACHE_FILE.exists(): - return {} - with _CACHE_FILE.open('rb') as f: - return tomllib.load(f) - - -def _write_toml_cache(data): - if not tomli_w or not _CACHE_FILE.parent.exists(): - return - with _CACHE_FILE.open('wb') as f: - tomli_w.dump(data, f) +def get_secret(name: str, + region: str = 'us-east-1', + ttl: int = DEFAULT_CACHE_TTL, + force_refresh: bool = False, + raw: bool = False): + """Get secret from AWS Secrets Manager with optional caching.""" + value = cached_fetch('secretsmanager', name, region, _fetch_secret, ttl, force_refresh) + if raw: + return value -def get_secret(name: str, region: str = 'us-east-1', ttl: int = 7 * 24 * 3600): - """Get secret from AWS Secrets Manager with optional caching.""" - return cached_fetch(_secret_cache, name, region, _fetch_secret, ttl) + # Try to parse JSON (most Secrets Manager use case) + try: + return loads(value) + except (ValueError, TypeError): + return value -def get_param(name: str, region: str = 'us-east-1', ttl: int = 7 * 24 * 3600): +def get_param(name: str, + region: str = 'us-east-1', + ttl: int = DEFAULT_CACHE_TTL, + force_refresh: bool = False): """Get parameter from AWS SSM Parameter Store with optional caching.""" - return cached_fetch(_param_cache, name, region, _fetch_param, ttl) + return cached_fetch('ssm', name, region, _fetch_param, ttl, force_refresh) def _fetch_secret(name: str, region: str = 'us-east-1'): client = get_boto_client('secretsmanager', region) resp = client.get_secret_value(SecretId=name) - value = resp['SecretString'] - - # Update local TOML cache if available - data = _read_toml_cache() - data[name] = value - _write_toml_cache(data) - return value + return resp['SecretString'] def _fetch_param(name: str, region: str = 'us-east-1'): client = get_boto_client('ssm', region) resp = client.get_parameter(Name=name, WithDecryption=True) - value = resp['Parameter']['Value'] - - # Update local TOML cache if available - data = _read_toml_cache() - data[name] = value - _write_toml_cache(data) - return value + return resp['Parameter']['Value'] diff --git a/src/secrets_cache/_cache.py b/src/secrets_cache/_cache.py index bc69d71..18d7328 100644 --- a/src/secrets_cache/_cache.py +++ b/src/secrets_cache/_cache.py @@ -1,13 +1,89 @@ +from pathlib import Path from time import time -def cached_fetch(cache: dict, key: str, region: str, fetcher, ttl: int): - """Fetch from cache or call fetcher.""" - now = time() - entry = cache.get(key) +# TOML cache file (optional) +_CACHE_FILE = Path.home() / '.secrets_cache.toml' + + +# Lazy-loaded cache +_cache = None + + +# Optional: import TOML only if available +try: + import tomllib # Python 3.11+ +except ImportError: + try: + # noinspection SpellCheckingInspection + import tomli as tomllib # Python 3.10 + except ImportError: + # noinspection SpellCheckingInspection + tomllib = None + +try: + import tomli_w +except ImportError: + tomli_w = None + + +def _load_cache() -> dict: + global _cache + + if _cache is None: + if tomllib and _CACHE_FILE.exists(): + print('WALLEN') + with _CACHE_FILE.open('rb') as f: + _cache = tomllib.load(f) + else: + _cache = {} + + return _cache + + +def _save_cache() -> None: + if (_cache is None + or not tomli_w + or not _CACHE_FILE.parent.exists()): + return + + with _CACHE_FILE.open('wb') as f: + tomli_w.dump(_cache, f) # type: ignore + + +def get_cached_value(service: str, name: str, now: float, ttl: int): + data = _load_cache() + service_cache = data.get(service, {}) + entry = service_cache.get(name) + if entry and (now - entry['fetched_at'] < ttl): return entry['value'] + return None + + +def update_cache(service: str, name: str, value: str, now: float): + data = _load_cache() + final_value = {'value': value, 'fetched_at': now} + + service_cache = data.get(service) + + if service_cache is None: + data[service] = {name: final_value} + else: + service_cache[name] = final_value + + _save_cache() + + +def cached_fetch(service: str, key: str, region: str, fetcher, ttl: int, force_refresh: bool): + """Fetch from cache or call fetcher.""" + now = int(time()) + + if not force_refresh: + if (value := get_cached_value(service, key, now, ttl)) is not None: + return value + value = fetcher(key, region) - cache[key] = {'value': value, 'fetched_at': now} + update_cache(service, key, value, now) return value diff --git a/src/secrets_cache/_constants.py b/src/secrets_cache/_constants.py new file mode 100644 index 0000000..ffe1b4d --- /dev/null +++ b/src/secrets_cache/_constants.py @@ -0,0 +1,5 @@ +"""Constants""" +import os + + +DEFAULT_CACHE_TTL = int(os.getenv('SECRETS_CACHE_TTL', 7 * 24 * 3600)) From 3e474284673c1a2d6979ab496ff5ebc988bb30bf Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 21:00:10 -0400 Subject: [PATCH 02/10] fixes --- src/secrets_cache/_aws.py | 2 +- src/secrets_cache/_cache.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/secrets_cache/_aws.py b/src/secrets_cache/_aws.py index a5e9717..5999a5b 100644 --- a/src/secrets_cache/_aws.py +++ b/src/secrets_cache/_aws.py @@ -1,4 +1,4 @@ -"""Main module.""" +"""AWS module.""" from json import loads import boto3 diff --git a/src/secrets_cache/_cache.py b/src/secrets_cache/_cache.py index 18d7328..f12eb54 100644 --- a/src/secrets_cache/_cache.py +++ b/src/secrets_cache/_cache.py @@ -1,3 +1,4 @@ +"""Caching module.""" from pathlib import Path from time import time @@ -32,7 +33,6 @@ def _load_cache() -> dict: if _cache is None: if tomllib and _CACHE_FILE.exists(): - print('WALLEN') with _CACHE_FILE.open('rb') as f: _cache = tomllib.load(f) else: From 492dfc2d15230d6372aae4882043ca1eb0535b2a Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 21:22:42 -0400 Subject: [PATCH 03/10] fixes --- .idea/inspectionProfiles/Project_Default.xml | 9 ++-- .../inspectionProfiles/profiles_settings.xml | 6 --- src/secrets_cache/_aws.py | 52 ++++++++++++++----- src/secrets_cache/_constants.py | 2 + 4 files changed, 45 insertions(+), 24 deletions(-) delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 880c590..3b0dd67 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -5,12 +5,11 @@ diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/src/secrets_cache/_aws.py b/src/secrets_cache/_aws.py index 5999a5b..f4ec05d 100644 --- a/src/secrets_cache/_aws.py +++ b/src/secrets_cache/_aws.py @@ -2,16 +2,16 @@ from json import loads import boto3 +from botocore.exceptions import ClientError from ._cache import cached_fetch -from ._constants import DEFAULT_CACHE_TTL - +from ._constants import DEFAULT_CACHE_TTL, DEFAULT_AWS_REGION # Module-level caches _boto_clients = {} -def get_boto_client(service: str, region: str = 'us-east-1'): +def get_boto_client(service: str, region: str): """Cache boto3 clients per service/region.""" key = service, region _client = _boto_clients.get(key) @@ -21,10 +21,10 @@ def get_boto_client(service: str, region: str = 'us-east-1'): def get_secret(name: str, - region: str = 'us-east-1', + region: str = DEFAULT_AWS_REGION, ttl: int = DEFAULT_CACHE_TTL, force_refresh: bool = False, - raw: bool = False): + raw: bool = False) -> str | bytes | dict: """Get secret from AWS Secrets Manager with optional caching.""" value = cached_fetch('secretsmanager', name, region, _fetch_secret, ttl, force_refresh) @@ -39,20 +39,46 @@ def get_secret(name: str, def get_param(name: str, - region: str = 'us-east-1', + region: str = DEFAULT_AWS_REGION, ttl: int = DEFAULT_CACHE_TTL, - force_refresh: bool = False): + force_refresh: bool = False) -> str: """Get parameter from AWS SSM Parameter Store with optional caching.""" return cached_fetch('ssm', name, region, _fetch_param, ttl, force_refresh) -def _fetch_secret(name: str, region: str = 'us-east-1'): +def _fetch_secret(name: str, region: str) -> str | bytes | None: client = get_boto_client('secretsmanager', region) - resp = client.get_secret_value(SecretId=name) - return resp['SecretString'] - -def _fetch_param(name: str, region: str = 'us-east-1'): + try: + resp = client.get_secret_value( + SecretId=name + ) + except ClientError as e: + # For a list of exceptions thrown, see + # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html + err_code = e.response['Error']['Code'] + if err_code == 'ResourceNotFoundException': + print(f'The requested secret {name} was not found') + elif err_code == 'InvalidRequestException': + print('The request was invalid due to:', e) + elif err_code == 'InvalidParameterException': + print('The request had invalid params:', e) + elif err_code == 'DecryptionFailure': + print('The requested secret can\'t be decrypted using the provided KMS key:', e) + elif err_code == 'InternalServiceError': + print('An error occurred on service side:', e) + else: + # Secrets Manager decrypts the secret value using the associated KMS CMK + # Depending on whether the secret was a string or binary, only one of these fields will be populated + if (secret := resp.get('SecretString')) is not None: + return secret + return resp['SecretBinary'] + + +def _fetch_param(name: str, region: str): client = get_boto_client('ssm', region) + resp = client.get_parameter(Name=name, WithDecryption=True) - return resp['Parameter']['Value'] + + param = resp['Parameter']['Value'] + return param diff --git a/src/secrets_cache/_constants.py b/src/secrets_cache/_constants.py index ffe1b4d..9d8ba10 100644 --- a/src/secrets_cache/_constants.py +++ b/src/secrets_cache/_constants.py @@ -2,4 +2,6 @@ import os +DEFAULT_AWS_REGION = os.getenv('AWS_REGION', 'us-east-1') + DEFAULT_CACHE_TTL = int(os.getenv('SECRETS_CACHE_TTL', 7 * 24 * 3600)) From 7b2449efe95d0ab29b897733b6ad3665d17e4eb2 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 21:24:52 -0400 Subject: [PATCH 04/10] fixes --- src/secrets_cache/_cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/secrets_cache/_cache.py b/src/secrets_cache/_cache.py index f12eb54..061dc09 100644 --- a/src/secrets_cache/_cache.py +++ b/src/secrets_cache/_cache.py @@ -85,5 +85,6 @@ def cached_fetch(service: str, key: str, region: str, fetcher, ttl: int, force_r return value value = fetcher(key, region) - update_cache(service, key, value, now) + if value is not None: + update_cache(service, key, value, now) return value From bd99bd5a1eaac67197f6807337d7162977d0b762 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 22:01:05 -0400 Subject: [PATCH 05/10] UPDATE README.md --- README.md | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5edfd72..e4b647e 100644 --- a/README.md +++ b/README.md @@ -17,40 +17,48 @@ Install the base package (minimal, Lambda-friendly): pip install secrets-cache[lambda] ```` -Install with **local cache support** (TOML) for testing / development: +For local development or testing (with local TOML caching, AWS SDK): ```bash pip install secrets-cache[local] ``` -Install with CLI support: +Optional CLI tools: ```bash pip install secrets-cache[cli] ``` -You can also combine extras: +## Usage -```bash -pip install "secrets-cache[local,cli]" +### Fetch a secret from AWS Secrets Manager + +```python +from secrets_cache import get_secret + +# Returns JSON-decoded dict if possible +db_creds = get_secret("prod/AppBeta/MySQL") + +# Returns raw string +raw_value = get_secret("prod/AppBeta/MySQL", raw=True) + +# Force refresh from AWS, ignoring cache +fresh_value = get_secret("prod/AppBeta/MySQL", force_refresh=True) ``` -## Usage +### Fetch a parameter from AWS SSM Parameter Store ```python -from secrets_cache import get_secret, get_param - -# Get a secret from AWS Secrets Manager -my_secret = get_secret("my-secret-name", region="us-east-1") +from secrets_cache import get_param -# Get a parameter from AWS SSM Parameter Store -my_param = get_param("/my/parameter/name", region="us-east-1") +api_url = get_param("prod/AppBeta/API_URL") ``` **Notes:** -* By default, secrets are cached **in memory** to reduce repeated AWS calls. -* If `local` extra is installed, secrets are also stored in `~/.secrets_cache.toml` for local caching. +* Secrets and parameters are **cached in-memory** and optionally in a **local TOML file** (`~/.secrets_cache.toml`) for repeated calls. +* Default cache TTL is **1 week** (configurable via `SECRETS_CACHE_TTL` environment variable). +* AWS region defaults to `AWS_REGION` environment variable or `us-east-1`. * Module-level caches persist across **warm AWS Lambda invocations**, so repeated calls in the same container are very fast. ## Features From 42fb482df2a011a738025816338c3bfa4626a484 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 22:05:11 -0400 Subject: [PATCH 06/10] UPDATE README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e4b647e..c3b05e6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # Secrets Cache -![PyPI version](https://img.shields.io/pypi/v/secrets-cache.svg) +[![PyPI version](https://img.shields.io/pypi/v/secrets-cache.svg)](https://pypi.org/project/secrets-cache/) +[![PyPI license](https://img.shields.io/pypi/l/secrets-cache.svg)](https://pypi.org/project/secrets-cache/) +[![PyPI Python versions](https://img.shields.io/pypi/pyversions/secrets-cache.svg)](https://pypi.org/project/secrets-cache/) +[![GitHub Actions](https://github.com/rnag/secrets-cache/actions/workflows/release.yml/badge.svg)](https://github.com/rnag/secrets-cache/actions/workflows/release.yml) [![Documentation Status](https://readthedocs.org/projects/secrets-cache/badge/?version=latest)](https://secrets-cache.readthedocs.io/en/latest/?version=latest) Cache secrets locally from AWS Secrets Manager and other secret stores, with optional local caching for development or Lambda-friendly usage. From ff4a8003b8df3c2c4051e59bcba1a7459b0cd886 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 22:06:01 -0400 Subject: [PATCH 07/10] UPDATE README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c3b05e6..5d829e6 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![PyPI version](https://img.shields.io/pypi/v/secrets-cache.svg)](https://pypi.org/project/secrets-cache/) [![PyPI license](https://img.shields.io/pypi/l/secrets-cache.svg)](https://pypi.org/project/secrets-cache/) [![PyPI Python versions](https://img.shields.io/pypi/pyversions/secrets-cache.svg)](https://pypi.org/project/secrets-cache/) -[![GitHub Actions](https://github.com/rnag/secrets-cache/actions/workflows/release.yml/badge.svg)](https://github.com/rnag/secrets-cache/actions/workflows/release.yml) +[![GitHub Actions](https://github.com/rnag/py-secrets-cache/actions/workflows/release.yml/badge.svg)](https://github.com/rnag/py-secrets-cache/actions/workflows/release.yml) [![Documentation Status](https://readthedocs.org/projects/secrets-cache/badge/?version=latest)](https://secrets-cache.readthedocs.io/en/latest/?version=latest) Cache secrets locally from AWS Secrets Manager and other secret stores, with optional local caching for development or Lambda-friendly usage. From 9f48ee303453fce5d767bf5a61c4aeb1c5c4e20c Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 22:07:55 -0400 Subject: [PATCH 08/10] UPDATE README.md --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b206841..25e0e38 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ # Publish package on main branch if it's tagged with 'v*' # Ref: https://github.community/t/run-workflow-on-push-tag-on-specific-branch/17519 -name: build & release +name: release # Controls when the action will run. on: From 2363c186d78ce5e32bae1e57cb817bc001901c04 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 22:15:05 -0400 Subject: [PATCH 09/10] UPDATE HISTORY.md --- HISTORY.md | 6 ++++++ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 5f6822d..5fc3750 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # History +## 0.3.0 (2025-09-13) + +* Fix so local caching via TOML works. +* `get_secret()`: Add `force_refresh` and `raw` parameters +* Attempt to `JSON.loads` secret string from AWS Secrets Manager by default. + ## 0.2.0 (2025-09-13) * Add core caching logic. diff --git a/pyproject.toml b/pyproject.toml index a9915cd..172ed30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "secrets-cache" -version = "0.2.0" +version = "0.3.0" description = "Cache secrets locally from AWS Secrets Manager and other secret stores." readme = "README.md" authors = [ From 624b55ac89154f2beb86ef2e153d9fbc613bdba5 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sat, 13 Sep 2025 22:17:32 -0400 Subject: [PATCH 10/10] Fix tests --- .github/workflows/test.yml | 2 +- tests/test_secrets_cache.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ceb691..fd67f1b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install .[test] + pip install .[local,test] - name: Lint with ruff run: | # stop the build if there are Python syntax errors or undefined names diff --git a/tests/test_secrets_cache.py b/tests/test_secrets_cache.py index f15c270..740f386 100644 --- a/tests/test_secrets_cache.py +++ b/tests/test_secrets_cache.py @@ -8,8 +8,10 @@ from secrets_cache import get_secret, get_param -secret = get_secret("my-secret") -param = get_param("my-param") + +# secret = get_secret("my-secret") +# param = get_param("my-param") + @pytest.fixture def response():