Skip to content

Commit 5f72fc7

Browse files
authored
Merge pull request #257 from jdkandersson/enhancement/251-cache-validation
Enhancement/251 cache validation
2 parents 1f4ff46 + ddf3bae commit 5f72fc7

15 files changed

Lines changed: 613 additions & 10 deletions

File tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,4 @@ venv36
2222
coverage.xml
2323
docs/source/_*
2424
__pycache__
25+
__open_alchemy_*_cache__

CHANGELOG.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
99

1010
### Fixed
1111

12-
- Removed unnecessary imports in `__init__.py` files.
12+
- Removed unnecessary imports in `__init__.py` files. [#255]
13+
14+
### Added
15+
16+
- Caching validation results to speed up startup. [#251]
1317

1418
## [v2.1.0] - 2020-12-20
1519

@@ -496,3 +500,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
496500
[#201]: https://github.com/jdkandersson/OpenAlchemy/issues/201
497501
[#202]: https://github.com/jdkandersson/OpenAlchemy/issues/202
498502
[#236]: https://github.com/jdkandersson/OpenAlchemy/issues/236
503+
[#251]: https://github.com/jdkandersson/OpenAlchemy/issues/251
504+
[#255]: https://github.com/jdkandersson/OpenAlchemy/issues/255

open_alchemy/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ def init_model_factory(
6464
schemas = components.get("schemas", {})
6565

6666
# Pre-processing schemas
67-
_schemas_module.process(schemas=schemas)
67+
_schemas_module.process(schemas=schemas, spec_filename=spec_path)
6868

6969
# Getting artifacts
7070
schemas_artifacts = _schemas_artifacts.get_from_schemas(

open_alchemy/build/MANIFEST.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
recursive-include {{ name }} *.json
1+
recursive-include {{ name }} *.json __open_alchemy_*_cache__
22
remove .*
33

open_alchemy/build/__init__.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
import jinja2
1212

13+
from .. import cache
1314
from .. import exceptions
1415
from .. import models_file as models_file_module
1516
from .. import schemas as schemas_module
@@ -336,7 +337,9 @@ def dump(
336337
# Write files in the package directory.
337338
package = directory / name
338339
package.mkdir(parents=True, exist_ok=True)
339-
(package / "spec.json").write_text(spec_str)
340+
spec_file = package / "spec.json"
341+
spec_file.write_text(spec_str)
342+
cache.schemas_are_valid(str(spec_file))
340343
(package / "__init__.py").write_text(init)
341344
except OSError as exc:
342345
raise exceptions.BuildError(str(exc)) from exc

open_alchemy/build/setup.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ setuptools.setup(
44
name="{{ name }}",
55
version="{{ version }}",
66
packages=setuptools.find_packages(),
7-
python_requires=">=3.6",
7+
python_requires=">=3.7",
88
install_requires=[
99
"OpenAlchemy",
1010
],

open_alchemy/cache.py

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
"""
2+
Cache for OpenAlchemy.
3+
4+
The name of the file is:
5+
__open_alchemy_<sha256 of spec filename>_cache__
6+
7+
The structure of the file is:
8+
9+
{
10+
"hash": "<sha256 hash of the file contents>",
11+
"data": {
12+
"schemas": {
13+
"valid": true/false
14+
}
15+
}
16+
}
17+
"""
18+
19+
import hashlib
20+
import json
21+
import pathlib
22+
import shutil
23+
24+
from . import exceptions
25+
26+
27+
def calculate_hash(value: str) -> str:
28+
"""Create hash of a value."""
29+
sha256 = hashlib.sha256()
30+
sha256.update(value.encode())
31+
return sha256.hexdigest()
32+
33+
34+
def calculate_cache_path(path: pathlib.Path) -> pathlib.Path:
35+
"""
36+
Calculate the name of the cache file.
37+
38+
Args:
39+
path: The path to the spec file.
40+
41+
Returns:
42+
The path to the cache file.
43+
44+
"""
45+
return path.parent / f"__open_alchemy_{calculate_hash(path.name)}_cache__"
46+
47+
48+
_HASH_KEY = "hash"
49+
_DATA_KEY = "data"
50+
_DATA_SCHEMAS_KEY = "schemas"
51+
_DATA_SCHEMAS_VALID_KEY = "valid"
52+
53+
54+
def schemas_valid(filename: str) -> bool:
55+
"""
56+
Calculate whether the cache indicates that the schemas in the file are valid.
57+
58+
Algorithm:
59+
1. If the file does not exist, return False.
60+
2. If the file is actually a folder, return False.
61+
3. If the spec file is actually a folder, return False.
62+
4. If the spec file does not exist, return False.
63+
5. Calculate the hash of the spec file contents.
64+
6. Try to load the cache, if it fails or it is not a dictionary, return False.
65+
7. Try to retrieve the hash key, if it does not exist, return False.
66+
8. If the value of the hash key is different to the hash of the file, return False.
67+
9. Look for the data.schemas.valid key, if it does not exist, return False.
68+
12. If the value of data.schemas.valid is True return True, otherwise return False.
69+
70+
Args:
71+
filename: The name of the OpenAPI specification file.
72+
73+
Returns:
74+
Whether the cache indicates that the schemas in the file are valid.
75+
76+
"""
77+
path = pathlib.Path(filename)
78+
cache_path = calculate_cache_path(path)
79+
80+
# Check that both file and cache exists and are files
81+
if (
82+
not path.exists()
83+
or not path.is_file()
84+
or not cache_path.exists()
85+
or not cache_path.is_file()
86+
):
87+
return False
88+
89+
file_hash = calculate_hash(path.read_text())
90+
91+
try:
92+
cache = json.loads(cache_path.read_text())
93+
except json.JSONDecodeError:
94+
return False
95+
96+
cache_valid = (
97+
isinstance(cache, dict)
98+
and _HASH_KEY in cache
99+
and _DATA_KEY in cache
100+
and isinstance(cache[_DATA_KEY], dict)
101+
and _DATA_SCHEMAS_KEY in cache[_DATA_KEY]
102+
and isinstance(cache[_DATA_KEY][_DATA_SCHEMAS_KEY], dict)
103+
and _DATA_SCHEMAS_VALID_KEY in cache[_DATA_KEY][_DATA_SCHEMAS_KEY]
104+
)
105+
if not cache_valid:
106+
return False
107+
108+
cache_file_hash = cache[_HASH_KEY]
109+
if file_hash != cache_file_hash:
110+
return False
111+
112+
return cache[_DATA_KEY][_DATA_SCHEMAS_KEY][_DATA_SCHEMAS_VALID_KEY] is True
113+
114+
115+
def schemas_are_valid(filename: str) -> None:
116+
"""
117+
Update the cache to indicate that the filename is valid.
118+
119+
Algorithm:
120+
1. If the spec filename is actually a folder, raise a CacheError.
121+
2. If the spec filename does not exist, raise a CacheError.
122+
3. Calculate the hash of the spec file contents.
123+
4. If the chache is actually a folder, delete the folder.
124+
5. If the cache does not exist, create the cache.
125+
6. Read the contents of the cache. If it is not a dictionary, throw the contents
126+
away and create an empty dictionary.
127+
7. Create or update the hash key in the cache dictionary to be the calculated value.
128+
8. Look for the data key in the cache dictionary. If it does not exist or is not a
129+
dictionary, make it an empty dictionary.
130+
9. Look for the schemas key under data in the cache dictionary. If it does not exist
131+
or is not a dictionary, set it to be an empty dictionary.
132+
10. Create or update the valid key under data.schemas and set it to True.
133+
11. Write the dictionary to the file as JSON.
134+
135+
Args:
136+
filename: The name of the spec file.
137+
138+
"""
139+
path = pathlib.Path(filename)
140+
if not path.exists():
141+
raise exceptions.CacheError(
142+
f"the spec file does not exists, filename={filename}"
143+
)
144+
if not path.is_file():
145+
raise exceptions.CacheError(f"the spec file is not a file, filename={filename}")
146+
file_hash = calculate_hash(path.read_text())
147+
148+
cache_path = calculate_cache_path(path)
149+
if cache_path.exists() and not cache_path.is_file():
150+
shutil.rmtree(cache_path)
151+
if not cache_path.exists():
152+
cache_path.write_text("", encoding="utf-8")
153+
154+
try:
155+
cache = json.loads(cache_path.read_text())
156+
except json.JSONDecodeError:
157+
cache = {}
158+
if not isinstance(cache, dict):
159+
cache = {}
160+
161+
cache[_HASH_KEY] = file_hash
162+
163+
if _DATA_KEY not in cache or not isinstance(cache[_DATA_KEY], dict):
164+
cache[_DATA_KEY] = {}
165+
cache_data = cache[_DATA_KEY]
166+
if _DATA_SCHEMAS_KEY not in cache_data or not isinstance(
167+
cache_data[_DATA_SCHEMAS_KEY], dict
168+
):
169+
cache_data[_DATA_SCHEMAS_KEY] = {}
170+
cache_data_schemas = cache_data[_DATA_SCHEMAS_KEY]
171+
cache_data_schemas[_DATA_SCHEMAS_VALID_KEY] = True
172+
173+
cache_path.write_text(json.dumps(cache), encoding="utf-8")

open_alchemy/exceptions.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,7 @@ class BuildError(BaseError):
7474

7575
class CLIError(BaseError):
7676
"""Raised when an error occurs when the CLI is used."""
77+
78+
79+
class CacheError(BaseError):
80+
"""Raised when an error occurs when the cache is used."""

open_alchemy/schemas/__init__.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,17 @@
11
"""Performs operations on the schemas to prepare them for further processing."""
22

3+
import typing
4+
35
from .. import types as _types
46
from . import association
57
from . import backref
68
from . import foreign_key
79
from . import validation
810

911

10-
def process(*, schemas: _types.Schemas) -> None:
12+
def process(
13+
*, schemas: _types.Schemas, spec_filename: typing.Optional[str] = None
14+
) -> None:
1115
"""
1216
Pre-process schemas.
1317
@@ -18,7 +22,7 @@ def process(*, schemas: _types.Schemas) -> None:
1822
schemas: The schemas to pre-process in place.
1923
2024
"""
21-
validation.process(schemas=schemas)
25+
validation.process(schemas=schemas, spec_filename=spec_filename)
2226
backref.process(schemas=schemas)
2327
foreign_key.process(schemas=schemas)
2428
association.process(schemas=schemas)

open_alchemy/schemas/validation/__init__.py

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import typing
44

5+
from ... import cache
56
from ... import exceptions as _exceptions
67
from ... import types as _oa_types
78
from ..helpers import iterate
@@ -87,14 +88,21 @@ def _other_schemas_checks(*, schemas: _oa_types.Schemas) -> types.Result:
8788
return types.Result(valid=True, reason=None)
8889

8990

90-
def process(*, schemas: _oa_types.Schemas) -> None:
91+
def process(
92+
*, schemas: _oa_types.Schemas, spec_filename: typing.Optional[str] = None
93+
) -> None:
9194
"""
9295
Validate schemas.
9396
9497
Args:
9598
schemas: The schemas to validate.
99+
spec_filename: The filename of the spec, used to cache the result.
96100
97101
"""
102+
if spec_filename is not None:
103+
if cache.schemas_valid(spec_filename):
104+
return
105+
98106
schemas_result = schemas_validation.check(schemas=schemas)
99107
if not schemas_result.valid:
100108
raise _exceptions.MalformedSchemaError(schemas_result.reason)
@@ -121,6 +129,9 @@ def process(*, schemas: _oa_types.Schemas) -> None:
121129
if not other_results_result.valid:
122130
raise _exceptions.MalformedSchemaError(other_results_result.reason)
123131

132+
if spec_filename is not None:
133+
cache.schemas_are_valid(spec_filename)
134+
124135

125136
def check_one_model(*, schemas: _oa_types.Schemas) -> types.Result:
126137
"""

0 commit comments

Comments
 (0)