Skip to content

Commit c8f4074

Browse files
authored
Always disable sync in SQLite cache (#21184)
Fixes #21176 This seems to resolve all the HDD performance issues. Although this makes cache less durable, this is fine, as FS is not perfect in this sense either (and it doesn't really need to be that durable).
1 parent a7bdffd commit c8f4074

File tree

2 files changed

+10
-14
lines changed

2 files changed

+10
-14
lines changed

mypy/build.py

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -877,7 +877,7 @@ def __init__(
877877
]
878878
)
879879

880-
self.metastore = create_metastore(options, parallel_worker)
880+
self.metastore = create_metastore(options)
881881

882882
# a mapping from source files to their corresponding shadow files
883883
# for efficient lookup
@@ -1616,13 +1616,10 @@ def exclude_from_backups(target_dir: str) -> None:
16161616
pass
16171617

16181618

1619-
def create_metastore(options: Options, parallel_worker: bool = False) -> MetadataStore:
1619+
def create_metastore(options: Options) -> MetadataStore:
16201620
"""Create the appropriate metadata store."""
16211621
if options.sqlite_cache:
1622-
# We use this flag in both coordinator and workers to speed up commits,
1623-
# see mypy.metastore.connect_db() for details.
1624-
sync_off = options.num_workers > 0 or parallel_worker
1625-
mds: MetadataStore = SqliteMetadataStore(_cache_dir_prefix(options), sync_off=sync_off)
1622+
mds: MetadataStore = SqliteMetadataStore(_cache_dir_prefix(options))
16261623
else:
16271624
mds = FilesystemMetadataStore(_cache_dir_prefix(options))
16281625
return mds

mypy/metastore.py

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -154,21 +154,20 @@ def close(self) -> None:
154154
"""
155155

156156

157-
def connect_db(db_file: str, sync_off: bool = False) -> sqlite3.Connection:
157+
def connect_db(db_file: str) -> sqlite3.Connection:
158158
import sqlite3.dbapi2
159159

160160
db = sqlite3.dbapi2.connect(db_file)
161-
if sync_off:
162-
# This is a bit unfortunate (as we may get corrupt cache after e.g. Ctrl + C),
163-
# but without this flag, commits are *very* slow, especially when using HDDs,
164-
# see https://www.sqlite.org/faq.html#q19 for details.
165-
db.execute("PRAGMA synchronous=OFF")
161+
# This is a bit unfortunate (as we may get corrupt cache after e.g. Ctrl + C),
162+
# but without this flag, commits are *very* slow, especially when using HDDs,
163+
# see https://www.sqlite.org/faq.html#q19 for details.
164+
db.execute("PRAGMA synchronous=OFF")
166165
db.executescript(SCHEMA)
167166
return db
168167

169168

170169
class SqliteMetadataStore(MetadataStore):
171-
def __init__(self, cache_dir_prefix: str, sync_off: bool = False) -> None:
170+
def __init__(self, cache_dir_prefix: str) -> None:
172171
# We check startswith instead of equality because the version
173172
# will have already been appended by the time the cache dir is
174173
# passed here.
@@ -177,7 +176,7 @@ def __init__(self, cache_dir_prefix: str, sync_off: bool = False) -> None:
177176
return
178177

179178
os.makedirs(cache_dir_prefix, exist_ok=True)
180-
self.db = connect_db(os_path_join(cache_dir_prefix, "cache.db"), sync_off=sync_off)
179+
self.db = connect_db(os_path_join(cache_dir_prefix, "cache.db"))
181180

182181
def _query(self, name: str, field: str) -> Any:
183182
# Raises FileNotFound for consistency with the file system version

0 commit comments

Comments
 (0)