diff --git a/bbconf/bbagent.py b/bbconf/bbagent.py index 6248f6d..34e444e 100644 --- a/bbconf/bbagent.py +++ b/bbconf/bbagent.py @@ -1,9 +1,11 @@ import logging import statistics +import threading from functools import cached_property from pathlib import Path import numpy as np +from cachetools import TTLCache from sqlalchemy.engine import ScalarResult from sqlalchemy.orm import Session from sqlalchemy.sql import and_, distinct, func, or_, select @@ -63,6 +65,14 @@ def __init__( self._bedset = BedAgentBedSet(self.config) self._objects = BBObjects(self.config) + # get_stats() runs three uncached COUNT queries on the multi-hundred- + # thousand-row bed table and is called on hot paths (the stats endpoint + # plus the neighbours/list/search result builders). Cache the result + # with a TTL so those paths do not hit the database on every request. + # The lock guards the cache dict only, never the DB query itself. + self._stats_cache = TTLCache(maxsize=1, ttl=3600) + self._stats_lock = threading.Lock() + @property def bed(self) -> BedAgentBedFile: return self._bed @@ -86,9 +96,17 @@ def get_stats(self) -> StatsReturn: """ Get statistics for a bed file. + The result is cached with a TTL because this runs three COUNT queries + against the large bed table and is called on hot API paths. + Returns: Statistics. """ + with self._stats_lock: + cached = self._stats_cache.get("stats") + if cached is not None: + return cached + with Session(self.config.db_engine.engine) as session: number_of_bed = session.execute(select(func.count(Bed.id))).one()[0] number_of_bedset = session.execute(select(func.count(BedSets.id))).one()[0] @@ -97,12 +115,17 @@ def get_stats(self) -> StatsReturn: select(func.count(distinct(Bed.genome_alias))) ).one()[0] - return StatsReturn( + stats = StatsReturn( bedfiles_number=number_of_bed, bedsets_number=number_of_bedset, genomes_number=number_of_genomes, ) + with self._stats_lock: + self._stats_cache["stats"] = stats + + return stats + def get_detailed_stats(self, concise: bool = False) -> FileStats: """ Get comprehensive statistics for all bed files. diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index c45d0be..8d7a195 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -15,7 +15,7 @@ from sqlalchemy import and_, cast, delete, func, or_, select from sqlalchemy.dialects import postgresql from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session, aliased +from sqlalchemy.orm import Session, aliased, selectinload from sqlalchemy.orm.attributes import flag_modified from tqdm import tqdm @@ -107,20 +107,58 @@ def get(self, identifier: str, full: bool = False) -> BedMetadataAll: """ statement = select(Bed).where(and_(Bed.id == identifier)) - bed_plots = BedPlots() - bed_files = BedFiles() - with Session(self._sa_engine) as session: bed_object = session.scalar(statement) if not bed_object: raise BEDFileNotFoundError(f"Bed file with id: {identifier} not found.") - if full: - for result in bed_object.files: - # PLOTS - if result.name in BedPlots.model_fields: + return self._build_metadata(bed_object, full=full) + + def _build_metadata( + self, bed_object: Bed, full: bool = False + ) -> BedMetadataAll: + """ + Build a BedMetadataAll model from a Bed ORM object. + + For ``full=True`` this assembles plots, files, stats, bedsets, and + universe metadata (which lazy-load relationships, so the caller must + keep the SQLAlchemy session open). For ``full=False`` only scalar + columns and the (joined-loaded) annotations are accessed, so the + Bed object may be detached from its session. + + Args: + bed_object: Bed ORM object to build metadata from. + full: If True, return full metadata, including statistics, files, + and raw metadata from pephub. + + Returns: + BED file metadata. + """ + identifier = bed_object.id + + bed_plots = BedPlots() + bed_files = BedFiles() + + if full: + for result in bed_object.files: + # PLOTS + if result.name in BedPlots.model_fields: + setattr( + bed_plots, + result.name, + FileModel( + **result.__dict__, + object_id=f"bed.{identifier}.{result.name}", + access_methods=self.config.construct_access_method_list( + result.path + ), + ), + ) + # FILES + elif result.name in BedFiles.model_fields: + ( setattr( - bed_plots, + bed_files, result.name, FileModel( **result.__dict__, @@ -129,48 +167,34 @@ def get(self, identifier: str, full: bool = False) -> BedMetadataAll: result.path ), ), - ) - # FILES - elif result.name in BedFiles.model_fields: - ( - setattr( - bed_files, - result.name, - FileModel( - **result.__dict__, - object_id=f"bed.{identifier}.{result.name}", - access_methods=self.config.construct_access_method_list( - result.path - ), - ), - ), - ) - - else: - _LOGGER.error( - f"Unknown file type: {result.name}. And is not in the model fields. Skipping.." - ) - bed_stats = BedStatsModel(**bed_object.stats.__dict__) - bed_bedsets = [] - for relation in bed_object.bedsets: - bed_bedsets.append( - BedSetMinimal( - id=relation.bedset.id, - description=relation.bedset.description, - name=relation.bedset.name, - ) + ), ) - if bed_object.universe: - universe_meta = UniverseMetadata(**bed_object.universe.__dict__) else: - universe_meta = UniverseMetadata() + _LOGGER.error( + f"Unknown file type: {result.name}. And is not in the model fields. Skipping.." + ) + bed_stats = BedStatsModel(**bed_object.stats.__dict__) + bed_bedsets = [] + for relation in bed_object.bedsets: + bed_bedsets.append( + BedSetMinimal( + id=relation.bedset.id, + description=relation.bedset.description, + name=relation.bedset.name, + ) + ) + + if bed_object.universe: + universe_meta = UniverseMetadata(**bed_object.universe.__dict__) else: - bed_plots = None - bed_files = None - bed_stats = None - universe_meta = None - bed_bedsets = [] + universe_meta = UniverseMetadata() + else: + bed_plots = None + bed_files = None + bed_stats = None + universe_meta = None + bed_bedsets = [] try: if full: @@ -290,17 +314,32 @@ def get_neighbours( limit=limit, offset=offset, ) - result_list = [] - for result in results.points: - result_id = result.id.replace("-", "") - result_list.append( - QdrantSearchResult( - id=result_id, - payload=result.payload, - score=result.score, - metadata=self.get(result_id, full=False), - ) + # Hydrate all neighbours with a single batched query instead of one + # SELECT per neighbour (was an N+1). annotations is joined-loaded, + # but selectinload keeps that explicit for this detached-object path. + ids = [result.id.replace("-", "") for result in results.points] + with Session(self._sa_engine) as session: + beds = { + bed.id: bed + for bed in session.scalars( + select(Bed) + .where(Bed.id.in_(ids)) + .options(selectinload(Bed.annotations)) + ).all() + } + result_list = [ + QdrantSearchResult( + id=result.id.replace("-", ""), + payload=result.payload, + score=result.score, + metadata=self._build_metadata( + beds[result.id.replace("-", "")], full=False + ), ) + for result in results.points + # skip stale Qdrant points that no longer exist in the database + if result.id.replace("-", "") in beds + ] except UnexpectedResponse as err: _LOGGER.error( f"Qdrant request failed. Error: {err}. Returning empty result set." diff --git a/docs/changelog.md b/docs/changelog.md index a5fe3fc..33b6a4d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,16 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +### [0.14.14] - 2026-07-13 +### Fixed: +- Eliminated an N+1 query in `get_neighbours()` by fetching all neighbour metadata in a single batched query (with annotations eager-loaded) instead of one query per neighbour; stale Qdrant points are now skipped rather than raising + + +### [0.14.13] - 2026-07-13 +### Fixed: +- Cache `get_stats()` with a TTL to avoid running uncached COUNT queries on the bed table on every request to hot API paths (stats, neighbours, list, search) + + ### [0.14.12] - 2026-04-22 ### Changed: - Updated yacman version to 2.0.0 diff --git a/pyproject.toml b/pyproject.toml index fba05bd..9542f8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bbconf" -version = "0.14.12" +version = "0.14.14" description = "Configuration and data management tool for BEDbase" readme = "README.md" license = "BSD-2-Clause" @@ -37,6 +37,7 @@ dependencies = [ "umap-learn >= 0.5.8", "qdrant_client >= 1.16.1", "setuptools < 70.0.0", + "cachetools >= 4.2.4", ] [project.urls]