Cache get_stats() with a TTL to fix uncached COUNT on hot API paths

bbconf/bbagent.py

@@ -1,9 +1,11 @@
 import logging
 import statistics
+import threading
 from functools import cached_property
 from pathlib import Path
 
 import numpy as np
+from cachetools import TTLCache
 from sqlalchemy.engine import ScalarResult
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import and_, distinct, func, or_, select
@@ -63,6 +65,14 @@ def __init__(
         self._bedset = BedAgentBedSet(self.config)
         self._objects = BBObjects(self.config)
 
+        # get_stats() runs three uncached COUNT queries on the multi-hundred-
+        # thousand-row bed table and is called on hot paths (the stats endpoint
+        # plus the neighbours/list/search result builders). Cache the result
+        # with a TTL so those paths do not hit the database on every request.
+        # The lock guards the cache dict only, never the DB query itself.
+        self._stats_cache = TTLCache(maxsize=1, ttl=3600)
+        self._stats_lock = threading.Lock()
+
     @property
     def bed(self) -> BedAgentBedFile:
         return self._bed
@@ -86,9 +96,17 @@ def get_stats(self) -> StatsReturn:
         """
         Get statistics for a bed file.
 
+        The result is cached with a TTL because this runs three COUNT queries
+        against the large bed table and is called on hot API paths.
+
         Returns:
             Statistics.
         """
+        with self._stats_lock:
+            cached = self._stats_cache.get("stats")
+        if cached is not None:
+            return cached
+
         with Session(self.config.db_engine.engine) as session:
             number_of_bed = session.execute(select(func.count(Bed.id))).one()[0]
             number_of_bedset = session.execute(select(func.count(BedSets.id))).one()[0]
@@ -97,12 +115,17 @@ def get_stats(self) -> StatsReturn:
                 select(func.count(distinct(Bed.genome_alias)))
             ).one()[0]
 
-        return StatsReturn(
+        stats = StatsReturn(
             bedfiles_number=number_of_bed,
             bedsets_number=number_of_bedset,
             genomes_number=number_of_genomes,
         )
 
+        with self._stats_lock:
+            self._stats_cache["stats"] = stats
+
+        return stats
+
     def get_detailed_stats(self, concise: bool = False) -> FileStats:
         """
         Get comprehensive statistics for all bed files.

bbconf/modules/bedfiles.py

@@ -15,7 +15,7 @@
 from sqlalchemy import and_, cast, delete, func, or_, select
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.exc import IntegrityError
-from sqlalchemy.orm import Session, aliased
+from sqlalchemy.orm import Session, aliased, selectinload
 from sqlalchemy.orm.attributes import flag_modified
 from tqdm import tqdm
 
@@ -107,20 +107,58 @@ def get(self, identifier: str, full: bool = False) -> BedMetadataAll:
         """
         statement = select(Bed).where(and_(Bed.id == identifier))
 
-        bed_plots = BedPlots()
-        bed_files = BedFiles()
-
         with Session(self._sa_engine) as session:
             bed_object = session.scalar(statement)
             if not bed_object:
                 raise BEDFileNotFoundError(f"Bed file with id: {identifier} not found.")
 
-            if full:
-                for result in bed_object.files:
-                    # PLOTS
-                    if result.name in BedPlots.model_fields:
+            return self._build_metadata(bed_object, full=full)
+
+    def _build_metadata(
+        self, bed_object: Bed, full: bool = False
+    ) -> BedMetadataAll:
+        """
+        Build a BedMetadataAll model from a Bed ORM object.
+
+        For ``full=True`` this assembles plots, files, stats, bedsets, and
+        universe metadata (which lazy-load relationships, so the caller must
+        keep the SQLAlchemy session open). For ``full=False`` only scalar
+        columns and the (joined-loaded) annotations are accessed, so the
+        Bed object may be detached from its session.
+
+        Args:
+            bed_object: Bed ORM object to build metadata from.
+            full: If True, return full metadata, including statistics, files,
+                and raw metadata from pephub.
+
+        Returns:
+            BED file metadata.
+        """
+        identifier = bed_object.id
+
+        bed_plots = BedPlots()
+        bed_files = BedFiles()
+
+        if full:
+            for result in bed_object.files:
+                # PLOTS
+                if result.name in BedPlots.model_fields:
+                    setattr(
+                        bed_plots,
+                        result.name,
+                        FileModel(
+                            **result.__dict__,
+                            object_id=f"bed.{identifier}.{result.name}",
+                            access_methods=self.config.construct_access_method_list(
+                                result.path
+                            ),
+                        ),
+                    )
+                # FILES
+                elif result.name in BedFiles.model_fields:
+                    (
                         setattr(
-                            bed_plots,
+                            bed_files,
                             result.name,
                             FileModel(
                                 **result.__dict__,
@@ -129,48 +167,34 @@ def get(self, identifier: str, full: bool = False) -> BedMetadataAll:
                                     result.path
                                 ),
                             ),
-                        )
-                    # FILES
-                    elif result.name in BedFiles.model_fields:
-                        (
-                            setattr(
-                                bed_files,
-                                result.name,
-                                FileModel(
-                                    **result.__dict__,
-                                    object_id=f"bed.{identifier}.{result.name}",
-                                    access_methods=self.config.construct_access_method_list(
-                                        result.path
-                                    ),
-                                ),
-                            ),
-                        )
-
-                    else:
-                        _LOGGER.error(
-                            f"Unknown file type: {result.name}. And is not in the model fields. Skipping.."
-                        )
-                bed_stats = BedStatsModel(**bed_object.stats.__dict__)
-                bed_bedsets = []
-                for relation in bed_object.bedsets:
-                    bed_bedsets.append(
-                        BedSetMinimal(
-                            id=relation.bedset.id,
-                            description=relation.bedset.description,
-                            name=relation.bedset.name,
-                        )
+                        ),
                     )
 
-                if bed_object.universe:
-                    universe_meta = UniverseMetadata(**bed_object.universe.__dict__)
                 else:
-                    universe_meta = UniverseMetadata()
+                    _LOGGER.error(
+                        f"Unknown file type: {result.name}. And is not in the model fields. Skipping.."
+                    )
+            bed_stats = BedStatsModel(**bed_object.stats.__dict__)
+            bed_bedsets = []
+            for relation in bed_object.bedsets:
+                bed_bedsets.append(
+                    BedSetMinimal(
+                        id=relation.bedset.id,
+                        description=relation.bedset.description,
+                        name=relation.bedset.name,
+                    )
+                )
+
+            if bed_object.universe:
+                universe_meta = UniverseMetadata(**bed_object.universe.__dict__)
             else:
-                bed_plots = None
-                bed_files = None
-                bed_stats = None
-                universe_meta = None
-                bed_bedsets = []
+                universe_meta = UniverseMetadata()
+        else:
+            bed_plots = None
+            bed_files = None
+            bed_stats = None
+            universe_meta = None
+            bed_bedsets = []
 
         try:
             if full:
@@ -290,17 +314,32 @@ def get_neighbours(
                 limit=limit,
                 offset=offset,
             )
-            result_list = []
-            for result in results.points:
-                result_id = result.id.replace("-", "")
-                result_list.append(
-                    QdrantSearchResult(
-                        id=result_id,
-                        payload=result.payload,
-                        score=result.score,
-                        metadata=self.get(result_id, full=False),
-                    )
+            # Hydrate all neighbours with a single batched query instead of one
+            # SELECT per neighbour (was an N+1). annotations is joined-loaded,
+            # but selectinload keeps that explicit for this detached-object path.
+            ids = [result.id.replace("-", "") for result in results.points]
+            with Session(self._sa_engine) as session:
+                beds = {
+                    bed.id: bed
+                    for bed in session.scalars(
+                        select(Bed)
+                        .where(Bed.id.in_(ids))
+                        .options(selectinload(Bed.annotations))
+                    ).all()
+                }
+            result_list = [
+                QdrantSearchResult(
+                    id=result.id.replace("-", ""),
+                    payload=result.payload,
+                    score=result.score,
+                    metadata=self._build_metadata(
+                        beds[result.id.replace("-", "")], full=False
+                    ),
                 )
+                for result in results.points
+                # skip stale Qdrant points that no longer exist in the database
+                if result.id.replace("-", "") in beds
+            ]
         except UnexpectedResponse as err:
             _LOGGER.error(
                 f"Qdrant request failed. Error: {err}. Returning empty result set."

docs/changelog.md

@@ -3,6 +3,16 @@
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
 
+### [0.14.14] - 2026-07-13
+### Fixed:
+- Eliminated an N+1 query in `get_neighbours()` by fetching all neighbour metadata in a single batched query (with annotations eager-loaded) instead of one query per neighbour; stale Qdrant points are now skipped rather than raising
+
+
+### [0.14.13] - 2026-07-13
+### Fixed:
+- Cache `get_stats()` with a TTL to avoid running uncached COUNT queries on the bed table on every request to hot API paths (stats, neighbours, list, search)
+
+
 ### [0.14.12] - 2026-04-22
 ### Changed:
 - Updated yacman version to 2.0.0

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bbconf"
-version = "0.14.12"
+version = "0.14.14"
 description = "Configuration and data management tool for BEDbase"
 readme = "README.md"
 license = "BSD-2-Clause"
@@ -37,6 +37,7 @@ dependencies = [
     "umap-learn >= 0.5.8",
     "qdrant_client >= 1.16.1",
     "setuptools < 70.0.0",
+    "cachetools >= 4.2.4",
 ]
 
 [project.urls]