From c5dd331be745859f938ea04e04b2c6e0845fb0ed Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 23 Jun 2025 22:40:58 -0400 Subject: [PATCH 01/30] Added more stats in comprehensive stats --- bbconf/bbagent.py | 203 ++++++++++++++++++++++++++++++++++- bbconf/models/base_models.py | 25 +++++ manual_testing.py | 19 +++- 3 files changed, 239 insertions(+), 8 deletions(-) diff --git a/bbconf/bbagent.py b/bbconf/bbagent.py index 68c318e2..ba5879d4 100644 --- a/bbconf/bbagent.py +++ b/bbconf/bbagent.py @@ -1,10 +1,11 @@ import logging from functools import cached_property from pathlib import Path -from typing import List, Union +from typing import List, Union, Dict +import numpy as np from sqlalchemy.orm import Session -from sqlalchemy.sql import distinct, func, select +from sqlalchemy.sql import distinct, func, select, and_, or_ from bbconf.config_parser.bedbaseconfig import BedBaseConfig from bbconf.db_utils import ( @@ -16,8 +17,18 @@ UsageBedMeta, UsageFiles, UsageSearch, + GeoGsmStatus, + BedStats, + Files, +) +from bbconf.models.base_models import ( + StatsReturn, + UsageModel, + FileStats, + UsageStats, + AllFilesInfo, + FileInfo, ) -from bbconf.models.base_models import StatsReturn, UsageModel, FileStats, UsageStats from bbconf.modules.bedfiles import BedAgentBedFile from bbconf.modules.bedsets import BedAgentBedSet from bbconf.modules.objects import BBObjects @@ -134,27 +145,47 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: slice_value = 20 + bed_comments = self._stats_comments(session) + geo_status = self._stats_geo_status(session) + if concise: bed_compliance_concise = dict(list(bed_compliance.items())[0:slice_value]) bed_compliance_concise["other"] = sum( list(bed_compliance.values())[slice_value:] ) + if "" in bed_compliance_concise: + bed_compliance_concise["other"] = ( + bed_compliance_concise["other"] + bed_compliance_concise[""] + ) + bed_compliance_concise.pop("") file_genomes_concise = dict(list(file_genomes.items())[0:slice_value]) file_genomes_concise["other"] = sum( list(file_genomes.values())[slice_value:] ) + file_genomes.get("other", 0) + if "" in file_genomes_concise: + file_genomes_concise["other"] = ( + file_genomes_concise["other"] + file_genomes_concise[""] + ) + file_genomes_concise.pop("") file_organism_concise = dict(list(file_organism.items())[0:slice_value]) file_organism_concise["other"] = sum( list(file_organism.values())[slice_value:] ) + file_organism.get("other", 0) + if "" in file_organism_concise: + file_organism_concise["other"] = ( + file_organism_concise["other"] + file_organism_concise[""] + ) + file_organism_concise.pop("") return FileStats( data_format=data_format, bed_compliance=bed_compliance_concise, file_genome=file_genomes_concise, file_organism=file_organism_concise, + bed_comments=bed_comments, + geo_status=geo_status, ) return FileStats( @@ -162,6 +193,8 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: bed_compliance=bed_compliance, file_genome=file_genomes, file_organism=file_organism, + bed_comments=bed_comments, + geo_status=geo_status, ) def get_detailed_usage(self) -> UsageStats: @@ -364,3 +397,167 @@ def add_usage(self, stats: UsageModel) -> None: session.add(new_stats) session.commit() + + def _stats_comments(self, sa_session: Session) -> Dict[str, int]: + """ + Get statistics about comments that are present in bed files. + + :param sa_session: SQLAlchemy session + + :return: Dict[str, int] + """ + _LOGGER.info("Analyzing bed table for comments in bed files...") + + total_statement = select(func.count(Bed.id)).where(Bed.header.is_not(None)) + correct_statement = select(func.count(Bed.id)).where(Bed.header.like("#%")) + track_name_statement = select(func.count(Bed.id)).where( + Bed.header.like("track name%") + ) + track_type_statement = select(func.count(Bed.id)).where( + Bed.header.like("track type%") + ) + browser_statement = select(func.count(Bed.id)).where( + Bed.header.like("browser%") + ) + + total_bed_comments = ( + sa_session.execute(total_statement).one_or_none() or (0,) + )[0] + correct_bed_comments = ( + sa_session.execute(correct_statement).one_or_none() or (0,) + )[0] + track_name_comments = ( + sa_session.execute(track_name_statement).one_or_none() or (0,) + )[0] + track_type_comments = ( + sa_session.execute(track_type_statement).one_or_none() or (0,) + )[0] + browser_comments = ( + sa_session.execute(browser_statement).one_or_none() or (0,) + )[0] + + header_comments = ( + total_bed_comments + - correct_bed_comments + - track_name_comments + - track_type_comments + - browser_comments + ) + + return { + "correct_bed_comments": correct_bed_comments, + "track_name_comments": track_name_comments, + "track_type_comments": track_type_comments, + "browser_comments": browser_comments, + "header_comments": header_comments, + } + + def _stats_geo_status(self, sa_session: Session) -> Dict[str, int]: + """ + Get statistics about status of GEO bed file processing. + + :param sa_session: SQLAlchemy session + :return Dict[str, int] + """ + + success_statement = select( + func.count(distinct(GeoGsmStatus.sample_name)) + ).where(GeoGsmStatus.status == "SUCCESS") + + failed_count_statement = select( + func.count(distinct(GeoGsmStatus.sample_name)) + ).where(and_(GeoGsmStatus.status == "FAIL")) + mean_region_width_statement = select( + func.count(distinct(GeoGsmStatus.sample_name)) + ).where( + or_( + GeoGsmStatus.error.like("%Initial QC failed for%"), + GeoGsmStatus.error.like("%Quality control failed%"), + ) + ) + size_greater_then_statement = select( + func.count(distinct(GeoGsmStatus.sample_name)) + ).where(GeoGsmStatus.error.like("%File size is too big.%")) + + success_count = (sa_session.execute(success_statement).one_or_none() or (0,))[0] + failed_count = ( + sa_session.execute(failed_count_statement).one_or_none() or (0,) + )[0] + + mean_region_width_count = ( + sa_session.execute(mean_region_width_statement).one_or_none() or (0,) + )[0] + size_greater_than_count = ( + sa_session.execute(size_greater_then_statement).one_or_none() or (0,) + )[0] + corrupted_file_count = ( + failed_count - mean_region_width_count - size_greater_than_count + ) + + return { + "success": success_count, + "mean_region_width_lower_10": mean_region_width_count, + "size_greater_20": size_greater_than_count, + "corrupted_files": corrupted_file_count, + } + + def bed_files_info(self) -> AllFilesInfo: + """ + Get information about all bed files in bedbase. + :param sa_session: SQLAlchemy session + :return AllFilesInfo: + { + "total": int," + "files": [ + { id: str + bed_compliance: str + data_format: str + mean_region_width: float + file_size: int + number_of_regions: int + }, + ... ] + } + """ + + all_files_statement = ( + select( + Bed.id, + Bed.bed_compliance, + Bed.data_format, + BedStats.mean_region_width, + Files.size, + BedStats.number_of_regions, + ) + .join(BedStats, BedStats.id == Bed.id) # Explicit join condition + .join(Files, Files.bedfile_id == Bed.id) # Explicit join condition + .where(Files.name == "bed_file") + ) + + results = [] + error_list = [] + with Session(self.config.db_engine.engine) as session: + sql_result = session.execute(all_files_statement).all() + + for single_bed in sql_result: + try: + results.append( + FileInfo( + id=single_bed[0], + bed_compliance=single_bed[1], + data_format=single_bed[2], + mean_region_width=single_bed[3], + file_size=single_bed[4], + number_of_regions=single_bed[5], + ) + ) + except Exception: + error_list.append(single_bed[0]) + + _LOGGER.info( + f"Number of bed records with unknown regions and region width {len(error_list)}" + ) + return AllFilesInfo( + total=len(results), + files=results, + ) diff --git a/bbconf/models/base_models.py b/bbconf/models/base_models.py index 69fe02ec..feca72ab 100644 --- a/bbconf/models/base_models.py +++ b/bbconf/models/base_models.py @@ -31,6 +31,8 @@ class FileStats(BaseModel): data_format: Dict[str, int] file_genome: Dict[str, int] file_organism: Dict[str, int] + geo_status: Dict[str, int] + bed_comments: Dict[str, int] class UsageStats(BaseModel): @@ -55,3 +57,26 @@ class UsageModel(BaseModel): date_from: datetime.datetime date_to: Union[datetime.datetime, None] = None + + +class FileInfo(BaseModel): + """ + Main information about a file used for BEDbase verse statistics. + """ + + id: str + bed_compliance: str + data_format: str + mean_region_width: float + file_size: int + number_of_regions: int + + +class AllFilesInfo(BaseModel): + """ + Information about all files. e.g. file sizes, mean region width, etc. + + """ + + total: int + files: List[FileInfo] diff --git a/manual_testing.py b/manual_testing.py index 36884b28..12d96b04 100644 --- a/manual_testing.py +++ b/manual_testing.py @@ -211,11 +211,20 @@ def config_t(): def compreh_stats(): from bbconf import BedBaseAgent + import time agent = BedBaseAgent(config="/home/bnt4me/virginia/repos/bedhost/config.yaml") - results = agent.get_detailed_stats() - results = agent.get_detailed_usage() - print(results) + + time1 = time.time() + + results = agent.bed_files_info() + + time2 = time.time() + print(time2 - time1) + + # results = agent.get_detailed_stats() + # results = agent.get_detailed_usage() + # print(results) def get_unprocessed_files(): @@ -243,8 +252,8 @@ def get_genomes(): # get_pep() # get_id_plots_missing() # neighbour_beds() - sql_search() + # sql_search() # config_t() - # compreh_stats() + compreh_stats() # get_unprocessed_files() # get_genomes() From 85ba7d00635c3700f85deeadee9f90c8af1c68b5 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 24 Jun 2025 16:28:30 -0400 Subject: [PATCH 02/30] added more statistics about data in comprehensive stats --- bbconf/bbagent.py | 101 +++++++++++++++++++++++++++++++++++ bbconf/models/base_models.py | 10 ++++ manual_testing.py | 81 +++++++++++++++++++++++++++- 3 files changed, 190 insertions(+), 2 deletions(-) diff --git a/bbconf/bbagent.py b/bbconf/bbagent.py index ba5879d4..3796bf3f 100644 --- a/bbconf/bbagent.py +++ b/bbconf/bbagent.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import List, Union, Dict import numpy as np +import statistics from sqlalchemy.orm import Session from sqlalchemy.sql import distinct, func, select, and_, or_ @@ -28,6 +29,7 @@ UsageStats, AllFilesInfo, FileInfo, + BinValues, ) from bbconf.modules.bedfiles import BedAgentBedFile from bbconf.modules.bedsets import BedAgentBedSet @@ -148,6 +150,16 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: bed_comments = self._stats_comments(session) geo_status = self._stats_geo_status(session) + bedfiles_info = self.bed_files_info() + + number_of_regions = [bed.number_of_regions for bed in bedfiles_info.files] + list_mean_width = [bed.mean_region_width for bed in bedfiles_info.files] + list_file_size = [bed.file_size for bed in bedfiles_info.files] + + number_of_regions_bins = self._bin_number_of_regions(number_of_regions) + list_mean_width_bins = self._bin_mean_region_width(list_mean_width) + list_file_size_bins = self._bin_file_size(list_file_size) + if concise: bed_compliance_concise = dict(list(bed_compliance.items())[0:slice_value]) bed_compliance_concise["other"] = sum( @@ -186,6 +198,9 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: file_organism=file_organism_concise, bed_comments=bed_comments, geo_status=geo_status, + mean_region_width=list_mean_width_bins, + file_size=list_file_size_bins, + number_of_regions=number_of_regions_bins, ) return FileStats( @@ -195,6 +210,9 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: file_organism=file_organism, bed_comments=bed_comments, geo_status=geo_status, + mean_region_width=list_mean_width_bins, + file_size=list_file_size_bins, + number_of_regions=number_of_regions_bins, ) def get_detailed_usage(self) -> UsageStats: @@ -504,6 +522,7 @@ def _stats_geo_status(self, sa_session: Session) -> Dict[str, int]: def bed_files_info(self) -> AllFilesInfo: """ Get information about all bed files in bedbase. + :param sa_session: SQLAlchemy session :return AllFilesInfo: { @@ -561,3 +580,85 @@ def bed_files_info(self) -> AllFilesInfo: total=len(results), files=results, ) + + def _bin_number_of_regions(self, number_of_regions: list) -> BinValues: + """ + Create bins for number of regions in bed files + + :param number_of_regions: list of number of regions in bed files + :return: BinValues object containing bins and values + """ + + max_value_threshold = 400_000 # set a threshold for maximum value to avoid outliers in the histogram + + filtered_number_of_regions = [ + x if x <= max_value_threshold else max_value_threshold + 1 + for x in number_of_regions + ] + + n_region_counts, n_region_bin_edges = np.histogram( + filtered_number_of_regions, bins=100 + ) + n_region_counts = n_region_counts.astype(int).tolist() + n_region_bin_edges = n_region_bin_edges.astype(int).tolist() + + return BinValues( + bins=n_region_bin_edges, + counts=n_region_counts, + mean=statistics.mean(number_of_regions), + meadian=statistics.median(number_of_regions), + ) + + def _bin_mean_region_width(self, mean_region_widths: list) -> BinValues: + """ + Create bins for number of regions in bed files + + :param mean_region_widths: list of mean region widths in bed files + :return: BinValues object containing bins and values + """ + + max_value_threshold = 5_000 # set a threshold for maximum value to avoid outliers in the histogram + + filtered_mean_region_widths = [ + x if x <= max_value_threshold else max_value_threshold + 1 + for x in mean_region_widths + ] + + mean_reg_width_counts, mean_reg_width_bin_edges = np.histogram( + filtered_mean_region_widths, bins=100 + ) + mean_reg_width_counts = mean_reg_width_counts.tolist() + mean_reg_width_bin_edges = mean_reg_width_bin_edges.tolist() + + return BinValues( + bins=mean_reg_width_bin_edges, + counts=mean_reg_width_counts, + mean=statistics.mean(mean_region_widths), + meadian=statistics.median(mean_region_widths), + ) + + def _bin_file_size(self, list_file_size: list) -> BinValues: + """ + Create bins for number of regions in bed files + + :param list_file_size: list of bed file sizes in bytes + :return: BinValues object containing bins and values + """ + max_value_threshold = 10_000_000 + + filtered_list_file_size = [ + x for x in list_file_size if x <= max_value_threshold + ] + + file_size_counts, file_size_bin_edges = np.histogram( + filtered_list_file_size, bins=100 + ) + file_size_counts = file_size_counts.astype(int).tolist() + file_size_bin_edges = file_size_bin_edges.astype(int).tolist() + + return BinValues( + bins=file_size_bin_edges, + counts=file_size_counts, + mean=statistics.mean(filtered_list_file_size), + meadian=statistics.median(filtered_list_file_size), + ) diff --git a/bbconf/models/base_models.py b/bbconf/models/base_models.py index feca72ab..c2c23f73 100644 --- a/bbconf/models/base_models.py +++ b/bbconf/models/base_models.py @@ -26,6 +26,13 @@ class StatsReturn(BaseModel): genomes_number: int = 0 +class BinValues(BaseModel): + bins: List[Union[int, float, str]] + counts: List[int] + mean: Optional[float] + meadian: Optional[float] + + class FileStats(BaseModel): bed_compliance: Dict[str, int] data_format: Dict[str, int] @@ -33,6 +40,9 @@ class FileStats(BaseModel): file_organism: Dict[str, int] geo_status: Dict[str, int] bed_comments: Dict[str, int] + mean_region_width: BinValues + file_size: BinValues + number_of_regions: BinValues class UsageStats(BaseModel): diff --git a/manual_testing.py b/manual_testing.py index 12d96b04..99e53721 100644 --- a/manual_testing.py +++ b/manual_testing.py @@ -6,6 +6,8 @@ # from dotenv import load_dotenv from geniml.io import RegionSet from gtars.utils import read_tokens_from_gtok +import matplotlib.pyplot as plt +import numpy as np # from gtars.tokenizers import RegionSet @@ -217,14 +219,89 @@ def compreh_stats(): time1 = time.time() - results = agent.bed_files_info() + results = agent.get_detailed_stats() time2 = time.time() print(time2 - time1) # results = agent.get_detailed_stats() # results = agent.get_detailed_usage() - # print(results) + + def plot_file_sizes(file_size_counts, file_size_bin_edges): + + # Plot the histogram + plt.figure(figsize=(10, 6)) + plt.bar( + file_size_bin_edges[:-1], + file_size_counts, + width=np.diff(file_size_bin_edges), + edgecolor="black", + align="edge", + ) + + # Add labels and title + plt.xlabel("File Size Bin Edges (bytes)") + plt.ylabel("Counts") + plt.title("Histogram of File Sizes") + plt.grid(axis="y", linestyle="--", alpha=0.7) + + # Show the plot + plt.show() + + def plot_region_width(mean_reg_width_counts, mean_reg_width_bin_edges): + + # Plot the histogram + plt.figure(figsize=(10, 6)) + plt.bar( + mean_reg_width_bin_edges[:-1], + mean_reg_width_counts, + width=np.diff(mean_reg_width_bin_edges), + edgecolor="black", + align="edge", + ) + + # Add labels and title + plt.xlabel("Mean Region Width Bin Edges") + plt.ylabel("Counts") + plt.title("Histogram of Mean Region Widths") + plt.grid(axis="y", linestyle="--", alpha=0.7) + + # Show the plot + plt.show() + + def plot_number_of_regions(n_region_counts, n_region_bin_edges): + + # Plot the histogram + plt.figure(figsize=(10, 6)) + plt.bar( + n_region_bin_edges[:-1], + n_region_counts, + width=np.diff(n_region_bin_edges), + edgecolor="black", + align="edge", + ) + + # Add labels and title + plt.xlabel("Region Bin Edges") + plt.ylabel("Counts") + plt.title("Histogram of Number of Regions") + plt.grid(axis="y", linestyle="--", alpha=0.7) + + # Show the plot + plt.show() + + plot_file_sizes( + results.file_size.counts, + results.file_size.bins, + ) + plot_region_width( + results.mean_region_width.counts, + results.mean_region_width.bins, + ) + plot_number_of_regions( + results.number_of_regions.counts, + results.number_of_regions.bins, + ) def get_unprocessed_files(): From 01d4d163272d90f8a6388c93754755dc95d130a4 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 26 Jun 2025 15:32:34 -0400 Subject: [PATCH 03/30] new qdrant search added --- bbconf/config_parser/bedbaseconfig.py | 56 ++++++++ bbconf/models/bed_models.py | 17 +++ bbconf/modules/bedfiles.py | 176 +++++++++++++++++++++++++- manual_testing.py | 18 ++- 4 files changed, 262 insertions(+), 5 deletions(-) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index 879eb821..bc7d1be1 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -6,6 +6,7 @@ import boto3 import qdrant_client +from qdrant_client import QdrantClient, models import s3fs import yacman import zarr @@ -16,6 +17,7 @@ from geniml.search.interfaces import BiVectorSearchInterface from geniml.search.query2vec import BED2Vec from pephubclient import PEPHubClient +from qdrant_client.http.models import VectorsConfig from zarr import Group as Z_GROUP from bbconf.config_parser.const import ( @@ -60,6 +62,7 @@ def __init__(self, config: Union[Path, str], init_ml: bool = True): self._qdrant_engine = self._init_qdrant_backend() self._qdrant_text_engine = self._init_qdrant_text_backend() + self._qdrant_advanced_engine = self._init_qdrant_advanced_backend() if init_ml: self._b2bsi = self._init_b2bsi_object() @@ -264,6 +267,59 @@ def _init_qdrant_text_backend(self) -> Union[QdrantBackend, None]: ) return None + def _init_qdrant_advanced_backend(self) -> QdrantClient: + """ + Create qdrant client text embedding object using credentials provided in config file + + :return: QdrantClient + """ + + COLLECTION_NAME = "bedbase_query_search" + # DIMENTIONS = 1024 # normal: 384 + DIMENTIONS = 384 # normal: 384 + + _LOGGER.info(f"Initializing qdrant text advanced engine...") + + qdrant_cl = QdrantClient( + url=self.config.qdrant.host, + # port=self.port, + api_key=self.config.qdrant.api_key, + ) + + if not qdrant_cl.collection_exists(COLLECTION_NAME): + _LOGGER.info( + f"Collection 'bedbase_query_search' does not exist, creating it." + ) + qdrant_cl.create_collection( + collection_name=COLLECTION_NAME, + vectors_config=models.VectorParams( + size=DIMENTIONS, distance=models.Distance.COSINE + ), + quantization_config=models.ScalarQuantization( + scalar=models.ScalarQuantizationConfig( + type=models.ScalarType.INT8, + quantile=0.99, + always_ram=True, + ), + ), + ) + + return qdrant_cl + + # # Create collection only if it does not exist + # try: + # collection_info = self.qd_client.get_collection(collection_name=self.collection) + # _LOGGER.info( + # f"Using collection {self.collection} with {collection_info.points_count} points." + # ) + # except Exception: # qdrant_client.http.exceptions.UnexpectedResponse + # _LOGGER.info(f"Collection {self.collection} does not exist, creating it.") + # self.qd_client.recreate_collection( + # collection_name=self.collection, + # vectors_config=self.config, + # quantization_config=DEFAULT_QUANTIZATION_CONFIG, + # ) + def _init_bivec_object(self) -> Union[BiVectorSearchInterface, None]: """ Create BiVectorSearchInterface object using credentials provided in config file diff --git a/bbconf/models/bed_models.py b/bbconf/models/bed_models.py index f101c299..e39fbde7 100644 --- a/bbconf/models/bed_models.py +++ b/bbconf/models/bed_models.py @@ -257,3 +257,20 @@ class RefGenValidReturnModel(BaseModel): id: str provided_genome: Union[str, None] = None compared_genome: List[RefGenValidModel] + + +class VectorMetadata(BaseModel): + id: str + name: str + description: str + cell_line: str + cell_type: str + tissue: str + target: str + treatment: str + assay: str + genome_alias: str + genome_digest: Union[str, None] = None + species_name: str + + # embedding: List[float] diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index 68831e51..ae2b5d9d 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -17,6 +17,8 @@ from sqlalchemy.orm import Session, aliased from sqlalchemy.orm.attributes import flag_modified from tqdm import tqdm +from fastembed import TextEmbedding +from qdrant_client import models from bbconf.config_parser.bedbaseconfig import BedBaseConfig from bbconf.const import DEFAULT_LICENSE, PKG_NAME, ZARR_TOKENIZED_FOLDER @@ -59,12 +61,15 @@ TokenizedBedResponse, TokenizedPathResponse, UniverseMetadata, + VectorMetadata, ) _LOGGER = getLogger(PKG_NAME) QDRANT_GENOME = "hg38" +QDRANT_TEXT_SEARCH_COLLECTION = "bedbase_query_search" + class BedAgentBedFile: """ @@ -85,6 +90,9 @@ def __init__(self, config: BedBaseConfig, bbagent_obj=None): self._config = config self.bb_agent = bbagent_obj + self._embedding_model = TextEmbedding(config.config.path.text2vec) + # self._embedding_model = TextEmbedding("BAAI/bge-large-en-v1.5") + def get(self, identifier: str, full: bool = False) -> BedMetadataAll: """ Get file metadata by identifier. @@ -1221,6 +1229,7 @@ def sql_search( self, query: str, genome: str = None, + assay: str = None, limit: int = 10, offset: int = 0, ) -> BedListSearchResult: @@ -1249,13 +1258,19 @@ def sql_search( BedMetadata.cell_type.ilike(sql_search_str), ) + condition_statement = or_statement + if genome_alias := genome: _LOGGER.debug(f"Filtering by genome: {genome_alias}") - condition_statement = and_(Bed.genome_alias == genome_alias, or_statement) + condition_statement = and_( + Bed.genome_alias == genome_alias, condition_statement + ) - else: - condition_statement = or_statement + if assay: + _LOGGER.debug(f"Filtering by assay: {assay}") + + condition_statement = and_(BedMetadata.assay == assay, condition_statement) statement = statement.where(condition_statement) @@ -1287,6 +1302,37 @@ def sql_search( results=result_list, ) + # def comprehensive_sql_search( + # self, + # query: str, + # genome: str = None, + # assay: str = None, + # bed_compliance: str = None, + # limit: int = 10, + # offset: int = 0, + # ) -> BedListSearchResult: + # """ + # Comprehensive SQL search for bed files with multiple filters. + # + # :param query: text query -looking at: name, description, cell_type, cell_line, tissue, target, treatment. + # + # :param genome: genome alias to filter results. Default is None, which means no filtering by genome. + # :param assay: filter by assay type. Default is None, which means no filtering by assay. + # :param bed_compliance: filter by bed compliance type. Default is None, which means no filtering by bed compliance. + # :param limit: number of results to return. Default is 10. + # :param offset: offset to start from + # + # :return: + # """ + # + # query = f"%{query.strip()}%" + # query_search = or_(Bed.name.ilike(query), + # Bed.description.ilike(query), + # BedMetadata.cell_type.ilike(query), + # BedMetadata.cell_line.ilike(query), + # BedMetadata.tissue.ilike(query), + # BedMetadata) + def _sql_search_count(self, condition_statement) -> int: """ Get number of total found files in the database. @@ -1853,3 +1899,127 @@ def _update_sources( flag_modified(bedmetadata_object, "global_experiment_id") session.commit() + + def _get_search_metadata(self, batch: int = 1000) -> None: + """ + Get metadata for vector database. + + :param batch: number of files to upload in one batch + + :return: metadata for vector database + """ + + statement = ( + select(Bed).join(BedMetadata, Bed.id == BedMetadata.id).limit(150000) + ) + + with Session(self._sa_engine) as session: + results = session.scalars(statement) + + points = [] + results = [result for result in results] + + with tqdm(total=len(results), position=0, leave=True) as pbar: + processed_number = 0 + for result in results: + text = f"{result.name}. {result.description}. {result.annotations.cell_line}. {result.annotations.cell_type}. {result.annotations.tissue}. {result.annotations.target}. {result.annotations.treatment}. {result.annotations.assay}. {result.annotations.species_name}." + embeddings_list = list(self._embedding_model.embed(text)) + # result_list.append( + data = VectorMetadata( + id=result.id, + name=result.name, + description=result.description, + genome_alias=result.genome_alias, + genome_digest=result.genome_digest, + cell_line=result.annotations.cell_line, + cell_type=result.annotations.cell_type, + tissue=result.annotations.tissue, + target=result.annotations.target, + treatment=result.annotations.treatment, + assay=result.annotations.assay, + species_name=result.annotations.species_name, + ) + + points.append( + PointStruct( + id=result.id, + vector=list(embeddings_list[0]), + payload=data.model_dump(), + ) + ) + + if processed_number % batch == 0: + pbar.set_description( + f"Uploading points to qdrant using batch..." + ) + operation_info = self._config._qdrant_advanced_engine.upsert( + collection_name=QDRANT_TEXT_SEARCH_COLLECTION, + points=points, + ) + pbar.write("Uploaded batch to qdrant.") + points = [] + assert operation_info.status == "completed" + + pbar.write(f"File: {result.id} successfully indexed.") + pbar.update(1) + + return None + + def comp_search( + self, + query: str = "liver", + genome_alias: str = "", + assay: str = "", + limit: int = 100, + offset: int = 0, + ) -> BedListSearchResult: + """ """ + + should_statement = [] + + if genome_alias: + should_statement.append( + models.FieldCondition( + key="genome_alias", + match=models.MatchValue(value=genome_alias), + ) + ) + if assay: + should_statement.append( + models.FieldCondition( + key="assay", + match=models.MatchValue(value=assay), + ) + ) + + embeddings_list = list(self._embedding_model.embed(query))[0] + + results = self._config._qdrant_advanced_engine.search( + collection_name=QDRANT_TEXT_SEARCH_COLLECTION, + query_vector=list(embeddings_list), + limit=limit, + offset=offset, + search_params=models.SearchParams( + exact=True, + ), + query_filter=models.Filter(should=should_statement), + with_payload=True, + with_vectors=True, + ) + result_list = [] + for result in results: + result_id = result.id.replace("-", "") + result_list.append( + QdrantSearchResult( + id=result_id, + payload=result.payload, + score=result.score, + metadata=self.get(result_id, full=False), + ) + ) + return BedListSearchResult( + count=self.bb_agent.get_stats().bedfiles_number, + limit=limit, + offset=offset, + results=result_list, + ) diff --git a/manual_testing.py b/manual_testing.py index 99e53721..bb47829c 100644 --- a/manual_testing.py +++ b/manual_testing.py @@ -8,6 +8,7 @@ from gtars.utils import read_tokens_from_gtok import matplotlib.pyplot as plt import numpy as np +import time # from gtars.tokenizers import RegionSet @@ -213,7 +214,6 @@ def config_t(): def compreh_stats(): from bbconf import BedBaseAgent - import time agent = BedBaseAgent(config="/home/bnt4me/virginia/repos/bedhost/config.yaml") @@ -321,6 +321,19 @@ def get_genomes(): print(results) +def new_search(): + from bbconf import BedBaseAgent + + agent = BedBaseAgent(config="/home/bnt4me/virginia/repos/bedhost/config.yaml") + time1 = time.time() + + results = agent.bed._get_search_metadata() + # results = agent.bed.comp_search() + time2 = time.time() + + print(f"Time taken: {time2 - time1} seconds") + + if __name__ == "__main__": # zarr_s3() # add_s3() @@ -331,6 +344,7 @@ def get_genomes(): # neighbour_beds() # sql_search() # config_t() - compreh_stats() + # compreh_stats() # get_unprocessed_files() # get_genomes() + new_search() From 6ca3aed78238e3bef5fd5d158ec2e32da96143f3 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Fri, 27 Jun 2025 15:59:44 -0400 Subject: [PATCH 04/30] Fixed qdrant filtering and added list assays function --- bbconf/bbagent.py | 16 ++++++++++++++++ bbconf/modules/bedfiles.py | 5 ++++- manual_testing.py | 12 +++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/bbconf/bbagent.py b/bbconf/bbagent.py index 3796bf3f..bfbd84d0 100644 --- a/bbconf/bbagent.py +++ b/bbconf/bbagent.py @@ -289,6 +289,22 @@ def get_list_genomes(self) -> List[str]: genomes = session.execute(statement).all() return [result[0] for result in genomes if result[0]] + def get_list_assays(self): + """ + Get list of genomes from the database + + :return: list of genomes + """ + + with Session(self.config.db_engine.engine) as session: + statement = ( + select(BedMetadata.assay) + .group_by(BedMetadata.assay) + .order_by(func.count(BedMetadata.assay).desc()) + ) + results = session.execute(statement).all() + return [result[0] for result in results if result[0]] + @cached_property def list_of_licenses(self) -> List[str]: """ diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index ae2b5d9d..ec64b1b3 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -2002,7 +2002,10 @@ def comp_search( search_params=models.SearchParams( exact=True, ), - query_filter=models.Filter(should=should_statement), + # query_filter=models.Filter(should=should_statement) if should_statement else None, + query_filter=( + models.Filter(must=should_statement) if should_statement else None + ), with_payload=True, with_vectors=True, ) diff --git a/manual_testing.py b/manual_testing.py index bb47829c..e0a25e61 100644 --- a/manual_testing.py +++ b/manual_testing.py @@ -334,6 +334,14 @@ def new_search(): print(f"Time taken: {time2 - time1} seconds") +def get_assay_list(): + from bbconf import BedBaseAgent + + agent = BedBaseAgent(config="/home/bnt4me/virginia/repos/bedhost/config.yaml") + results = agent.get_list_assays() + print(results) + + if __name__ == "__main__": # zarr_s3() # add_s3() @@ -347,4 +355,6 @@ def new_search(): # compreh_stats() # get_unprocessed_files() # get_genomes() - new_search() + # new_search() + + get_assay_list() From 3cadafb2109653101f40d3ebe6c193ba25adadbc Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Sun, 29 Jun 2025 23:31:13 -0400 Subject: [PATCH 05/30] Added more plots and statistics --- bbconf/bbagent.py | 92 ++++++++++++++++++++++++++++++++---- bbconf/models/base_models.py | 50 ++++++++++++-------- bbconf/models/bed_models.py | 5 +- manual_testing.py | 8 ++-- 4 files changed, 123 insertions(+), 32 deletions(-) diff --git a/bbconf/bbagent.py b/bbconf/bbagent.py index bfbd84d0..b5e8b6cd 100644 --- a/bbconf/bbagent.py +++ b/bbconf/bbagent.py @@ -30,6 +30,7 @@ AllFilesInfo, FileInfo, BinValues, + GEOStatistics, ) from bbconf.modules.bedfiles import BedAgentBedFile from bbconf.modules.bedsets import BedAgentBedSet @@ -110,6 +111,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: _LOGGER.info("Getting detailed statistics for all bed files") with Session(self.config.db_engine.engine) as session: + bed_compliance = { f[0]: f[1] for f in session.execute( @@ -160,6 +162,8 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: list_mean_width_bins = self._bin_mean_region_width(list_mean_width) list_file_size_bins = self._bin_file_size(list_file_size) + geo_stats = self._get_geo_stats(session) + if concise: bed_compliance_concise = dict(list(bed_compliance.items())[0:slice_value]) bed_compliance_concise["other"] = sum( @@ -201,6 +205,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: mean_region_width=list_mean_width_bins, file_size=list_file_size_bins, number_of_regions=number_of_regions_bins, + geo=geo_stats, ) return FileStats( @@ -213,6 +218,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: mean_region_width=list_mean_width_bins, file_size=list_file_size_bins, number_of_regions=number_of_regions_bins, + geo=geo_stats, ) def get_detailed_usage(self) -> UsageStats: @@ -254,6 +260,7 @@ def get_detailed_usage(self) -> UsageStats: .order_by(func.sum(UsageSearch.count).desc()) .limit(20) ).all() + if f[0] } bedset_search_terms = { @@ -265,6 +272,17 @@ def get_detailed_usage(self) -> UsageStats: .order_by(func.sum(UsageSearch.count).desc()) .limit(20) ).all() + if f[0] + } + + bed_downloads = { + f[0].split("/")[-1].split(".")[0]: f[1] + for f in session.execute( + select(UsageFiles.file_path, func.sum(UsageFiles.count)) + .group_by(UsageFiles.file_path) + .order_by(func.sum(UsageFiles.count).desc()) + .limit(20) + ).all() } return UsageStats( @@ -272,6 +290,7 @@ def get_detailed_usage(self) -> UsageStats: bedset_metadata=bedset_metadata, bed_search_terms=bed_search_terms, bedset_search_terms=bedset_search_terms, + bed_downloads=bed_downloads, ) def get_list_genomes(self) -> List[str]: @@ -621,8 +640,8 @@ def _bin_number_of_regions(self, number_of_regions: list) -> BinValues: return BinValues( bins=n_region_bin_edges, counts=n_region_counts, - mean=statistics.mean(number_of_regions), - meadian=statistics.median(number_of_regions), + mean=round(statistics.mean(number_of_regions), 2), + median=round(statistics.median(number_of_regions), 2), ) def _bin_mean_region_width(self, mean_region_widths: list) -> BinValues: @@ -649,8 +668,8 @@ def _bin_mean_region_width(self, mean_region_widths: list) -> BinValues: return BinValues( bins=mean_reg_width_bin_edges, counts=mean_reg_width_counts, - mean=statistics.mean(mean_region_widths), - meadian=statistics.median(mean_region_widths), + mean=round(statistics.mean(mean_region_widths), 2), + median=round(statistics.median(mean_region_widths), 2), ) def _bin_file_size(self, list_file_size: list) -> BinValues: @@ -660,21 +679,78 @@ def _bin_file_size(self, list_file_size: list) -> BinValues: :param list_file_size: list of bed file sizes in bytes :return: BinValues object containing bins and values """ - max_value_threshold = 10_000_000 + + max_value_threshold = 10 * 1024 * 1024 filtered_list_file_size = [ x for x in list_file_size if x <= max_value_threshold ] + filtered_list_file_size = [x / (1024 * 1024) for x in filtered_list_file_size] + file_size_counts, file_size_bin_edges = np.histogram( filtered_list_file_size, bins=100 ) file_size_counts = file_size_counts.astype(int).tolist() - file_size_bin_edges = file_size_bin_edges.astype(int).tolist() + file_size_bin_edges = file_size_bin_edges.astype(float).tolist() return BinValues( bins=file_size_bin_edges, counts=file_size_counts, - mean=statistics.mean(filtered_list_file_size), - meadian=statistics.median(filtered_list_file_size), + mean=round(statistics.mean(filtered_list_file_size), 2), + median=round(statistics.median(filtered_list_file_size), 2), + ) + + def _get_geo_stats(self, sa_session: Session) -> GEOStatistics: + """ + Get GEO statistics for the bedbase platform. + + :return: GEOStatistics + """ + + _LOGGER.info("Getting GEO statistics.") + + statement = select( + GeoGsmStatus.bed_id, + GeoGsmStatus.source_submission_date, + GeoGsmStatus.file_size, + ).distinct(GeoGsmStatus.sample_name) + + results = sa_session.execute(statement).all() + + years = [] + file_sizes = [] + + for result in results: + if result[1]: + years.append(result[1].year) + if result[2]: + file_sizes.append(result[2]) + + years_array = np.array([y for y in years if y is not None]) + unique_years, counts = np.unique(years_array, return_counts=True) + + cumulative_counts = np.cumsum(counts) + unique_years = unique_years.astype(str).tolist() + + years_cumulative = dict(zip(unique_years, cumulative_counts)) + years_numbers = dict(zip(unique_years, counts)) + + MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB + file_size_filter = [ + x if x <= MAX_FILE_SIZE else MAX_FILE_SIZE + 1 for x in file_sizes + ] + + file_size_filter = [x / (1024 * 1024) for x in file_size_filter] + file_size_counts, file_size_bin_edges = np.histogram(file_size_filter, bins=100) + + return GEOStatistics( + number_of_files=years_numbers, + cumulative_number_of_files=years_cumulative, + file_sizes=BinValues( + bins=list(file_size_bin_edges), + counts=file_size_counts.astype(int).tolist(), + mean=round(statistics.mean(file_sizes), 2), + median=round(statistics.median(file_sizes), 2), + ), ) diff --git a/bbconf/models/base_models.py b/bbconf/models/base_models.py index c2c23f73..4ce3d0f1 100644 --- a/bbconf/models/base_models.py +++ b/bbconf/models/base_models.py @@ -26,31 +26,13 @@ class StatsReturn(BaseModel): genomes_number: int = 0 -class BinValues(BaseModel): - bins: List[Union[int, float, str]] - counts: List[int] - mean: Optional[float] - meadian: Optional[float] - - -class FileStats(BaseModel): - bed_compliance: Dict[str, int] - data_format: Dict[str, int] - file_genome: Dict[str, int] - file_organism: Dict[str, int] - geo_status: Dict[str, int] - bed_comments: Dict[str, int] - mean_region_width: BinValues - file_size: BinValues - number_of_regions: BinValues - - class UsageStats(BaseModel): # file_downloads: Dict[str, int] # Placeholder for tracking file download statistics in the future. bed_metadata: Dict[str, int] bedset_metadata: Dict[str, int] bed_search_terms: Dict[str, int] bedset_search_terms: Dict[str, int] + bed_downloads: Dict[str, int] class UsageModel(BaseModel): @@ -90,3 +72,33 @@ class AllFilesInfo(BaseModel): total: int files: List[FileInfo] + + +class BinValues(BaseModel): + bins: List[Union[int, float, str]] + counts: List[int] + mean: float + median: float + + +class GEOStatistics(BaseModel): + """ + GEO statistics for files. + """ + + number_of_files: Dict[str, int] + cumulative_number_of_files: Dict[str, int] + file_sizes: BinValues + + +class FileStats(BaseModel): + bed_compliance: Dict[str, int] + data_format: Dict[str, int] + file_genome: Dict[str, int] + file_organism: Dict[str, int] + geo_status: Dict[str, int] + bed_comments: Dict[str, int] + mean_region_width: BinValues + file_size: BinValues + number_of_regions: BinValues + geo: GEOStatistics diff --git a/bbconf/models/bed_models.py b/bbconf/models/bed_models.py index e39fbde7..4aaf50c5 100644 --- a/bbconf/models/bed_models.py +++ b/bbconf/models/bed_models.py @@ -85,7 +85,7 @@ class BedPEPHub(BaseModel): species_id: str = "" cell_type: str = "" cell_line: str = "" - exp_protocol: str = Field("", description="Experimental protocol (e.g. ChIP-seq)") + assay: str = Field("", description="Experimental protocol (e.g. ChIP-seq)") library_source: str = Field( "", description="Library source (e.g. genomic, transcriptomic)" ) @@ -130,7 +130,8 @@ class StandardMeta(BaseModel): "", description="Library source (e.g. genomic, transcriptomic)" ) assay: str = Field( - "", description="Experimental protocol (e.g. ChIP-seq)", alias="exp_protocol" + "", + description="Experimental protocol (e.g. ChIP-seq)", ) antibody: str = Field("", description="Antibody used in the assay") target: str = Field("", description="Target of the assay (e.g. H3K4me3)") diff --git a/manual_testing.py b/manual_testing.py index e0a25e61..2eb313be 100644 --- a/manual_testing.py +++ b/manual_testing.py @@ -219,7 +219,9 @@ def compreh_stats(): time1 = time.time() - results = agent.get_detailed_stats() + # results = agent.get_detailed_stats() + + results = agent.get_detailed_usage() time2 = time.time() print(time2 - time1) @@ -352,9 +354,9 @@ def get_assay_list(): # neighbour_beds() # sql_search() # config_t() - # compreh_stats() + compreh_stats() # get_unprocessed_files() # get_genomes() # new_search() - get_assay_list() + # get_assay_list() From b230770ebb71eee8896143d1900e5268936f7093 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 1 Jul 2025 17:13:56 -0400 Subject: [PATCH 06/30] search improvements --- bbconf/config_parser/bedbaseconfig.py | 18 +- bbconf/config_parser/const.py | 1 + bbconf/config_parser/models.py | 3 + bbconf/models/bed_models.py | 4 +- bbconf/modules/bedfiles.py | 229 ++++++++++++++++++++++++-- manual_testing.py | 14 +- 6 files changed, 252 insertions(+), 17 deletions(-) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index bc7d1be1..27c00a6c 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -274,15 +274,14 @@ def _init_qdrant_advanced_backend(self) -> QdrantClient: :return: QdrantClient """ - COLLECTION_NAME = "bedbase_query_search" - # DIMENTIONS = 1024 # normal: 384 - DIMENTIONS = 384 # normal: 384 + COLLECTION_NAME = self.config.qdrant.search_collection + DIMENTIONS = 384 _LOGGER.info(f"Initializing qdrant text advanced engine...") qdrant_cl = QdrantClient( url=self.config.qdrant.host, - # port=self.port, + port=self.config.qdrant.port, api_key=self.config.qdrant.api_key, ) @@ -304,6 +303,17 @@ def _init_qdrant_advanced_backend(self) -> QdrantClient: ), ) + qdrant_cl.create_payload_index( + collection_name=COLLECTION_NAME, + field_name="assay", + field_schema="keyword", + ) + qdrant_cl.create_payload_index( + collection_name=COLLECTION_NAME, + field_name="genome_alias", + field_schema="keyword", + ) + return qdrant_cl # # Create collection only if it does not exist diff --git a/bbconf/config_parser/const.py b/bbconf/config_parser/const.py index 1c692880..6c0f8fb9 100644 --- a/bbconf/config_parser/const.py +++ b/bbconf/config_parser/const.py @@ -7,6 +7,7 @@ DEFAULT_QDRANT_PORT = 6333 DEFAULT_QDRANT_COLLECTION_NAME = "bedbase" DEFAULT_QDRANT_TEXT_COLLECTION_NAME = "bed_text" +DEFAULT_QDRANT_SEARCH_COLLECTION_NAME = "bedbase_query_search2" DEFAULT_QDRANT_API_KEY = None DEFAULT_SERVER_PORT = 80 diff --git a/bbconf/config_parser/models.py b/bbconf/config_parser/models.py index 5eb75838..f8d8adfe 100644 --- a/bbconf/config_parser/models.py +++ b/bbconf/config_parser/models.py @@ -14,6 +14,7 @@ DEFAULT_PEPHUB_NAMESPACE, DEFAULT_PEPHUB_TAG, DEFAULT_QDRANT_COLLECTION_NAME, + DEFAULT_QDRANT_TEXT_COLLECTION_NAME, DEFAULT_QDRANT_PORT, DEFAULT_QDRANT_TEXT_COLLECTION_NAME, DEFAULT_REGION2_VEC_MODEL, @@ -21,6 +22,7 @@ DEFAULT_SERVER_HOST, DEFAULT_SERVER_PORT, DEFAULT_TEXT2VEC_MODEL, + DEFAULT_QDRANT_SEARCH_COLLECTION_NAME, ) _LOGGER = logging.getLogger(__name__) @@ -54,6 +56,7 @@ class ConfigQdrant(BaseModel): api_key: Optional[str] = None file_collection: str = DEFAULT_QDRANT_COLLECTION_NAME text_collection: Optional[str] = DEFAULT_QDRANT_TEXT_COLLECTION_NAME + search_collection: Optional[str] = DEFAULT_QDRANT_SEARCH_COLLECTION_NAME class ConfigServer(BaseModel): diff --git a/bbconf/models/bed_models.py b/bbconf/models/bed_models.py index 4aaf50c5..eb59f1e2 100644 --- a/bbconf/models/bed_models.py +++ b/bbconf/models/bed_models.py @@ -273,5 +273,7 @@ class VectorMetadata(BaseModel): genome_alias: str genome_digest: Union[str, None] = None species_name: str - + # summary: str + # global_sample_id: str + # original_file_name: str # embedding: List[float] diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index ec64b1b3..b04ab03b 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -12,10 +12,11 @@ from pydantic import BaseModel from qdrant_client.http.models import PointStruct from qdrant_client.models import Distance, PointIdsList, VectorParams -from sqlalchemy import and_, delete, func, or_, select +from sqlalchemy import and_, delete, func, or_, select, cast from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, aliased from sqlalchemy.orm.attributes import flag_modified +from sqlalchemy.dialects import postgresql from tqdm import tqdm from fastembed import TextEmbedding from qdrant_client import models @@ -26,10 +27,12 @@ Bed, BedMetadata, BedStats, + BedSets, Files, GenomeRefStats, TokenizedBed, Universes, + BedFileBedSetRelation, ) from bbconf.exceptions import ( BedBaseConfError, @@ -68,8 +71,6 @@ QDRANT_GENOME = "hg38" -QDRANT_TEXT_SEARCH_COLLECTION = "bedbase_query_search" - class BedAgentBedFile: """ @@ -1900,29 +1901,48 @@ def _update_sources( session.commit() - def _get_search_metadata(self, batch: int = 1000) -> None: + def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> None: """ - Get metadata for vector database. + Reindex all bed files for semantic database :param batch: number of files to upload in one batch + :param purge: resets indexed in database for all files to False :return: metadata for vector database """ + # Add column that will indicate if this file is indexed or not statement = ( - select(Bed).join(BedMetadata, Bed.id == BedMetadata.id).limit(150000) + select(Bed) + .join(BedMetadata, Bed.id == BedMetadata.id) + .where(Bed.indexed == False) + .limit(150000) ) with Session(self._sa_engine) as session: + + if purge: + _LOGGER.info("Purging indexed files in the database ...") + session.query(Bed).update({Bed.indexed: False}) + session.commit() + _LOGGER.info("Purged indexed files in the database successfully!") + + _LOGGER.info("Fetching data from the database ...") results = session.scalars(statement) + _LOGGER.info("Fetch data successfully!") + points = [] results = [result for result in results] with tqdm(total=len(results), position=0, leave=True) as pbar: processed_number = 0 for result in results: - text = f"{result.name}. {result.description}. {result.annotations.cell_line}. {result.annotations.cell_type}. {result.annotations.tissue}. {result.annotations.target}. {result.annotations.treatment}. {result.annotations.assay}. {result.annotations.species_name}." + text = ( + f"{result.name}. {result.description}. {result.annotations.cell_line}. {result.annotations.cell_type}." + f" {result.annotations.tissue}. {result.annotations.target}. {result.annotations.treatment}. " + f"{result.annotations.assay}. {result.annotations.species_name}." + ) embeddings_list = list(self._embedding_model.embed(text)) # result_list.append( data = VectorMetadata( @@ -1947,15 +1967,18 @@ def _get_search_metadata(self, batch: int = 1000) -> None: payload=data.model_dump(), ) ) + processed_number += 1 + result.indexed = True if processed_number % batch == 0: pbar.set_description( f"Uploading points to qdrant using batch..." ) operation_info = self._config._qdrant_advanced_engine.upsert( - collection_name=QDRANT_TEXT_SEARCH_COLLECTION, + collection_name=self._config.config.qdrant.search_collection, points=points, ) + session.commit() pbar.write("Uploaded batch to qdrant.") points = [] assert operation_info.status == "completed" @@ -1963,7 +1986,131 @@ def _get_search_metadata(self, batch: int = 1000) -> None: pbar.write(f"File: {result.id} successfully indexed.") pbar.update(1) - return None + return None + + ###### More metadata + # def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> None: + # """ + # Reindex all bed files for semantic database + # + # :param batch: number of files to upload in one batch + # :param purge: resets indexed in database for all files to False + # + # :return: metadata for vector database + # """ + # + # # Add column that will indicate if this file is indexed or not + # + # statement = ( + # select( + # Bed + # # Bed.id, + # # Bed.name, + # # Bed.description, + # # Bed.genome_alias, + # # Bed.genome_digest, + # # BedMetadata.assay, + # # BedMetadata.cell_line, + # # BedMetadata.species_name, + # # BedMetadata.cell_type, + # # BedMetadata.antibody, + # # BedMetadata.tissue, + # # BedMetadata.target, + # # BedMetadata.treatment, + # # BedMetadata.original_file_name, + # # BedMetadata.global_sample_id, + # # BedSets.summary, + # ) + # .join(BedMetadata, Bed.id == BedMetadata.id) + # .outerjoin( + # BedFileBedSetRelation, Bed.id == BedFileBedSetRelation.bedfile_id + # ) + # .outerjoin( + # BedSets, + # BedSets.id == BedFileBedSetRelation.bedset_id + # ).where( + # and_( + # BedFileBedSetRelation.bedset_id == BedSets.id, + # or_( + # BedSets.summary == None, + # ~BedSets.summary.ilike("This SuperSeries%"), + # ), + # Bed.indexed == False, + # ), + # ) + # .limit(150000) + # ) + # + # with Session(self._sa_engine) as session: + # + # if purge: + # session.query(Bed).update({Bed.indexed: False}) + # session.commit() + # + # _LOGGER.info("Fetching data from the database ...") + # results = session.scalars(statement) + # + # _LOGGER.info("Fetch data successfully!") + # + # points = [] + # results = [result for result in results] + # + # with tqdm(total=len(results), position=0, leave=True) as pbar: + # processed_number = 0 + # for result in results: + # + # data = VectorMetadata( + # id=result.id, + # name=result.name, + # description=result.description, + # genome_alias=result.genome_alias, + # genome_digest=result.genome_digest, + # cell_line=result.annotations.cell_line, + # cell_type=result.annotations.cell_type, + # tissue=result.annotations.tissue, + # target=result.annotations.target, + # treatment=result.annotations.treatment, + # assay=result.annotations.assay, + # species_name=result.annotations.species_name, + # summary=result.bedsets[0].bedset.summary or "" if result.bedsets else "", + # global_sample_id="; ".join(result.annotations.global_sample_id).replace("geo:", "").replace("encode:", ""), + # original_file_name=result.annotations.original_file_name, + # ) + # + # text = (f"{data.id}. {data.description}. {data.name}. {data.cell_line}. {data.cell_type}. " + # f"{data.tissue}. {data.target}. {data.treatment}. {data.assay}. {data.global_sample_id}. " + # f"{data.summary}. {data.original_file_name}") + # embeddings_list = list(self._embedding_model.embed(text)) + # + # + # points.append( + # PointStruct( + # id=result.id, + # vector=list(embeddings_list[0]), + # payload=data.model_dump(exclude={"summary"}), + # ) + # ) + # processed_number += 1 + # + # result.indexed = True + # + # if processed_number % batch == 0: + # pbar.set_description( + # f"Uploading points to qdrant using batch..." + # ) + # operation_info = self._config._qdrant_advanced_engine.upsert( + # collection_name=self._config.config.qdrant.search_collection, + # points=points, + # ) + # session.commit() + # pbar.write("Uploaded batch to qdrant.") + # points = [] + # assert operation_info.status == "completed" + # + # pbar.write(f"File: {result.id} successfully indexed.") + # pbar.update(1) + # + # return None def comp_search( self, @@ -1995,7 +2142,7 @@ def comp_search( embeddings_list = list(self._embedding_model.embed(query))[0] results = self._config._qdrant_advanced_engine.search( - collection_name=QDRANT_TEXT_SEARCH_COLLECTION, + collection_name=self._config.config.qdrant.search_collection, query_vector=list(embeddings_list), limit=limit, offset=offset, @@ -2026,3 +2173,65 @@ def comp_search( offset=offset, results=result_list, ) + + def search_external_file(self, source: str, accession: str) -> BedListSearchResult: + """ """ + if source not in ["geo", "encode"]: + raise BedBaseConfError( + f"Source {source} is not supported. Supported sources are: 'geo', 'encode'." + ) + + if source == "geo" and accession.startswith("gse"): + statement = ( + select(Bed) + .join(BedMetadata, Bed.id == BedMetadata.id) + .where( + BedMetadata.global_experiment_id.contains( + cast( + [f"{source}:{accession}"], + postgresql.ARRAY(postgresql.VARCHAR), + ) + ) + ) + ) + else: + statement = ( + select(Bed) + .join(BedMetadata, Bed.id == BedMetadata.id) + .where( + BedMetadata.global_sample_id.contains( + cast( + [f"{source}:{accession}"], + postgresql.ARRAY(postgresql.VARCHAR), + ) + ) + ) + ) + + with Session(self._sa_engine) as session: + + bed_objects = session.scalars(statement) + results = [ + BedMetadataBasic( + **bedfile_obj.__dict__, + annotation=StandardMeta( + **( + bedfile_obj.annotations.__dict__ + if bedfile_obj.annotations + else {} + ) + ), + ) + for bedfile_obj in bed_objects + ] + result_list = [ + QdrantSearchResult(id=result.id, score=1, metadata=result) + for result in results + ] + + return BedListSearchResult( + count=len(result_list), + limit=len(result_list), + offset=0, + results=result_list, + ) diff --git a/manual_testing.py b/manual_testing.py index 2eb313be..e14a1eba 100644 --- a/manual_testing.py +++ b/manual_testing.py @@ -329,7 +329,7 @@ def new_search(): agent = BedBaseAgent(config="/home/bnt4me/virginia/repos/bedhost/config.yaml") time1 = time.time() - results = agent.bed._get_search_metadata() + results = agent.bed.reindex_semantic_search() # results = agent.bed.comp_search() time2 = time.time() @@ -344,6 +344,15 @@ def get_assay_list(): print(results) +def external_search(): + from bbconf import BedBaseAgent + + agent = BedBaseAgent(config="/home/bnt4me/virginia/repos/bedhost/config.yaml") + + result = agent.bed.search_external_file("geo", "gsm1399546") + result + + if __name__ == "__main__": # zarr_s3() # add_s3() @@ -354,9 +363,10 @@ def get_assay_list(): # neighbour_beds() # sql_search() # config_t() - compreh_stats() + # compreh_stats() # get_unprocessed_files() # get_genomes() # new_search() + external_search() # get_assay_list() From 5c61b5cba2596b0f487445f610ba7c34dbe752d4 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 17 Jul 2025 22:04:13 -0400 Subject: [PATCH 07/30] Fixed update bed file method --- bbconf/modules/bedfiles.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index b04ab03b..e515793c 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -804,6 +804,8 @@ def update( bed_object.processed = processed bed_object.indexed = upload_qdrant bed_object.last_update_date = datetime.datetime.now(datetime.timezone.utc) + if bed_metadata.description: + bed_object.description = bed_metadata.description session.commit() From 4ad46d06eacd9be29992819bba648bfdf546e0b6 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 17 Jul 2025 22:30:13 -0400 Subject: [PATCH 08/30] Added docstring and cleaning --- bbconf/modules/bedfiles.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index e515793c..9bd442bd 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -1206,6 +1206,12 @@ def bed_to_bed_search( limit: int = 10, offset: int = 0, ) -> BedListSearchResult: + """ + Search for bed files by using region set in qdrant database. + + + + """ results = self._config.b2bsi.query_search( region_set, limit=limit, offset=offset ) @@ -1242,6 +1248,7 @@ def sql_search( :param query: text query :param genome: genome alias to filter results + :param assay: filter by assay type :param limit: number of results to return :param offset: offset to start from @@ -1603,6 +1610,7 @@ def get_tokenized(self, bed_id: str, universe_id: str) -> TokenizedBedResponse: :return: zarr path """ + if not self.exist_tokenized(bed_id, universe_id): raise TokenizeFileNotExistError("Tokenized file not found in the database.") univers_group = self._config.zarr_root.require_group(universe_id) @@ -1910,7 +1918,7 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non :param batch: number of files to upload in one batch :param purge: resets indexed in database for all files to False - :return: metadata for vector database + :return: None """ # Add column that will indicate if this file is indexed or not @@ -2114,6 +2122,7 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non # # return None + # TODO: should be renamed def comp_search( self, query: str = "liver", @@ -2122,7 +2131,18 @@ def comp_search( limit: int = 100, offset: int = 0, ) -> BedListSearchResult: - """ """ + """ + Run semantic search for bed files using qdrant. + This is not bivec search, but ususal qdrant search with embeddings. + + :param query: text query to search for + :param genome_alias: genome alias to filter results + :param assay: filter by assay type + :param limit: number of results to return + :param offset: offset to start from + + :return: list of bed file metadata + """ should_statement = [] @@ -2177,7 +2197,15 @@ def comp_search( ) def search_external_file(self, source: str, accession: str) -> BedListSearchResult: - """ """ + """ + Search for bed files by external source and accession number. + e.g. source='geo', accession='GSE12345' + + :param source: external source, e.g. 'geo' or 'encode' + :param accession: accession number, e.g. 'GSE12345' or 'ENCSR12345' + + :return: list of bed file metadata + """ if source not in ["geo", "encode"]: raise BedBaseConfError( f"Source {source} is not supported. Supported sources are: 'geo', 'encode'." From 4647d4d0960e89f617b79d0ac190058e5ce285e6 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 17 Jul 2025 23:39:56 -0400 Subject: [PATCH 09/30] updated reindexing --- bbconf/modules/bedfiles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index 9bd442bd..461d6e1d 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -1949,9 +1949,9 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non processed_number = 0 for result in results: text = ( - f"{result.name}. {result.description}. {result.annotations.cell_line}. {result.annotations.cell_type}." + f"{result.description}. {result.annotations.cell_line}. {result.annotations.cell_type}." f" {result.annotations.tissue}. {result.annotations.target}. {result.annotations.treatment}. " - f"{result.annotations.assay}. {result.annotations.species_name}." + f"{result.annotations.assay}. {result.name}." ) embeddings_list = list(self._embedding_model.embed(text)) # result_list.append( From 8d963e503c35c2f7c2432e9c3f719bd92d60712c Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 21 Jul 2025 15:51:30 -0400 Subject: [PATCH 10/30] updated text for embeddings --- bbconf/config_parser/const.py | 2 +- bbconf/modules/bedfiles.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bbconf/config_parser/const.py b/bbconf/config_parser/const.py index 6c0f8fb9..b9541f51 100644 --- a/bbconf/config_parser/const.py +++ b/bbconf/config_parser/const.py @@ -7,7 +7,7 @@ DEFAULT_QDRANT_PORT = 6333 DEFAULT_QDRANT_COLLECTION_NAME = "bedbase" DEFAULT_QDRANT_TEXT_COLLECTION_NAME = "bed_text" -DEFAULT_QDRANT_SEARCH_COLLECTION_NAME = "bedbase_query_search2" +DEFAULT_QDRANT_SEARCH_COLLECTION_NAME = "bedbase_query_search" DEFAULT_QDRANT_API_KEY = None DEFAULT_SERVER_PORT = 80 diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index 461d6e1d..24591e18 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -741,7 +741,7 @@ def update( bed_metadata = StandardMeta(**metadata if metadata else {}) classification = BedClassification(**classification if classification else {}) - if upload_pephub: + if upload_pephub and metadata: metadata = BedPEPHub(**metadata) try: self.update_pephub(identifier, metadata.model_dump(), overwrite) @@ -1171,6 +1171,7 @@ def text_to_bed_search( ) -> BedListSearchResult: """ Search for bed files by text query in qdrant database + This is bivec_search :param query: text query :param limit: number of results to return @@ -1254,6 +1255,7 @@ def sql_search( :return: list of bed file metadata """ + _LOGGER.debug(f"Looking for: {query}") statement = select(Bed).join(BedMetadata) @@ -1949,12 +1951,14 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non processed_number = 0 for result in results: text = ( - f"{result.description}. {result.annotations.cell_line}. {result.annotations.cell_type}." - f" {result.annotations.tissue}. {result.annotations.target}. {result.annotations.treatment}. " - f"{result.annotations.assay}. {result.name}." + f"biosample is {result.annotations.cell_line} / {result.annotations.cell_type} / " + f"{result.annotations.tissue} with target {result.annotations.target} " + f"assay {result.annotations.assay}." + f"File name {result.name} with summary {result.description}" ) + embeddings_list = list(self._embedding_model.embed(text)) - # result_list.append( + data = VectorMetadata( id=result.id, name=result.name, From 8ebed01ce5fd49801ddf4c389eb70e3bc926da02 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 21 Jul 2025 16:08:29 -0400 Subject: [PATCH 11/30] Added option to not include metadata in search results --- bbconf/modules/bedfiles.py | 51 ++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index 24591e18..de801586 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -1167,7 +1167,11 @@ def _embed_file(self, bed_file: Union[str, RegionSet]) -> np.ndarray: return bed_embedding.reshape(1, vec_dim) def text_to_bed_search( - self, query: str, limit: int = 10, offset: int = 0 + self, + query: str, + limit: int = 10, + offset: int = 0, + with_metadata: bool = True, ) -> BedListSearchResult: """ Search for bed files by text query in qdrant database @@ -1176,6 +1180,7 @@ def text_to_bed_search( :param query: text query :param limit: number of results to return :param offset: offset to start from + :param with_metadata: if True, will return metadata for each result :return: list of bed file metadata """ @@ -1186,7 +1191,10 @@ def text_to_bed_search( for result in results: result_id = result["id"].replace("-", "") try: - result_meta = self.get(result_id) + if with_metadata: + result_meta = self.get(result_id) + else: + result_meta = None except BEDFileNotFoundError as e: _LOGGER.warning( f"Could not retrieve metadata for bed file: {result_id}. Error: {e}" @@ -1194,8 +1202,13 @@ def text_to_bed_search( continue if result_meta: results_list.append(QdrantSearchResult(**result, metadata=result_meta)) + + if with_metadata: + count = self.bb_agent.get_detailed_stats().file_genome.get("hg38", 0) + else: + count = 0 return BedListSearchResult( - count=self.bb_agent.get_detailed_stats().file_genome.get("hg38", 0), + count=count, limit=limit, offset=offset, results=results_list, @@ -1210,8 +1223,11 @@ def bed_to_bed_search( """ Search for bed files by using region set in qdrant database. + :param region_set: RegionSet object to search for (bed file) + :param limit: number of results to return + :param offset: offset to start from - + :return: BedListSetResults """ results = self._config.b2bsi.query_search( region_set, limit=limit, offset=offset @@ -1950,6 +1966,12 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non with tqdm(total=len(results), position=0, leave=True) as pbar: processed_number = 0 for result in results: + # text = ( + # f"{result.description}. {result.annotations.cell_line}. {result.annotations.cell_type}." + # f" {result.annotations.tissue}. {result.annotations.target}." + # f"{result.annotations.assay}. {result.name}." + # ) + text = ( f"biosample is {result.annotations.cell_line} / {result.annotations.cell_type} / " f"{result.annotations.tissue} with target {result.annotations.target} " @@ -1958,7 +1980,7 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non ) embeddings_list = list(self._embedding_model.embed(text)) - + # result_list.append( data = VectorMetadata( id=result.id, name=result.name, @@ -2134,6 +2156,7 @@ def comp_search( assay: str = "", limit: int = 100, offset: int = 0, + with_metadata: bool = True, ) -> BedListSearchResult: """ Run semantic search for bed files using qdrant. @@ -2144,6 +2167,7 @@ def comp_search( :param assay: filter by assay type :param limit: number of results to return :param offset: offset to start from + :param with_metadata: if True, metadata will be returned in the results. Default is True. :return: list of bed file metadata """ @@ -2182,19 +2206,32 @@ def comp_search( with_payload=True, with_vectors=True, ) + result_list = [] for result in results: result_id = result.id.replace("-", "") + + if with_metadata: + metadata = self.get(result_id, full=False) + else: + metadata = None + result_list.append( QdrantSearchResult( id=result_id, payload=result.payload, score=result.score, - metadata=self.get(result_id, full=False), + metadata=metadata, ) ) + + if with_metadata: + count = self.bb_agent.get_stats().bedfiles_number + else: + count = 0 + return BedListSearchResult( - count=self.bb_agent.get_stats().bedfiles_number, + count=count, limit=limit, offset=offset, results=result_list, From f9cea35b8a01086baa3198122d699847d46dd952 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 21 Jul 2025 16:12:28 -0400 Subject: [PATCH 12/30] removed old semantic search and renamed search method --- bbconf/modules/bedfiles.py | 133 ++----------------------------------- 1 file changed, 4 insertions(+), 129 deletions(-) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index de801586..d62653e5 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -1136,9 +1136,9 @@ def upload_file_qdrant( def _embed_file(self, bed_file: Union[str, RegionSet]) -> np.ndarray: """ - Create embeding for bed file + Create embedding for bed file - :param bed_file: bed file path or regionset + :param bed_file: bed file path or regionet :param bed_file: path to the bed file, or RegionSet object :return np array of embeddings @@ -2024,132 +2024,7 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non return None - ###### More metadata - # def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> None: - # """ - # Reindex all bed files for semantic database - # - # :param batch: number of files to upload in one batch - # :param purge: resets indexed in database for all files to False - # - # :return: metadata for vector database - # """ - # - # # Add column that will indicate if this file is indexed or not - # - # statement = ( - # select( - # Bed - # # Bed.id, - # # Bed.name, - # # Bed.description, - # # Bed.genome_alias, - # # Bed.genome_digest, - # # BedMetadata.assay, - # # BedMetadata.cell_line, - # # BedMetadata.species_name, - # # BedMetadata.cell_type, - # # BedMetadata.antibody, - # # BedMetadata.tissue, - # # BedMetadata.target, - # # BedMetadata.treatment, - # # BedMetadata.original_file_name, - # # BedMetadata.global_sample_id, - # # BedSets.summary, - # ) - # .join(BedMetadata, Bed.id == BedMetadata.id) - # .outerjoin( - # BedFileBedSetRelation, Bed.id == BedFileBedSetRelation.bedfile_id - # ) - # .outerjoin( - # BedSets, - # BedSets.id == BedFileBedSetRelation.bedset_id - # ).where( - # and_( - # BedFileBedSetRelation.bedset_id == BedSets.id, - # or_( - # BedSets.summary == None, - # ~BedSets.summary.ilike("This SuperSeries%"), - # ), - # Bed.indexed == False, - # ), - # ) - # .limit(150000) - # ) - # - # with Session(self._sa_engine) as session: - # - # if purge: - # session.query(Bed).update({Bed.indexed: False}) - # session.commit() - # - # _LOGGER.info("Fetching data from the database ...") - # results = session.scalars(statement) - # - # _LOGGER.info("Fetch data successfully!") - # - # points = [] - # results = [result for result in results] - # - # with tqdm(total=len(results), position=0, leave=True) as pbar: - # processed_number = 0 - # for result in results: - # - # data = VectorMetadata( - # id=result.id, - # name=result.name, - # description=result.description, - # genome_alias=result.genome_alias, - # genome_digest=result.genome_digest, - # cell_line=result.annotations.cell_line, - # cell_type=result.annotations.cell_type, - # tissue=result.annotations.tissue, - # target=result.annotations.target, - # treatment=result.annotations.treatment, - # assay=result.annotations.assay, - # species_name=result.annotations.species_name, - # summary=result.bedsets[0].bedset.summary or "" if result.bedsets else "", - # global_sample_id="; ".join(result.annotations.global_sample_id).replace("geo:", "").replace("encode:", ""), - # original_file_name=result.annotations.original_file_name, - # ) - # - # text = (f"{data.id}. {data.description}. {data.name}. {data.cell_line}. {data.cell_type}. " - # f"{data.tissue}. {data.target}. {data.treatment}. {data.assay}. {data.global_sample_id}. " - # f"{data.summary}. {data.original_file_name}") - # embeddings_list = list(self._embedding_model.embed(text)) - # - # - # points.append( - # PointStruct( - # id=result.id, - # vector=list(embeddings_list[0]), - # payload=data.model_dump(exclude={"summary"}), - # ) - # ) - # processed_number += 1 - # - # result.indexed = True - # - # if processed_number % batch == 0: - # pbar.set_description( - # f"Uploading points to qdrant using batch..." - # ) - # operation_info = self._config._qdrant_advanced_engine.upsert( - # collection_name=self._config.config.qdrant.search_collection, - # points=points, - # ) - # session.commit() - # pbar.write("Uploaded batch to qdrant.") - # points = [] - # assert operation_info.status == "completed" - # - # pbar.write(f"File: {result.id} successfully indexed.") - # pbar.update(1) - # - # return None - - # TODO: should be renamed - def comp_search( + def semantic_search( self, query: str = "liver", genome_alias: str = "", @@ -2160,7 +2035,7 @@ def comp_search( ) -> BedListSearchResult: """ Run semantic search for bed files using qdrant. - This is not bivec search, but ususal qdrant search with embeddings. + This is not bivec search, but usual qdrant search with embeddings. :param query: text query to search for :param genome_alias: genome alias to filter results From adc411686ec85189cc905a696dd953f3f5d2fc1c Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 7 Aug 2025 18:07:38 -0400 Subject: [PATCH 13/30] fixed uploading to semantic search qdarnt --- bbconf/modules/bedfiles.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index d62653e5..f96e0a8a 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -2022,7 +2022,14 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non pbar.write(f"File: {result.id} successfully indexed.") pbar.update(1) - return None + operation_info = self._config._qdrant_advanced_engine.upsert( + collection_name=self._config.config.qdrant.search_collection, + points=points, + ) + assert operation_info.status == "completed" + session.commit() + + return None def semantic_search( self, From 4f4e55d2c2df8810361f9786726d8e48589da2a6 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 13 Aug 2025 14:46:40 -0400 Subject: [PATCH 14/30] updated bivec search --- bbconf/modules/bedfiles.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index f96e0a8a..8895f185 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -1200,11 +1200,10 @@ def text_to_bed_search( f"Could not retrieve metadata for bed file: {result_id}. Error: {e}" ) continue - if result_meta: - results_list.append(QdrantSearchResult(**result, metadata=result_meta)) + results_list.append(QdrantSearchResult(**result, metadata=result_meta if with_metadata else None)) if with_metadata: - count = self.bb_agent.get_detailed_stats().file_genome.get("hg38", 0) + count = 21000 #TODO: This is a placeholder, we need to find a way to get the actual count else: count = 0 return BedListSearchResult( From e4fe8aa8988f1661de2689eec3472c3d46563969 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 13 Aug 2025 17:52:55 -0400 Subject: [PATCH 15/30] reference genome compat updates --- bbconf/config_parser/bedbaseconfig.py | 1 + bbconf/db_utils.py | 2 +- bbconf/modules/bedfiles.py | 11 +++++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index 27c00a6c..e6d90c89 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -375,6 +375,7 @@ def _init_pephubclient() -> Union[PEPHubClient, None]: :return: PephubClient """ + try: _LOGGER.info(f"Initializing PEPHub client...") return PEPHubClient() diff --git a/bbconf/db_utils.py b/bbconf/db_utils.py index aea952d2..25f0368d 100644 --- a/bbconf/db_utils.py +++ b/bbconf/db_utils.py @@ -141,7 +141,7 @@ class Bed(Base): ) license_mapping: Mapped["License"] = relationship("License", back_populates="bed") - ref_classifier: Mapped["GenomeRefStats"] = relationship( + ref_classifier: Mapped[List["GenomeRefStats"]] = relationship( "GenomeRefStats", back_populates="bed", cascade="all, delete-orphan" ) processed: Mapped[bool] = mapped_column( diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index 8895f185..40f016c3 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -1009,6 +1009,9 @@ def _update_ref_validation( _LOGGER.info("Updating reference validation data..") + for exiting_ref_validation in bed_object.ref_classifier: + sa_session.delete(exiting_ref_validation) + for ref_gen_check, data in ref_validation.items(): new_gen_ref = GenomeRefStats( **RefGenValidModel( @@ -1200,10 +1203,14 @@ def text_to_bed_search( f"Could not retrieve metadata for bed file: {result_id}. Error: {e}" ) continue - results_list.append(QdrantSearchResult(**result, metadata=result_meta if with_metadata else None)) + results_list.append( + QdrantSearchResult( + **result, metadata=result_meta if with_metadata else None + ) + ) if with_metadata: - count = 21000 #TODO: This is a placeholder, we need to find a way to get the actual count + count = 21000 # TODO: This is a placeholder, we need to find a way to get the actual count else: count = 0 return BedListSearchResult( From 0846e152db1325428987367ef14fb24703b49806 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 13 Aug 2025 18:39:51 -0400 Subject: [PATCH 16/30] updated reindexing method --- bbconf/db_utils.py | 4 ++ bbconf/modules/bedfiles.py | 121 ++++++++++++++++++++++++------------- manual_testing.py | 11 +++- 3 files changed, 92 insertions(+), 44 deletions(-) diff --git a/bbconf/db_utils.py b/bbconf/db_utils.py index 25f0368d..8afecd17 100644 --- a/bbconf/db_utils.py +++ b/bbconf/db_utils.py @@ -101,6 +101,10 @@ class Bed(Base): indexed: Mapped[bool] = mapped_column( default=False, comment="Whether sample was added to qdrant" ) + file_indexed: Mapped[bool] = mapped_column( + default=False, + comment="Whether file was tokenized and added to the vector database", + ) pephub: Mapped[bool] = mapped_column( default=False, comment="Whether sample was added to pephub" ) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index 40f016c3..3699dc00 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -1386,7 +1386,7 @@ def _sql_search_count(self, condition_statement) -> int: count = session.execute(statement).one() return count[0] - def reindex_qdrant(self, batch: int = 100) -> None: + def reindex_qdrant(self, batch: int = 100, purge: bool = False) -> None: """ Re-upload all files to quadrant. !Warning: only hg38 genome can be added to qdrant! @@ -1399,57 +1399,92 @@ def reindex_qdrant(self, batch: int = 100) -> None: """ bb_client = BBClient() - annotation_result = self.get_ids_list( - limit=100000, genome=QDRANT_GENOME, offset=0 - ) + with Session(self._sa_engine) as session: + if purge: + _LOGGER.info("Purging indexed files in the database ...") + session.query(Bed).update({Bed.file_indexed: False}) + session.commit() + _LOGGER.info("Purged indexed files in the database successfully!") - if not annotation_result.results: - _LOGGER.error("No bed files found.") - return None - results = annotation_result.results + statement = ( + select(Bed) + .join(BedMetadata, Bed.id == BedMetadata.id) + .where( + and_(Bed.file_indexed == False, Bed.genome_alias == QDRANT_GENOME) + ) + .limit(150000) + ) - with tqdm(total=len(results), position=0, leave=True) as pbar: - points_list = [] - processed_number = 0 - for record in results: - try: - bed_region_set_obj = GRegionSet(bb_client.seek(record.id)) - except FileNotFoundError: - bed_region_set_obj = bb_client.load_bed(record.id) + annotation_results = session.scalars(statement) - pbar.set_description(f"Processing file: {record.id}") + results: List[Bed] = [result for result in annotation_results] + if not results: + _LOGGER.info("No files to reindex in qdrant.") + return None - file_embedding = self._embed_file(bed_region_set_obj) - points_list.append( - PointStruct( + with tqdm(total=len(results), position=0, leave=True) as pbar: + points_list = [] + processed_number = 0 + for record in results: + try: + bed_region_set_obj = GRegionSet(bb_client.seek(record.id)) + except FileNotFoundError: + bed_region_set_obj = bb_client.load_bed(record.id) + + pbar.set_description(f"Processing file: {record.id}") + + file_embedding = self._embed_file(bed_region_set_obj) + + bed_metadata = VectorMetadata( id=record.id, - vector=file_embedding.tolist()[0], - payload=( - record.annotation.model_dump() if record.annotation else {} - ), + name=record.name, + description=record.description, + genome_alias=record.genome_alias, + genome_digest=record.genome_digest, + cell_line=record.annotations.cell_line, + cell_type=record.annotations.cell_type, + tissue=record.annotations.tissue, + target=record.annotations.target, + treatment=record.annotations.treatment, + assay=record.annotations.assay, + species_name=record.annotations.species_name, ) - ) - processed_number += 1 - if processed_number % batch == 0: - pbar.set_description(f"Uploading points to qdrant using batch...") - operation_info = self._config.qdrant_engine.qd_client.upsert( - collection_name=self._config.config.qdrant.file_collection, - points=points_list, + + record.file_indexed = True + + points_list.append( + PointStruct( + id=record.id, + vector=file_embedding.tolist()[0], + payload=(bed_metadata.model_dump()), + ) ) - pbar.write("Uploaded batch to qdrant.") - points_list = [] - assert operation_info.status == "completed" + processed_number += 1 + + if processed_number % batch == 0: + pbar.set_description( + f"Uploading points to qdrant using batch..." + ) + operation_info = self._config.qdrant_engine.qd_client.upsert( + collection_name=self._config.config.qdrant.file_collection, + points=points_list, + ) + pbar.write("Uploaded batch to qdrant.") + points_list = [] + assert operation_info.status == "completed" - pbar.write(f"File: {record.id} successfully indexed.") - pbar.update(1) + session.commit() - _LOGGER.info(f"Uploading points to qdrant using batches...") - operation_info = self._config.qdrant_engine.qd_client.upsert( - collection_name=self._config.config.qdrant.file_collection, - points=points_list, - ) - assert operation_info.status == "completed" - return None + pbar.write(f"File: {record.id} successfully indexed.") + pbar.update(1) + + _LOGGER.info(f"Uploading points to qdrant using batches...") + operation_info = self._config.qdrant_engine.qd_client.upsert( + collection_name=self._config.config.qdrant.file_collection, + points=points_list, + ) + assert operation_info.status == "completed" + return None def delete_qdrant_point(self, identifier: str) -> None: """ diff --git a/manual_testing.py b/manual_testing.py index e14a1eba..3563884c 100644 --- a/manual_testing.py +++ b/manual_testing.py @@ -353,6 +353,13 @@ def external_search(): result +def reindex_files(): + from bbconf import BedBaseAgent + + agent = BedBaseAgent(config="/home/bnt4me/virginia/repos/bedhost/config.yaml") + agent.bed.reindex_qdrant(purge=True, batch=10) + + if __name__ == "__main__": # zarr_s3() # add_s3() @@ -368,5 +375,7 @@ def external_search(): # get_genomes() # new_search() - external_search() + # external_search() # get_assay_list() + + reindex_files() From 31f02bca0b40ca8933a7c888331332e9551d3aaf Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 18 Aug 2025 17:27:01 -0400 Subject: [PATCH 17/30] improved ref compatibility update --- bbconf/db_utils.py | 25 +++++++++++++++++++++++++ bbconf/modules/bedfiles.py | 21 +++++++++++++-------- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/bbconf/db_utils.py b/bbconf/db_utils.py index 8afecd17..2bcdf6ee 100644 --- a/bbconf/db_utils.py +++ b/bbconf/db_utils.py @@ -417,6 +417,23 @@ class License(Base): bed: Mapped[List["Bed"]] = relationship("Bed", back_populates="license_mapping") +# +# +# class ReferenceGenome(Base): +# __tablename__ = "reference_genomes" +# +# digest: Mapped[str] = mapped_column(primary_key=True, index=True) +# alias: Mapped[str] = mapped_column( +# nullable=False, comment="Name of the reference genome" +# ) +# +# bed_reference: Mapped[List["GenomeRefStats"]] = relationship( +# "GenomeRefStats", +# back_populates="genome_object", +# cascade="all, delete-orphan", +# ) + + class GenomeRefStats(Base): __tablename__ = "genome_ref_stats" @@ -431,6 +448,9 @@ class GenomeRefStats(Base): compared_genome: Mapped[str] = mapped_column( nullable=False, comment="Compared Genome" ) + # genome_digest: Mapped[str] = mapped_column( + # ForeignKey("reference_genomes.digest", ondelete="CASCADE"), + # ) xs: Mapped[float] = mapped_column(nullable=True, default=None) oobr: Mapped[float] = mapped_column(nullable=True, default=None) @@ -440,6 +460,11 @@ class GenomeRefStats(Base): bed: Mapped["Bed"] = relationship("Bed", back_populates="ref_classifier") + # genome_object: Mapped["ReferenceGenome"] = relationship( + # "ReferenceGenome", + # back_populates="bed_ref_stats", + # ) + __table_args__ = (UniqueConstraint("bed_id", "compared_genome"),) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index 3699dc00..b8fb2d01 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -478,6 +478,15 @@ def get_reference_validation(self, identifier: str) -> RefGenValidReturnModel: result_list = [] + results = [item for item in results] + + if not results: + return RefGenValidReturnModel( + id=identifier, + provided_genome=None, + compared_genome=[], + ) + for result in results: result_list.append( RefGenValidModel( @@ -1012,6 +1021,8 @@ def _update_ref_validation( for exiting_ref_validation in bed_object.ref_classifier: sa_session.delete(exiting_ref_validation) + sa_session.commit() + for ref_gen_check, data in ref_validation.items(): new_gen_ref = GenomeRefStats( **RefGenValidModel( @@ -1021,15 +1032,9 @@ def _update_ref_validation( ).model_dump(), bed_id=bed_object.id, ) - try: - sa_session.add(new_gen_ref) - sa_session.commit() - except IntegrityError as _: - sa_session.rollback() - _LOGGER.info( - f"Reference validation exists for BED id: {bed_object.id} and ref_gen_check." - ) + sa_session.add(new_gen_ref) + sa_session.commit() return None def delete(self, identifier: str) -> None: From 1069092746121828775a161ec1517796d79987d8 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 18 Aug 2025 17:53:13 -0400 Subject: [PATCH 18/30] Fixed qdrant connection --- bbconf/config_parser/bedbaseconfig.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index e6d90c89..323fb21a 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -267,7 +267,7 @@ def _init_qdrant_text_backend(self) -> Union[QdrantBackend, None]: ) return None - def _init_qdrant_advanced_backend(self) -> QdrantClient: + def _init_qdrant_advanced_backend(self) -> Union[QdrantClient, None]: """ Create qdrant client text embedding object using credentials provided in config file @@ -279,11 +279,20 @@ def _init_qdrant_advanced_backend(self) -> QdrantClient: _LOGGER.info(f"Initializing qdrant text advanced engine...") - qdrant_cl = QdrantClient( - url=self.config.qdrant.host, - port=self.config.qdrant.port, - api_key=self.config.qdrant.api_key, - ) + try: + qdrant_cl = QdrantClient( + url=self.config.qdrant.host, + port=self.config.qdrant.port, + api_key=self.config.qdrant.api_key, + ) + except qdrant_client.http.exceptions.ResponseHandlingException as err: + _LOGGER.error( + f"Error in Connection to qdrant! skipping... Error: {err}. Qdrant host: {self._config.qdrant.host}" + ) + warnings.warn( + f"error in Connection to qdrant! skipping... Error: {err}", UserWarning + ) + return None if not qdrant_cl.collection_exists(COLLECTION_NAME): _LOGGER.info( From 77daf72b3bef242a5ca107e58f868869faf1d1c1 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 18 Aug 2025 17:57:51 -0400 Subject: [PATCH 19/30] Fixed qdrant connection 2 --- bbconf/config_parser/bedbaseconfig.py | 77 +++++++++++---------------- 1 file changed, 32 insertions(+), 45 deletions(-) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index 323fb21a..05f68b76 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -285,6 +285,38 @@ def _init_qdrant_advanced_backend(self) -> Union[QdrantClient, None]: port=self.config.qdrant.port, api_key=self.config.qdrant.api_key, ) + + if not qdrant_cl.collection_exists(COLLECTION_NAME): + _LOGGER.info( + f"Collection 'bedbase_query_search' does not exist, creating it." + ) + qdrant_cl.create_collection( + collection_name=COLLECTION_NAME, + vectors_config=models.VectorParams( + size=DIMENTIONS, distance=models.Distance.COSINE + ), + quantization_config=models.ScalarQuantization( + scalar=models.ScalarQuantizationConfig( + type=models.ScalarType.INT8, + quantile=0.99, + always_ram=True, + ), + ), + ) + + qdrant_cl.create_payload_index( + collection_name=COLLECTION_NAME, + field_name="assay", + field_schema="keyword", + ) + qdrant_cl.create_payload_index( + collection_name=COLLECTION_NAME, + field_name="genome_alias", + field_schema="keyword", + ) + + return qdrant_cl + except qdrant_client.http.exceptions.ResponseHandlingException as err: _LOGGER.error( f"Error in Connection to qdrant! skipping... Error: {err}. Qdrant host: {self._config.qdrant.host}" @@ -294,51 +326,6 @@ def _init_qdrant_advanced_backend(self) -> Union[QdrantClient, None]: ) return None - if not qdrant_cl.collection_exists(COLLECTION_NAME): - _LOGGER.info( - f"Collection 'bedbase_query_search' does not exist, creating it." - ) - qdrant_cl.create_collection( - collection_name=COLLECTION_NAME, - vectors_config=models.VectorParams( - size=DIMENTIONS, distance=models.Distance.COSINE - ), - quantization_config=models.ScalarQuantization( - scalar=models.ScalarQuantizationConfig( - type=models.ScalarType.INT8, - quantile=0.99, - always_ram=True, - ), - ), - ) - - qdrant_cl.create_payload_index( - collection_name=COLLECTION_NAME, - field_name="assay", - field_schema="keyword", - ) - qdrant_cl.create_payload_index( - collection_name=COLLECTION_NAME, - field_name="genome_alias", - field_schema="keyword", - ) - - return qdrant_cl - - # # Create collection only if it does not exist - # try: - # collection_info = self.qd_client.get_collection(collection_name=self.collection) - # _LOGGER.info( - # f"Using collection {self.collection} with {collection_info.points_count} points." - # ) - # except Exception: # qdrant_client.http.exceptions.UnexpectedResponse - # _LOGGER.info(f"Collection {self.collection} does not exist, creating it.") - # self.qd_client.recreate_collection( - # collection_name=self.collection, - # vectors_config=self.config, - # quantization_config=DEFAULT_QUANTIZATION_CONFIG, - # ) - def _init_bivec_object(self) -> Union[BiVectorSearchInterface, None]: """ Create BiVectorSearchInterface object using credentials provided in config file From 63a9158f179a8a99cece30a75d7f899e863aa050 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 19 Aug 2025 17:56:32 -0400 Subject: [PATCH 20/30] Added reference genome information table --- bbconf/db_utils.py | 43 ++++++++++++++++++------------------- bbconf/models/bed_models.py | 3 ++- bbconf/modules/bedfiles.py | 8 ++++++- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/bbconf/db_utils.py b/bbconf/db_utils.py index 2bcdf6ee..0674add0 100644 --- a/bbconf/db_utils.py +++ b/bbconf/db_utils.py @@ -417,21 +417,19 @@ class License(Base): bed: Mapped[List["Bed"]] = relationship("Bed", back_populates="license_mapping") -# -# -# class ReferenceGenome(Base): -# __tablename__ = "reference_genomes" -# -# digest: Mapped[str] = mapped_column(primary_key=True, index=True) -# alias: Mapped[str] = mapped_column( -# nullable=False, comment="Name of the reference genome" -# ) -# -# bed_reference: Mapped[List["GenomeRefStats"]] = relationship( -# "GenomeRefStats", -# back_populates="genome_object", -# cascade="all, delete-orphan", -# ) +class ReferenceGenome(Base): + __tablename__ = "reference_genomes" + + digest: Mapped[str] = mapped_column(primary_key=True, index=True) + alias: Mapped[str] = mapped_column( + nullable=False, comment="Name of the reference genome" + ) + + bed_reference: Mapped[List["GenomeRefStats"]] = relationship( + "GenomeRefStats", + back_populates="genome_object", + cascade="all, delete-orphan", + ) class GenomeRefStats(Base): @@ -448,9 +446,9 @@ class GenomeRefStats(Base): compared_genome: Mapped[str] = mapped_column( nullable=False, comment="Compared Genome" ) - # genome_digest: Mapped[str] = mapped_column( - # ForeignKey("reference_genomes.digest", ondelete="CASCADE"), - # ) + genome_digest: Mapped[str] = mapped_column( + ForeignKey("reference_genomes.digest", ondelete="CASCADE"), + ) xs: Mapped[float] = mapped_column(nullable=True, default=None) oobr: Mapped[float] = mapped_column(nullable=True, default=None) @@ -460,10 +458,11 @@ class GenomeRefStats(Base): bed: Mapped["Bed"] = relationship("Bed", back_populates="ref_classifier") - # genome_object: Mapped["ReferenceGenome"] = relationship( - # "ReferenceGenome", - # back_populates="bed_ref_stats", - # ) + genome_object: Mapped["ReferenceGenome"] = relationship( + "ReferenceGenome", + back_populates="bed_reference", + lazy="joined", + ) __table_args__ = (UniqueConstraint("bed_id", "compared_genome"),) diff --git a/bbconf/models/bed_models.py b/bbconf/models/bed_models.py index eb59f1e2..a6059568 100644 --- a/bbconf/models/bed_models.py +++ b/bbconf/models/bed_models.py @@ -244,7 +244,8 @@ class TokenizedPathResponse(BaseModel): class RefGenValidModel(BaseModel): provided_genome: str - compared_genome: str + compared_genome: Union[str, None] + genome_digest: Union[str, None] xs: float = 0.0 oobr: Union[float, None] = None sequence_fit: Union[float, None] = None diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index b8fb2d01..e9e321c0 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -491,7 +491,10 @@ def get_reference_validation(self, identifier: str) -> RefGenValidReturnModel: result_list.append( RefGenValidModel( provided_genome=result.provided_genome, - compared_genome=result.compared_genome, + compared_genome=( + result.genome_object.alias if result.genome_object else None + ), + genome_digest=result.genome_digest, xs=result.xs, oobr=result.oobr, sequence_fit=result.sequence_fit, @@ -545,6 +548,7 @@ def add( :param processed: true if bedfile was processed and statistics and plots were calculated :return: None """ + _LOGGER.info(f"Adding bed file to database. bed_id: {identifier}") if self.exists(identifier): @@ -686,6 +690,7 @@ def add( **data.model_dump(), provided_genome=classification.genome_alias, compared_genome=ref_gen_check, + genome_digest=ref_gen_check, ).model_dump(), bed_id=identifier, ) @@ -1029,6 +1034,7 @@ def _update_ref_validation( **data.model_dump(), provided_genome=bed_object.genome_alias, compared_genome=ref_gen_check, + genome_digest=ref_gen_check, ).model_dump(), bed_id=bed_object.id, ) From 22a2676e333b764dd2cb6aa02cccc000e40604fd Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 19 Aug 2025 18:05:51 -0400 Subject: [PATCH 21/30] reindex changed to encode only --- bbconf/modules/bedfiles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index e9e321c0..68b0c482 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -1421,7 +1421,9 @@ def reindex_qdrant(self, batch: int = 100, purge: bool = False) -> None: select(Bed) .join(BedMetadata, Bed.id == BedMetadata.id) .where( - and_(Bed.file_indexed == False, Bed.genome_alias == QDRANT_GENOME) + and_(Bed.file_indexed == False, Bed.genome_alias == QDRANT_GENOME, + BedMetadata.global_experiment_id.contains(['encode'] #TODO: delete this line of code + )) ) .limit(150000) ) From 4e66dcf6ea815ffb2050a06176f0e47e0d6e60a3 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 19 Aug 2025 18:36:58 -0400 Subject: [PATCH 22/30] updated version --- bbconf/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbconf/_version.py b/bbconf/_version.py index 350cbe9d..9e5065a1 100644 --- a/bbconf/_version.py +++ b/bbconf/_version.py @@ -1 +1 @@ -__version__ = "0.11.4" +__version__ = "0.12.0a1" From 19ad6c2a90d33a3784f0e63524d8a3d86098e5f4 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 21 Aug 2025 11:48:19 -0400 Subject: [PATCH 23/30] cleaning --- bbconf/config_parser/bedbaseconfig.py | 23 +++++++------ bbconf/config_parser/models.py | 1 - bbconf/db_utils.py | 1 - bbconf/modules/bedfiles.py | 47 +++++---------------------- bbconf/modules/bedsets.py | 2 +- 5 files changed, 20 insertions(+), 54 deletions(-) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index 05f68b76..65affd87 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -17,7 +17,6 @@ from geniml.search.interfaces import BiVectorSearchInterface from geniml.search.query2vec import BED2Vec from pephubclient import PEPHubClient -from qdrant_client.http.models import VectorsConfig from zarr import Group as Z_GROUP from bbconf.config_parser.const import ( @@ -70,7 +69,7 @@ def __init__(self, config: Union[Path, str], init_ml: bool = True): self._bivec = self._init_bivec_object() else: _LOGGER.info( - f"Skipping initialization of ML models, init_ml parameter set to False." + "Skipping initialization of ML models, init_ml parameter set to False." ) self._b2bsi = None @@ -210,7 +209,7 @@ def _init_db_engine(self) -> BaseEngine: Create database engine object using credentials provided in config file """ - _LOGGER.info(f"Initializing database engine...") + _LOGGER.info("Initializing database engine...") return BaseEngine( host=self._config.database.host, port=self._config.database.port, @@ -227,7 +226,7 @@ def _init_qdrant_backend(self) -> QdrantBackend: :return: QdrantClient """ - _LOGGER.info(f"Initializing qdrant engine...") + _LOGGER.info("Initializing qdrant engine...") try: return QdrantBackend( collection=self._config.qdrant.file_collection, @@ -250,7 +249,7 @@ def _init_qdrant_text_backend(self) -> Union[QdrantBackend, None]: :return: QdrantClient """ - _LOGGER.info(f"Initializing qdrant text engine...") + _LOGGER.info("Initializing qdrant text engine...") try: return QdrantBackend( dim=TEXT_EMBEDDING_DIMENSION, @@ -277,7 +276,7 @@ def _init_qdrant_advanced_backend(self) -> Union[QdrantClient, None]: COLLECTION_NAME = self.config.qdrant.search_collection DIMENTIONS = 384 - _LOGGER.info(f"Initializing qdrant text advanced engine...") + _LOGGER.info("Initializing qdrant text advanced engine...") try: qdrant_cl = QdrantClient( @@ -288,7 +287,7 @@ def _init_qdrant_advanced_backend(self) -> Union[QdrantClient, None]: if not qdrant_cl.collection_exists(COLLECTION_NAME): _LOGGER.info( - f"Collection 'bedbase_query_search' does not exist, creating it." + "Collection 'bedbase_query_search' does not exist, creating it." ) qdrant_cl.create_collection( collection_name=COLLECTION_NAME, @@ -333,11 +332,11 @@ def _init_bivec_object(self) -> Union[BiVectorSearchInterface, None]: :return: BiVectorSearchInterface """ - _LOGGER.info(f"Initializing BiVectorBackend...") + _LOGGER.info("Initializing BiVectorBackend...") search_backend = BiVectorBackend( metadata_backend=self._qdrant_text_engine, bed_backend=self._qdrant_engine ) - _LOGGER.info(f"Initializing BiVectorSearchInterface...") + _LOGGER.info("Initializing BiVectorSearchInterface...") search_interface = BiVectorSearchInterface( backend=search_backend, query2vec=self.config.path.text2vec, @@ -351,7 +350,7 @@ def _init_b2bsi_object(self) -> Union[BED2BEDSearchInterface, None]: :return: Bed2BEDSearchInterface object """ try: - _LOGGER.info(f"Initializing search interfaces...") + _LOGGER.info("Initializing search interfaces...") return BED2BEDSearchInterface( backend=self.qdrant_engine, query2vec=BED2Vec(model=self._config.path.region2vec), @@ -373,7 +372,7 @@ def _init_pephubclient() -> Union[PEPHubClient, None]: """ try: - _LOGGER.info(f"Initializing PEPHub client...") + _LOGGER.info("Initializing PEPHub client...") return PEPHubClient() except Exception as e: _LOGGER.error(f"Error in creating PephubClient object: {e}") @@ -405,7 +404,7 @@ def _init_r2v_object(self) -> Union[Region2VecExModel, None]: Create Region2VecExModel object using credentials provided in config file """ try: - _LOGGER.info(f"Initializing R2V object...") + _LOGGER.info("Initializing R2V object...") return Region2VecExModel(self.config.path.region2vec) except Exception as e: _LOGGER.error(f"Error in creating Region2VecExModel object: {e}") diff --git a/bbconf/config_parser/models.py b/bbconf/config_parser/models.py index f8d8adfe..13726a62 100644 --- a/bbconf/config_parser/models.py +++ b/bbconf/config_parser/models.py @@ -14,7 +14,6 @@ DEFAULT_PEPHUB_NAMESPACE, DEFAULT_PEPHUB_TAG, DEFAULT_QDRANT_COLLECTION_NAME, - DEFAULT_QDRANT_TEXT_COLLECTION_NAME, DEFAULT_QDRANT_PORT, DEFAULT_QDRANT_TEXT_COLLECTION_NAME, DEFAULT_REGION2_VEC_MODEL, diff --git a/bbconf/db_utils.py b/bbconf/db_utils.py index 0674add0..d49902a8 100644 --- a/bbconf/db_utils.py +++ b/bbconf/db_utils.py @@ -1,6 +1,5 @@ import datetime import logging -import os from typing import List, Optional import pandas as pd diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index 68b0c482..a3827180 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -27,12 +27,10 @@ Bed, BedMetadata, BedStats, - BedSets, Files, GenomeRefStats, TokenizedBed, Universes, - BedFileBedSetRelation, ) from bbconf.exceptions import ( BedBaseConfError, @@ -1347,37 +1345,6 @@ def sql_search( results=result_list, ) - # def comprehensive_sql_search( - # self, - # query: str, - # genome: str = None, - # assay: str = None, - # bed_compliance: str = None, - # limit: int = 10, - # offset: int = 0, - # ) -> BedListSearchResult: - # """ - # Comprehensive SQL search for bed files with multiple filters. - # - # :param query: text query -looking at: name, description, cell_type, cell_line, tissue, target, treatment. - # - # :param genome: genome alias to filter results. Default is None, which means no filtering by genome. - # :param assay: filter by assay type. Default is None, which means no filtering by assay. - # :param bed_compliance: filter by bed compliance type. Default is None, which means no filtering by bed compliance. - # :param limit: number of results to return. Default is 10. - # :param offset: offset to start from - # - # :return: - # """ - # - # query = f"%{query.strip()}%" - # query_search = or_(Bed.name.ilike(query), - # Bed.description.ilike(query), - # BedMetadata.cell_type.ilike(query), - # BedMetadata.cell_line.ilike(query), - # BedMetadata.tissue.ilike(query), - # BedMetadata) - def _sql_search_count(self, condition_statement) -> int: """ Get number of total found files in the database. @@ -1421,9 +1388,11 @@ def reindex_qdrant(self, batch: int = 100, purge: bool = False) -> None: select(Bed) .join(BedMetadata, Bed.id == BedMetadata.id) .where( - and_(Bed.file_indexed == False, Bed.genome_alias == QDRANT_GENOME, - BedMetadata.global_experiment_id.contains(['encode'] #TODO: delete this line of code - )) + and_( + Bed.file_indexed == False, + Bed.genome_alias == QDRANT_GENOME, + # BedMetadata.global_experiment_id.contains(['encode']) # If we want only encode data + ) ) .limit(150000) ) @@ -1476,7 +1445,7 @@ def reindex_qdrant(self, batch: int = 100, purge: bool = False) -> None: if processed_number % batch == 0: pbar.set_description( - f"Uploading points to qdrant using batch..." + "Uploading points to qdrant using batch..." ) operation_info = self._config.qdrant_engine.qd_client.upsert( collection_name=self._config.config.qdrant.file_collection, @@ -1491,7 +1460,7 @@ def reindex_qdrant(self, batch: int = 100, purge: bool = False) -> None: pbar.write(f"File: {record.id} successfully indexed.") pbar.update(1) - _LOGGER.info(f"Uploading points to qdrant using batches...") + _LOGGER.info("Uploading points to qdrant using batches...") operation_info = self._config.qdrant_engine.qd_client.upsert( collection_name=self._config.config.qdrant.file_collection, points=points_list, @@ -2062,7 +2031,7 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non if processed_number % batch == 0: pbar.set_description( - f"Uploading points to qdrant using batch..." + "Uploading points to qdrant using batch..." ) operation_info = self._config._qdrant_advanced_engine.upsert( collection_name=self._config.config.qdrant.search_collection, diff --git a/bbconf/modules/bedsets.py b/bbconf/modules/bedsets.py index 74f22a7e..6179ee98 100644 --- a/bbconf/modules/bedsets.py +++ b/bbconf/modules/bedsets.py @@ -4,7 +4,7 @@ from geniml.io.utils import compute_md5sum_bedset from sqlalchemy import Float, Numeric, func, or_, select from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session, relationship +from sqlalchemy.orm import Session from bbconf.config_parser import BedBaseConfig from bbconf.const import PKG_NAME From 5c716b8e6342583adff4c63aa1a4d4c1850673bb Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 21 Aug 2025 12:24:23 -0400 Subject: [PATCH 24/30] Fixed pr comments --- bbconf/modules/bedfiles.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index a3827180..b0dfaec4 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -1150,7 +1150,7 @@ def _embed_file(self, bed_file: Union[str, RegionSet]) -> np.ndarray: """ Create embedding for bed file - :param bed_file: bed file path or regionet + :param bed_file: bed file path or region set :param bed_file: path to the bed file, or RegionSet object :return np array of embeddings @@ -1219,7 +1219,9 @@ def text_to_bed_search( ) if with_metadata: - count = 21000 # TODO: This is a placeholder, we need to find a way to get the actual count + count = self._config._qdrant_advanced_engine.get_collection( + collection_name=self._config.config.qdrant.file_collection + ).points_count else: count = 0 return BedListSearchResult( @@ -1989,12 +1991,6 @@ def reindex_semantic_search(self, batch: int = 1000, purge: bool = False) -> Non with tqdm(total=len(results), position=0, leave=True) as pbar: processed_number = 0 for result in results: - # text = ( - # f"{result.description}. {result.annotations.cell_line}. {result.annotations.cell_type}." - # f" {result.annotations.tissue}. {result.annotations.target}." - # f"{result.annotations.assay}. {result.name}." - # ) - text = ( f"biosample is {result.annotations.cell_line} / {result.annotations.cell_type} / " f"{result.annotations.tissue} with target {result.annotations.target} " From 7ee080df5a717df5d74e7a20921ac7c62ff831c1 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 27 Aug 2025 12:59:48 -0400 Subject: [PATCH 25/30] fixed comments of PR --- bbconf/config_parser/bedbaseconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index 65affd87..56cb6d29 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -274,7 +274,7 @@ def _init_qdrant_advanced_backend(self) -> Union[QdrantClient, None]: """ COLLECTION_NAME = self.config.qdrant.search_collection - DIMENTIONS = 384 + DIMENSIONS = 384 _LOGGER.info("Initializing qdrant text advanced engine...") @@ -292,7 +292,7 @@ def _init_qdrant_advanced_backend(self) -> Union[QdrantClient, None]: qdrant_cl.create_collection( collection_name=COLLECTION_NAME, vectors_config=models.VectorParams( - size=DIMENTIONS, distance=models.Distance.COSINE + size=DIMENSIONS, distance=models.Distance.COSINE ), quantization_config=models.ScalarQuantization( scalar=models.ScalarQuantizationConfig( From 4cfadd3334e176e1da286a09135b5a55c1644303 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 28 Aug 2025 14:27:50 -0400 Subject: [PATCH 26/30] ref gen validation improvements + disabled pephub --- bbconf/config_parser/bedbaseconfig.py | 15 +++-- bbconf/modules/bedfiles.py | 95 +++++++++++++++++++-------- 2 files changed, 74 insertions(+), 36 deletions(-) diff --git a/bbconf/config_parser/bedbaseconfig.py b/bbconf/config_parser/bedbaseconfig.py index 56cb6d29..a6cedc37 100644 --- a/bbconf/config_parser/bedbaseconfig.py +++ b/bbconf/config_parser/bedbaseconfig.py @@ -371,13 +371,14 @@ def _init_pephubclient() -> Union[PEPHubClient, None]: :return: PephubClient """ - try: - _LOGGER.info("Initializing PEPHub client...") - return PEPHubClient() - except Exception as e: - _LOGGER.error(f"Error in creating PephubClient object: {e}") - warnings.warn(f"Error in creating PephubClient object: {e}", UserWarning) - return None + # try: + # _LOGGER.info("Initializing PEPHub client...") + # return PEPHubClient() + # except Exception as e: + # _LOGGER.error(f"Error in creating PephubClient object: {e}") + # warnings.warn(f"Error in creating PephubClient object: {e}", UserWarning) + # return None + return None def _init_boto3_client( self, diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index b0dfaec4..48ad904f 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -682,17 +682,25 @@ def add( session.add(new_metadata) if ref_validation: - for ref_gen_check, data in ref_validation.items(): - new_gen_ref = GenomeRefStats( - **RefGenValidModel( - **data.model_dump(), - provided_genome=classification.genome_alias, - compared_genome=ref_gen_check, - genome_digest=ref_gen_check, - ).model_dump(), - bed_id=identifier, - ) - session.add(new_gen_ref) + + new_gen_refs = self._create_ref_validation_models( + ref_validation=ref_validation, + bed_id=identifier, + provided_genome=classification.genome_alias, + ) + session.add_all(new_gen_refs) + + # for ref_gen_check, data in ref_validation.items(): + # new_gen_ref = GenomeRefStats( + # **RefGenValidModel( + # **data.model_dump(), + # provided_genome=classification.genome_alias, + # compared_genome=ref_gen_check, + # genome_digest=ref_gen_check, + # ).model_dump(), + # bed_id=identifier, + # ) + # session.add(new_gen_ref) session.commit() return None @@ -707,13 +715,13 @@ def update( classification: Union[dict, None] = None, ref_validation: Union[Dict[str, BaseModel], None] = None, license_id: str = DEFAULT_LICENSE, - upload_qdrant: bool = True, - upload_pephub: bool = True, + upload_qdrant: bool = False, + upload_pephub: bool = False, upload_s3: bool = True, local_path: str = None, overwrite: bool = False, nofail: bool = False, - processed: bool = True, + processed: bool = False, ) -> None: """ Update bed file to the database. @@ -810,7 +818,10 @@ def update( ) self._update_ref_validation( - sa_session=session, bed_object=bed_object, ref_validation=ref_validation + sa_session=session, + bed_id=identifier, + ref_validation=ref_validation, + provided_genome=bed_object.genome_alias or "", ) bed_object.processed = processed @@ -1002,9 +1013,12 @@ def _update_files( f"File with name: {v.name} already exists. Updating.." ) - @staticmethod def _update_ref_validation( - sa_session: Session, bed_object: Bed, ref_validation: Dict[str, BaseModel] + self, + sa_session: Session, + bed_id: str, + ref_validation: Dict[str, BaseModel], + provided_genome: str = "", ) -> None: """ Update reference validation data @@ -1012,34 +1026,57 @@ def _update_ref_validation( ! This function won't update the reference validation data, if it exists, it will skip it. :param sa_session: sqlalchemy session - :param bed_object: bed sqlalchemy object + :param bed_id: bed sqlalchemy object :param ref_validation: bed file metadata + :param provided_genome: genome reference that was provided by user """ if not ref_validation: return None - _LOGGER.info("Updating reference validation data..") + sa_session.execute( + delete(GenomeRefStats).where(GenomeRefStats.bed_id == bed_id) + ) - for exiting_ref_validation in bed_object.ref_classifier: - sa_session.delete(exiting_ref_validation) + new_gen_refs = self._create_ref_validation_models( + ref_validation=ref_validation, + bed_id=bed_id, + provided_genome=provided_genome, + ) + sa_session.add_all(new_gen_refs) + # One commit for both deletes and inserts sa_session.commit() - for ref_gen_check, data in ref_validation.items(): - new_gen_ref = GenomeRefStats( + return None + + def _create_ref_validation_models( + self, + ref_validation: Dict[str, BaseModel], + bed_id: str, + provided_genome: str = None, + ) -> list[GenomeRefStats]: + + compatibility = {} + + for k, v in ref_validation.items(): + if v.tier_ranking < 4: + compatibility[k] = v + + # Add all new GenomeRefStats objects in one go + new_gen_refs: list[GenomeRefStats] = [ + GenomeRefStats( **RefGenValidModel( **data.model_dump(), - provided_genome=bed_object.genome_alias, + provided_genome=provided_genome, compared_genome=ref_gen_check, genome_digest=ref_gen_check, ).model_dump(), - bed_id=bed_object.id, + bed_id=bed_id, ) - sa_session.add(new_gen_ref) - - sa_session.commit() - return None + for ref_gen_check, data in compatibility.items() + ] + return new_gen_refs def delete(self, identifier: str) -> None: """ From e772271aad7a9ce1c828ae04ecdd0d8e9fdc9ec8 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 10 Sep 2025 20:45:26 -0400 Subject: [PATCH 27/30] fixed https://github.com/databio/bedhost/issues/216 ; Fixed tests --- bbconf/bbagent.py | 24 ++++++++++++++++++++++++ bbconf/models/base_models.py | 1 + manual_testing.py | 8 ++++---- tests/test_bedfile.py | 7 ++++++- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/bbconf/bbagent.py b/bbconf/bbagent.py index b5e8b6cd..59a58b6d 100644 --- a/bbconf/bbagent.py +++ b/bbconf/bbagent.py @@ -146,6 +146,14 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: .order_by(func.count(BedMetadata.species_name).desc()) ).all() } + file_assay = { + f[0]: f[1] + for f in session.execute( + select(BedMetadata.assay, func.count(BedMetadata.assay)) + .group_by(BedMetadata.assay) + .order_by(func.count(BedMetadata.assay).desc()) + ).all() + } slice_value = 20 @@ -194,12 +202,27 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: file_organism_concise["other"] + file_organism_concise[""] ) file_organism_concise.pop("") + file_assay_concise = dict(list(file_assay.items())[0:slice_value]) + file_assay_concise["other"] = sum( + list(file_assay.values())[slice_value:] + ) + file_assay.get("other", 0) + if "" in file_assay_concise: + file_assay_concise["other"] = ( + file_assay_concise["other"] + file_assay_concise[""] + ) + file_assay_concise.pop("") + if "OTHER" in file_assay_concise: + file_assay_concise["other"] = ( + file_assay_concise["other"] + file_assay_concise["OTHER"] + ) + file_assay_concise.pop("OTHER") return FileStats( data_format=data_format, bed_compliance=bed_compliance_concise, file_genome=file_genomes_concise, file_organism=file_organism_concise, + file_assay=file_assay_concise, bed_comments=bed_comments, geo_status=geo_status, mean_region_width=list_mean_width_bins, @@ -213,6 +236,7 @@ def get_detailed_stats(self, concise: bool = False) -> FileStats: bed_compliance=bed_compliance, file_genome=file_genomes, file_organism=file_organism, + file_assay=file_assay, bed_comments=bed_comments, geo_status=geo_status, mean_region_width=list_mean_width_bins, diff --git a/bbconf/models/base_models.py b/bbconf/models/base_models.py index 4ce3d0f1..a7711857 100644 --- a/bbconf/models/base_models.py +++ b/bbconf/models/base_models.py @@ -96,6 +96,7 @@ class FileStats(BaseModel): data_format: Dict[str, int] file_genome: Dict[str, int] file_organism: Dict[str, int] + file_assay: Dict[str, int] geo_status: Dict[str, int] bed_comments: Dict[str, int] mean_region_width: BinValues diff --git a/manual_testing.py b/manual_testing.py index 3563884c..2fc2205d 100644 --- a/manual_testing.py +++ b/manual_testing.py @@ -219,9 +219,9 @@ def compreh_stats(): time1 = time.time() - # results = agent.get_detailed_stats() + results = agent.get_detailed_stats(concise=True) - results = agent.get_detailed_usage() + # results = agent.get_detailed_usage() time2 = time.time() print(time2 - time1) @@ -370,7 +370,7 @@ def reindex_files(): # neighbour_beds() # sql_search() # config_t() - # compreh_stats() + compreh_stats() # get_unprocessed_files() # get_genomes() # new_search() @@ -378,4 +378,4 @@ def reindex_files(): # external_search() # get_assay_list() - reindex_files() + # reindex_files() diff --git a/tests/test_bedfile.py b/tests/test_bedfile.py index 974e6795..c0ed1d1d 100644 --- a/tests/test_bedfile.py +++ b/tests/test_bedfile.py @@ -58,7 +58,9 @@ def test_get_all(self, bbagent_obj, mocked_phc): assert return_result is not None assert return_result.files is not None assert return_result.plots is not None - assert return_result.raw_metadata is not None + + # TODO: PEPhub is disabled + # assert return_result.raw_metadata is not None assert return_result.genome_alias == "hg38" assert return_result.stats.number_of_regions == 1 @@ -80,6 +82,9 @@ def test_get_all_not_found(self, bbagent_obj): assert return_result.genome_alias == "hg38" assert return_result.id == BED_TEST_ID + @pytest.mark.skip( + "Skipped, because PHC is disabled" + ) # TODO: should we disable PHC everywhere? def test_get_raw_metadata(self, bbagent_obj, mocked_phc): with ContextManagerDBTesting(config=bbagent_obj.config, add_data=True): return_result = bbagent_obj.bed.get_raw_metadata(BED_TEST_ID) From 8c5b9a15d343857a6de3f6c0eb44754223ef6caf Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 10 Sep 2025 22:53:46 -0400 Subject: [PATCH 28/30] Added gtars to requirements --- requirements/requirements-all.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index 91e2bada..ce2fca39 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -1,5 +1,6 @@ yacman >= 0.9.1 sqlalchemy >= 2.0.0 +gtars >= 0.4.0 geniml[ml] >= 0.7.1 psycopg >= 3.1.15 coloredlogs From 033f68dcb3b663e245ad86d7fbe9d38a08daf2cb Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 11 Sep 2025 10:09:54 -0400 Subject: [PATCH 29/30] Versio 0.12.0 + changelog --- bbconf/_version.py | 2 +- docs/changelog.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bbconf/_version.py b/bbconf/_version.py index 9e5065a1..ea370a8e 100644 --- a/bbconf/_version.py +++ b/bbconf/_version.py @@ -1 +1 @@ -__version__ = "0.12.0a1" +__version__ = "0.12.0" diff --git a/docs/changelog.md b/docs/changelog.md index 9df3a992..44bbe791 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,20 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +### [0.12.0] - 2025-09-11 +### Added: +- New qdrant semantic search +- Added more plots to bedbase summary page +- New reference genome compatibility that supports new refgenie +- Genomes table, with ability of automatic updates from refgenie +- Added filter in search (Assay and genome) + +### Changed: +- Improved reindexing methods + +### Fixed: +- Issues in bedfile update method + ### [0.11.4] - 2025-06-01 ### Fixed: - SQL search From 02aae8879be36636b3a65ebd331d0879fc15d9ed Mon Sep 17 00:00:00 2001 From: Oleksandr <41573628+khoroshevskyi@users.noreply.github.com> Date: Thu, 11 Sep 2025 11:39:52 -0400 Subject: [PATCH 30/30] Update bbconf/modules/bedfiles.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- bbconf/modules/bedfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbconf/modules/bedfiles.py b/bbconf/modules/bedfiles.py index 48ad904f..cad3e1a0 100644 --- a/bbconf/modules/bedfiles.py +++ b/bbconf/modules/bedfiles.py @@ -2190,7 +2190,7 @@ def search_external_file(self, source: str, accession: str) -> BedListSearchResu f"Source {source} is not supported. Supported sources are: 'geo', 'encode'." ) - if source == "geo" and accession.startswith("gse"): + if source == "geo" and accession.upper().startswith("GSE"): statement = ( select(Bed) .join(BedMetadata, Bed.id == BedMetadata.id)