Skip to content

support new ftp path layout #157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
vinay-ebi wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

support new ftp path layout #157

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
da465d9
support new ftp path layout
vinay-ebi Aug 5, 2025
3d0da02
support new ftp path layout: Update test cases
vinay-ebi Aug 6, 2025
f231acf
Merge branch 'main' into feature/new_ftplayout
dpopleton Sep 25, 2025
4bcca54
Major update for new path structure:
dpopleton Sep 25, 2025
6277bd0
Updated json generator for new path and added parquet generation
dpopleton Oct 2, 2025
9028dd8
minor fixes to json generator
dpopleton Oct 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ dependencies = [
"duckdb-engine >= 0.17.0",
"pymysql",
"mysqlclient",
"pandas", #for ftp_index_generator
"pyarrow" #for ftp_index_generator
]

[project.urls]
Expand Down
167 changes: 139 additions & 28 deletions src/ensembl/production/metadata/api/adaptors/genome.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

from ensembl.production.metadata.api.adaptors.base import BaseAdaptor, check_parameter, cfg
from ensembl.production.metadata.api.exceptions import TypeNotFoundException
from ensembl.production.metadata.api.factories.utils import format_accession_path
from ensembl.production.metadata.api.models import Genome, Organism, Assembly, OrganismGroup, OrganismGroupMember, \
GenomeRelease, EnsemblRelease, EnsemblSite, AssemblySequence, GenomeDataset, Dataset, DatasetType, DatasetSource, \
ReleaseStatus, DatasetStatus, utils, DatasetAttribute, Attribute
Expand Down Expand Up @@ -54,12 +55,16 @@ class GenomeDatasetsListItem(NamedTuple):


class GenomeAdaptor(BaseAdaptor):
def __init__(self, metadata_uri: str, taxonomy_uri: str):
def __init__(self, metadata_uri: str, taxonomy_uri=None):
super().__init__(metadata_uri)
self.taxonomy_db = DBConnection(taxonomy_uri, pool_size=cfg.pool_size, pool_recycle=cfg.pool_recycle)
if taxonomy_uri is None:
self.taxonomy_db = None
else:
self.taxonomy_db = DBConnection(taxonomy_uri, pool_size=cfg.pool_size, pool_recycle=cfg.pool_recycle)

def fetch_taxonomy_names(self, taxonomy_ids, synonyms=None):

if self.taxonomy_db is None:
raise ValueError("Taxonomy DB not provided")
if synonyms is None:
synonyms = []
taxonomy_ids = check_parameter(taxonomy_ids)
Expand Down Expand Up @@ -92,6 +97,8 @@ def fetch_taxonomy_names(self, taxonomy_ids, synonyms=None):
return taxons

def fetch_taxonomy_ids(self, taxonomy_names):
if self.taxonomy_db is None:
raise ValueError("Taxonomy DB not provided")
taxids = []
taxonomy_names = check_parameter(taxonomy_names)
for taxon in taxonomy_names:
Expand Down Expand Up @@ -876,13 +883,44 @@ def fetch_assemblies_count(self, species_taxonomy_id: int, release_version: floa
return session.execute(query).scalar()

def get_public_path(self, genome_uuid, dataset_type='all', release=None):
"""
Retrieve public file paths for genomic datasets.

Args:
genome_uuid (str): Unique identifier for the genome
dataset_type (str): Type of dataset ('genebuild', 'assembly', 'homologies', 'variation', or 'all')
release (str, optional): Specific Ensembl release label. If None, uses current release.

Returns:
list: List of dictionaries containing dataset_type and path information

Raises:
ValueError: If genome_uuid or release not found, or required metadata missing
TypeNotFoundException: If requested dataset_type is not available
"""
paths = []
scientific_name = None
accession = None
genebuild_source_name = None
last_geneset_update = None

with self.metadata_db.session_scope() as session:
# Single combined query to get all required data
# === VALIDATION SECTION ===
genome_exists = session.execute(
select(Genome.genome_uuid).where(Genome.genome_uuid == genome_uuid)
).first()
if not genome_exists:
raise ValueError(f"Genome with UUID {genome_uuid} not found")

if release is not None:
release_exists = session.execute(
select(EnsemblRelease.label).where(EnsemblRelease.label == release)
).first()
if not release_exists:
raise ValueError(f"Ensembl release with label '{release}' not found")

# === METADATA RETRIEVAL ===
# Get core genome metadata: organism name, assembly accession, and genebuild info
query = select(
Organism.scientific_name,
Assembly.accession,
Expand Down Expand Up @@ -912,57 +950,130 @@ def get_public_path(self, genome_uuid, dataset_type='all', release=None):
else:
scientific_name = accession = genebuild_source_name = last_geneset_update = None

# Query for which of the 5 supported dataset types exist for this genome
supported_types = ['genebuild', 'assembly', 'homologies', 'regulatory_features', 'variation']
# === DATASET TYPE DISCOVERY ===
supported_types = ['genebuild', 'assembly', 'homologies', 'variation']
unique_dataset_types_query = select(DatasetType.name).distinct().join(
Dataset
).join(GenomeDataset).join(Genome).where(
Genome.genome_uuid == genome_uuid,
DatasetType.name.in_(supported_types)
DatasetType.name.in_(supported_types),
Dataset.status == DatasetStatus.RELEASED
)
unique_dataset_types = session.execute(unique_dataset_types_query).scalars().all()

variation_release = None
homology_release = None

# === RELEASE HANDLING ===
if release is None:
if 'variation' in unique_dataset_types:
variation_release = session.execute(
select(EnsemblRelease.label).join(GenomeDataset).join(Dataset).join(DatasetType).join(
Genome).where(
Genome.genome_uuid == genome_uuid,
DatasetType.name == 'variation',
Dataset.status == DatasetStatus.RELEASED,
GenomeDataset.is_current == True,
EnsemblRelease.release_type == 'partial'
)
).scalar_one_or_none()

if 'homologies' in unique_dataset_types:
homology_release = session.execute(
select(EnsemblRelease.label).join(GenomeDataset).join(Dataset).join(DatasetType).join(
Genome).where(
Genome.genome_uuid == genome_uuid,
DatasetType.name == 'homologies',
Dataset.status == DatasetStatus.RELEASED,
GenomeDataset.is_current == True,
EnsemblRelease.release_type == 'partial'
)
).scalar_one_or_none()
else:
if 'variation' in unique_dataset_types:
variation_release = session.execute(
select(EnsemblRelease.label).join(GenomeDataset).join(Dataset).join(DatasetType).join(
Genome).where(
Genome.genome_uuid == genome_uuid,
DatasetType.name == 'variation',
Dataset.status == DatasetStatus.RELEASED,
EnsemblRelease.release_type == 'partial',
EnsemblRelease.release_id <= (
select(EnsemblRelease.release_id).where(
EnsemblRelease.label == release).scalar_subquery()
)
).order_by(EnsemblRelease.release_id.desc())
).scalar_one_or_none()

if variation_release is None:
unique_dataset_types.remove('variation')

if 'homologies' in unique_dataset_types:
homology_release = session.execute(
select(EnsemblRelease.label).join(GenomeDataset).join(Dataset).join(DatasetType).join(
Genome).where(
Genome.genome_uuid == genome_uuid,
DatasetType.name == 'homologies',
Dataset.status == DatasetStatus.RELEASED,
EnsemblRelease.release_type == 'partial',
EnsemblRelease.release_id <= (
select(EnsemblRelease.release_id).where(
EnsemblRelease.label == release).scalar_subquery()
)
).order_by(EnsemblRelease.release_id.desc())
).scalar_one_or_none()

if homology_release is None:
unique_dataset_types.remove('homologies')

# === DATA VALIDATION ===
if 'genebuild' not in unique_dataset_types or 'assembly' not in unique_dataset_types:
raise ValueError(
f"Missing genebuild or assembly dataset types. Something is seriously wrong with {genome_uuid}")

if scientific_name is None or accession is None or genebuild_source_name is None or last_geneset_update is None:
raise ValueError("Required metadata fields are missing. Please check the database entries.")
unique_dataset_types = ['regulation' if t == 'regulatory_features' else t for t in unique_dataset_types]
if dataset_type == 'regulatory_features':
dataset_type = 'regulation'
match = re.match(r'^(\d{4}-\d{2})', last_geneset_update) # Match format YYYY-MM

# === PATH CONSTRUCTION ===
match = re.match(r'^(\d{4}-\d{2})', last_geneset_update)
if not match:
raise ValueError(f"Invalid last_geneset_update format: {last_geneset_update}")
last_geneset_update = match.group(1).replace('-', '_')
scientific_name = re.sub(r'[^a-zA-Z0-9]+', ' ', scientific_name)
scientific_name = scientific_name.replace(' ', '_')
scientific_name = re.sub(r'^_+|_+$', '', scientific_name)

genebuild_source_name = genebuild_source_name.lower()
base_path = f"{scientific_name}/{accession}"
common_path = f"{base_path}/{genebuild_source_name}"

base_path = format_accession_path(accession)
common_path = f"{base_path}/{genebuild_source_name}/{last_geneset_update}"

if 'homologies' in unique_dataset_types and homology_release:
homology_release = homology_release.replace('-', '_')
if 'variation' in unique_dataset_types and variation_release:
variation_release = variation_release.replace('-', '_')

path_templates = {
'genebuild': f"{common_path}/geneset/{last_geneset_update}",
'genebuild': f"{common_path}/geneset",
'assembly': f"{base_path}/genome",
'homologies': f"{common_path}/homology/{last_geneset_update}",
'regulation': f"{common_path}/regulation",
'variation': f"{common_path}/variation/{last_geneset_update}",
'homologies': f"{common_path}/homology/{homology_release}",
'variation': f"{common_path}/variation/{variation_release}",
}

# Check for invalid dataset type early
# === REQUEST VALIDATION ===
if dataset_type not in unique_dataset_types and dataset_type != 'all':
raise TypeNotFoundException(f"Dataset Type : {dataset_type} not found in metadata.")

# If 'all', add paths for all unique dataset types
# === PATH GENERATION ===
if dataset_type == 'all':
for t in unique_dataset_types:
for dataset_type_name in unique_dataset_types:
paths.append({
"dataset_type": t,
"path": path_templates[t]
"dataset_type": dataset_type_name,
"path": path_templates[dataset_type_name]
})
elif dataset_type in path_templates:
# Add path for the specific dataset type
paths.append({
"dataset_type": dataset_type,
"path": path_templates[dataset_type]
})
else:
# If the code reaches here, it means there is a logic error
raise TypeNotFoundException(f"Dataset Type : {dataset_type} has no associated path.")

return paths
return paths
89 changes: 17 additions & 72 deletions src/ensembl/production/metadata/api/adaptors/vep.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re

from sqlalchemy import and_
from sqlalchemy.orm import aliased

from ensembl.production.metadata.api.adaptors.base import BaseAdaptor
from ensembl.production.metadata.api.models import Organism, Assembly, DatasetAttribute, Genome, GenomeDataset, Dataset

from ensembl.production.metadata.api.adaptors.genome import GenomeAdaptor

class VepAdaptor(BaseAdaptor):
def __init__(self, metadata_uri: str, file="all"):
super().__init__(metadata_uri)
self.metadata_uri = metadata_uri
self.file = file

def fetch_vep_locations(self, genome_uuid):
Expand All @@ -30,69 +26,18 @@ def fetch_vep_locations(self, genome_uuid):
:param genome_uuid: The UUID of the genome to fetch locations for.
:return: A dictionary containing the FAA and GFF locations or a specific location string if 'file' is set.
"""
with self.metadata_db.session_scope() as session:
# Aliases for clarity and distinct filtering
annotation_source_attr = aliased(DatasetAttribute)
last_geneset_update_attr = aliased(DatasetAttribute)

query = (
session.query(
Organism.scientific_name,
Assembly.accession.label("assembly_accession"),
annotation_source_attr.value.label("annotation_source"),
last_geneset_update_attr.value.label("last_geneset_update"),
)
.join(Genome, Genome.organism_id == Organism.organism_id)
.join(Assembly, Assembly.assembly_id == Genome.assembly_id)
.join(GenomeDataset, GenomeDataset.genome_id == Genome.genome_id)
.join(Dataset, Dataset.dataset_id == GenomeDataset.dataset_id)
.outerjoin(
annotation_source_attr,
and_(
annotation_source_attr.dataset_id == Dataset.dataset_id,
annotation_source_attr.attribute.has(name="genebuild.annotation_source"),
),
)
.outerjoin(
last_geneset_update_attr,
and_(
last_geneset_update_attr.dataset_id == Dataset.dataset_id,
last_geneset_update_attr.attribute.has(name="genebuild.last_geneset_update"),
),
)
.filter(
Genome.genome_uuid == genome_uuid,
Dataset.name == "genebuild",
)
.distinct()
)

result = query.one_or_none()

if not result:
raise ValueError(f"No data found for genome UUID: {genome_uuid}")
elif not result.annotation_source or not result.last_geneset_update:
raise ValueError(f"Missing annotation source or last geneset update for genome UUID: {genome_uuid}")

# Format the scientific name
scientific_name = result.scientific_name
scientific_name = re.sub(r"[^a-zA-Z0-9]+", " ", scientific_name)
scientific_name = re.sub(r" +", "_", scientific_name).strip("_")

# Format last geneset update
last_geneset_update = re.sub(r"-", "_", result.last_geneset_update)

# Construct the locations
faa_location = f"{scientific_name}/{result.assembly_accession}/vep/genome/softmasked.fa.bgz"
gff_location = (
f"{scientific_name}/{result.assembly_accession}/vep/"
f"{result.annotation_source}/geneset/{last_geneset_update}/genes.gff3.bgz"
)

# Return based on the `file` argument
if self.file == "faa_location":
return faa_location
elif self.file == "gff_location":
return gff_location
else:
return {"faa_location": faa_location, "gff_location": gff_location}
genome_adaptor = GenomeAdaptor(self.metadata_uri)
genebuild_path = genome_adaptor.get_public_path(genome_uuid, dataset_type='genebuild')
genebuild_path = genebuild_path[0]['path']
assembly_path = genome_adaptor.get_public_path(genome_uuid, dataset_type='assembly')
assembly_path = assembly_path[0]['path']

faa_location = f"{assembly_path}/unmasked.fa.bgz"
gff_location = f"{genebuild_path}/genes.gff3.bgz"
# Return based on the `file` argument
if self.file == "faa_location":
return faa_location
elif self.file == "gff_location":
return gff_location
else:
return {"faa_location": faa_location, "gff_location": gff_location}
Loading