diff --git a/ci/scripts/low-vers.py b/ci/scripts/low-vers.py index 8c735c063f..c8f52418bb 100755 --- a/ci/scripts/low-vers.py +++ b/ci/scripts/low-vers.py @@ -14,7 +14,7 @@ from contextlib import ExitStack from functools import cached_property from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, assert_never import dependency_groups from packaging.requirements import Requirement @@ -93,6 +93,7 @@ class Args(argparse.Namespace): output: Path | None _extras: list[str] _all_extras: bool + _skip_extras: list[str] _groups: list[str] _all_groups: bool @@ -121,7 +122,7 @@ def parser(cls) -> argparse.ArgumentParser: dest="_extras", metavar="EXTRA", type=str, - nargs="*", + nargs="+", default=(), help="extras to install", ) @@ -131,6 +132,15 @@ def parser(cls) -> argparse.ArgumentParser: action="store_true", help="get all extras", ) + parser.add_argument( + "--skip-extras", + dest="_skip_extras", + metavar="EXTRA", + type=str, + nargs="+", + default=(), + help="extras to skip when `--all-extras` is set", + ) parser.add_argument( "--groups", dest="_groups", @@ -167,13 +177,20 @@ def pyproject(self) -> dict[str, Any]: @cached_property def extras(self) -> AbstractSet[str]: """Return the extras to install.""" - if self._extras: - if self._all_extras: + match self._extras, self._all_extras, self._skip_extras: + case [], True, skip: + return dict.fromkeys( + self.pyproject["project"]["optional-dependencies"].keys() + - set(skip) + ).keys() + case extras, False, []: + return dict.fromkeys(extras).keys() + case _, True, _: sys.exit("Cannot specify both --extras and --all-extras") - return dict.fromkeys(self._extras).keys() - if not self._all_extras: - return set() - return self.pyproject["project"]["optional-dependencies"].keys() + case _, False, _: + sys.exit("Cannot specify --skip-extras without --all-extras") + case never: + assert_never(never) @cached_property def groups(self) -> AbstractSet[str]: diff --git a/docs/conf.py b/docs/conf.py index e9ffddb05c..5e9aa583ed 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import shutil import sys from datetime import datetime @@ -126,6 +127,7 @@ nb_execution_excludepatterns = [ f"{d}{'/*' * n}" for d in ["tutorials", "how-to"] for n in (1, 2, 3) ] +nb_execution_show_tb = bool(os.environ.get("READTHEDOCS")) nb_merge_streams = True ogp_site_url = "https://scanpy.scverse.org/en/stable/" @@ -140,7 +142,8 @@ katex_prerender = shutil.which(NODEJS_BINARY) is not None intersphinx_mapping = dict( - anndata=("https://anndata.scverse.org/en/stable/", None), + # Needs latest until `.acc` is released in 0.13 + anndata=("https://anndata.scverse.org/en/latest/", None), bbknn=("https://bbknn.readthedocs.io/en/latest/", None), cuml=("https://docs.rapids.ai/api/cuml/stable/", None), cycler=("https://matplotlib.org/cycler/", None), diff --git a/docs/release-notes/1.6.0.md b/docs/release-notes/1.6.0.md index 19b227fc05..6118ec3ee1 100644 --- a/docs/release-notes/1.6.0.md +++ b/docs/release-notes/1.6.0.md @@ -40,7 +40,7 @@ This release includes an overhaul of {func}`~scanpy.pl.dotplot`, {func}`~scanpy. #### Additions -- {func}`~anndata.concat` is now exported from scanpy, see {doc}`anndata:concatenation` for more info. {pr}`1338` {smaller}`I Virshup` +- {func}`~anndata.concat` is now exported from scanpy, see {doc}`anndata:tutorials/concatenation` for more info. {pr}`1338` {smaller}`I Virshup` - Added highly variable gene selection strategy from Seurat v3 {pr}`1204` {smaller}`A Gayoso` - Added [CellRank](https://github.com/theislab/cellrank/) to scanpy ecosystem {pr}`1304` {smaller}`giovp` - Added `backup_url` param to {func}`~scanpy.read_10x_h5` {pr}`1296` {smaller}`A Gayoso` diff --git a/docs/release-notes/4199.feat.md b/docs/release-notes/4199.feat.md new file mode 100644 index 0000000000..25da840acc --- /dev/null +++ b/docs/release-notes/4199.feat.md @@ -0,0 +1 @@ +Add {mod}`anndata.acc` support to {func}`scanpy.get.aggregate`. {smaller}`P Angerer` diff --git a/hatch.toml b/hatch.toml index febdf1a48c..0af1829c08 100644 --- a/hatch.toml +++ b/hatch.toml @@ -26,13 +26,14 @@ overrides.matrix.deps.env-vars = [ overrides.matrix.deps.pre-install-commands = [ { if = [ "low-vers", - ], value = "uv run ci/scripts/low-vers.py pyproject.toml --all-extras --groups=test -o ci/scanpy-low-vers.txt" }, + ], value = "uv run ci/scripts/low-vers.py pyproject.toml --all-extras --skip-extras=scanpy2 --groups=test -o ci/scanpy-low-vers.txt" }, ] overrides.matrix.deps.python = [ { if = [ "low-vers" ], value = "3.12" }, ] overrides.matrix.deps.extra-dependencies = [ { if = [ "stable" ], value = "scipy>=1.17" }, + { if = [ "pre" ], value = "scanpy[scanpy2]" }, { if = [ "pre" ], value = "anndata @ git+https://github.com/scverse/anndata.git" }, { if = [ "pre" ], value = "pandas>=3" }, ] diff --git a/pyproject.toml b/pyproject.toml index 3c73b926b7..482ec4eeb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,7 +99,7 @@ scrublet = [ "scikit-image>=0.25" ] # highly_variable_genes method 'seurat_v3' skmisc = [ "scikit-misc>=0.5.1" ] illico = [ "illico>=0.6" ] -scanpy2 = [ "igraph>=0.10.8", "scanpy[illico]", "scikit-misc>=0.5.1" ] +scanpy2 = [ "anndata>=0.13", "igraph>=0.10.8", "scanpy[illico]", "scikit-misc>=0.5.1" ] [dependency-groups] dev = [ @@ -118,14 +118,14 @@ test = [ { include-group = "test-min" }, ] docs = [ - "ipython>=8.27", # for nbsphinx code highlighting + "ipython>=8.27", # for nbsphinx code highlighting "myst-nb>=1.4", "myst-parser>=2", "nbsphinx>=0.9", - "numpy>=2.4", # type aliases + "numpy>=2.4", # type aliases "sam-algorithm", # TODO: remove necessity for being able to import doc-linked classes - "scanpy[dask-ml,leiden,paga,plotting]", + "scanpy[dask-ml,leiden,paga,plotting,scanpy2,scrublet]", "scanpydoc>=0.16.1", "scverse-misc[sphinx]", "sphinx>=9.1", @@ -137,7 +137,7 @@ docs = [ "sphinx-issues>=5.0.1", "sphinxcontrib-bibtex", "sphinxcontrib-katex", - "sphinxext-opengraph", # for nice cards when sharing on social + "sphinxext-opengraph", # for nice cards when sharing on social ] test-min = [ "dependency-groups", # for CI scripts doctests @@ -274,6 +274,8 @@ filterwarnings = [ "ignore:is_categorical_dtype is deprecated:FutureWarning", # Ignore numba PEP 456 warning specific to ARM machines "ignore:FNV hashing is not implemented in Numba.*:UserWarning", + # Ignore numba macOS warnings + "ignore:Detected unsupported threading environment:UserWarning", # we want to see and eventually fix these "default::numba.core.errors.NumbaPerformanceWarning", # we should set init=obsm["X_pca"] or so diff --git a/src/scanpy/_settings/presets.py b/src/scanpy/_settings/presets.py index d83560946b..1302d12df0 100644 --- a/src/scanpy/_settings/presets.py +++ b/src/scanpy/_settings/presets.py @@ -5,7 +5,7 @@ import re from contextlib import contextmanager from dataclasses import dataclass -from functools import cached_property, partial, wraps +from functools import cache, cached_property, partial, wraps from importlib.metadata import distributions, requires from typing import TYPE_CHECKING, Literal, NamedTuple @@ -17,8 +17,11 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator, Mapping + from collections.abc import Set as AbstractSet from typing import Self + from packaging.utils import NormalizedName + __all__ = [ "DETest", @@ -312,14 +315,18 @@ def check(self) -> Self: return self +@cache +def dist_names() -> AbstractSet[NormalizedName]: + return dict.fromkeys(canonicalize_name(d.name) for d in distributions()).keys() + + def _missing_scanpy2_deps() -> list[Requirement]: - dist_names = {canonicalize_name(d.name) for d in distributions()} return [ r for r in map(Requirement, requires("scanpy") or ()) if r.marker and r.marker.evaluate({"extra": "scanpy2"}, "requirement") - and canonicalize_name(r.name) not in dist_names + and canonicalize_name(r.name) not in dist_names() ] diff --git a/src/scanpy/get/_aggregated.py b/src/scanpy/get/_aggregated.py index 3468543a8c..7d933ec5d4 100644 --- a/src/scanpy/get/_aggregated.py +++ b/src/scanpy/get/_aggregated.py @@ -1,6 +1,8 @@ from __future__ import annotations +from collections.abc import Collection from functools import partial, singledispatch +from importlib.util import find_spec from typing import TYPE_CHECKING, Literal, TypedDict, get_args import numba @@ -11,9 +13,9 @@ from scipy import sparse from sklearn.utils.sparsefuncs import csc_median_axis_0 -from scanpy._compat import CSBase, CSRBase, DaskArray, warn - +from .._compat import CSBase, CSRBase, DaskArray, warn from .._utils import _resolve_axis, get_literal_vals +from .._utils._doctests import doctest_needs from ._kernels import ( agg_sum_csc, agg_sum_csr, @@ -24,10 +26,24 @@ from .get import _check_mask if TYPE_CHECKING: - from collections.abc import Collection, Iterable + import sys + from collections.abc import Iterable from numpy.typing import NDArray + if sys.version_info >= (3, 13): + from typing import TypeIs + else: + from typing_extensions import TypeIs + +if TYPE_CHECKING or find_spec("anndata.acc"): + from anndata.acc import A, AdRef, Idx2D, LayerAcc, MultiAcc +else: + AdRef = type("AdRef", (), dict(__module__="anndata.acc")) + type Idx2D = object + LayerAcc = type("LayerAcc", (), dict(__module__="anndata.acc")) + MultiAcc = type("MultiAcc", (), dict(__module__="anndata.acc")) + type Array = np.ndarray | CSBase | DaskArray type ConstantDtypeAgg = Literal["count_nonzero", "sum", "median"] type AggType = ConstantDtypeAgg | Literal["mean", "var"] @@ -199,19 +215,93 @@ def _power(x: Array, power: float) -> Array: return x**power if isinstance(x, np.ndarray) else x.power(power) +def _collection_of[T](thing: object, typ: type[T]) -> TypeIs[Collection[T]]: + return ( + isinstance(thing, Collection) + and not isinstance(thing, typ) + and len(thing) > 0 + and all(isinstance(e, typ) for e in thing) + ) + + +def _validate_by( + by: AdRef | Collection[AdRef] | str | Collection[str], +) -> list[AdRef] | None: + """Normalize `by` to a list of :class:`~anndata.acc.AdRef` if possible, else `None`.""" + if isinstance(by, AdRef): + return [by] + if _collection_of(by, AdRef): + return list(by) + if not isinstance(by, str) and not _collection_of(by, str): + msg = ( + "`by` must be a single `AdRef`, a collection of `AdRef`, " + f"or a collection of strings, was {by!r}" + ) + raise TypeError(msg) + return None + + +def _resolve_by_and_axis[I: Idx2D | int]( + by: AdRef[I, AnnData] | Collection[AdRef[I, AnnData]] | str | Collection[str], + axis: Literal["obs", 0, "var", 1] | None, + *, + acc: LayerAcc | MultiAcc | None, + layer: str | None, + obsm: str | None, + varm: str | None, +) -> tuple[list[AdRef[I, AnnData]] | None, Literal["obs", "var"]]: + """Resolve `axis_name` based on the accessor.""" + n_old_vec = sum(p is not None for p in [varm, obsm, layer]) + if (by_refs := _validate_by(by)) is None: + if n_old_vec > 1: + msg = "Please only provide one (or none) of varm, obsm, or layer" + raise TypeError(msg) + if axis is None: + axis = 1 if varm else 0 + _, axis_name = _resolve_axis(axis) + if axis_name != (ax_wanted := "var" if varm else "obs" if obsm else axis_name): + msg = f"`{ax_wanted}m` can only be used when grouping over `{ax_wanted}`" + raise ValueError(msg) + return None, axis_name + + if axis is not None: + msg = "`axis` cannot be used when `by` is given as AdRef(s); the axis is inferred from `by`" + raise TypeError(msg) + if n_old_vec: + msg = "`layer`, `obsm`, and `varm` cannot be used when `by` is given as AdRef(s); use `acc` instead" + raise TypeError(msg) + dims = {d for ref in by_refs for d in ref.dims} + if len(dims) != 1: + msg = f"All `by` accessors must refer to the same single axis (`obs` or `var`), got {dims}" + raise ValueError(msg) + axis_name = next(iter(dims)) + if isinstance(acc, MultiAcc) and axis_name != acc.dim: + msg = f"`by`’s axis ({axis_name}) must match `acc`’s ({acc.dim})" + raise ValueError(msg) + return by_refs, axis_name + + +@doctest_needs("anndata_acc") def aggregate( # noqa: PLR0912 adata: AnnData, - by: str | Collection[str], + by: ( + str + | Collection[str] + | AdRef[Idx2D | int, AnnData] + | Collection[AdRef[Idx2D | int, AnnData]] + ), func: AggType | Iterable[AggType], *, - axis: Literal["obs", 0, "var", 1] | None = None, - mask: NDArray[np.bool] | str | None = None, + acc: LayerAcc | MultiAcc | None = None, + mask: NDArray[np.bool] | AdRef[Idx2D | int, AnnData] | str | None = None, dof: int = 1, + # old API + axis: Literal["obs", 0, "var", 1] | None = None, layer: str | None = None, obsm: str | None = None, varm: str | None = None, ) -> AnnData: - """Aggregate data matrix based on some categorical grouping. + r"""Aggregate data matrix based on some categorical grouping. This function is useful for pseudobulking as well as plotting. @@ -219,8 +309,6 @@ def aggregate( # noqa: PLR0912 list of metrics. Each metric is computed over the group and results in a new layer in the output `AnnData` object. - If none of `layer`, `obsm`, or `varm` are passed in, `X` will be used for aggregation data. - .. array-support:: get.aggregate Params @@ -228,21 +316,26 @@ def aggregate( # noqa: PLR0912 adata :class:`~anndata.AnnData` to be aggregated. by - Key of the column to be grouped-by. + References to the vectors to be grouped-by. + Passing a str means using a `obs`/`var` column. func How to aggregate. - axis - Axis on which to find group by column. mask - Boolean mask (or key to column containing mask) to apply along the axis. + Boolean mask (or reference to a mask vector) to apply along the axis. + Passing a str means using a `obs`/`var` column. dof Degrees of freedom for variance. Defaults to 1. + acc + If not None, accessor for aggregation data. + Replaces `layer`, `obsm`, and `varm`. + axis + Axis on which to find group by column. + (inferred from `by` if it is an :class:`~anndata.acc.AdRef`) layer - If not None, key for aggregation data. obsm - If not None, key for aggregation data. varm If not None, key for aggregation data. + Use `acc` instead. Returns ------- @@ -278,6 +371,15 @@ def aggregate( # noqa: PLR0912 Note that this filters out any combination of groups that wasn't present in the original data. + The same computation using the new (:mod:`anndata.acc`-based) API: + + >>> from anndata.acc import A + >>> sc.get.aggregate(pbmc, by=A.obs["louvain"], func=["mean", "count_nonzero"]) + AnnData object with n_obs × n_vars = 8 × 13714 + obs: 'louvain', 'n_obs_aggregated' + var: 'n_cells' + layers: 'mean', 'count_nonzero' + """ if not isinstance(adata, AnnData): msg = ( @@ -285,67 +387,67 @@ def aggregate( # noqa: PLR0912 f"was passed {type(adata)}." ) raise NotImplementedError(msg) - if axis is None: - axis = 1 if varm else 0 - axis, axis_name = _resolve_axis(axis) + + by_refs, axis_name = _resolve_by_and_axis( + by, axis, layer=layer, obsm=obsm, varm=varm, acc=acc + ) + del axis mask = _check_mask(adata, mask, axis_name) - data = adata.X - if sum(p is not None for p in [varm, obsm, layer]) > 1: - msg = "Please only provide one (or none) of varm, obsm, or layer" - raise TypeError(msg) - if varm is not None: - if axis != 1: - msg = "varm can only be used when axis is 1" - raise ValueError(msg) - data = adata.varm[varm] - elif obsm is not None: - if axis != 0: - msg = "obsm can only be used when axis is 0" - raise ValueError(msg) - data = adata.obsm[obsm] - elif layer is not None: - data = adata.layers[layer] - if axis == 1: + if by_refs is not None: + if acc is None: + acc = A.X + if not isinstance(acc, LayerAcc | MultiAcc): + msg = ( + "`acc` must be a `LayerAcc` (e.g. `A.X`, `A.layers[...]`) or " + f"`MultiAcc` (e.g. `A.obsm[...]`, `A.varm[...]`), was {acc!r}" + ) + raise TypeError(msg) + data = adata[acc] + if isinstance(acc, LayerAcc) and axis_name == "var": data = data.T - elif axis == 1: - # i.e., all of `varm`, `obsm`, `layers` are None so we use `X` which must be transposed - data = data.T - - dim_df = getattr(adata, axis_name) - categorical, new_label_df = _combine_categories(dim_df, by) + dim_df = pd.DataFrame({ + ref.idx if isinstance(ref.idx, str) else str(ref): adata[ref] + for ref in by_refs + }) + else: + match layer, obsm, varm, axis_name: + case None, None, None, "obs": + data = adata.X + case None, None, None, "var": + data = adata.X.T + case str(), None, None, "obs": + data = adata.layers[layer] + case str(), None, None, "var": + data = adata.layers[layer].T + case None, str(), None, "obs": + data = adata.obsm[obsm] + case None, None, str(), "var": + data = adata.varm[varm] + dim_df = getattr(adata, axis_name)[[by] if isinstance(by, str) else list(by)] + categorical, new_label_df = _combine_categories(dim_df) # Add number of obs aggregated into each group (respecting the mask) new_label_df["n_obs_aggregated"] = pd.Series( _group_counts(categorical, mask), index=categorical.categories ).reindex(new_label_df.index) # Actual computation - layers = _aggregate( - data, - by=categorical, - func=func, - mask=mask, - dof=dof, - ) + layers = _aggregate(data, by=categorical, func=func, mask=mask, dof=dof) # Define new var dataframe - if obsm or varm: - if isinstance(data, pd.DataFrame): - # Check if there could be labels - var = pd.DataFrame(index=data.columns) - else: - # Create them otherwise - var = pd.DataFrame(index=pd.RangeIndex(data.shape[1]).astype(str)) + if obsm or varm or isinstance(acc, MultiAcc): + var = pd.DataFrame( # Check if there could be labels, create them otherwise + index=data.columns + if isinstance(data, pd.DataFrame) + else pd.RangeIndex(data.shape[1]).astype(str) + ) else: - var = getattr(adata, "var" if axis == 0 else "obs") + var = getattr(adata, "var" if axis_name == "obs" else "obs") # It's all coming together result = AnnData(layers=layers, obs=new_label_df, var=var) - if axis == 1: - return result.T - else: - return result + return result if axis_name == "obs" else result.T @singledispatch @@ -642,42 +744,42 @@ def aggregate_array( return result -def _combine_categories( - label_df: pd.DataFrame, cols: Collection[str] | str -) -> tuple[pd.Categorical, pd.DataFrame]: +def _combine_categories(label_df: pd.DataFrame) -> tuple[pd.Categorical, pd.DataFrame]: """Return both the result categories and a dataframe labelling each row.""" from itertools import product - if isinstance(cols, str): - cols = [cols] - df = pd.DataFrame( - {c: pd.Categorical(label_df[c]).remove_unused_categories() for c in cols}, + { + c: pd.Categorical(label_df[c]).remove_unused_categories() + for c in label_df.columns + }, ) - n_categories = [len(df[c].cat.categories) for c in cols] + n_categories = [len(df[c].cat.categories) for c in label_df.columns] # It's like np.concatenate([x for x in product(*[range(n) for n in n_categories])]) code_combinations = np.indices(n_categories).reshape(len(n_categories), -1) result_categories = pd.Index([ - "_".join(map(str, x)) for x in product(*[df[c].cat.categories for c in cols]) + "_".join(map(str, x)) + for x in product(*[df[c].cat.categories for c in label_df.columns]) ]) # Dataframe with unique combination of categories for each row new_label_df = pd.DataFrame( { c: pd.Categorical.from_codes(code_combinations[i], df[c].cat.categories) - for i, c in enumerate(cols) + for i, c in enumerate(label_df.columns) }, index=result_categories, ) # Calculating result codes - factors = np.ones(len(cols) + 1, dtype=np.int32) # First factor needs to be 1 + # First factor needs to be 1 + factors = np.ones(len(label_df.columns) + 1, dtype=np.int32) np.cumprod(n_categories[::-1], out=factors[1:]) factors = factors[:-1][::-1] - code_array = np.zeros((len(cols), df.shape[0]), dtype=np.int32) - for i, c in enumerate(cols): + code_array = np.zeros((len(label_df.columns), df.shape[0]), dtype=np.int32) + for i, c in enumerate(label_df.columns): code_array[i] = df[c].cat.codes code_array *= factors[:, None] diff --git a/src/scanpy/get/get.py b/src/scanpy/get/get.py index 86fda4a906..1b2bf9d66c 100644 --- a/src/scanpy/get/get.py +++ b/src/scanpy/get/get.py @@ -2,6 +2,7 @@ from __future__ import annotations +from importlib.util import find_spec from typing import TYPE_CHECKING, TypedDict import numpy as np @@ -17,10 +18,16 @@ from anndata._core.sparse_dataset import BaseCompressedSparseDataset from anndata._core.views import ArrayView + from anndata.acc import AdRef, Idx2D from .._compat import DaskArray +if TYPE_CHECKING or find_spec("anndata.acc"): + from anndata.acc import AdRef +else: + AdRef = type("AdRef", (), dict(__module__="anndata.acc")) + # -------------------------------------------------------------------------------- # Plotting data helpers # -------------------------------------------------------------------------------- @@ -477,7 +484,7 @@ def _set_obs_rep( def _check_mask[M: NDArray[np.bool] | NDArray[np.floating] | pd.Series | None]( data: AnnData | np.ndarray | CSBase | DaskArray, - mask: str | M, + mask: str | AdRef[Idx2D | int, AnnData] | M, dim: Literal["obs", "var"], *, allow_probabilities: bool = False, @@ -500,20 +507,11 @@ def _check_mask[M: NDArray[np.bool] | NDArray[np.floating] | pd.Series | None]( return mask desc = "mask/probabilities" if allow_probabilities else "mask" - if isinstance(mask, str): + if isinstance(mask, str | AdRef): if not isinstance(data, AnnData): - msg = f"Cannot refer to {desc} with string without providing anndata object as argument" - raise ValueError(msg) - - annot: pd.DataFrame = getattr(data, dim) - if mask not in annot.columns: - msg = ( - f"Did not find `adata.{dim}[{mask!r}]`. " - f"Either add the {desc} first to `adata.{dim}`" - f"or consider using the {desc} argument with an array." - ) + msg = f"Cannot use refererence for {desc} without providing anndata object as argument" raise ValueError(msg) - mask_array = annot[mask].to_numpy() + mask_array = _get_mask_by_ref(data, mask, dim, desc=desc) else: if len(mask) != data.shape[0 if dim == "obs" else 1]: msg = f"The shape of the {desc} do not match the data." @@ -531,3 +529,27 @@ def _check_mask[M: NDArray[np.bool] | NDArray[np.floating] | pd.Series | None]( raise ValueError(msg) return mask_array + + +def _get_mask_by_ref( + adata: AnnData, mask: AdRef | str, dim: Literal["obs", "var"], *, desc: str +) -> NDArray[np.bool] | NDArray[np.floating]: + if isinstance(mask, AdRef): + if next(iter(mask.dims)) != dim: + msg = f"Dimension of {desc} does not match {dim}." + raise ValueError(msg) + try: + return np.asarray(adata[mask]) + except KeyError: + msg = f"Did not find `{mask}` in `adata`. " + else: + annot: pd.DataFrame = getattr(adata, dim) + if mask not in annot.columns: + msg = f"Did not find `adata.{dim}[{mask!r}]`. " + raise ValueError(msg) + return annot[mask].to_numpy() + msg += ( + f"Either add the {desc} first to `adata.{dim}`" + f"or consider using the {desc} argument with an array." + ) + raise ValueError(msg) diff --git a/src/testing/scanpy/_pytest/marks.py b/src/testing/scanpy/_pytest/marks.py index 654340f604..f2c41dc3fd 100644 --- a/src/testing/scanpy/_pytest/marks.py +++ b/src/testing/scanpy/_pytest/marks.py @@ -1,9 +1,33 @@ from __future__ import annotations from enum import Enum, auto -from importlib.util import find_spec +from functools import cache +from importlib.metadata import distributions, requires, version +from typing import TYPE_CHECKING import pytest +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +if TYPE_CHECKING: + from collections.abc import Set as AbstractSet + + from packaging.utils import NormalizedName + + +def _missing_scanpy2_deps() -> list[Requirement]: + return [ + r + for r in map(Requirement, requires("scanpy") or ()) + if r.marker + and r.marker.evaluate({"extra": "scanpy2"}, "requirement") + and canonicalize_name(r.name) not in dist_names() + ] + + +@cache +def dist_names() -> AbstractSet[NormalizedName]: + return dict.fromkeys(canonicalize_name(d.name) for d in distributions()).keys() class QuietMarkDecorator(pytest.MarkDecorator): @@ -26,7 +50,10 @@ def _generate_next_value_( """Distribution name for matching modules.""" return name.replace("_", "-") - mod: str + req: Requirement + + scanpy2 = "scanpy[scanpy2]" + anndata_acc = "anndata>=0.13" colour = "colour-science" dask = auto() @@ -56,17 +83,22 @@ def _generate_next_value_( trimap = auto() wishbone = "wishbone-dev" - def __init__(self, mod: str) -> None: - self.mod = mod + def __init__(self, req: str) -> None: + self.req = Requirement(req) reason = self.skip_reason dec = pytest.mark.skipif(bool(reason), reason=reason or "") super().__init__(dec.mark) @property def skip_reason(self) -> str | None: - if find_spec(self._name_): + if self._name_ == "scanpy2": + if not (missing := _missing_scanpy2_deps()): + return None + return f"scanpy 2 deps missing: {', '.join(m.name for m in missing)}" + + if ( + canonicalize_name(self.req.name) in dist_names() + and version(self.req.name) in self.req.specifier + ): return None - reason = f"needs module `{self._name_}`" - if self._name_.casefold() != self.mod.casefold().replace("-", "_"): - reason = f"{reason} (`pip install {self.mod}`)" - return reason + return f"needs `{self.req}`" diff --git a/tests/test_aggregated.py b/tests/test_aggregated.py index 459c6e85df..b2c3b7bd99 100644 --- a/tests/test_aggregated.py +++ b/tests/test_aggregated.py @@ -22,6 +22,7 @@ from collections.abc import Callable from typing import Literal + from anndata.acc import AdAcc from numpy.typing import NDArray from scanpy._compat import CSRBase @@ -53,18 +54,25 @@ def xfail_dask_median( @pytest.mark.parametrize("axis", [0, 1]) -def test_mask(axis: Literal[0, 1]) -> None: +@pytest.mark.parametrize("typ", ["str", pytest.param("ref", marks=needs.anndata_acc)]) +def test_mask(axis: Literal[0, 1] | None, typ: Literal["str", "ref"]) -> None: blobs = sc.datasets.blobs() mask = blobs.obs["blobs"] == 0 blobs.obs["mask_col"] = mask if axis == 1: blobs = blobs.T - by_name = sc.get.aggregate(blobs, "blobs", "sum", axis=axis, mask="mask_col") - by_value = sc.get.aggregate(blobs, "blobs", "sum", axis=axis, mask=mask) + if typ == "str": + ref = "mask_col" + elif typ == "ref": + from anndata.acc import A + + ref = A.obs["mask_col"] if axis == 0 else A.var["mask_col"] - assert_equal(by_name, by_value) + by_ref = sc.get.aggregate(blobs, "blobs", "sum", axis=axis, mask=ref) + by_value = sc.get.aggregate(blobs, "blobs", "sum", axis=axis, mask=mask) - assert np.all(by_name["0"].layers["sum"] == 0) + assert_equal(by_ref, by_value) + assert np.all(by_ref["0"].layers["sum"] == 0) @pytest.mark.parametrize("array_type", VALID_ARRAY_TYPES) @@ -189,13 +197,6 @@ def test_aggregate_entry() -> None: assert_equal(x_result.layers, varm_result.T.layers) -def test_aggregate_incorrect_dim() -> None: - adata = pbmc3k_processed().raw.to_adata() - - with pytest.raises(ValueError, match="was 'foo'"): - sc.get.aggregate(adata, ["louvain"], "sum", axis="foo") - - def to_bad_chunking(x: CSRBase) -> DaskArray: import dask.array as da @@ -394,56 +395,50 @@ def test_aggregate_examples( @pytest.mark.parametrize( - ("label_cols", "cols", "expected"), + ("label_cols", "expected"), [ pytest.param( dict( a=pd.Categorical(["a", "b", "c"]), b=pd.Categorical(["d", "d", "f"]), + c=pd.Categorical(["g", "h", "h"]), ), - ["a", "b"], - pd.Categorical(["a_d", "b_d", "c_f"]), - id="two_of_two", + pd.Categorical(["a_d_g", "b_d_h", "c_f_h"]), + id="three", ), pytest.param( dict( a=pd.Categorical(["a", "b", "c"]), b=pd.Categorical(["d", "d", "f"]), - c=pd.Categorical(["g", "h", "h"]), ), - ["a", "b", "c"], - pd.Categorical(["a_d_g", "b_d_h", "c_f_h"]), - id="three_of_three", + pd.Categorical(["a_d", "b_d", "c_f"]), + id="two-1", ), pytest.param( dict( a=pd.Categorical(["a", "b", "c"]), - b=pd.Categorical(["d", "d", "f"]), c=pd.Categorical(["g", "h", "h"]), ), - ["a", "c"], pd.Categorical(["a_g", "b_h", "c_h"]), - id="two_of_three-1", + id="two-2", ), pytest.param( dict( - a=pd.Categorical(["a", "b", "c"]), b=pd.Categorical(["d", "d", "f"]), c=pd.Categorical(["g", "h", "h"]), ), - ["b", "c"], pd.Categorical(["d_g", "d_h", "f_h"]), - id="two_of_three-2", + id="two-3", ), ], ) def test_combine_categories( - label_cols: dict[str, pd.Categorical], cols: list[str], expected: pd.Categorical + label_cols: dict[str, pd.Categorical], expected: pd.Categorical ) -> None: from scanpy.get._aggregated import _combine_categories label_df = pd.DataFrame(label_cols) - result, result_label_df = _combine_categories(label_df, cols) + result, result_label_df = _combine_categories(label_df) assert isinstance(result, pd.Categorical) @@ -454,7 +449,9 @@ def test_combine_categories( ) reconstructed_df = pd.DataFrame( - [x.split("_") for x in result], columns=cols, index=result.astype(str) + [x.split("_") for x in result], + columns=list(label_cols), + index=result.astype(str), ).astype("category") pd.testing.assert_frame_equal(reconstructed_df, result_label_df) @@ -548,10 +545,154 @@ def test_aggregate_obsm_labels() -> None: assert_equal(expected, result) -def test_dispatch_not_implemented() -> None: +@needs.anndata_acc +@pytest.mark.parametrize("axis", ["obs", "var"]) +@pytest.mark.parametrize("attr", [pytest.param(None, id="x"), "layers", "obsm", "varm"]) +@pytest.mark.parametrize("by", ["blobs", ["blobs", "extra"]], ids=["single", "multi"]) +def test_acc_api( + *, + axis: Literal["obs", "var"], + attr: Literal["obsm", "varm", "layers"] | None, + by: str | list[str], +) -> None: + if (attr == "obsm" and axis == "var") or (attr == "varm" and axis == "obs"): + pytest.skip() + + from anndata.acc import A + + adata = sc.datasets.blobs() + adata.obs["blobs"] = adata.obs["blobs"].astype(str) + adata.obs["extra"] = np.tile(["a", "b"], adata.n_obs)[: adata.n_obs] + if attr == "layers": + adata.layers["test"] = adata.X.copy() + del adata.X + elif attr in {"obsm", "varm"}: + adata.obsm["test"] = adata.X[:, ::2].copy() + del adata.X + if axis == "var": + adata = adata.T.copy() + + old = sc.get.aggregate( + *(adata, by, ["sum", "mean"]), + axis=axis, + **({} if attr is None else {attr.removesuffix("s"): "test"}), + ) + new = sc.get.aggregate( + *(adata, getattr(A, axis)[by], ["sum", "mean"]), + **({} if attr is None else dict(acc=getattr(A, attr)["test"])), + ) + + assert_equal(old, new) + + +@needs.anndata_acc +@pytest.mark.parametrize( + ("mk_args", "exc_cls", "pat"), + [ + pytest.param( + lambda _: dict(axis=0), TypeError, r"axis.*cannot be used", id="axis" + ), + pytest.param( + lambda _: dict(layer="x"), TypeError, r"layer.*cannot be used", id="layer" + ), + pytest.param( + lambda a: dict(acc=a.obsp["connectivities"]), + TypeError, + r"`acc` must be a `LayerAcc`.*or.*`MultiAcc`", + id="acc-type", + ), + pytest.param( + lambda a: dict(by=[a.obs["blobs"], a.var.index]), + ValueError, + "same single axis", + id="by-dims", + ), + pytest.param( + lambda a: dict(acc=a.varm["test"]), + ValueError, + r"`by`.*(obs).*`acc`.*(var)", + id="acc-dim", + ), + ], +) +def test_acc_api_errors( + mk_args: Callable[[AdAcc], dict], exc_cls: type[Exception], pat: str +) -> None: + from anndata.acc import A + + adata = sc.datasets.blobs() + adata.obs["blobs"] = adata.obs["blobs"].astype(str) + adata.varm["test"] = adata.X.T[:, ::2].copy() + adata.obsp["connectivities"] = np.eye(adata.n_obs) + kwargs = mk_args(A) + kwargs.setdefault("by", A.obs["blobs"]) + + with pytest.raises(exc_cls, match=pat): + sc.get.aggregate(adata, func="sum", **kwargs) + + +@pytest.mark.parametrize( + ("kwargs", "match"), + [ + pytest.param( + dict(layer="test", obsm="test"), + r"only provide one \(or none\) of varm, obsm, or layer", + id="layer-and-obsm", + ), + pytest.param( + dict(obsm="test", axis=1), + r"`obsm` can only be used when grouping over `obs`", + id="obsm-axis-var", + ), + pytest.param( + dict(varm="test", axis=0), + r"`varm` can only be used when grouping over `var`", + id="varm-axis-obs", + ), + pytest.param(dict(axis="foo"), r"was 'foo'", id="bad-axis-value"), + ], +) +def test_old_api_errors(kwargs: dict, match: str) -> None: + adata = sc.datasets.blobs() + adata.layers["test"] = adata.X.copy() + adata.obsm["test"] = adata.X.copy() + adata.varm["test"] = np.column_stack([adata.X[0], adata.X[1]]) + with pytest.raises((TypeError, ValueError), match=match): + sc.get.aggregate(adata, by="blobs", func="sum", **kwargs) + + +def test_error_by_invalid_type() -> None: + adata = sc.datasets.blobs() + with pytest.raises(TypeError, match=r"`by` must be.*AdRef.*str"): + sc.get.aggregate(adata, 123, "sum") # type: ignore[arg-type] + + +def test_error_dispatch_not_implemented() -> None: adata = sc.datasets.blobs() with pytest.raises(NotImplementedError): - sc.get.aggregate(adata.X, adata.obs["blobs"], "sum") + sc.get.aggregate(adata.X, adata.obs["blobs"], "sum") # type: ignore[arg-type] + + +@needs.anndata_acc +def test_by_obsm_slice() -> None: + """Test that not only `.obs`/`.var` are supported.""" + from anndata.acc import A + + adata = sc.datasets.blobs() + adata.obs["blobs"] = adata.obs["blobs"].astype(str) + adata.obsm["thing"] = np.column_stack([ + adata.obs["blobs"].astype(int).to_numpy(), + np.zeros(adata.n_obs, dtype=int), + ]) + + result = sc.get.aggregate(adata, by=A.obsm["thing"][:, 0], func=["sum", "mean"]) + expected = sc.get.aggregate(adata, by="blobs", func=["sum", "mean"]) + + np.testing.assert_allclose(result.layers["sum"], expected.layers["sum"]) + np.testing.assert_allclose(result.layers["mean"], expected.layers["mean"]) + pd.testing.assert_series_equal( + result.obs["n_obs_aggregated"], expected.obs["n_obs_aggregated"] + ) def test_factors() -> None: diff --git a/tests/test_scaling.py b/tests/test_scaling.py index 0c665f2d39..61ee930067 100644 --- a/tests/test_scaling.py +++ b/tests/test_scaling.py @@ -118,7 +118,7 @@ def test_scale( def test_mask_string(): - with pytest.raises(ValueError, match=r"Cannot refer to mask.* without.*anndata"): + with pytest.raises(ValueError, match=r"Cannot.*refer.*mask.*without.*anndata"): sc.pp.scale(np.array(X_original), mask_obs="mask") adata = AnnData(np.array(X_for_mask, dtype="float32")) adata.obs["some cells"] = np.array((0, 0, 1, 1, 1, 0, 0), dtype=bool) diff --git a/tests/test_settings.py b/tests/test_settings.py index 3ea4290b14..da3f91af2f 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,9 +1,12 @@ from __future__ import annotations +import inspect + import pytest import scanpy as sc -from scanpy._settings.presets import _missing_scanpy2_deps +from scanpy._settings import presets +from testing.scanpy._pytest import marks # TODO: reset everything @@ -21,7 +24,7 @@ def test_set_figure_params_warns() -> None: def test_preset_scanpy_v2_preview_checks_deps() -> None: - if _missing_scanpy2_deps(): + if presets._missing_scanpy2_deps(): with pytest.raises(ImportError, match=r"scanpy\[scanpy2\]"): sc.settings.preset = sc.Preset.ScanpyV2Preview else: @@ -29,3 +32,14 @@ def test_preset_scanpy_v2_preview_checks_deps() -> None: assert sc.settings.preset is sc.Preset.ScanpyV2Preview sc.settings.preset = sc.Preset.ScanpyV1 assert sc.settings.preset is sc.Preset.ScanpyV1 + + +@pytest.mark.parametrize("func", ["_missing_scanpy2_deps", "dist_names"]) +def test_no_divergence(func: str) -> None: + """Unfortunately this function has to be duplicated. + + - we can’t import `scanpy` too early for coverage + - we can’t import `testing.scanpy` in `scanpy` + """ + a, b = (inspect.getsource(getattr(mod, func)) for mod in [presets, marks]) + assert a == b