diff --git a/pyproject.toml b/pyproject.toml index 23853bac..7542f173 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dynamic = [ "version", # allow version to be set by git tags ] dependencies = [ - "anndata", + "anndata>=0.12", "click", "dask-image", "h5py", diff --git a/src/spatialdata_io/_constants/_constants.py b/src/spatialdata_io/_constants/_constants.py index b4add6eb..479682d9 100644 --- a/src/spatialdata_io/_constants/_constants.py +++ b/src/spatialdata_io/_constants/_constants.py @@ -190,15 +190,10 @@ class AteraKeys(ModeEnum): # cell_feature_matrix.zarr.zip groups/keys X_GROUP = "X" - X_DATA = "data" - X_INDICES = "indices" - X_INDPTR = "indptr" OBS_GROUP = "obs" VAR_GROUP = "var" OBSM_GROUP = "obsm" VARM_GROUP = "varm" - CATEGORIES = "categories" - CODES = "codes" COLOR = "color" # root `.zattrs` manifest (RFC "cell feature anndata", schema anndata_zattrs/v0.1.0) @@ -217,7 +212,6 @@ class AteraKeys(ModeEnum): LABEL_ID = "label_id" CENTROID_X = "centroid_column" CENTROID_Y = "centroid_row" - CELL_AREA = "cell_area" # var / features FEATURE_ID = "feature_id" diff --git a/src/spatialdata_io/converters/atera_to_parquet.py b/src/spatialdata_io/converters/atera_to_parquet.py index cafcbcce..ed83d1ed 100644 --- a/src/spatialdata_io/converters/atera_to_parquet.py +++ b/src/spatialdata_io/converters/atera_to_parquet.py @@ -6,11 +6,16 @@ import numpy as np import pyarrow as pa import pyarrow.parquet as pq -import zarr from spatialdata_io._constants._constants import AteraKeys -from spatialdata_io.readers._atera_common import _get_array, _get_group, _read_transcript_tile, _tile_row_ranges -from spatialdata_io.readers.atera import read_var +from spatialdata_io.readers._atera_common import ( + _get_array, + _get_group, + _open_zip, + _read_feature_names, + _read_transcript_tile, + _tile_row_ranges, +) __all__ = ["atera_to_parquet"] @@ -57,9 +62,9 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path: ``transcript_id`` is taken from each tile's stored ``id`` array when every tile has one, and is otherwise assigned contiguously in the order of the grid group's ``grid_keys`` attribute. - The input is read and written one spatial tile at a time (mirroring how `atera`'s own transcripts - reader is built), so converting even a dataset with tens of millions of transcripts does not require - holding the whole table in memory at once. + The input is read and written one row range of a spatial tile at a time (one parquet row group each, + as `atera`'s own transcripts reader partitions it), so converting even a dataset with hundreds of millions + of transcripts does not require holding the whole table in memory at once. Parameters ---------- @@ -75,55 +80,57 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path: path = Path(path) output = Path(output) - feature_names = read_var(path)[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() + feature_names = _read_feature_names(path) - store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True) - writer = pq.ParquetWriter(output, _PROSEG_SCHEMA) + # written next to `output` and moved there when complete, so a failed run leaves no truncated (but valid) + # parquet file and keeps an earlier complete one + partial = output.with_name(output.name + ".partial") try: - grid = _get_group(zarr.open_group(store, mode="r"), str(AteraKeys.GRID_GROUP)) - tile_keys = [str(key) for key in cast("list[str]", grid.attrs[str(AteraKeys.GRID_KEYS)])] - # Stored IDs are used only if every tile has them; otherwise every transcript is numbered - # contiguously in `grid_keys` order, so stored and generated IDs never mix. - use_stored_ids = all(str(AteraKeys.TRANSCRIPTS_ID) in _get_group(grid, key) for key in tile_keys) - - offset = 0 - for tile_key in tile_keys: - tile = _get_group(grid, tile_key) - n_tile = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] - # A tile can hold 100M+ transcripts, so it is read and written in row ranges (one parquet row group each). - for start, stop in _tile_row_ranges(n_tile): - tile_df = _read_transcript_tile(path, tile_key, feature_names, start, stop) - n = len(tile_df) - if n == 0: - continue - - if use_stored_ids: - stored_ids = _get_array(tile, str(AteraKeys.TRANSCRIPTS_ID))[start:stop] - transcript_id = np.asarray(stored_ids, dtype=np.uint64).reshape(-1) - else: - transcript_id = np.arange(offset, offset + n, dtype=np.uint64) - - cell_id = tile_df[str(AteraKeys.CELL_ID)].to_numpy() - cell_id_str = np.where(cell_id == -1, _CELL_ID_UNASSIGNED, cell_id.astype(str)) - - table = pa.table( - { - "transcript_id": transcript_id, - "cell_id": cell_id_str, - "overlaps_nucleus": tile_df["overlaps_nucleus"].to_numpy().astype(np.uint8), - "feature_name": tile_df[str(AteraKeys.FEATURE_NAME)].astype(str).to_numpy(), - "x_location": tile_df[str(AteraKeys.TRANSCRIPTS_X)].to_numpy().astype(np.float32), - "y_location": tile_df[str(AteraKeys.TRANSCRIPTS_Y)].to_numpy().astype(np.float32), - "z_location": tile_df[str(AteraKeys.TRANSCRIPTS_Z)].to_numpy().astype(np.float32), - "qv": tile_df["quality_score"].to_numpy().astype(np.float32), - "fov_name": np.full(n, tile_key), - }, - schema=_PROSEG_SCHEMA, - ) - writer.write_table(table) - offset += n - finally: - writer.close() - store.close() - + with _open_zip(path / AteraKeys.TRANSCRIPTS_FILE) as group, pq.ParquetWriter(partial, _PROSEG_SCHEMA) as writer: + grid = _get_group(group, str(AteraKeys.GRID_GROUP)) + tile_keys = [str(key) for key in cast("list[str]", grid.attrs[str(AteraKeys.GRID_KEYS)])] + # Stored IDs are used only if every tile has them; otherwise every transcript is numbered + # contiguously in `grid_keys` order, so stored and generated IDs never mix. + use_stored_ids = all(str(AteraKeys.TRANSCRIPTS_ID) in _get_group(grid, key) for key in tile_keys) + + offset = 0 + for tile_key in tile_keys: + tile = _get_group(grid, tile_key) + n_tile = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] + # A tile can hold 100M+ transcripts, so it is read and written in row ranges (one parquet row group each). + for start, stop in _tile_row_ranges(n_tile): + tile_df = _read_transcript_tile(grid, tile_key, feature_names, start, stop) + n = len(tile_df) + if n == 0: + continue + + if use_stored_ids: + stored_ids = _get_array(tile, str(AteraKeys.TRANSCRIPTS_ID))[start:stop] + transcript_id = np.asarray(stored_ids, dtype=np.uint64).reshape(-1) + else: + transcript_id = np.arange(offset, offset + n, dtype=np.uint64) + + cell_id = tile_df[str(AteraKeys.CELL_ID)].to_numpy() + cell_id_str = np.where(cell_id == -1, _CELL_ID_UNASSIGNED, cell_id.astype(str)) + + table = pa.table( + { + "transcript_id": transcript_id, + "cell_id": cell_id_str, + "overlaps_nucleus": tile_df["overlaps_nucleus"].to_numpy().astype(np.uint8), + "feature_name": tile_df[str(AteraKeys.FEATURE_NAME)].astype(str).to_numpy(), + "x_location": tile_df[str(AteraKeys.TRANSCRIPTS_X)].to_numpy().astype(np.float32), + "y_location": tile_df[str(AteraKeys.TRANSCRIPTS_Y)].to_numpy().astype(np.float32), + "z_location": tile_df[str(AteraKeys.TRANSCRIPTS_Z)].to_numpy().astype(np.float32), + "qv": tile_df["quality_score"].to_numpy().astype(np.float32), + "fov_name": np.full(n, tile_key), + }, + schema=_PROSEG_SCHEMA, + ) + writer.write_table(table) + offset += n + except BaseException: + partial.unlink(missing_ok=True) + raise + partial.replace(output) return output diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 849d933b..9c17f044 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import contextlib import json import re @@ -17,23 +16,27 @@ from geopandas import GeoDataFrame from shapely import GeometryType, from_ragged_array from spatialdata import SpatialData +from spatialdata._logging import logger from spatialdata.models import ( Image2DModel, Image3DModel, - Labels2DModel, PointsModel, ShapesModel, ) -from spatialdata.transformations import set_transformation from spatialdata.transformations.transformations import Identity, Scale from xarray import DataTree from spatialdata_io._constants._constants import AteraKeys +from spatialdata_io.readers._utils._utils import parse_physical_size if TYPE_CHECKING: - from collections.abc import Iterator, Mapping + from collections.abc import Iterator, Mapping, Sequence + from numpy.typing import ArrayLike from xarray import DataArray + from zarr.abc.buffer import Buffer + from zarr.abc.store import ByteRequest + from zarr.core.buffer import BufferPrototype # `zarr.Group.get_array` / `zarr.Group.get_group` (typed lookups that raise on a missing node or on the @@ -75,98 +78,82 @@ def _get_group(group: zarr.Group, path: str) -> zarr.Group: # Max transcripts per `dask` partition / `atera_to_parquet` batch (~130 MB of decoded columns). DEFAULT_TRANSCRIPT_CHUNK_ROWS = 5_000_000 -_UINT32_SENTINEL = np.iinfo(np.uint32).max +class _PadRaggedVlenStore(zarr.storage.WrapperStore[zarr.storage.ZipStore]): + """Pad the short trailing chunk of a 1-D ``vlen-utf8`` zarr v2 array to the nominal chunk size. -@contextlib.contextmanager -def _patched_ragged_vlen_chunk_decode() -> Iterator[None]: - """Work around ``zarr``'s `V2Codec` failing to decode a vlen (string) array's ragged trailing chunk. - - `obs`/`var`'s string columns (e.g. `barcode`) are stored as 1-D zarr v2 arrays with a fixed - chunk size chosen independently of the row count (e.g. `1_000_000`), so the trailing chunk is - usually smaller than that nominal size. For fixed-width dtypes this is harmless: `zarr` pads an - incomplete edge chunk to the full chunk shape (with the fill value) before compression, so it - always decodes back to the full nominal size. Variable-length (`vlen-utf8`/`StringDType`) - chunks aren't padded this way -- only the elements actually written are stored -- but - `V2Codec._decode_single` unconditionally reshapes the decoded chunk to the nominal chunk shape - regardless of dtype, so reading across that trailing chunk raises e.g. ``ValueError: cannot - reshape array of size 376951 into shape (1000000,)``. Reproduced against `zarr` 3.1.6 and - 3.4.0 (the latest release as of writing); no matching issue found upstream yet. - - This patches `V2Codec._decode_single` (the abstract-method entry point present on every - `zarr` 3.x release, unlike the private `_decode_sync`/`_decode_single` split introduced partway - through the 3.x series) for the duration of the context, falling back to the chunk's actual - (smaller) size instead of raising, only for the 1-D case this affects; any other reshape failure - (e.g. a genuinely corrupt chunk, or a ragged chunk in more than one dimension) is still raised. + 10x writes the last chunk of the `obs`/`var` string columns (e.g. `barcode`, chunks of 1,000,000) with only the + items it holds. zarr decodes every v2 chunk at the nominal size and raises "cannot reshape array of size 376951 + into shape (1000000,)"; zarr's own writer pads that chunk. Only a trailing chunk that holds exactly the remaining + items is padded; any other chunk is passed on unchanged, so zarr still raises on a corrupt one. """ - try: - from numcodecs.compat import ensure_ndarray_like - from zarr.codecs._v2 import V2Codec - from zarr.registry import get_ndbuffer_class - except ImportError: - # `zarr`'s internal layout changed enough that this patch no longer applies; read without - # it rather than fail outright (bundles without a ragged vlen chunk are unaffected either way). - yield - return - - original_decode_single = V2Codec._decode_single - - async def patched_decode_single(self: V2Codec, chunk_bytes: Any, chunk_spec: Any) -> Any: - def decode() -> Any: - cdata = chunk_bytes.as_array_like() - chunk = self.compressor.decode(cdata) if self.compressor else cdata - if self.filters: - for f in reversed(self.filters): - chunk = f.decode(chunk) - chunk = ensure_ndarray_like(chunk) - if chunk_spec.dtype.dtype_cls is not np.dtypes.ObjectDType: - try: - chunk = chunk.view(chunk_spec.dtype.to_native_dtype()) - except TypeError: - chunk = np.array(chunk).astype(chunk_spec.dtype.to_native_dtype()) - elif chunk.dtype != object: - raise RuntimeError("cannot read object array without object codec") - - chunk = chunk.reshape(-1, order="A") - nominal_size = int(np.prod(chunk_spec.shape)) - if chunk.size == nominal_size: - chunk = chunk.reshape(chunk_spec.shape, order=chunk_spec.order) - elif len(chunk_spec.shape) != 1: - raise ValueError(f"cannot reshape array of size {chunk.size} into shape {chunk_spec.shape}") - # else: ragged trailing chunk of a 1-D vlen array -- keep its natural, smaller size. - return get_ndbuffer_class().from_ndarray_like(chunk) - - return await asyncio.to_thread(decode) - - V2Codec._decode_single = patched_decode_single # type: ignore[method-assign] - try: - yield - finally: - V2Codec._decode_single = original_decode_single # type: ignore[method-assign] + + async def get(self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None) -> Buffer | None: + buf = await super().get(key, prototype, byte_range) + parent, _, name = key.rpartition("/") + if buf is None or byte_range is not None or not name.isdigit(): + return buf + meta_buf = await super().get(f"{parent}/.zarray" if parent else ".zarray", prototype) + meta = json.loads(meta_buf.to_bytes()) if meta_buf is not None else {} + if len(meta.get("shape", ())) != 1 or [f["id"] for f in meta.get("filters") or []] != ["vlen-utf8"]: + return buf + (size,), (chunk,) = meta["shape"], meta["chunks"] + n_items = size - int(name) * chunk + if not 0 < n_items < chunk: + return buf + from numcodecs import VLenUTF8, get_codec + + compressor = get_codec(meta["compressor"]) if meta.get("compressor") else None + raw = buf.to_bytes() + items = VLenUTF8().decode(compressor.decode(raw) if compressor else raw) + if len(items) != n_items: + return buf + padded = np.full(chunk, meta.get("fill_value") or "", dtype=object) + padded[:n_items] = items + encoded = VLenUTF8().encode(padded) + return prototype.buffer.from_bytes(compressor.encode(encoded) if compressor else encoded) + + +@contextlib.contextmanager +def _open_zip(path: Path) -> Iterator[zarr.Group]: + """Open a zipped zarr store read-only (see `_PadRaggedVlenStore`).""" + with zarr.storage.ZipStore(path, read_only=True) as store: + yield zarr.open_group(_PadRaggedVlenStore(store), mode="r") + + +def _unique_feature_names(names: Any) -> pd.Index: + """Feature names made unique as anndata does (``G``, ``G-1``), so they can index `var` and label categories.""" + from anndata.utils import make_index_unique + + names = pd.Index([str(name) for name in names]) # numpy can't cast its StringDType, which zarr may return + if not names.is_unique: + logger.warning(f"{int(names.duplicated().sum())} duplicate feature names; made unique with a suffix.") + return make_index_unique(names) + + +def _read_feature_names(path: Path) -> list[str]: + """The unique feature names of the bundle in store order, which is the order of each tile's ``gene_offset``.""" + from anndata.io import read_elem + + with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: + return _unique_feature_names(read_elem(group[f"{AteraKeys.VAR_GROUP}/{AteraKeys.FEATURE_NAME}"])).tolist() + + +def _read_cell_ids(path: Path) -> np.ndarray: + """The cell ids of the bundle in store order, which is the row order of ``cells.zarr.zip``.""" + from anndata.io import read_elem + + with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: + return np.asarray(read_elem(group[f"{AteraKeys.OBS_GROUP}/{AteraKeys.CELL_ID}"])) def _decode_packed_cell_id(raw: np.ndarray) -> np.ndarray: """Decode a (N, 2) uint32 packed cell id array into the int64 ids used in the cell-feature-matrix obs. - The two uint32 columns pack into an int64 as ``low + (high << 32)``. The sentinel - ``(uint32 max, uint32 max)`` marks "no cell assigned" and is decoded to ``-1``. + The two uint32 columns pack into an int64 as ``low + (high << 32)``, which is the little-endian int64 view of + the pair. The sentinel ``(uint32 max, uint32 max)`` marks "no cell assigned" and so views as ``-1``. """ - low = raw[:, 0].astype(np.int64) - high = raw[:, 1].astype(np.int64) - packed = low + (high << 32) - unassigned = (raw[:, 0] == _UINT32_SENTINEL) & (raw[:, 1] == _UINT32_SENTINEL) - packed[unassigned] = -1 - return packed - - -def _get_labels( - cells_group: zarr.Group, - mask_index: int, - labels_models_kwargs: Mapping[str, Any] = MappingProxyType({}), -) -> DataArray | DataTree: - """Read the labels raster from cells.zarr.zip masks/{mask_index} (0 = nucleus, 1 = cell).""" - masks = da.from_array(_get_array(cells_group, f"{AteraKeys.MASKS_GROUP}/{mask_index}")) - return Labels2DModel.parse(masks, dims=("y", "x"), transformations={"global": Identity()}, **labels_models_kwargs) + return np.ascontiguousarray(raw, dtype=" 0) & (num_vertices < 3) # shapely needs 3 vertices for a ring + if degenerate.any(): + logger.warning(f"{int(degenerate.sum())} polygon(s) with 1 or 2 vertices kept as empty polygons.") + coords = coords[np.repeat(~degenerate, num_vertices)] + num_vertices = np.where(degenerate, 0, num_vertices) ring_offsets = np.concatenate([[0], np.cumsum(num_vertices)]) geom_offsets = np.arange(n_polygons + 1) geoms = from_ragged_array(GeometryType.POLYGON, coords, offsets=(ring_offsets, geom_offsets)) @@ -243,82 +235,79 @@ def _tile_row_ranges(n_rows: int, chunk_rows: int = DEFAULT_TRANSCRIPT_CHUNK_ROW return [(start, min(start + chunk_rows, n_rows)) for start in range(0, n_rows, chunk_rows)] or [(0, 0)] -def _read_transcript_tile( - path: Path, tile_key: str, feature_names: list[str], start: int = 0, stop: int | None = None -) -> pd.DataFrame: - """Read and decode rows ``[start, stop)`` of a single spatial tile of transcripts.zarr.zip (default: the whole tile). - - Opens its own zip store (rather than sharing one across tiles) so this function can safely - be called from independent `dask` tasks. - """ - store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - tile = _get_group(group, f"{AteraKeys.GRID_GROUP}/{tile_key}") - location_array = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)) - stop = location_array.shape[0] if stop is None else min(stop, location_array.shape[0]) - location = np.asarray(location_array[start:stop]) - gene_offset = np.asarray(_get_array(tile, str(AteraKeys.TRANSCRIPTS_GENE_OFFSET))[...]) - cell_id_raw = np.asarray(_get_array(tile, str(AteraKeys.CELL_ID))[start:stop]) - overlaps_nucleus = np.asarray( - _get_array(tile, str(AteraKeys.TRANSCRIPTS_OVERLAPS_NUCLEUS))[start:stop] - ).reshape(-1) - quality_score = np.asarray(_get_array(tile, str(AteraKeys.TRANSCRIPTS_QUALITY_SCORE))[start:stop]).reshape(-1) - finally: - store.close() +def _transcript_rows(tile: zarr.Group, start: int, stop: int, feature_name: pd.Categorical[Any]) -> pd.DataFrame: + """Decode rows ``[start, stop)`` of one tile of transcripts.zarr.zip, labelled with `feature_name`.""" - # Rows within a tile are sorted by gene; gene_offset[g] = [start, end) gives the row range for - # gene g, so the per-row gene index must be reconstructed rather than read directly: row r belongs to - # the first gene whose range ends after r. - gene_index = np.searchsorted(gene_offset[:, 1], np.arange(start, stop), side="right") - feature_name = pd.Categorical.from_codes(gene_index, categories=pd.Index(feature_names)) + def rows(key: AteraKeys) -> np.ndarray: + return np.asarray(_get_array(tile, str(key))[start:stop]) + location = rows(AteraKeys.TRANSCRIPTS_LOCATION) return pd.DataFrame( { str(AteraKeys.TRANSCRIPTS_X): location[:, 0].astype(np.float32), str(AteraKeys.TRANSCRIPTS_Y): location[:, 1].astype(np.float32), str(AteraKeys.TRANSCRIPTS_Z): location[:, 2].astype(np.float32), str(AteraKeys.FEATURE_NAME): feature_name, - str(AteraKeys.CELL_ID): _decode_packed_cell_id(cell_id_raw), - "overlaps_nucleus": overlaps_nucleus.astype(bool), - "quality_score": quality_score.astype(np.float32), + str(AteraKeys.CELL_ID): _decode_packed_cell_id(rows(AteraKeys.CELL_ID)), + "overlaps_nucleus": rows(AteraKeys.TRANSCRIPTS_OVERLAPS_NUCLEUS).reshape(-1).astype(bool), + "quality_score": rows(AteraKeys.TRANSCRIPTS_QUALITY_SCORE).reshape(-1).astype(np.float32), } ) -def _get_points(path: Path, pixel_size: float, feature_names: list[str]) -> dd.DataFrame: - """Lazily read transcripts.zarr.zip, one `dask` task per spatial tile.""" - store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - grid = _get_group(group, str(AteraKeys.GRID_GROUP)) - tile_keys = sorted(grid.group_keys()) - tile_sizes = { - key: _get_array(_get_group(grid, key), str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] for key in tile_keys - } - finally: - store.close() +def _read_transcript_tile( + grid: zarr.Group, tile_key: str, feature_names: list[str], start: int = 0, stop: int | None = None +) -> pd.DataFrame: + """Read and decode rows ``[start, stop)`` of a single spatial tile of transcripts.zarr.zip (default: the whole tile). + + `grid` is the store's ``grid`` group. All `dask` tasks share it, so the zip is opened once per process, not once + per task. + """ + tile = _get_group(grid, tile_key) + n_rows = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] + stop = n_rows if stop is None else min(stop, n_rows) + gene_offset = np.asarray(_get_array(tile, str(AteraKeys.TRANSCRIPTS_GENE_OFFSET))[...]) + # Rows within a tile are sorted by gene; gene_offset[g] = [start, end) gives the row range for + # gene g, so the per-row gene index must be reconstructed rather than read directly: row r belongs to + # the first gene whose range ends after r. + gene_index = np.searchsorted(gene_offset[:, 1], np.arange(start, stop), side="right") + feature_name = pd.Categorical.from_codes(gene_index, categories=pd.Index(feature_names)) + return _transcript_rows(tile, start, stop, feature_name) + - cat_dtype = pd.CategoricalDtype(categories=feature_names) - meta = pd.DataFrame( +def _transcripts_meta(categories: list[str]) -> pd.DataFrame: + """An empty transcripts table with the column dtypes of `_transcript_rows`.""" + return pd.DataFrame( { str(AteraKeys.TRANSCRIPTS_X): pd.array([], dtype="float32"), str(AteraKeys.TRANSCRIPTS_Y): pd.array([], dtype="float32"), str(AteraKeys.TRANSCRIPTS_Z): pd.array([], dtype="float32"), - str(AteraKeys.FEATURE_NAME): pd.array([], dtype=cat_dtype), + str(AteraKeys.FEATURE_NAME): pd.Categorical([], categories=categories), str(AteraKeys.CELL_ID): pd.array([], dtype="int64"), "overlaps_nucleus": pd.array([], dtype="bool"), "quality_score": pd.array([], dtype="float32"), } ) + + +def _get_points(path: Path, pixel_size: float, feature_names: list[str]) -> dd.DataFrame: + """Lazily read transcripts.zarr.zip, one `dask` task per row range of a spatial tile (see `_tile_row_ranges`).""" + # the store stays referenced by the graph; it reopens on first use, also after pickling + grid = zarr.open_group(zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True), mode="r") + grid = _get_group(grid, str(AteraKeys.GRID_GROUP)) + tile_keys = sorted(grid.group_keys()) + tile_sizes = { + key: _get_array(_get_group(grid, key), str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] for key in tile_keys + } + # One partition per `chunk_rows`-row slice of a tile (not per whole tile), so peak memory is bounded by # `dask`'s number of concurrent tasks times the chunk size, however large a tile is. delayed_tiles = [ - delayed(_read_transcript_tile)(path, key, feature_names, start, stop) + delayed(_read_transcript_tile)(grid, key, feature_names, start, stop) for key in tile_keys for start, stop in _tile_row_ranges(tile_sizes[key]) ] - table = dd.from_delayed(delayed_tiles, meta=meta) + table = dd.from_delayed(delayed_tiles, meta=_transcripts_meta(feature_names)) transform = Scale([1.0 / pixel_size, 1.0 / pixel_size], axes=("x", "y")) return PointsModel.parse( @@ -376,8 +365,8 @@ def read_transcripts_for_genes( plotting a handful of genes across the whole dataset much cheaper than loading and filtering the full transcripts table. - Requires ``sdata`` to have been read with ``cells_table=True``, to look up gene indices (via - the table's `var`) and `pixel_size` (via ``sdata.attrs``). + Gene positions come from the bundle's own `var`, so ``sdata`` can hold a filtered table or none; its + ``sdata.attrs["pixel_size"]`` is used. Parameters ---------- @@ -403,7 +392,10 @@ def read_transcripts_for_genes( if isinstance(genes, str): genes = [genes] path = Path(path) - feature_names = sdata.tables["table"].var[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() + feature_names = _read_feature_names(path) + missing = [gene for gene in genes if gene not in feature_names] + if missing: + raise ValueError(f"gene(s) not found in {AteraKeys.CELL_FEATURE_MATRIX_FILE!s} var: {missing}") gene_indices = {gene: feature_names.index(gene) for gene in genes} pixel_size = cast(float, sdata.attrs[str(AteraKeys.PIXEL_SIZE)]) @@ -413,60 +405,24 @@ def read_transcripts_for_genes( else: categories = genes - columns = [ - str(AteraKeys.TRANSCRIPTS_X), - str(AteraKeys.TRANSCRIPTS_Y), - str(AteraKeys.TRANSCRIPTS_Z), - str(AteraKeys.FEATURE_NAME), - str(AteraKeys.CELL_ID), - "overlaps_nucleus", - "quality_score", - ] - store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - tile_keys = sorted(_get_group(group, str(AteraKeys.GRID_GROUP)).group_keys()) - - frames = [] - for tile_key in tile_keys: - tile = _get_group(group, f"{AteraKeys.GRID_GROUP}/{tile_key}") + frames = [] + with _open_zip(path / AteraKeys.TRANSCRIPTS_FILE) as group: + grid = _get_group(group, str(AteraKeys.GRID_GROUP)) + for tile_key in sorted(grid.group_keys()): + tile = _get_group(grid, tile_key) gene_offset = _get_array(tile, str(AteraKeys.TRANSCRIPTS_GENE_OFFSET)) for gene, gene_idx in gene_indices.items(): - start, end = np.asarray(gene_offset[gene_idx]) + start, end = (int(v) for v in np.asarray(gene_offset[gene_idx])) if end <= start: continue - location = np.asarray(_get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION))[start:end]) - cell_id_raw = np.asarray(_get_array(tile, str(AteraKeys.CELL_ID))[start:end]) - overlaps_nucleus = np.asarray( - _get_array(tile, str(AteraKeys.TRANSCRIPTS_OVERLAPS_NUCLEUS))[start:end] - ).reshape(-1) - quality_score = np.asarray( - _get_array(tile, str(AteraKeys.TRANSCRIPTS_QUALITY_SCORE))[start:end] - ).reshape(-1) if by_codeword: - codeword_identity = np.asarray( - _get_array(tile, str(AteraKeys.CODEWORD_IDENTITY))[start:end] - ).reshape(-1) - feature_name = pd.Categorical([f"{gene}_cw{cw}" for cw in codeword_identity], categories=categories) + codeword = np.asarray(_get_array(tile, str(AteraKeys.CODEWORD_IDENTITY))[start:end]).reshape(-1) + labels = [f"{gene}_cw{cw}" for cw in codeword] else: - feature_name = pd.Categorical([gene] * (end - start), categories=categories) - frames.append( - pd.DataFrame( - { - str(AteraKeys.TRANSCRIPTS_X): location[:, 0].astype(np.float32), - str(AteraKeys.TRANSCRIPTS_Y): location[:, 1].astype(np.float32), - str(AteraKeys.TRANSCRIPTS_Z): location[:, 2].astype(np.float32), - str(AteraKeys.FEATURE_NAME): feature_name, - str(AteraKeys.CELL_ID): _decode_packed_cell_id(cell_id_raw), - "overlaps_nucleus": overlaps_nucleus.astype(bool), - "quality_score": quality_score.astype(np.float32), - } - ) - ) - finally: - store.close() - - df = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame(columns=columns) + labels = [gene] * (end - start) + frames.append(_transcript_rows(tile, start, end, pd.Categorical(labels, categories=categories))) + + df = pd.concat(frames, ignore_index=True) if frames else _transcripts_meta(categories) transform = Scale([1.0 / pixel_size, 1.0 / pixel_size], axes=("x", "y")) sdata.points[points_key] = PointsModel.parse( @@ -489,8 +445,10 @@ def _tile_bounds(tile_key: str, tile_size: float) -> tuple[float, float, float, return tile_x * tile_size, tile_y * tile_size, (tile_x + 1) * tile_size, (tile_y + 1) * tile_size -def _bbox_overlaps(a: tuple[float, float, float, float], b: tuple[float, float, float, float]) -> bool: - return a[0] < b[2] and b[0] < a[2] and a[1] < b[3] and b[1] < a[3] +def _bbox_overlaps(boxes: ArrayLike, bbox: Sequence[float]) -> np.ndarray: + """Which of the ``(..., 4)`` (xmin, ymin, xmax, ymax) `boxes` overlap `bbox`.""" + boxes = np.asarray(boxes) + return (boxes[..., 0] < bbox[2]) & (bbox[0] < boxes[..., 2]) & (boxes[..., 1] < bbox[3]) & (bbox[1] < boxes[..., 3]) def _cell_row_positions_in_bbox(path: Path, bbox: tuple[float, float, float, float]) -> np.ndarray: @@ -502,22 +460,9 @@ def _cell_row_positions_in_bbox(path: Path, bbox: tuple[float, float, float, flo fact `read_cell_boundaries` relies on for its own `bbox` filtering). """ pixel_size = _get_pixel_size(path) - xmin, ymin, xmax, ymax = (v * pixel_size for v in bbox) - - store = zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") + with _open_zip(path / AteraKeys.CELLS_FILE) as group: cell_bboxes = np.asarray(_get_array(group, str(AteraKeys.BBOXES))[...]).reshape(-1, 4) - finally: - store.close() - - keep = ( - (cell_bboxes[:, 0] < xmax) - & (xmin < cell_bboxes[:, 2]) - & (cell_bboxes[:, 1] < ymax) - & (ymin < cell_bboxes[:, 3]) - ) - return np.nonzero(keep)[0] + return np.flatnonzero(_bbox_overlaps(cell_bboxes, [v * pixel_size for v in bbox])) def read_cell_boundaries( @@ -552,9 +497,8 @@ def read_cell_boundaries( the top-level ``bboxes`` array); this un-quantizes them via ``absolute = cell_min + relative / 255 * (cell_max - cell_min)``. - Requires ``sdata`` to have been read with ``cells_table=True``, to map row positions in - ``gridded_polygon_sets`` to cell ids (via the table's `obs`), and `pixel_size` (via - ``sdata.attrs``). + Row positions in ``gridded_polygon_sets`` are mapped to cell ids through the bundle's own `obs`, so ``sdata`` + can hold a filtered table or none; its ``sdata.attrs["pixel_size"]`` is used. Parameters ---------- @@ -580,17 +524,12 @@ def read_cell_boundaries( """ path = Path(path) pixel_size = cast(float, sdata.attrs[str(AteraKeys.PIXEL_SIZE)]) - cell_ids = sdata.tables["table"].obs[str(AteraKeys.CELL_ID)].to_numpy() + cell_ids = _read_cell_ids(path) is_nucleus = mask_index == 0 - raw_bbox = None - if bbox is not None: - xmin, ymin, xmax, ymax = bbox - raw_bbox = (xmin * pixel_size, ymin * pixel_size, xmax * pixel_size, ymax * pixel_size) + raw_bbox = None if bbox is None else [v * pixel_size for v in bbox] - store = zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") + with _open_zip(path / AteraKeys.CELLS_FILE) as group: cell_bboxes = np.asarray(_get_array(group, str(AteraKeys.BBOXES))[...]).reshape(-1, 4) mask_group = _get_group(group, f"{AteraKeys.GRIDDED_POLYGON_SETS_GROUP}/{mask_index}") # Each pyramid level merges 2x2 blocks of tiles from the level below, so a tile's spatial @@ -614,13 +553,7 @@ def read_cell_boundaries( relative_vertices = np.asarray(_get_array(tile, str(AteraKeys.RELATIVE_VERTICES))[...]).reshape(-1, 2) if raw_bbox is not None: - tile_cell_bboxes = cell_bboxes[cell_index] - keep = ( - (tile_cell_bboxes[:, 0] < raw_bbox[2]) - & (raw_bbox[0] < tile_cell_bboxes[:, 2]) - & (tile_cell_bboxes[:, 1] < raw_bbox[3]) - & (raw_bbox[1] < tile_cell_bboxes[:, 3]) - ) + keep = _bbox_overlaps(cell_bboxes[cell_index], raw_bbox) if not keep.any(): continue relative_vertices = relative_vertices[np.repeat(keep, num_vertices)] @@ -634,33 +567,15 @@ def read_cell_boundaries( coords_parts.append((vertex_mins + relative_vertices / 255 * vertex_spans).astype(np.float32)) num_vertices_parts.append(num_vertices) cell_index_parts.append(cell_index) - finally: - store.close() - - coords = np.concatenate(coords_parts) if coords_parts else np.empty((0, 2), dtype=np.float32) - num_vertices = np.concatenate(num_vertices_parts) if num_vertices_parts else np.empty((0,), dtype=np.int32) - cell_index = np.concatenate(cell_index_parts) if cell_index_parts else np.empty((0,), dtype=np.uint32) + if not coords_parts: # spatialdata has no empty shapes element: ShapesModel rejects an empty GeoDataFrame + raise ValueError(f"No polygons of {AteraKeys.CELLS_FILE!s} overlap bbox={bbox}.") + coords, num_vertices = np.concatenate(coords_parts), np.concatenate(num_vertices_parts) + cell_index = np.concatenate(cell_index_parts) sdata.shapes[shapes_key] = _polygons_to_shapes(coords, num_vertices, cell_index, cell_ids, pixel_size, is_nucleus) return sdata -def _parse_morphology_channel_name(filename: str) -> str: - """Parse the channel name out of a ``chNNNN_.ome.tif`` morphology image filename.""" - match = re.match(r"ch\d+_(.+)\.ome\.tif+$", filename) - if match is None: - raise ValueError(f"Expected a morphology image filename of the form 'chNNNN_.ome.tif', found {filename}") - return match.group(1) - - -def _parse_morphology_channel_index(filename: str) -> int: - """Parse the channel index out of a ``chNNNN_.ome.tif`` morphology image filename.""" - match = re.match(r"ch(\d+)_.+\.ome\.tif+$", filename) - if match is None: - raise ValueError(f"Expected a morphology image filename of the form 'chNNNN_.ome.tif', found {filename}") - return int(match.group(1)) - - def _get_pixel_size(path: Path) -> float: """Return the dataset's pixel size (microns/pixel). @@ -674,8 +589,6 @@ def _get_pixel_size(path: Path) -> float: specs = json.load(f) return specs[str(AteraKeys.PIXEL_SIZE)] - from ome_types import from_tiff - morphology_dir = path / AteraKeys.MORPHOLOGY_2D_DIR files = ( sorted(f for f in morphology_dir.iterdir() if f.name.endswith(".ome.tif") and not f.name.startswith("._")) @@ -687,59 +600,60 @@ def _get_pixel_size(path: Path) -> float: f"Cannot determine the pixel size: {path!s} contains neither {AteraKeys.SPECS_FILE!s} nor any " f"morphology OME-TIFF files in {morphology_dir.name!s}/." ) - physical_size_x = from_tiff(files[0]).images[0].pixels.physical_size_x - if physical_size_x is None: - raise ValueError(f"{files[0]!s} does not specify `PhysicalSizeX` in its OME-XML metadata.") - return physical_size_x + return parse_physical_size(files[0]) -def _tiled_imread( - path: Path, imread_kwargs: Mapping[str, Any] = MappingProxyType({}), n_levels: int = 1 -) -> list[da.Array]: - """Read an OME-TIFF as `dask` arrays (one per pyramid level) chunked by the file's own on-disk tile grid. - - `dask_image.imread.imread` gives each page (e.g. each channel or z-plane) exactly one `dask` - chunk -- the whole plane, which for these whole-slide-image morphology files is tens of GB. - That single giant chunk then has to be fully decoded before `Image2DModel.parse`/`Image3DModel.parse` - can rechunk it into tiles or build a downsampled pyramid level, so even plotting one channel at - the lowest pyramid resolution requires materializing the entire full-resolution plane in memory. - - This instead reads through `tifffile.TiffPageSeries.aszarr(level=i)`, which exposes the file's - native tile grid (e.g. 1024x1024, matching `p.is_tiled`/`p.tilewidth`/`p.tilelength`) as `zarr` - chunks, so only the tiles actually touched downstream get decoded. - - These OME-TIFFs embed their own multi-level pyramid (each level half the size of the previous one). - Reading up to `n_levels` of those stored levels, rather than downsampling `level=0` lazily, means - a low-resolution level only costs reading that (small) level from disk, not every full-resolution - tile. If the file has fewer levels than requested, only the available ones are returned. +class _TiffTiles: + """Level 0 of a TIFF's first series as an array that `dask` slices one tile per task. + + Each slice opens the file itself, so a `dask` graph holds only the path and pickles (tifffile's zarr store keeps + an open file handle and does not), and a task decodes only the tiles it covers, not the whole plane as + `dask_image.imread` does. Chunks follow the file's tile grid (strips for an untiled file). """ - import tifffile - tf = tifffile.TiffFile(path, **imread_kwargs) - series = tf.series[0] - n_levels = min(n_levels, len(series.levels)) - return [da.from_zarr(zarr.open(series.aszarr(level=i), mode="r")) for i in range(n_levels)] + def __init__(self, path: Path, imread_kwargs: Mapping[str, Any]) -> None: + import tifffile + + self.path, self.imread_kwargs = path, dict(imread_kwargs) + with tifffile.TiffFile(path, **self.imread_kwargs) as tf: + series, page = tf.series[0], tf.series[0].keyframe + self.shape, self.dtype, self.axes = tuple(series.shape), series.dtype, series.axes + tile = (page.tilelength, page.tilewidth) if page.is_tiled else (page.rowsperstrip, page.imagewidth) + self.ndim = len(self.shape) + # one tile in y and x, every sample (S) of a pixel, one plane of any other axis + self.chunks = tuple( + {"Y": tile[0], "X": tile[1], "S": n}.get(axis, 1) for axis, n in zip(self.axes, self.shape, strict=True) + ) + def __getitem__(self, key: Any) -> np.ndarray: + import tifffile -def _parse_morphology_pyramid( - levels: list[da.Array], - dims: tuple[str, ...], - c_coords: list[str], - image_models_kwargs: Mapping[str, Any], - model: type[Image2DModel] | type[Image3DModel], -) -> DataArray | DataTree: - """Build a (multiscale) image model from pre-computed pyramid levels instead of downsampling `level=0`.""" - from xarray import Dataset + with tifffile.TiffFile(self.path, **self.imread_kwargs) as tf: + return cast(np.ndarray, zarr.open_array(tf.series[0].aszarr(level=0), mode="r")[key]) - kwargs = {k: v for k, v in image_models_kwargs.items() if k != "scale_factors"} - parsed = [model.parse(level, dims=dims, c_coords=c_coords, **kwargs) for level in levels] - if len(parsed) == 1: - tree: DataArray | DataTree = parsed[0] - else: - tree = DataTree.from_dict({f"scale{i}": Dataset({"image": p}) for i, p in enumerate(parsed)}) - set_transformation(tree, Identity(), "global") - model.validate(tree) - return tree + +def _tiled_imread( + path: Path, imread_kwargs: Mapping[str, Any] = MappingProxyType({}) +) -> tuple[da.Array, tuple[str, ...]]: + """Read level 0 of an OME-TIFF lazily, one `dask` task per tile; return it with its dims. + + Stored pyramid levels are not read: the image models build the pyramid from level 0 with `scale_factors`. + Axes of length 1 other than y and x are dropped, samples (``S``) and channels (``C``) both become ``c``. + """ + tiles = _TiffTiles(path, imread_kwargs) + image = da.from_array(tiles, chunks=tiles.chunks, meta=np.empty((0,) * tiles.ndim, tiles.dtype), lock=False) + keep = [i for i, (axis, n) in enumerate(zip(tiles.axes, tiles.shape, strict=True)) if n > 1 or axis in "YX"] + image = image.squeeze(axis=tuple(i for i in range(tiles.ndim) if i not in keep)) + return image, tuple("c" if tiles.axes[i] in "SC" else tiles.axes[i].lower() for i in keep) + + +def _ome_channel_name(path: Path, index: int) -> str: + from ome_types import from_tiff + + name = from_tiff(path).images[0].pixels.channels[index].name + if name is None: + raise ValueError(f"Channel {index} in {path!s} has no name in its OME-XML metadata.") + return name def _get_morphology_images( @@ -748,37 +662,31 @@ def _get_morphology_images( image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), three_d: bool = False, ) -> DataArray | DataTree: - """Read the morphology OME-TIFF files in a directory as a single multi-channel image.""" + """Read the ``chNNNN_.ome.tif`` morphology files in a directory as one multi-channel image. + + A 2D file holds a plane per channel of the panel, of which only plane ``NNNN`` carries this file's channel; a 3D + file holds the z-stack of its one channel. Channels are named from the OME-XML in both cases. + """ files = sorted(f for f in dir_path.iterdir() if f.name.endswith(".ome.tif") and not f.name.startswith("._")) if not files: raise FileNotFoundError(f"No morphology OME-TIFF files found in {dir_path}.") - # `scale_factors=[2, 2, 2, 2]` (the default) means 5 levels in total; use the file's stored levels for them. - n_levels = len(image_models_kwargs.get("scale_factors") or []) + 1 - - if three_d: - # one file per channel, each file has shape (z, y, x); stack along a new channel axis - channel_names = [_parse_morphology_channel_name(f.name) for f in files] - per_file_levels = [_tiled_imread(f, imread_kwargs, n_levels) for f in files] - n = min(len(lv) for lv in per_file_levels) - levels = [da.stack([lv[i] for lv in per_file_levels], axis=0) for i in range(n)] - return _parse_morphology_pyramid(levels, ("c", "z", "y", "x"), channel_names, image_models_kwargs, Image3DModel) - - # Each 2D file is itself a multi-page OME-TIFF containing every channel of the panel (shape - # (n_channels, y, x)), but only the plane at the index given by the file's own "chNNNN" prefix - # holds that channel's real image data; take just that plane from each file. - from ome_types import from_tiff - - channel_indices = [_parse_morphology_channel_index(f.name) for f in files] - channel_names = [] - for f, i in zip(files, channel_indices, strict=True): - name = from_tiff(f).images[0].pixels.channels[i].name - if name is None: - raise ValueError(f"Channel {i} in {f!s} has no name in its OME-XML metadata.") - channel_names.append(name) - per_file_levels = [ - [level[i] for level in _tiled_imread(f, imread_kwargs, n_levels)] - for f, i in zip(files, channel_indices, strict=True) - ] - n = min(len(lv) for lv in per_file_levels) - levels = [da.stack([lv[j] for lv in per_file_levels], axis=0) for j in range(n)] - return _parse_morphology_pyramid(levels, ("c", "y", "x"), channel_names, image_models_kwargs, Image2DModel) + indices = [] + for f in files: + match = re.match(r"ch(\d+)_.+\.ome\.tif$", f.name) + if match is None: + raise ValueError( + f"Expected a morphology image filename of the form 'chNNNN_.ome.tif', found {f.name}" + ) + indices.append(0 if three_d else int(match.group(1))) + planes = [] + for f, i in zip(files, indices, strict=True): + image, dims = _tiled_imread(f, imread_kwargs) + planes.append(image if three_d or "c" not in dims else image[i]) + model = Image3DModel if three_d else Image2DModel + return model.parse( + da.stack(planes), + dims=("c", "z", "y", "x") if three_d else ("c", "y", "x"), + c_coords=[_ome_channel_name(f, i) for f, i in zip(files, indices, strict=True)], + transformations={"global": Identity()}, + **image_models_kwargs, + ) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 19f6a9ab..f8045313 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -1,19 +1,19 @@ from __future__ import annotations import json -import logging +import warnings from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, cast -import dask.array as da import numpy as np import pandas as pd import zarr -from dask import delayed from scipy.sparse import csc_matrix, csr_matrix from spatialdata import SpatialData -from spatialdata.models import TableModel +from spatialdata._logging import logger +from spatialdata.models import Image2DModel, TableModel +from spatialdata.transformations import Affine, BaseTransformation, Identity from spatialdata_io._constants._constants import AteraKeys from spatialdata_io._docs import inject_docs @@ -22,20 +22,26 @@ DEFAULT_VAR_COLUMNS, _cell_row_positions_in_bbox, _get_array, - _get_labels, + _get_group, _get_morphology_images, _get_pixel_size, _get_points, _get_polygons, - _patched_ragged_vlen_chunk_decode, + _open_zip, + _read_cell_ids, + _read_feature_names, + _tiled_imread, + _unique_feature_names, read_cell_boundaries, read_transcripts_for_genes, ) from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata +from spatialdata_io.readers.xenium import _affine_from_keypoints, _get_labels if TYPE_CHECKING: from collections.abc import Mapping, Sequence + import dask.array as da from anndata import AnnData from xarray import DataArray, DataTree @@ -52,9 +58,6 @@ # `anndata`'s own zarr reading functions (`anndata.io.read_elem`/`anndata.io.sparse_dataset`). This # only works on bundles whose zarr stores carry the standard AnnData `encoding-type`/ # `encoding-version`/`shape` attrs. -# -# Every non-table element (labels/shapes/points/morphology images) lives in `_atera_common.py` -# rather than duplicated here. @inject_docs(xx=AteraKeys) @@ -69,6 +72,7 @@ def atera( transcripts: bool = False, morphology_images: bool = True, morphology_3d_images: bool = False, + aligned_images: bool = True, var_columns: list[str] | Literal["all"] = DEFAULT_VAR_COLUMNS, table_row_chunk_size: int = DEFAULT_TABLE_ROW_CHUNK_SIZE, imread_kwargs: Mapping[str, Any] = MappingProxyType({}), @@ -98,7 +102,10 @@ def atera( path Path to the dataset. cells_table - Whether to read the cell annotations in the `AnnData` table. + Whether to read the cell annotations in the `AnnData` table. As in `xenium`, `X` and `var` hold the Gene + Expression features. The counts of the other features (negative control probes and codewords, unassigned + codewords) go to a second table, ``table_controls``, with the same cells, `obs` index and annotation, + a sparse `X` and those features in `var`. cells_boundaries Whether to read cell boundaries (polygons). nucleus_boundaries @@ -115,6 +122,12 @@ def atera( morphology_3d_images Whether to read the 3D morphology images. This is opt-in and defaults to `False` because these images can be very large. + aligned_images + Whether to read the post-run images aligned to the morphology images (``_he_image.ome.tif`` / + ``_if_image.ome.tif`` in the bundle root), placed with their ``*imagealignment.csv`` affine. A + warning is logged if that affine does not map ``_keypoints.csv`` onto itself. An H&E without an + alignment CSV is placed by an affine fit to its keypoints, or left unaligned (with a warning) if they are + unusable. var_columns Which columns of `var` to keep, out of the full table (which includes ~135 per-cluster differential-expression columns). Defaults to a small subset (`{xx.FEATURE_NAME!r}`, @@ -126,9 +139,9 @@ def atera( Larger values mean fewer, larger `dask` tasks (faster, but more peak memory per chunk); smaller values mean more, smaller ones. imread_kwargs - Keyword arguments passed to `tifffile.TiffFile` when reading the morphology images (see - `_tiled_imread`); the images are read tile-by-tile off `tifffile`'s own `zarr` store rather - than via `dask_image.imread.imread`, so this only accepts `TiffFile` constructor kwargs. + Keyword arguments passed to `tifffile.TiffFile` when reading the morphology and aligned images (see + `_tiled_imread`). The images are read one tile per `dask` task rather than with `dask_image.imread.imread`, + so this only accepts `TiffFile` constructor kwargs. image_models_kwargs Keyword arguments to pass to the image models. labels_models_kwargs @@ -150,34 +163,21 @@ def atera( pixel_size = _get_pixel_size(path) - needs_cells_zarr = cells_boundaries or nucleus_boundaries or cells_labels or nucleus_labels - if not cells_table and (needs_cells_zarr or transcripts): - logging.info("Reading the table is required for the requested elements; setting cells_table=True.") - cells_table = True - - # Prefer `cell_labels` as the table's annotation target (matching the `xenium` reader's - # convention; instance ids are `obs["label_id"]`), but fall back to `cell_boundaries` (instance ids - # are `obs["cell_id"]`) when labels aren't being loaded, so the table doesn't end up annotating an - # element that isn't actually present in `sdata`. - default_region = "cell_labels" if cells_labels else "cell_boundaries" if cells_boundaries else "cell_labels" - - table: AnnData | None = None - feature_names: list[str] | None = None + # The table annotates `cell_labels` (instance ids `obs["label_id"]`, as in `xenium`), else `cell_boundaries` + # (`obs["cell_id"]`), else nothing: the nucleus elements are numbered per nucleus, not per cell. + region = "cell_labels" if cells_labels else "cell_boundaries" if cells_boundaries else None + tables: dict[str, AnnData] = {} if cells_table: - table, feature_names = _get_table( - path, var_columns=var_columns, row_chunk_size=table_row_chunk_size, region=default_region - ) + tables["table"], tables["table_controls"], _ = _get_table(path, var_columns, table_row_chunk_size, region) shapes: dict[str, Any] = {} labels: dict[str, DataArray | DataTree] = {} points: dict[str, Any] = {} images: dict[str, DataArray | DataTree] = {} - if needs_cells_zarr: - cells_store = zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, read_only=True) - cells_group = zarr.open_group(cells_store, mode="r") - - cell_ids = table.obs[str(AteraKeys.CELL_ID)].to_numpy() if table is not None else None + if cells_boundaries or nucleus_boundaries or cells_labels or nucleus_labels: + cells_group = zarr.open_group(zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, read_only=True), mode="r") + cell_ids = _read_cell_ids(path) if nucleus_labels: labels["nucleus_labels"] = _get_labels(cells_group, mask_index=0, labels_models_kwargs=labels_models_kwargs) @@ -193,57 +193,99 @@ def atera( ) if transcripts: - if feature_names is None: - raise ValueError("Reading transcripts requires `cells_table=True` (to map gene indices to names).") - points["transcripts"] = _get_points(path, pixel_size, feature_names) + points["transcripts"] = _get_points(path, pixel_size, _read_feature_names(path)) if morphology_images: images["morphology"] = _get_morphology_images( path / AteraKeys.MORPHOLOGY_2D_DIR, imread_kwargs, image_models_kwargs ) if morphology_3d_images: + # (c, z, y, x): the 3-tuple default chunks gain one z plane per chunk, and z is not downsampled + kwargs_3d = dict(image_models_kwargs) + if isinstance(chunks := kwargs_3d.get("chunks"), tuple | list) and len(chunks) == 3: + kwargs_3d["chunks"] = (chunks[0], 1, *chunks[1:]) + kwargs_3d["scale_factors"] = [ + f if isinstance(f, dict) else {"z": 1, "y": f, "x": f} for f in kwargs_3d.get("scale_factors") or [] + ] images["morphology_3d"] = _get_morphology_images( - path / AteraKeys.MORPHOLOGY_3D_DIR, imread_kwargs, image_models_kwargs, three_d=True + path / AteraKeys.MORPHOLOGY_3D_DIR, imread_kwargs, kwargs_3d, three_d=True ) - tables = {"table": table} if table is not None else {} + if aligned_images: + images |= _get_aligned_images(path, imread_kwargs, image_models_kwargs) + sdata = SpatialData(images=images, labels=labels, points=points, tables=tables, shapes=shapes) sdata = _set_reader_metadata(sdata, "atera") sdata.attrs[str(AteraKeys.PIXEL_SIZE)] = pixel_size return sdata -def _read_x_chunk(zarr_path: Path, zarr_key: AteraKeys, start: int, stop: int) -> csr_matrix: - """Read one contiguous row-range of `X` via `anndata.io.sparse_dataset`'s own row slicing. +KEYPOINTS_MAX_RESIDUAL_PX = 10.0 - Opens its own zip store (rather than sharing one across chunks) so this function can safely be - called from independent `dask` tasks. - """ - from anndata.io import sparse_dataset - - store = zarr.storage.ZipStore(zarr_path / zarr_key, read_only=True) - try: - group = zarr.open_group(store, mode="r") - return cast(csr_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[start:stop]) - finally: - store.close() +def _get_aligned_images( + path: Path, imread_kwargs: Mapping[str, Any], image_models_kwargs: Mapping[str, Any] +) -> dict[str, DataArray | DataTree]: + """The post-run H&E and IF images, read like the morphology images, in the global (morphology px) system. -def _lazy_x( - zarr_path: Path, zarr_key: AteraKeys, n_obs: int, n_var: int, dtype: np.dtype, row_chunk_size: int -) -> da.Array: - """Build a `dask`-backed `n_obs x n_var` CSR array, delegating each chunk's read to `sparse_dataset`.""" - meta = csr_matrix((0, n_var), dtype=dtype) - blocks = [] - for start in range(0, n_obs, row_chunk_size): - stop = min(start + row_chunk_size, n_obs) - block = delayed(_read_x_chunk)(zarr_path, AteraKeys.CELL_FEATURE_MATRIX_FILE, start, stop) - blocks.append(da.from_delayed(block, shape=(stop - start, n_var), dtype=dtype, meta=meta)) - return da.concatenate(blocks, axis=0) + Each is placed by its alignment CSV or, for an H&E without one, by an affine fit to its keypoints (as in `xenium`). + """ + images = {} + for file in sorted(path.glob("*.ome.tif")): + name = next((n for n in ["he_image", "if_image"] if file.name.endswith(f"{n}.ome.tif")), None) + if name is None or file.name.startswith("._"): # macOS sidecars + continue + alignment_file = file.with_name(file.name.replace(".ome.tif", "alignment.csv")) + kp_file = file.with_name(file.name.removesuffix("he_image.ome.tif") + "keypoints.csv") + image, dims = _tiled_imread(file, imread_kwargs) + transformation: BaseTransformation = Identity() + if alignment_file.is_file(): + affine = pd.read_csv(alignment_file, header=None).to_numpy(dtype=float) + transformation = Affine(affine, input_axes=("x", "y"), output_axes=("x", "y")) + if name == "he_image": + _check_keypoints(alignment_file, kp_file) + elif name == "he_image": + transformation = _affine_from_keypoints(kp_file) or transformation + if "c" not in dims: + image, dims = image[None], ("c", *dims) + n_channels = image.shape[dims.index("c")] + images[name] = Image2DModel.parse( + image, + dims=dims, + c_coords=["r", "g", "b", "a"][:n_channels] if name == "he_image" and n_channels in [3, 4] else None, + transformations={"global": transformation}, + **image_models_kwargs, + ) + return images + + +def _check_keypoints(alignment_file: Path, kp_file: Path) -> float | None: + """Warn when the alignment affine does not map the keypoints onto each other; return the median residual (px).""" + if not kp_file.is_file(): + return None + columns = ["fixedX", "fixedY", "alignmentX", "alignmentY"] + kp = pd.read_csv(kp_file) + if not set(columns) <= set(kp.columns): + warnings.warn(f"{kp_file.name}: expected columns {columns}, found {list(kp.columns)}.", stacklevel=2) + return None + xy = kp[columns].apply(pd.to_numeric, errors="coerce").to_numpy(dtype=float) + xy = xy[np.isfinite(xy).all(axis=1)] + if len(xy) < 3: + warnings.warn(f"{kp_file.name}: {len(xy)} finite keypoints, at least 3 are needed.", stacklevel=2) + return None + affine = pd.read_csv(alignment_file, header=None).to_numpy(dtype=float) + src, dst = xy[:, 2:], xy[:, :2] + median = float(np.median(np.linalg.norm((np.c_[src, np.ones(len(src))] @ affine.T)[:, :2] - dst, axis=1))) + if median > KEYPOINTS_MAX_RESIDUAL_PX: + logger.warning( + f"{alignment_file.name} maps {kp_file.name} with a median residual of {median:.1f} px " + f"(> {KEYPOINTS_MAX_RESIDUAL_PX}); the aligned image may be misplaced." + ) + return median def _align_index_column(df: pd.DataFrame) -> pd.DataFrame: - """Make a column that shares the index's name have the index's dtype. + """Make a column that shares the index's name have the index's dtype, and leave the index unnamed. The bundles' `obs`/`var` list the index (`barcode`/`feature_id`) as a column too. Under pandas 3, `read_elem` returns that index as `str` but the column as `object`, and `anndata` refuses to write @@ -253,6 +295,9 @@ def _align_index_column(df: pd.DataFrame) -> pd.DataFrame: name = df.index.name if name is not None and name in df.columns: df[name] = df[name].astype(df.index.dtype) + # the column keeps the values; an index of the same name breaks `obs.reset_index()`, which spatialdata's + # table joins call (spatialdata 0.8.0 _core/query/relational_query.py:259): "cannot insert barcode" + df.index.name = None return df @@ -306,11 +351,72 @@ def _add_label_id(obs_df: pd.DataFrame) -> None: obs_df[str(AteraKeys.LABEL_ID)] = np.arange(1, len(obs_df) + 1, dtype=np.uint32) -def _parse_table(adata: AnnData, region: str) -> AnnData: - """Annotate `region` with `adata`, using whichever `obs` column holds that element's instance ids. +# store `var` columns that the `xenium` layout renames +_XENIUM_VAR_NAMES = {str(AteraKeys.VAR_FEATURE_TYPE): "feature_types"} + + +def _xenium_var(var: pd.DataFrame) -> pd.DataFrame: + """Lay out the store's `var` as the `xenium` reader does. + + Gene symbols are the index (made unique with a suffix), Ensembl ids go to ``gene_ids`` and the feature type to + ``feature_types`` (the bundles index by ``feature_id`` and name the type ``feature_type``). The type values are + already the Xenium ones (``Gene Expression``, ``Negative Control Probe``, ...). + """ + out = var.rename(columns=_XENIUM_VAR_NAMES) + out["gene_ids"] = var.index.astype(str).to_numpy() + out.index = _unique_feature_names(var[str(AteraKeys.FEATURE_NAME)]) + return out + + +def _is_gene(var: pd.DataFrame) -> np.ndarray: + """Which features are ``Gene Expression``, the only ones `xenium` keeps in `X` by default (``gex_only=True``).""" + if "feature_types" not in var.columns: + return np.ones(len(var), dtype=bool) + return (var["feature_types"] == "Gene Expression").to_numpy() + + +def _read_obs_var(group: zarr.Group) -> tuple[pd.DataFrame, pd.DataFrame]: + """`obs` with ``label_id`` and the full `var` in the `xenium` layout, in store order.""" + obs = _read_dataframe(group, AteraKeys.OBS_GROUP) + _add_label_id(obs) + return obs, _xenium_var(_read_dataframe(group, AteraKeys.VAR_GROUP)) + + +def _assemble_tables( + x: Any, + obs: pd.DataFrame, + var: pd.DataFrame, + controls: csr_matrix | csc_matrix, + var_columns: list[str] | Literal["all"], + region: str | None, + obsm: Mapping[str, Any] = MappingProxyType({}), +) -> tuple[AnnData, AnnData]: + """The Gene Expression table and the ``table_controls`` table of all other features, for the same cells. + + The other features are the negative control probes and codewords and the unassigned codewords, so `X` and + ``n_genes`` of the first table count genes only. The second has a sparse `X` (CSR, as stored), the same `obs` + index and instance id columns, and annotates the same element. `var_columns` name the store's columns. + """ + from anndata import AnnData + + is_gene = _is_gene(var) + columns = list( + var.columns if var_columns == "all" else [_XENIUM_VAR_NAMES.get(c, c) for c in var_columns] + ["gene_ids"] + ) + obsm = {**obsm, "spatial": obs[[str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)]].to_numpy()} + table = _parse_table(AnnData(X=x, obs=obs, var=var.loc[is_gene, columns], obsm=obsm), region) + ids = obs[[str(AteraKeys.CELL_ID), str(AteraKeys.LABEL_ID)]].copy() + controls_table = AnnData(X=csr_matrix(controls), obs=ids, var=var.loc[~is_gene, columns]) + return table, _parse_table(controls_table, region) + + +def _parse_table(adata: AnnData, region: str | None) -> AnnData: + """Annotate `region` (if any) with `adata`, using whichever `obs` column holds that element's instance ids. ``cell_labels`` raster values are ``label_id``; ``cell_boundaries`` shapes are indexed by ``cell_id``. """ + if region is None or adata.n_obs == 0: # spatialdata rejects an empty annotating table ("Regions ... do not match") + return TableModel.parse(adata) instance_key = AteraKeys.LABEL_ID if region == "cell_labels" else AteraKeys.CELL_ID adata.obs["region"] = pd.Categorical([region] * adata.n_obs) return TableModel.parse(adata, region=region, region_key="region", instance_key=str(instance_key)) @@ -320,66 +426,54 @@ def _get_table( path: Path, var_columns: list[str] | Literal["all"] = DEFAULT_VAR_COLUMNS, row_chunk_size: int = DEFAULT_TABLE_ROW_CHUNK_SIZE, - region: str = "cell_labels", -) -> tuple[AnnData, list[str]]: - """Read ``cell_feature_matrix.zarr.zip`` into an AnnData table using native `anndata` zarr IO. + region: str | None = "cell_labels", +) -> tuple[AnnData, AnnData, list[str]]: + """Read ``cell_feature_matrix.zarr.zip`` into the gene table and the ``table_controls`` table using native `anndata` zarr IO. `obs`/`var`/`obsm` are read with `anndata.io.read_elem` (which also means `obs`/`var` come back indexed by whichever column the store's ``_index`` attr names -- e.g. `barcode`/`feature_id`). - `X` is built as a lazily `dask`-backed array in row chunks (see `_lazy_x`), each chunk read via - `anndata.io.sparse_dataset` rather than manual `indptr` arithmetic. + `X` is a lazy `dask` array of `row_chunk_size`-row CSR chunks (`anndata.experimental.read_elem_lazy`). - `region` should be the name of a `SpatialElement` that will actually be present in the - `SpatialData` object the table is assembled into (see the ``default_region`` logic in - `atera`), so the table doesn't end up annotating an element that was never loaded. + `region` is the element the table annotates; it must be read too (see `atera`). None annotates nothing. """ - from anndata import AnnData + from anndata.experimental import read_elem_lazy from anndata.io import read_elem, sparse_dataset - store = zarr.storage.ZipStore(path / AteraKeys.CELL_FEATURE_MATRIX_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - - with _patched_ragged_vlen_chunk_decode(): - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) - feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() - if var_columns != "all": - var_df = var_df[list(var_columns)] - colors = _read_category_colors(group, AteraKeys.OBS_GROUP) - colors.update(_read_category_colors(group, AteraKeys.VAR_GROUP, list(var_df.columns))) - - obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) - _add_label_id(obs_df) - - obsm = ( - cast(dict[str, Any], read_elem(group[str(AteraKeys.OBSM_GROUP)])) - if str(AteraKeys.OBSM_GROUP) in group - else {} - ) - + with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: + obs, var = _read_obs_var(group) + colors = _read_category_colors(group, AteraKeys.OBS_GROUP) + # keyed by the column's name in `var` (`feature_types_colors`), as scanpy looks them up + var_colors = { + f"{_XENIUM_VAR_NAMES.get(k.removesuffix('_colors'), k.removesuffix('_colors'))}_colors": v + for k, v in _read_category_colors( + group, AteraKeys.VAR_GROUP, None if var_columns == "all" else list(var_columns) + ).items() + } + obsm = ( + cast(dict[str, Any], read_elem(group[str(AteraKeys.OBSM_GROUP)])) + if str(AteraKeys.OBSM_GROUP) in group + else {} + ) manifest_extras = _read_manifest_extras(group) - x_ds = sparse_dataset(group[str(AteraKeys.X_GROUP)]) - n_obs, n_var = x_ds.shape - dtype = x_ds.dtype - # Genes detected per cell is the number of stored entries per row of the CSR matrix, which `indptr` - # already encodes; this avoids a full pass over the (lazy) counts. - n_genes = np.diff(_get_array(group, f"{AteraKeys.X_GROUP}/indptr")[:]) if x_ds.format == "csr" else None - finally: - store.close() - - x = _lazy_x(path, AteraKeys.CELL_FEATURE_MATRIX_FILE, n_obs, n_var, dtype, row_chunk_size) - - if n_genes is not None: - obs_df["n_genes"] = n_genes - - adata = AnnData(X=x, obs=obs_df, var=var_df, obsm=obsm) - adata.obsm["spatial"] = cast(pd.DataFrame, adata.obs)[ - [str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)] - ].to_numpy() - adata.uns.update(colors) + # the graph keeps the store, which reopens on first use (also after pickling) + x_group = _get_group(group, str(AteraKeys.X_GROUP)) + x = cast("da.Array", read_elem_lazy(x_group, chunks=(row_chunk_size, -1))) + stored_per_row = np.diff(_get_array(x_group, "indptr")[:]) + is_gene = _is_gene(var) + controls: csr_matrix | csc_matrix + if (path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE).is_file(): # column reads are cheap in the CSC copy + with _open_zip(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE) as group: + controls = cast(csc_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[:, np.flatnonzero(~is_gene)]) + else: + controls = cast(csr_matrix, x[:, ~is_gene].compute()) + # genes detected per cell: the stored entries of each CSR row (its `indptr` step) minus those of the controls + obs["n_genes"] = stored_per_row - controls.getnnz(axis=1) + table, controls_table = _assemble_tables(x[:, is_gene], obs, var, controls, var_columns, region, obsm) + table.uns.update({**colors, **var_colors}) + controls_table.uns.update(var_colors) if manifest_extras is not None: - adata.uns[str(AteraKeys.UNS_MANIFEST)] = manifest_extras - return _parse_table(adata, region), feature_names + table.uns[str(AteraKeys.UNS_MANIFEST)] = manifest_extras + return table, controls_table, var.index.tolist() def read_var(path: str | Path) -> pd.DataFrame: @@ -388,8 +482,8 @@ def read_var(path: str | Path) -> pd.DataFrame: Includes the ~135 per-cluster differential-expression columns that `atera` drops by default (see `var_columns` on `atera`) to avoid holding that wider table in memory. - Use this to build your own `var`/`AnnData` with whichever columns you need, e.g. - ``sdata.tables["table"].var = read_var(path)[["feature_name", "de_leiden_res_1.0_c0_score"]]``. + It has the same index as the table's `var` (gene symbols), plus the non-gene features, so columns can be added + by name, e.g. ``table.var["score"] = read_var(path)["de_leiden_res_1.0_c0_score"]``. Parameters ---------- @@ -398,17 +492,10 @@ def read_var(path: str | Path) -> pd.DataFrame: Returns ------- - The complete `var` `DataFrame`, indexed by `feature_id`. + The complete `var` `DataFrame` of every feature, in the `xenium` layout (see `atera`). """ - path = Path(path) - store = zarr.storage.ZipStore(path / AteraKeys.CELL_FEATURE_MATRIX_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - with _patched_ragged_vlen_chunk_decode(): - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) - finally: - store.close() - return var_df + with _open_zip(Path(path) / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: + return _xenium_var(_read_dataframe(group, AteraKeys.VAR_GROUP)) def read_table_for_cells( @@ -419,8 +506,8 @@ def read_table_for_cells( ) -> AnnData: """Read the cell-by-gene table for only a subset of cells, without loading the full `X`. - Unlike ``atera(path, cells_table=True)``, which eagerly builds the whole `X`, this reads only - the requested rows directly off `anndata.io.sparse_dataset`'s own fancy-indexing support + Unlike ``atera(path, cells_table=True)``, whose `X` is lazy over every cell, this reads only + the requested rows, in memory, directly off `anndata.io.sparse_dataset`'s own fancy-indexing support (``x_ds[row_positions]``), which already handles arbitrary/unsorted row selections internally. Exactly one of `cell_ids`/`bbox` must be given. @@ -441,45 +528,28 @@ def read_table_for_cells( Returns ------- - An `AnnData` `TableModel` with only the requested cells. + An `AnnData` `TableModel` with only the requested cells and the Gene Expression features. """ if (cell_ids is None) == (bbox is None): raise ValueError("Exactly one of `cell_ids`/`bbox` must be given.") - from anndata import AnnData from anndata.io import sparse_dataset path = Path(path) - store = zarr.storage.ZipStore(path / AteraKeys.CELL_FEATURE_MATRIX_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - with _patched_ragged_vlen_chunk_decode(): - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) - if var_columns != "all": - var_df = var_df[list(var_columns)] - obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) - _add_label_id(obs_df) - + with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: + obs, var = _read_obs_var(group) if cell_ids is not None: - index = pd.Index(obs_df[str(AteraKeys.CELL_ID)].to_numpy()) - row_positions = index.get_indexer(pd.Index(np.asarray(cell_ids))) + row_positions = pd.Index(obs[str(AteraKeys.CELL_ID)].to_numpy()).get_indexer(pd.Index(np.asarray(cell_ids))) missing = np.asarray(cell_ids)[row_positions == -1] if missing.size: raise ValueError(f"cell_id(s) not found in {AteraKeys.CELL_FEATURE_MATRIX_FILE!s}: {missing.tolist()}") else: assert bbox is not None row_positions = _cell_row_positions_in_bbox(path, bbox) - - x_ds = sparse_dataset(group[str(AteraKeys.X_GROUP)]) - x = cast(csr_matrix, x_ds[row_positions]) - obs_subset = obs_df.iloc[row_positions] - finally: - store.close() - - adata = AnnData(X=x, obs=obs_subset, var=var_df) - adata.obsm["spatial"] = cast(pd.DataFrame, adata.obs)[ - [str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)] - ].to_numpy() - return _parse_table(adata, "cell_labels") + x = cast(csr_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[row_positions]) + obs = obs.iloc[row_positions].copy() + is_gene = _is_gene(var) + obs["n_genes"] = x[:, is_gene].getnnz(axis=1) + return _assemble_tables(x[:, is_gene], obs, var, x[:, ~is_gene], var_columns, "cell_labels")[0] def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnData: @@ -509,34 +579,14 @@ def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnDat genes = [genes] if isinstance(genes, str) else list(genes) path = Path(path) - store = zarr.storage.ZipStore(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - - with _patched_ragged_vlen_chunk_decode(): - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP)[ - [str(AteraKeys.FEATURE_ID), str(AteraKeys.FEATURE_NAME)] - ] - feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() - index = pd.Index(feature_names) - col_positions = index.get_indexer(pd.Index(np.asarray(genes))) - missing = np.asarray(genes)[col_positions == -1] - if missing.size: - raise ValueError( - f"gene(s) not found in {AteraKeys.VAR_GROUP!s}.{AteraKeys.FEATURE_NAME!s}: {missing.tolist()}" - ) - - var_subset = var_df.iloc[col_positions] - var_subset.index = pd.Index(np.asarray(genes), name=str(AteraKeys.FEATURE_NAME)) - _align_index_column(var_subset) - - obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) - _add_label_id(obs_df) - - x_ds = sparse_dataset(group[str(AteraKeys.X_GROUP)]) - x: csc_matrix = cast(csc_matrix, x_ds[:, col_positions]) - finally: - store.close() - - adata = AnnData(X=x, obs=obs_df, var=var_subset) - return _parse_table(adata, "cell_labels") + with _open_zip(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE) as group: + obs, var = _read_obs_var(group) + col_positions = var.index.get_indexer(pd.Index(genes)) + missing = np.asarray(genes)[col_positions == -1] + if missing.size: + raise ValueError( + f"gene(s) not found in {AteraKeys.VAR_GROUP!s}.{AteraKeys.FEATURE_NAME!s}: {missing.tolist()}" + ) + x = cast(csc_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[:, col_positions]) + obsm: dict[str, Any] = {"spatial": obs[[str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)]].to_numpy()} + return _parse_table(AnnData(X=x, obs=obs, var=var.iloc[col_positions], obsm=obsm), "cell_labels") diff --git a/src/spatialdata_io/readers/xenium.py b/src/spatialdata_io/readers/xenium.py index cd6700dd..c0daf024 100644 --- a/src/spatialdata_io/readers/xenium.py +++ b/src/spatialdata_io/readers/xenium.py @@ -38,6 +38,7 @@ ShapesModel, TableModel, ) +from spatialdata.transformations import set_transformation from spatialdata.transformations.transformations import Affine, Identity, Scale, Translation from xarray import DataArray, DataTree @@ -1082,6 +1083,8 @@ def _add_aligned_images( ome_tif_files = list(path.glob("*.ome.tif")) csv_files = list(path.glob("*.csv")) for file in ome_tif_files: + if file.name.startswith("._"): # macOS sidecar files + continue element_name = None for suffix in [ XeniumKeys.ALIGNED_HE_IMAGE_SUFFIX, @@ -1102,10 +1105,41 @@ def _add_aligned_images( # parse the image image = xenium_aligned_image(file, alignment_file, imread_kwargs, image_models_kwargs) + if alignment_file is None and element_name == "he_image": + keypoints = file.with_name(file.name.removesuffix(XeniumKeys.ALIGNED_HE_IMAGE_SUFFIX) + "keypoints.csv") + affine = _affine_from_keypoints(keypoints) + if affine is not None: + set_transformation(image, affine, "global") images[element_name] = image return images +def _affine_from_keypoints(kp_file: Path) -> Affine | None: + """Least-squares affine from Xenium Explorer keypoints (``alignmentX/Y`` -> ``fixedX/Y``), or None with a warning. + + The alignment CSV that Xenium Explorer writes is this fit. A full affine, because a similarity fit leaves about 22 px + of error on scans with 0.2% anisotropy. + """ + if not kp_file.is_file(): + return None + columns = ["fixedX", "fixedY", "alignmentX", "alignmentY"] + kp = pd.read_csv(kp_file) + if not set(columns) <= set(kp.columns): + warnings.warn(f"{kp_file.name}: expected columns {columns}, found {list(kp.columns)}.", stacklevel=2) + return None + xy = kp[columns].apply(pd.to_numeric, errors="coerce").to_numpy(dtype=float) + xy = xy[np.isfinite(xy).all(axis=1)] + if len(xy) < 3: + warnings.warn(f"{kp_file.name}: {len(xy)} finite keypoints, at least 3 are needed.", stacklevel=2) + return None + coef, _, rank, _ = np.linalg.lstsq(np.c_[xy[:, 2:], np.ones(len(xy))], xy[:, :2], rcond=None) + if rank < 3: + warnings.warn(f"{kp_file.name}: the keypoints are collinear, so they fix no affine.", stacklevel=2) + return None + logging.info(f"Placed the H&E image by an affine fit to {kp_file.name} (no alignment CSV).") + return Affine(np.r_[coef.T, [[0.0, 0.0, 1.0]]], input_axes=("x", "y"), output_axes=("x", "y")) + + def xenium_aligned_image( image_path: str | Path, alignment_file: str | Path | None, @@ -1158,9 +1192,12 @@ def xenium_aligned_image( if dims is None: if len(image.shape) == 4: assert image.shape[0] == 1 - assert image.shape[-1] == 3 image = image.squeeze(0) - dims = ("y", "x", "c") + if image.shape[-1] in [3, 4]: + dims = ("y", "x", "c") + else: + assert image.shape[0] in [3, 4] + dims = ("c", "y", "x") else: assert len(image.shape) == 3 assert image.shape[0] in [3, 4] diff --git a/src/spatialdata_io/writers/atera.py b/src/spatialdata_io/writers/atera.py index cc23c4d6..7733845e 100644 --- a/src/spatialdata_io/writers/atera.py +++ b/src/spatialdata_io/writers/atera.py @@ -52,6 +52,7 @@ def write_atera_cell_feature_matrix( path: str | Path, *, table_name: str = "table", + controls_table: str | AnnData | None = "table_controls", x_layer: str | None = None, major_axis: Literal["cell", "feature"] = "cell", chunk_size: int = 1_000_000, @@ -63,6 +64,14 @@ def write_atera_cell_feature_matrix( The table's `obs`/`var` columns are written as they are (names, order, dtypes), except for the columns `atera` adds on read (``label_id``, ``n_genes``, ``region``) and ``obsm["spatial"]``. + `atera` reads the Gene Expression features into ``table`` and the other features (negative control probes + and codewords, unassigned codewords) into ``table_controls``, with `var` in the `xenium` layout (symbols as + the index, ``gene_ids``, ``feature_types``). Both are written back into one matrix with the bundle's `var` + columns (``feature_name``, ``feature_id``, ``feature_type``): the genes first, then the controls, each in + its table's order. That is the bundles' own order where we checked it (the 10x tiny Atera bundle lists its + 53 Gene Expression features first); a bundle that interleaves the two is written in a different feature order. + A single table with a ``feature_type`` column (the bundle's own `var` layout) is written as it is. + Parameters ---------- sdata @@ -74,6 +83,10 @@ def write_atera_cell_feature_matrix( ``cell_feature_matrix.zarr.zip``). table_name Which table of `sdata` to write. + controls_table + The table of the non-gene features to append to `table_name`'s (same cells, in the same order): its name + in `sdata` (skipped if `sdata` has no such table, or is an `AnnData`), or the `AnnData` itself. `None` + writes `table_name` alone. x_layer Write ``table.layers[x_layer]`` as ``X`` instead of ``table.X``. The bundle's ``X`` is raw integer counts, so use this (e.g. ``x_layer="counts"``) when ``table.X`` has been normalised in place. @@ -96,6 +109,7 @@ def write_atera_cell_feature_matrix( if major_axis not in ("cell", "feature"): raise ValueError(f"`major_axis` must be 'cell' or 'feature', got {major_axis!r}.") table = sdata.tables[table_name] if isinstance(sdata, SpatialData) else sdata + controls = _controls_table(sdata, controls_table) path = Path(path) if path.suffix != ".zip": path = path / str(AteraKeys.CELL_FEATURE_MATRIX_FILE) @@ -104,9 +118,11 @@ def write_atera_cell_feature_matrix( if x_layer is not None and x_layer not in table.layers: raise KeyError(f"`x_layer={x_layer!r}` is not in the table's layers ({list(table.layers)}).") x: Any = table.X if x_layer is None else table.layers[x_layer] + if controls is not None: + x = _append_controls(table, controls, x) obs, obs_colors = _prepare_obs(table) - var, var_colors = _prepare_var(table) - n_obs, n_var = table.shape + var, var_colors = _prepare_var(table, controls) + n_obs, n_var = table.n_obs, len(var) if tuple(x.shape) != (n_obs, n_var): # pragma: no cover - AnnData enforces this raise ValueError("`X` shape does not match `obs`/`var`.") @@ -130,7 +146,7 @@ def write_atera_cell_feature_matrix( ) obsm = _embeddings(table.obsm, n_obs, "obsm") obsm_manifest = _write_embeddings(root, str(AteraKeys.OBSM_GROUP), obsm, chunk_size, compressor) - varm = _embeddings(table.varm, n_var, "varm") + varm = _embeddings(_varm(table, controls), n_var, "varm") varm_manifest = _write_embeddings(root, str(AteraKeys.VARM_GROUP), varm, chunk_size, compressor) _write_x(root, zf, x, (n_obs, n_var), major_axis, chunk_size, row_chunk_size, n_threads, compressor) @@ -196,11 +212,15 @@ def _clean_columns(df: pd.DataFrame) -> pd.DataFrame: return pd.DataFrame(columns) -def _colors(table: AnnData, df: pd.DataFrame) -> dict[str, np.ndarray]: +def _colors(table: AnnData, df: pd.DataFrame, sources: dict[str, str] | None = None) -> dict[str, np.ndarray]: colors = {} for column in df.columns: if isinstance(df[column].dtype, pd.CategoricalDtype): - found = _colors_for(table, str(column), len(df[column].cat.categories)) + k = len(df[column].cat.categories) + source = (sources or {}).get(str(column)) + found = _colors_for(table, source, k) if source is not None else None + if found is None: + found = _colors_for(table, str(column), k) if found is not None: colors[str(column)] = found return colors @@ -218,18 +238,66 @@ def _prepare_obs(table: AnnData) -> tuple[pd.DataFrame, dict[str, np.ndarray]]: return df, _colors(table, df) -def _prepare_var(table: AnnData) -> tuple[pd.DataFrame, dict[str, np.ndarray]]: - var = cast(pd.DataFrame, table.var).reset_index(drop=True) +def _controls_table(sdata: SpatialData | AnnData, controls_table: str | AnnData | None) -> AnnData | None: + if isinstance(controls_table, str): + return ( + sdata.tables[controls_table] if isinstance(sdata, SpatialData) and controls_table in sdata.tables else None + ) + return controls_table + + +def _append_controls(table: AnnData, controls: AnnData, x: Any) -> Any: + """`x` with the controls' `X` appended as columns, after checking both tables hold the same cells in order.""" + if not controls.obs_names.equals(table.obs_names): + raise ValueError("`table_controls` must hold the same cells as the table, in the same order (`obs_names`).") + cid = str(AteraKeys.CELL_ID) + if cid in controls.obs and cid in table.obs and not np.array_equal(controls.obs[cid], table.obs[cid]): + raise ValueError(f"`table_controls` and the table disagree on obs[{cid!r}].") + c: Any = controls.X + if isinstance(x, da.Array): + if not isinstance(c, da.Array): + c = da.from_array(sparse.csr_matrix(c), chunks=(x.chunks[0], -1), asarray=False) + return da.concatenate([x, c.rechunk({0: x.chunks[0]})], axis=1) + if isinstance(c, da.Array): + c = c.compute() + return sparse.hstack([sparse.csr_matrix(x), sparse.csr_matrix(c)], format="csr") + + +# `var` columns of the `xenium` layout that `atera` reads (symbol index) -> the bundle's own column +_BUNDLE_VAR_NAMES = {"feature_types": str(AteraKeys.VAR_FEATURE_TYPE), "gene_ids": str(AteraKeys.FEATURE_ID)} + + +def _varm(table: AnnData, controls: AnnData | None) -> dict[str, Any]: + if controls is None: + return dict(table.varm) + if set(table.varm) != set(controls.varm): + raise ValueError("The table and `table_controls` must have the same `varm` keys.") + return {k: np.concatenate([np.asarray(table.varm[k]), np.asarray(controls.varm[k])]) for k in table.varm} + + +def _prepare_var(table: AnnData, controls: AnnData | None = None) -> tuple[pd.DataFrame, dict[str, np.ndarray]]: + var = pd.concat([cast(pd.DataFrame, t.var) for t in (table, controls) if t is not None]) + index = var.index.astype(str) + var = var.reset_index(drop=True) + sources = {} # written column -> the table's column, whose `uns[f"{source}_colors"]` holds its colors + for column, bundle in _BUNDLE_VAR_NAMES.items(): + if column in var.columns: + var = var.drop(columns=[bundle], errors="ignore").rename(columns={column: bundle}) + sources[bundle] = column + if str(AteraKeys.FEATURE_NAME) not in var.columns and "feature_types" in sources.values(): + var.insert(0, str(AteraKeys.FEATURE_NAME), index) # the `xenium` layout's symbols if "feature_id" not in var.columns: - var.insert(0, "feature_id", table.var.index.astype(str)) + var.insert(0, "feature_id", index) for required in (str(AteraKeys.FEATURE_NAME), str(AteraKeys.VAR_FEATURE_TYPE)): if required not in var.columns: - raise ValueError(f"The table's var must have a {required!r} column.") + raise ValueError( + f"The table's var must have a {required!r} column (or `feature_types`, as `atera` reads it)." + ) df = _clean_columns(var) for name in (str(AteraKeys.VAR_FEATURE_TYPE), str(AteraKeys.VAR_GENOME)): if name in df.columns and not isinstance(df[name].dtype, pd.CategoricalDtype): df[name] = df[name].astype("category") - return df, _colors(table, df) + return df, _colors(table, df, sources) # -------------------------------------------------------------------------------------------------- @@ -551,7 +619,7 @@ def _build_manifest( str(AteraKeys.MANIFEST_VERSION): dict(_SCHEMA_VERSION), str(AteraKeys.MANIFEST_MATRIX_PROPERTIES): { "num_cells": int(table.n_obs), - "num_features": int(table.n_vars), + "num_features": len(var_df), "major_axis": major_axis, "feature_types": feature_types, }, diff --git a/tests/converters/test_atera_to_parquet.py b/tests/converters/test_atera_to_parquet.py index e3bade15..250805e2 100644 --- a/tests/converters/test_atera_to_parquet.py +++ b/tests/converters/test_atera_to_parquet.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Any import numpy as np import pandas as pd @@ -92,3 +93,22 @@ def test_chunked_conversion_matches_unchunked( assert chunked.equals(whole) assert chunked["feature_name"].to_pylist() == ["GENE_A"] * 3 + ["GENE_B"] * 4 + ["GENE_A"] * 2 + ["GENE_B"] * 3 + + +def test_failed_conversion_keeps_the_earlier_output(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _write_atera_dataset(tmp_path, {"0,0": 4, "1,0": 4}, ["0,0", "1,0"], ids={}) + output = atera_to_parquet(tmp_path, tmp_path / "transcripts.parquet") + before = output.read_bytes() + + read_tile = atera_module._read_transcript_tile + + def fail_on_second_tile(grid: zarr.Group, tile_key: str, *args: Any) -> pd.DataFrame: + if tile_key == "1,0": + raise OSError("disk full") + return read_tile(grid, tile_key, *args) + + monkeypatch.setattr(atera_module, "_read_transcript_tile", fail_on_second_tile) + with pytest.raises(OSError, match="disk full"): + atera_to_parquet(tmp_path, output) + assert output.read_bytes() == before + assert not (tmp_path / "transcripts.parquet.partial").exists() diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 6bc08de4..f132f4a8 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -1,27 +1,44 @@ from pathlib import Path -from typing import cast +from typing import Any, cast import numpy as np import pandas as pd import pytest +import shapely import zarr from anndata.io import write_elem from scipy.sparse import csr_matrix +from spatialdata import SpatialData from spatialdata.models import get_table_keys +from spatialdata.transformations import Affine from tests.converters.test_atera_to_parquet import _write_atera_dataset from xarray import DataTree from spatialdata_io._constants._constants import AteraKeys from spatialdata_io.readers import _atera_common as common from spatialdata_io.readers._atera_common import _read_transcript_tile, _tile_row_ranges -from spatialdata_io.readers.atera import _get_table, read_table_for_cells, read_table_for_genes, read_var +from spatialdata_io.readers.atera import ( + _check_keypoints, + _get_aligned_images, + _get_table, + read_table_for_cells, + read_table_for_genes, + read_var, +) # Atera `cell_id`s are large packed integers, unrelated to the `1..n_obs` values in the `cell_labels` raster. CELL_IDS = [4_320_252_178, 4_975_876_403, 5_016_677_209, 5_217_424_078] +NEG_COUNTS = [0.0, 1.0, 0.0, 2.0] # the negative control probe NEG_1, after the genes GENE_A and GENE_B @pytest.fixture def atera_path(tmp_path: Path) -> Path: + _write_matrices(tmp_path, ["GENE_A", "GENE_B", "NEG_1"]) + _write_cells_transcripts_and_images(tmp_path, len(CELL_IDS)) + return tmp_path + + +def _write_matrices(path: Path, feature_names: list[str]) -> None: n = len(CELL_IDS) obs = pd.DataFrame( { @@ -33,20 +50,20 @@ def atera_path(tmp_path: Path) -> Path: ) var = pd.DataFrame( { - str(AteraKeys.FEATURE_NAME): ["GENE_A", "GENE_B"], - str(AteraKeys.VAR_FILTERED): [False, False], - str(AteraKeys.VAR_HIGHLY_VARIABLE): [False, False], - str(AteraKeys.VAR_FEATURE_TYPE): ["Gene Expression", "Gene Expression"], - str(AteraKeys.VAR_GENOME): ["g", "g"], + str(AteraKeys.FEATURE_NAME): feature_names, + str(AteraKeys.VAR_FILTERED): [False] * 3, + str(AteraKeys.VAR_HIGHLY_VARIABLE): [False] * 3, + str(AteraKeys.VAR_FEATURE_TYPE): ["Gene Expression", "Gene Expression", "Negative Control Probe"], + str(AteraKeys.VAR_GENOME): ["g"] * 3, }, - index=pd.Index(["ID_A", "ID_B"], dtype=object, name=str(AteraKeys.FEATURE_ID)), + index=pd.Index(["ID_A", "ID_B", "ID_NEG"], dtype=object, name=str(AteraKeys.FEATURE_ID)), ) - x = csr_matrix(np.arange(n * 2, dtype=np.float32).reshape(n, 2)) + x = csr_matrix(np.c_[np.arange(n * 2, dtype=np.float32).reshape(n, 2), NEG_COUNTS]) for file in (AteraKeys.CELL_FEATURE_MATRIX_FILE, AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE): - store = zarr.storage.ZipStore(tmp_path / file, mode="w") + store = zarr.storage.ZipStore(path / file, mode="w") try: group = zarr.open_group(store, mode="w", zarr_format=2) - write_elem(group, str(AteraKeys.X_GROUP), x) + write_elem(group, str(AteraKeys.X_GROUP), x if file == AteraKeys.CELL_FEATURE_MATRIX_FILE else x.tocsc()) write_elem(group, str(AteraKeys.OBS_GROUP), obs) write_elem(group, str(AteraKeys.VAR_GROUP), var) # Real bundles also list the index (`barcode`/`feature_id`) in `column-order`, so it comes back from @@ -56,11 +73,86 @@ def atera_path(tmp_path: Path) -> Path: elem.attrs["column-order"] = [str(df.index.name), *cast("list[str]", elem.attrs["column-order"])] finally: store.close() - return tmp_path + + +PIXEL_SIZE = 0.5 + + +def _write_cells_transcripts_and_images(path: Path, n: int) -> None: + """The rest of a bundle: cell `i` is the 6 x 6 px square at ``10 i + 2``, with one 4-vertex polygon per mask.""" + import json + + import tifffile + + (path / AteraKeys.SPECS_FILE).write_text(json.dumps({"pixel_size": PIXEL_SIZE})) + store = zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, mode="w") + try: + root = zarr.open_group(store, mode="w", zarr_format=2) + mask = np.zeros((64, 96), dtype=np.uint32) + for c in range(n): + mask[10 * c + 2 : 10 * c + 8, 10 * c + 2 : 10 * c + 8] = c + 1 + bboxes = np.array([[10 * c + 2, 10 * c + 2, 10 * c + 8, 10 * c + 8] for c in range(n)], np.float32) * PIXEL_SIZE + root.create_array("bboxes", data=bboxes) + for i in ("0", "1"): + root.create_array(f"masks/{i}", data=mask) + square = bboxes[:, [0, 1, 2, 1, 2, 3, 0, 3]] + root.create_array(f"polygon_sets/{i}/vertices", data=np.c_[square, np.zeros((n, 2), np.float32)]) + root.create_array(f"polygon_sets/{i}/num_vertices", data=np.full(n, 4, np.int32)) + root.create_array(f"polygon_sets/{i}/cell_index", data=np.arange(n, dtype=np.uint32)) + gridded = root.create_group(f"gridded_polygon_sets/{i}") + gridded.attrs["grid_size"] = [100.0, 100.0] + for level in range(5): + gridded.create_array(f"{level}/0,0/num_vertices", data=np.full(n, 4, np.int32)) + gridded.create_array(f"{level}/0,0/cell_index", data=np.arange(n, dtype=np.uint32)) + corners = np.array([0, 0, 255, 0, 255, 255, 0, 255], np.uint8) + gridded.create_array(f"{level}/0,0/relative_vertices", data=np.tile(corners, n)) + finally: + store.close() + + # one tile; GENE_A owns rows 0-2 and GENE_B rows 3-5, the first transcript is unassigned + store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, mode="w") + try: + grid = zarr.open_group(store, mode="w", zarr_format=2).create_group(str(AteraKeys.GRID_GROUP)) + grid.attrs[str(AteraKeys.GRID_KEYS)] = ["0,0"] + cell_id = np.zeros((6, 2), np.uint32) + cell_id[0] = np.iinfo(np.uint32).max + arrays = { + "location": np.c_[np.arange(6), np.arange(6), np.zeros(6)].astype(np.float32), + "gene_offset": np.array([[0, 3], [3, 6], [6, 6]], np.uint32), + "cell_id": cell_id, + "overlaps_nucleus": np.zeros((6, 1), np.uint8), + "quality_score": np.full((6, 1), 30, np.float32), + "codeword_identity": np.full((6, 1), 7, np.uint16), + } + for key, data in arrays.items(): + grid.create_array(f"0,0/{key}", data=data) + finally: + store.close() + + # each 2D file holds every channel, the "chNNNN" one carries the data; one 3D file per channel + rng = np.random.default_rng(0) + planes = rng.integers(0, 255, (2, 64, 96), dtype=np.uint8) + (path / AteraKeys.MORPHOLOGY_2D_DIR).mkdir() + (path / AteraKeys.MORPHOLOGY_3D_DIR).mkdir() + for c, name in enumerate(["DAPI", "VIM"]): + tifffile.imwrite( + path / AteraKeys.MORPHOLOGY_2D_DIR / f"ch{c:04d}_{name.lower()}.ome.tif", + planes, + ome=True, + tile=(32, 32), + metadata={"axes": "CYX", "Channel": {"Name": ["DAPI", "VIM"]}}, + ) + tifffile.imwrite( + path / AteraKeys.MORPHOLOGY_3D_DIR / f"ch{c:04d}_{name.lower()}_3d.ome.tif", + np.stack([planes[c]] * 3), + ome=True, + tile=(32, 32), + metadata={"axes": "ZYX", "Channel": {"Name": [name]}}, + ) def test_cell_labels_table_uses_label_id(atera_path: Path) -> None: - table, _ = _get_table(atera_path, region="cell_labels") + table = _get_table(atera_path, region="cell_labels")[0] region, _, instance_key = get_table_keys(table) assert region == "cell_labels" @@ -68,12 +160,31 @@ def test_cell_labels_table_uses_label_id(atera_path: Path) -> None: # the `cell_labels` raster is numbered 1..n_obs by row position assert table.obs["label_id"].tolist() == [1, 2, 3, 4] assert table.obs["cell_id"].tolist() == CELL_IDS - # genes detected per cell, read from the CSR `indptr` (cell 0 only has a zero in its first column) + # genes detected per cell (cell 0 only has a zero in its first column); NEG_1 is not a gene assert table.obs["n_genes"].tolist() == [1, 2, 2, 2] +@pytest.mark.parametrize("region", ["cell_labels", "cell_boundaries"]) +def test_x_holds_genes_and_table_controls_the_controls(atera_path: Path, region: str) -> None: + from scipy.sparse import issparse + + adata, controls, _ = _get_table(atera_path, region=region) + assert adata.var_names.tolist() == ["GENE_A", "GENE_B"] + assert adata.obs["n_genes"].tolist() == [1, 2, 2, 2] + assert read_table_for_cells(atera_path, cell_ids=CELL_IDS).var_names.tolist() == ["GENE_A", "GENE_B"] + assert issparse(controls.X) + assert controls.X.dtype == adata.X.dtype # as stored + assert cast(csr_matrix, controls.X).toarray()[:, 0].tolist() == NEG_COUNTS + assert controls.var_names.tolist() == ["NEG_1"] + assert controls.var["feature_types"].tolist() == ["Negative Control Probe"] + assert controls.obs_names.equals(adata.obs_names) + assert get_table_keys(controls) == get_table_keys(adata) + _, _, instance_key = get_table_keys(adata) + assert controls.obs[instance_key].tolist() == adata.obs[instance_key].tolist() + + def test_cell_boundaries_table_uses_cell_id(atera_path: Path) -> None: - table, _ = _get_table(atera_path, region="cell_boundaries") + table = _get_table(atera_path, region="cell_boundaries")[0] assert get_table_keys(table) == ("cell_boundaries", "region", "cell_id") @@ -108,8 +219,26 @@ def test_table_obs_and_var_are_writable(atera_path: Path, table: str) -> None: write_elem(zarr.open_group(zarr.storage.MemoryStore(), mode="w", zarr_format=2), name, df) -def test_read_var_is_writable(atera_path: Path) -> None: - write_elem(zarr.open_group(zarr.storage.MemoryStore(), mode="w", zarr_format=2), "var", read_var(atera_path)) +@pytest.mark.parametrize("table", ["_get_table", "read_table_for_cells", "read_table_for_genes"]) +def test_table_obs_index_does_not_shadow_the_barcode_column(atera_path: Path, table: str) -> None: + # spatialdata's table joins call `obs.reset_index()` (spatialdata 0.8.0 _core/query/relational_query.py:259), + # which fails with "cannot insert barcode, already exists" when the index carries the column's name + if table == "_get_table": + adata = _get_table(atera_path)[0] + elif table == "read_table_for_cells": + adata = read_table_for_cells(atera_path, cell_ids=CELL_IDS[:2]) + else: + adata = read_table_for_genes(atera_path, "GENE_B") + + assert adata.obs.index.name is None + assert adata.obs["barcode"].tolist() == adata.obs_names.tolist() + cast(pd.DataFrame, adata.obs).reset_index() + + +def test_read_var_is_writable_and_indexed_like_the_table(atera_path: Path) -> None: + var = read_var(atera_path) + assert var.index.tolist() == ["GENE_A", "GENE_B", "NEG_1"] + write_elem(zarr.open_group(zarr.storage.MemoryStore(), mode="w", zarr_format=2), "var", var) def test_tile_row_ranges() -> None: @@ -121,9 +250,11 @@ def test_tile_row_ranges() -> None: def test_chunked_tile_reads_match_whole_tile(tmp_path: Path) -> None: _write_atera_dataset(tmp_path, {"0,0": 7}, ["0,0"], ids={}) feature_names = ["GENE_A", "GENE_B"] - whole = _read_transcript_tile(tmp_path, "0,0", feature_names) + root = zarr.open_group(zarr.storage.ZipStore(tmp_path / AteraKeys.TRANSCRIPTS_FILE, read_only=True), mode="r") + grid = root.get_group("grid") + whole = _read_transcript_tile(grid, "0,0", feature_names) # GENE_A owns rows 0-2 and GENE_B rows 3-6, so the 2-row chunks straddle the gene boundary. - parts = [_read_transcript_tile(tmp_path, "0,0", feature_names, a, b) for a, b in _tile_row_ranges(7, 2)] + parts = [_read_transcript_tile(grid, "0,0", feature_names, a, b) for a, b in _tile_row_ranges(7, 2)] assert pd.concat(parts, ignore_index=True).equals(whole) assert whole["feature_name"].tolist() == ["GENE_A"] * 3 + ["GENE_B"] * 4 @@ -138,28 +269,343 @@ def test_points_partitions_are_bounded(tmp_path: Path, monkeypatch: pytest.Monke assert len(points) == 7 -def test_morphology_images_use_embedded_pyramid(tmp_path: Path) -> None: - import tifffile - - rng = np.random.default_rng(0) - full = rng.integers(0, 255, (2, 64, 96), dtype=np.uint8) - for idx, name in enumerate(["dapi", "vim"]): - with tifffile.TiffWriter(tmp_path / f"ch{idx:04d}_{name}.ome.tif", ome=True) as tif: - levels = [full[:, :: 2**i, :: 2**i] for i in range(3)] - tif.write( - levels[0], - tile=(16, 16), - subifds=len(levels) - 1, - metadata={"axes": "CYX", "Channel": {"Name": ["dapi", "vim"]}}, - ) - for level in levels[1:]: - tif.write(level, tile=(16, 16), subfiletype=1) +@pytest.mark.parametrize("scale_factors", [[2, 2], [4]]) +def test_morphology_pyramid_is_built_from_level_0(atera_path: Path, scale_factors: list[int]) -> None: + from spatialdata import bounding_box_query + kwargs = {"chunks": (1, 32, 32), "scale_factors": scale_factors} image = cast( + DataTree, common._get_morphology_images(atera_path / AteraKeys.MORPHOLOGY_2D_DIR, image_models_kwargs=kwargs) + ) + factors = np.cumprod([1, *scale_factors]) + assert [image[f"scale{i}"]["image"].shape for i in range(len(factors))] == [(2, 64 // f, 96 // f) for f in factors] + assert image["scale0"]["image"].coords["c"].values.tolist() == ["DAPI", "VIM"] + # each scale has coordinates in scale0 px, so a query crops the same region at every scale + crop = cast( DataTree, - common._get_morphology_images(tmp_path, image_models_kwargs={"chunks": (1, 32, 32), "scale_factors": [2, 2]}), + bounding_box_query( + image, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[32, 32], target_coordinate_system="global" + ), + ) + assert [crop[k]["image"].shape[1:] for k in crop.children] == [(32 // f, 32 // f) for f in factors] + + +def test_morphology_3d_names_channels_like_2d(atera_path: Path) -> None: + image = common._get_morphology_images(atera_path / AteraKeys.MORPHOLOGY_3D_DIR, three_d=True) + assert image.coords["c"].values.tolist() == ["DAPI", "VIM"] + assert image.dims == ("c", "z", "y", "x") + + +def test_tiled_imread_reads_one_tile_per_task_and_pickles(atera_path: Path) -> None: + import pickle + + import dask + import tifffile + + file = atera_path / AteraKeys.MORPHOLOGY_2D_DIR / "ch0001_vim.ome.tif" + image, dims = common._tiled_imread(file) + assert dims == ("c", "y", "x") + assert image.chunks == ((1, 1), (32, 32), (32, 32, 32)) + graph = pickle.loads(pickle.dumps(image)) + with dask.config.set(scheduler="processes", num_workers=2): + np.testing.assert_array_equal(graph.compute(), tifffile.imread(file)) + + +# an H&E aligned by a rotation by 90 degrees, a scale of 2 and a translation +HE_AFFINE = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) + + +def _write_he(path: Path, alignment_csv: bool = True) -> None: + import tifffile + + tifffile.imwrite( + path / "S1_he_image.ome.tif", np.zeros((3, 40, 60), dtype=np.uint8), ome=True, metadata={"axes": "CYX"} ) - assert list(image.children) == ["scale0", "scale1", "scale2"] - assert [image[k]["image"].shape for k in image.children] == [(2, 64, 96), (2, 32, 48), (2, 16, 24)] - # The lowest level is read from the stored pyramid, not downsampled from `scale0`. - np.testing.assert_array_equal(np.asarray(image["scale2"]["image"])[0], full[0, ::4, ::4]) + if alignment_csv: + pd.DataFrame(HE_AFFINE).to_csv(path / "S1_he_imagealignment.csv", header=False, index=False) + + +def _write_keypoints(path: Path) -> pd.DataFrame: + src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) + dst = (np.c_[src, np.ones(3)] @ HE_AFFINE.T)[:, :2] + kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) + kp.to_csv(path / "S1_keypoints.csv", index=False) + return kp + + +def _global_affine(image: Any) -> np.ndarray: + from spatialdata.transformations import get_transformation + + t = cast(Affine, get_transformation(image, "global")) + return t.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) + + +def test_aligned_he_image_is_placed_by_its_affine(tmp_path: Path) -> None: + _write_he(tmp_path) + (tmp_path / "._S1_he_image.ome.tif").write_bytes(b"\x00" * 16) # a macOS sidecar, not a TIFF + images = _get_aligned_images(tmp_path, {}, {}) + assert list(images) == ["he_image"] + assert list(images["he_image"].coords["c"].values) == ["r", "g", "b"] + np.testing.assert_allclose(_global_affine(images["he_image"]), HE_AFFINE) + + +def test_keypoints_check(tmp_path: Path, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: + from spatialdata._logging import logger + + monkeypatch.setattr(logger, "propagate", True) + + _write_he(tmp_path) + kp = _write_keypoints(tmp_path) + csv, kp_file = tmp_path / "S1_he_imagealignment.csv", tmp_path / "S1_keypoints.csv" + assert _check_keypoints(csv, kp_file) == pytest.approx(0.0, abs=1e-9) + assert "misplaced" not in caplog.text + kp.assign(fixedX=kp["fixedX"] + 50).to_csv(kp_file, index=False) + assert _check_keypoints(csv, kp_file) == pytest.approx(50.0) + assert "misplaced" in caplog.text + pd.DataFrame({"a": [1]}).to_csv(kp_file, index=False) + with pytest.warns(UserWarning, match="expected columns"): + assert _check_keypoints(csv, kp_file) is None + + +def test_he_without_alignment_csv_is_placed_by_keypoints(tmp_path: Path) -> None: + _write_he(tmp_path, alignment_csv=False) + _write_keypoints(tmp_path) + he = _get_aligned_images(tmp_path, {}, {})["he_image"] + np.testing.assert_allclose(_global_affine(he), HE_AFFINE, atol=1e-9) + + +@pytest.mark.parametrize("planar", [False, True]) +def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool) -> None: + import tifffile + + rgb = np.arange(3 * 40 * 60, dtype=np.uint32).reshape(3, 40, 60).astype(np.uint8) + if planar: # OME axes SYX, as in the Atera v1 Onboard 1.1.1 H&E + tifffile.imwrite(tmp_path / "S1_he_image.ome.tif", rgb, ome=True, photometric="rgb", planarconfig="separate") + else: # YXS, as in the 1.0.4 kidney H&E + tifffile.imwrite(tmp_path / "S1_he_image.ome.tif", np.moveaxis(rgb, 0, -1), ome=True, photometric="rgb") + he = _get_aligned_images(tmp_path, {}, {})["he_image"] + assert list(he.coords["c"].values) == ["r", "g", "b"] + np.testing.assert_array_equal(np.asarray(he.transpose("c", "y", "x").data), rgb) + + +def test_atera_reads_writes_and_queries(atera_path: Path, tmp_path: Path) -> None: + from spatialdata import bounding_box_query, read_zarr + + from spatialdata_io import atera + + sdata = atera(atera_path, transcripts=True) + assert set(sdata.images) == {"morphology"} + assert set(sdata.labels) == {"cell_labels", "nucleus_labels"} + assert set(sdata.shapes) == {"cell_boundaries", "nucleus_boundaries"} + transcripts = sdata.points["transcripts"].compute() + assert transcripts["cell_id"].tolist() == [-1, 0, 0, 0, 0, 0] # the packed (uint32 max, uint32 max) is unassigned + sdata.write(tmp_path / "out.zarr") + back = read_zarr(tmp_path / "out.zarr") + table, table_back = sdata.tables["table"], back.tables["table"] + np.testing.assert_array_equal(cast(csr_matrix, table_back.X).toarray(), cast(Any, table.X).compute().toarray()) + controls, controls_back = sdata.tables["table_controls"], back.tables["table_controls"] + np.testing.assert_array_equal(cast(csr_matrix, controls_back.X).toarray(), cast(csr_matrix, controls.X).toarray()) + assert controls_back.var_names.tolist() == ["NEG_1"] + crop = cast( + SpatialData, + bounding_box_query( + back, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[15, 15], target_coordinate_system="global" + ), + ) + assert crop.tables["table"].obs["cell_id"].tolist() == CELL_IDS[:2] + + +def test_lazy_x_matches_the_store_and_pickles(atera_path: Path) -> None: + import pickle + + x = cast(Any, _get_table(atera_path, row_chunk_size=3)[0].X) + assert x.chunks[0] == (3, 1) + expected = np.arange(len(CELL_IDS) * 2, dtype=np.float32).reshape(-1, 2) + np.testing.assert_array_equal(pickle.loads(pickle.dumps(x)).compute().toarray(), expected) + + +def test_pixel_size_falls_back_to_the_ome_metadata_in_microns(tmp_path: Path) -> None: + import tifffile + + (tmp_path / AteraKeys.MORPHOLOGY_2D_DIR).mkdir() + tifffile.imwrite( + tmp_path / AteraKeys.MORPHOLOGY_2D_DIR / "ch0000_dapi.ome.tif", + np.zeros((1, 8, 8), np.uint8), + ome=True, + metadata={ + "axes": "CYX", + "PhysicalSizeX": 212.0, + "PhysicalSizeXUnit": "nm", + "PhysicalSizeY": 212.0, + "PhysicalSizeYUnit": "nm", + }, + ) + assert common._get_pixel_size(tmp_path) == pytest.approx(0.212) + + +@pytest.mark.parametrize("pipeline", ["BatchedCodecPipeline", "FusedCodecPipeline"]) +def test_short_trailing_vlen_chunk_is_padded(tmp_path: Path, pipeline: str) -> None: + import json + import zipfile + + import zarr.core.codec_pipeline + from numcodecs import VLenUTF8, get_codec + + if not hasattr(zarr.core.codec_pipeline, pipeline): + pytest.skip(f"zarr {zarr.__version__} has no {pipeline}") + root = tmp_path / "s.zarr" + zarr.create_array(zarr.storage.LocalStore(root), shape=(5,), chunks=(3,), dtype=str, zarr_format=2)[:] = np.array( + ["a", "bb", "ccc", "dd", "e"], dtype=object + ) + meta = json.loads((root / ".zarray").read_text()) + compressor = get_codec(meta["compressor"]) if meta["compressor"] else None + + def zip_with_trailing_chunk(items: list[str]) -> Path: + encoded = VLenUTF8().encode(np.array(items, dtype=object)) + (root / "1").write_bytes(compressor.encode(encoded) if compressor else encoded) + with zipfile.ZipFile(tmp_path / "s.zip", "w") as zf: + zf.writestr(".zgroup", json.dumps({"zarr_format": 2})) + for f in root.iterdir(): + zf.write(f, f"s/{f.name}") + return tmp_path / "s.zip" + + with zarr.config.set({"codec_pipeline.path": f"zarr.core.codec_pipeline.{pipeline}"}): + # 10x's writer stores only the 2 items of the trailing chunk; zarr reads it at the nominal size of 3 + path = zip_with_trailing_chunk(["dd", "e"]) + with pytest.raises(ValueError, match="reshape"): + zarr.open_group(zarr.storage.ZipStore(path, read_only=True), mode="r").get_array("s")[:] + with common._open_zip(path) as group: + assert np.asarray(group.get_array("s")[:]).tolist() == ["a", "bb", "ccc", "dd", "e"] + path = zip_with_trailing_chunk(["dd"]) # one item short: corrupt, not padded + with common._open_zip(path) as group, pytest.raises(ValueError, match="reshape"): + group.get_array("s")[:] + + +def test_atera_reads_3d_morphology_with_default_kwargs(atera_path: Path) -> None: + from spatialdata_io import atera + + image = atera(atera_path, morphology_3d_images=True).images["morphology_3d"] + assert image["scale0"]["image"].dims == ("c", "z", "y", "x") + assert image["scale0"]["image"].data.chunksize[:2] == (1, 1) + assert [image[k]["image"].shape for k in image.children] == [(2, 3, 64 // f, 96 // f) for f in (1, 2, 4, 8, 16)] + + +def test_helpers_map_positions_through_the_bundle_not_the_table(atera_path: Path) -> None: + from spatialdata_io import atera + from spatialdata_io.readers.atera import read_cell_boundaries, read_transcripts_for_genes + + sdata = atera(atera_path) + table = sdata.tables["table"] + # an HVG-style gene subset and a reordered cell subset of the table + sdata.tables["table"] = table[[2, 0], (table.var["feature_name"] == "GENE_B").to_numpy()].copy() + points = read_transcripts_for_genes(sdata, atera_path, "GENE_B").points["transcripts_subset"].compute() + assert points["x"].tolist() == [3.0, 4.0, 5.0] # GENE_B owns rows 3-5 of the tile + shapes = read_cell_boundaries(sdata, atera_path).shapes["cell_boundaries"] + assert shapes.index.tolist() == CELL_IDS + del sdata.tables["table"] # nor do they need a table + assert len(read_transcripts_for_genes(sdata, atera_path, ["GENE_A"]).points["transcripts_subset"]) == 3 + + +@pytest.mark.parametrize("table", ["_get_table", "read_table_for_cells", "read_table_for_genes"]) +def test_var_follows_the_xenium_layout(atera_path: Path, table: str) -> None: + # The xenium() reader indexes var by gene symbol, with `gene_ids` (Ensembl) and `feature_types`; downstream + # (gene search, QC) reads symbols from the index, ids from `gene_ids` and controls from `feature_types`. + if table == "_get_table": + adata = _get_table(atera_path)[0] + elif table == "read_table_for_cells": + adata = read_table_for_cells(atera_path, cell_ids=CELL_IDS[:2]) + else: + adata = read_table_for_genes(atera_path, ["GENE_A", "GENE_B"]) + assert adata.var.index.tolist() == ["GENE_A", "GENE_B"] + assert adata.var.index.name is None # a name shared with the `feature_name` column breaks writes once suffixed + assert adata.var["gene_ids"].tolist() == ["ID_A", "ID_B"] + assert adata.var["feature_types"].tolist() == ["Gene Expression", "Gene Expression"] + assert "feature_type" not in adata.var.columns + + +def test_duplicate_feature_names_are_made_unique_everywhere(atera_path: Path, tmp_path: Path) -> None: + import pyarrow.parquet as pq + + from spatialdata_io import atera + from spatialdata_io.converters.atera_to_parquet import atera_to_parquet + from spatialdata_io.readers.atera import read_transcripts_for_genes + + _write_matrices(atera_path, ["GENE_A", "GENE_A", "NEG_1"]) + sdata = atera(atera_path, transcripts=True) + assert sdata.tables["table"].var_names.tolist() == ["GENE_A", "GENE_A-1"] + transcripts = sdata.points["transcripts"].compute() + assert transcripts["feature_name"].tolist() == ["GENE_A"] * 3 + ["GENE_A-1"] * 3 + sdata.write(tmp_path / "out.zarr") + genes = read_table_for_genes(atera_path, ["GENE_A-1", "GENE_A"]) + np.testing.assert_array_equal(cast(csr_matrix, genes.X).toarray(), np.arange(8).reshape(4, 2)[:, ::-1]) + second = read_transcripts_for_genes(sdata, atera_path, "GENE_A-1").points["transcripts_subset"].compute() + assert second["x"].tolist() == [3.0, 4.0, 5.0] + table = pq.read_table(atera_to_parquet(atera_path, tmp_path / "t.parquet")) + assert table["feature_name"].to_pylist() == ["GENE_A"] * 3 + ["GENE_A-1"] * 3 + + +def test_empty_selections(atera_path: Path) -> None: + from spatialdata_io import atera + from spatialdata_io.readers.atera import read_cell_boundaries, read_transcripts_for_genes + + sdata = atera(atera_path) + points = read_transcripts_for_genes(sdata, atera_path, "NEG_1").points["transcripts_subset"] # no transcripts + assert len(points) == 0 + assert points["feature_name"].dtype == "category" + for table in [read_table_for_cells(atera_path, cell_ids=[]), read_table_for_cells(atera_path, bbox=(80, 0, 90, 5))]: + assert table.shape == (0, 2) + SpatialData(tables={"table": table}) # a valid element + with pytest.raises(ValueError, match="No polygons"): + read_cell_boundaries(sdata, atera_path, bbox=(80, 0, 90, 5)) + + +def test_polygons_with_one_or_two_vertices_are_kept_empty( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + import logging + + from spatialdata._logging import logger + + monkeypatch.setattr(logger, "propagate", True) + + num_vertices = np.array([3, 2, 1, 0, 4]) + coords = np.arange(2 * num_vertices.sum(), dtype=np.float32).reshape(-1, 2) + with caplog.at_level(logging.WARNING): + shapes = common._polygons_to_shapes(coords, num_vertices, np.arange(5), None, 1.0, is_nucleus=False) + assert shapes.geometry.is_empty.tolist() == [False, True, True, True, False] + np.testing.assert_array_equal(shapely.get_coordinates(shapes.geometry.iloc[4])[:4], coords[6:10]) + assert "2 polygon(s) with 1 or 2 vertices" in caplog.text + + +@pytest.mark.parametrize( + ("kwargs", "region"), + [ + ({"cells_table": False, "transcripts": True}, None), + ({"cells_labels": False, "cells_boundaries": False}, None), + ({"cells_labels": False}, "cell_boundaries"), + ({}, "cell_labels"), + ], +) +def test_table_and_its_region_follow_the_flags(atera_path: Path, kwargs: dict, region: str | None) -> None: + import warnings + + from spatialdata.models import TableModel + + from spatialdata_io import atera + + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + sdata = atera(atera_path, morphology_images=False, aligned_images=False, **kwargs) + assert not [w for w in record if "which is not present in the SpatialData object" in str(w.message)] + if kwargs.get("cells_table") is False: + assert not sdata.tables + assert len(sdata.points["transcripts"]) == 6 + assert sdata.shapes["cell_boundaries"].index.tolist() == CELL_IDS + else: + attrs = sdata.tables["table"].uns.get(TableModel.ATTRS_KEY, {}) + assert attrs.get("region") == region + + +def test_unique_feature_names_from_a_numpy_string_array() -> None: + names = np.array(["G", "G", "NEG_1"], dtype=np.dtypes.StringDType()) # what read_elem gives on real bundles + assert common._unique_feature_names(names).tolist() == ["G", "G-1", "NEG_1"] diff --git a/tests/readers/test_atera_writer.py b/tests/readers/test_atera_writer.py index f2b035ef..8327e27c 100644 --- a/tests/readers/test_atera_writer.py +++ b/tests/readers/test_atera_writer.py @@ -107,8 +107,10 @@ def test_roundtrip(tmp_path: Path, x_kind: str, chunk_size: int) -> None: path = write_atera_cell_feature_matrix(table, tmp_path, chunk_size=chunk_size, row_chunk_size=4) assert path == tmp_path / "cell_feature_matrix.zarr.zip" - back, feature_names = _get_table(tmp_path, var_columns="all") - assert np.array_equal(back.X.compute().toarray(), expected_x) + back, back_controls, feature_names = _get_table(tmp_path, var_columns="all") + # `atera` splits the features: Gene Expression in the table, the Negative Control Probe in `table_controls` + assert np.array_equal(back.X.compute().toarray(), expected_x[:, :-1]) + assert np.array_equal(back_controls.X.toarray(), expected_x[:, -1:]) assert feature_names == [f"G{i}" for i in range(N_VAR)] assert back.obs["cell_id"].tolist() == CELL_IDS.tolist() assert back.obs["label_id"].tolist() == list(range(1, N_OBS + 1)) @@ -118,7 +120,8 @@ def test_roundtrip(tmp_path: Path, x_kind: str, chunk_size: int) -> None: assert np.allclose(back.obs["cell_area"], table.obs["cell_area"]) assert np.allclose(back.obsm["spatial"], table.obsm["spatial"]) assert np.allclose(back.obsm["X_umap"], table.obsm["X_umap"]) - assert back.var["feature_type"].astype(str).tolist() == table.var["feature_type"].tolist() + feature_types = [*back.var["feature_types"].astype(str), *back_controls.var["feature_types"].astype(str)] + assert feature_types == table.var["feature_type"].tolist() assert "de_leiden_res_1.0_c0_logfc" in read_var(tmp_path).columns subset = read_table_for_cells(tmp_path, cell_ids=CELL_IDS[[3, 1]]) assert subset.obs["cell_id"].tolist() == CELL_IDS[[3, 1]].tolist() @@ -234,13 +237,13 @@ def test_x_layer_writes_raw_counts_when_x_is_normalised(tmp_path: Path) -> None: write_atera_cell_feature_matrix(table, tmp_path, x_layer="nope") write_atera_cell_feature_matrix(table, tmp_path, x_layer="counts") - back, _ = _get_table(tmp_path, var_columns="all") - assert np.array_equal(back.X.compute().toarray(), counts.toarray()) + back, _, _ = _get_table(tmp_path, var_columns="all") + assert np.array_equal(back.X.compute().toarray(), counts.toarray()[:, :-1]) def test_colors_survive_a_roundtrip(tmp_path: Path) -> None: write_atera_cell_feature_matrix(_make_table(), tmp_path) - back, _ = _get_table(tmp_path, var_columns="all") + back, _, _ = _get_table(tmp_path, var_columns="all") assert back.uns["leiden_res_1.0_colors"] == ["#ff0000ff", "#00ff00ff", "#0000ffff"] assert "many_colors" not in back.uns @@ -250,3 +253,53 @@ def test_non_integer_x_is_rejected(tmp_path: Path) -> None: table.X = table.X.astype(np.float32) * 0.5 with pytest.raises(ValueError, match="integer counts"): write_atera_cell_feature_matrix(table, tmp_path) + + +def _interleaved_table() -> ad.AnnData: + """`_make_table` with the features interleaved, as a bundle could store them: G0, NEG, G1, CW, G2, G3.""" + table = _make_table() + types = ["Gene Expression", "Negative Control Probe", "Gene Expression", "Negative Control Codeword"] + table.var["feature_type"] = pd.Categorical(types + ["Gene Expression"] * 2) + table.uns["feature_type_colors"] = ["#111111", "#222222", "#333333"] + return table + + +@pytest.mark.parametrize("lazy", [True, False]) +def test_atera_tables_round_trip_genes_first_then_controls(tmp_path: Path, lazy: bool) -> None: + source = _interleaved_table() + write_atera_cell_feature_matrix(source, tmp_path / "a") + table, controls, _ = _get_table(tmp_path / "a", var_columns="all") + if not lazy: + table.X = table.X.compute() + write_atera_cell_feature_matrix(SpatialData(tables={"table": table, "table_controls": controls}), tmp_path / "b") + + counts = source.X.toarray() + genes, others = [0, 2, 4, 5], [1, 3] + root = _open(tmp_path / "b" / "cell_feature_matrix.zarr.zip") + assert dict(root.attrs)["matrix_properties"]["num_features"] == N_VAR + assert np.array_equal(sparse_dataset(root["X"])[:].toarray(), counts[:, genes + others]) + var = ad.io.read_elem(root["var"]) + assert var.index.tolist() == [f"ENS{i}" for i in genes + others] + assert var["feature_name"].tolist() == [f"G{i}" for i in genes + others] + assert ( + var["feature_type"].astype(str).tolist() == source.var["feature_type"].astype(str).iloc[genes + others].tolist() + ) + assert "gene_ids" not in var.columns and "feature_types" not in var.columns + + table_back, controls_back, _ = _get_table(tmp_path / "b", var_columns="all") + assert table_back.var_names.tolist() == table.var_names.tolist() + assert controls_back.var_names.tolist() == controls.var_names.tolist() + assert np.array_equal(table_back.X.compute().toarray(), counts[:, genes]) + assert np.array_equal(controls_back.X.toarray(), counts[:, others]) + assert table_back.obs["cell_id"].tolist() == CELL_IDS.tolist() + # the var colors are keyed by the column `atera` names `feature_types`, and survive + assert table.uns["feature_types_colors"] == ["#111111ff", "#222222ff", "#333333ff"] + assert table_back.uns["feature_types_colors"] == table.uns["feature_types_colors"] + + +def test_table_controls_with_other_cells_is_rejected(tmp_path: Path) -> None: + write_atera_cell_feature_matrix(_interleaved_table(), tmp_path / "a") + table, controls, _ = _get_table(tmp_path / "a", var_columns="all") + with pytest.raises(ValueError, match="same cells"): + write_atera_cell_feature_matrix(table, tmp_path / "b", controls_table=controls[::-1].copy()) + assert not (tmp_path / "b").exists() or not list((tmp_path / "b").iterdir()) diff --git a/tests/test_xenium.py b/tests/test_xenium.py index 1c116e57..e514bac4 100644 --- a/tests/test_xenium.py +++ b/tests/test_xenium.py @@ -2,6 +2,7 @@ import warnings from pathlib import Path from tempfile import TemporaryDirectory +from typing import Any import numpy as np import pandas as pd @@ -10,14 +11,18 @@ from pytest_mock import MockerFixture from spatialdata import match_table_to_element, read_zarr from spatialdata.models import get_table_keys +from spatialdata.transformations import Affine, Identity, get_transformation from spatialdata_io.__main__ import xenium_wrapper from spatialdata_io.readers.xenium import ( + _add_aligned_images, + _affine_from_keypoints, _cell_id_str_from_prefix_suffix_uint32_reference, _warn_if_scaled_protein, cell_id_str_from_prefix_suffix_uint32, prefix_suffix_uint32_from_cell_id_str, xenium, + xenium_aligned_image, ) from tests._utils import skip_if_below_python_version @@ -453,3 +458,107 @@ def test_cli_xenium_valid_json_forwarded( assert result.exit_code == 0, result.output call_kwargs = mock_xenium.call_args.kwargs assert call_kwargs[kwarg_param] == {"chunks": 512} + + +@pytest.mark.parametrize("planar", [False, True]) +def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool) -> None: + import tifffile + + rgb = np.arange(3 * 40 * 60, dtype=np.uint32).reshape(3, 40, 60).astype(np.uint8) + path = tmp_path / "S1_he_image.ome.tif" + if planar: # OME axes SYX + tifffile.imwrite(path, rgb, ome=True, photometric="rgb", planarconfig="separate") + else: # YXS + tifffile.imwrite(path, np.moveaxis(rgb, 0, -1), ome=True, photometric="rgb") + he = xenium_aligned_image(path, None) + assert list(he.coords["c"].values) == ["r", "g", "b"] + np.testing.assert_array_equal(np.asarray(he.transpose("c", "y", "x").data), rgb) + + +# an H&E aligned by a rotation by 90 degrees, a scale of 2 and a translation +HE_AFFINE = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) + + +def _write_he(path: Path, affine: np.ndarray, alignment_csv: bool = True) -> None: + import tifffile + + tifffile.imwrite( + path / "S1_he_image.ome.tif", np.zeros((3, 40, 60), dtype=np.uint8), ome=True, metadata={"axes": "CYX"} + ) + if alignment_csv: + pd.DataFrame(affine).to_csv(path / "S1_he_imagealignment.csv", header=False, index=False) + + +def _write_keypoints(path: Path, affine: np.ndarray) -> pd.DataFrame: + src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) + dst = (np.c_[src, np.ones(3)] @ affine.T)[:, :2] + kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) + kp.to_csv(path / "S1_keypoints.csv", index=False) + return kp + + +def _global_affine(image: Any) -> np.ndarray: + t = get_transformation(image, "global") + assert isinstance(t, Affine) + return t.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) + + +def test_aligned_he_image_is_placed_by_its_alignment_csv(tmp_path: Path) -> None: + _write_he(tmp_path, HE_AFFINE) + shifted = HE_AFFINE.copy() + shifted[0, 2] += 100.0 + _write_keypoints(tmp_path, shifted) # the alignment CSV wins over the keypoints + images = _add_aligned_images(tmp_path) + assert list(images) == ["he_image"] + np.testing.assert_allclose(_global_affine(images["he_image"]), HE_AFFINE) + + +def test_he_without_alignment_csv_is_placed_by_keypoints(tmp_path: Path) -> None: + _write_he(tmp_path, HE_AFFINE, alignment_csv=False) + _write_keypoints(tmp_path, HE_AFFINE) + he = _add_aligned_images(tmp_path)["he_image"] + np.testing.assert_allclose(_global_affine(he), HE_AFFINE, atol=1e-9) + + +def test_affine_from_keypoints_reproduces_an_anisotropic_alignment(tmp_path: Path) -> None: + # x and y scales differ by 0.2%, which a similarity fit can't reproduce + affine = np.array([[0.0086, 1.2898, 914.1], [-1.2871, 0.0086, 32040.6], [0.0, 0.0, 1.0]]) + src = np.array([[16690.0, 14065.0], [22114.0, 25667.0], [13029.0, 25846.0], [16798.0, 35944.0]]) + dst = (np.c_[src, np.ones(len(src))] @ affine.T)[:, :2] + kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) + kp.to_csv(tmp_path / "S1_keypoints.csv", index=False) + fitted = _affine_from_keypoints(tmp_path / "S1_keypoints.csv") + assert fitted is not None + np.testing.assert_allclose( + fitted.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")), affine, atol=1e-6 + ) + + +@pytest.mark.parametrize( + "case", ["other_columns", "two_rows", "nan_rows", "collinear", "other_sample", "appledouble", "no_keypoints"] +) +def test_he_stays_unaligned_without_usable_keypoints(tmp_path: Path, case: str) -> None: + _write_he(tmp_path, HE_AFFINE, alignment_csv=False) + kp = _write_keypoints(tmp_path, HE_AFFINE) + kp_file = tmp_path / "S1_keypoints.csv" + if case == "other_columns": + kp.set_axis(["a", "b", "c", "d"], axis=1).to_csv(kp_file, index=False) + elif case == "two_rows": + kp.iloc[:2].to_csv(kp_file, index=False) + elif case == "nan_rows": + kp.assign(fixedX=[1.0, np.nan, "x"]).to_csv(kp_file, index=False) + elif case == "collinear": + kp.assign(alignmentX=[1.0, 2.0, 3.0], alignmentY=[1.0, 2.0, 3.0]).to_csv(kp_file, index=False) + elif case in ["other_sample", "appledouble"]: + kp_file.rename(tmp_path / ("S2_keypoints.csv" if case == "other_sample" else "._S1_keypoints.csv")) + else: + kp_file.unlink() + (tmp_path / "._S1_he_image.ome.tif").write_bytes(b"\x00" * 16) # a macOS sidecar, not a TIFF + + expect_warning = case in ["other_columns", "two_rows", "nan_rows", "collinear"] + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + images = _add_aligned_images(tmp_path) + assert list(images) == ["he_image"] + assert isinstance(get_transformation(images["he_image"], "global"), Identity) + assert any("keypoint" in str(w.message) for w in record) == expect_warning