From 3fbf4c64ee14c897ff1a68fcebdb11215ccc5378 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Tue, 10 Nov 2026 15:25:40 +0100 Subject: [PATCH 01/29] feat(atera): read the aligned post-run H&E/IF images and check their affine against keypoints.csv Atera bundles ship he_image.ome.tif with he_imagealignment.csv (3 x 3 image px -> morphology px, the Xenium convention) and keypoints.csv. The reader reuses the Xenium aligned-image parser (aligned_images=True), and warns when the affine maps the keypoints with a median residual above 10 px. --- src/spatialdata_io/readers/atera.py | 44 +++++++++++++++++++++++++++++ tests/readers/test_atera.py | 42 +++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 19f6a9ab..7f9a617a 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -32,6 +32,7 @@ read_transcripts_for_genes, ) from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata +from spatialdata_io.readers.xenium import _add_aligned_images if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -69,6 +70,7 @@ def atera( transcripts: bool = False, morphology_images: bool = True, morphology_3d_images: bool = False, + aligned_images: bool = True, var_columns: list[str] | Literal["all"] = DEFAULT_VAR_COLUMNS, table_row_chunk_size: int = DEFAULT_TABLE_ROW_CHUNK_SIZE, imread_kwargs: Mapping[str, Any] = MappingProxyType({}), @@ -115,6 +117,10 @@ def atera( morphology_3d_images Whether to read the 3D morphology images. This is opt-in and defaults to `False` because these images can be very large. + aligned_images + Whether to read the post-run images aligned to the morphology images (``*he_image.ome.tif`` / + ``*if_image.ome.tif`` in the bundle root), placed with their ``*imagealignment.csv`` affine. When a + ``keypoints.csv`` is present, a warning is logged if the affine does not map its keypoints onto each other. var_columns Which columns of `var` to keep, out of the full table (which includes ~135 per-cluster differential-expression columns). Defaults to a small subset (`{xx.FEATURE_NAME!r}`, @@ -206,6 +212,13 @@ def atera( path / AteraKeys.MORPHOLOGY_3D_DIR, imread_kwargs, image_models_kwargs, three_d=True ) + if aligned_images: + # same file conventions (and 3 x 3 image px -> morphology px affine) as the Xenium bundles + images |= _add_aligned_images( + path, image_models_kwargs=image_models_kwargs + ) # imread_kwargs are TiffFile kwargs here + _check_keypoints(path) + tables = {"table": table} if table is not None else {} sdata = SpatialData(images=images, labels=labels, points=points, tables=tables, shapes=shapes) sdata = _set_reader_metadata(sdata, "atera") @@ -213,6 +226,37 @@ def atera( return sdata +KEYPOINTS_FILE = "keypoints.csv" +KEYPOINTS_MAX_RESIDUAL_PX = 10.0 + + +def _check_keypoints(path: Path) -> float | None: + """Warn when an alignment affine does not map ``keypoints.csv`` onto itself; return the median residual (px). + + Xenium Explorer writes keypoints as ``fixedX, fixedY`` (morphology image, px) and ``alignmentX, alignmentY`` + (aligned image, px). Files with other columns are skipped. + """ + kp_file = path / KEYPOINTS_FILE + alignments = sorted(path.glob("*imagealignment.csv")) + if not kp_file.exists() or len(alignments) != 1: + return None + kp = pd.read_csv(kp_file) + cols = ["fixedX", "fixedY", "alignmentX", "alignmentY"] + if not set(cols) <= set(kp.columns) or kp.empty: + logging.info(f"{kp_file.name}: columns {list(kp.columns)} not recognised, keypoints not checked.") + return None + affine = pd.read_csv(alignments[0], header=None).to_numpy(dtype=float) + src = np.c_[kp[["alignmentX", "alignmentY"]].to_numpy(dtype=float), np.ones(len(kp))] + residual = np.linalg.norm((src @ affine.T)[:, :2] - kp[["fixedX", "fixedY"]].to_numpy(dtype=float), axis=1) + median = float(np.median(residual)) + if median > KEYPOINTS_MAX_RESIDUAL_PX: + logging.warning( + f"{alignments[0].name} maps {kp_file.name} with a median residual of {median:.1f} px " + f"(> {KEYPOINTS_MAX_RESIDUAL_PX}); the aligned image may be misplaced." + ) + return median + + def _read_x_chunk(zarr_path: Path, zarr_key: AteraKeys, start: int, stop: int) -> csr_matrix: """Read one contiguous row-range of `X` via `anndata.io.sparse_dataset`'s own row slicing. diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 6bc08de4..c5e05815 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -163,3 +163,45 @@ def test_morphology_images_use_embedded_pyramid(tmp_path: Path) -> None: assert [image[k]["image"].shape for k in image.children] == [(2, 64, 96), (2, 32, 48), (2, 16, 24)] # The lowest level is read from the stored pyramid, not downsampled from `scale0`. np.testing.assert_array_equal(np.asarray(image["scale2"]["image"])[0], full[0, ::4, ::4]) + + +def _write_he(path: Path, affine: np.ndarray) -> None: + import tifffile + + tifffile.imwrite( + path / "he_image.ome.tif", np.zeros((3, 40, 60), dtype=np.uint8), ome=True, metadata={"axes": "CYX"} + ) + pd.DataFrame(affine).to_csv(path / "he_imagealignment.csv", header=False, index=False) + + +def test_aligned_he_image_is_placed_by_its_affine(tmp_path: Path) -> None: + from spatialdata.transformations import get_transformation + + from spatialdata_io.readers.xenium import _add_aligned_images + + affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) + _write_he(tmp_path, affine) + images = _add_aligned_images(tmp_path) + assert list(images) == ["he_image"] + he = images["he_image"] + assert list(he.coords["c"].values) == ["r", "g", "b"] + m = get_transformation(he, "global").to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) + np.testing.assert_allclose(m, affine) + + +def test_keypoints_check(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + from spatialdata_io.readers.atera import _check_keypoints + + affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) + _write_he(tmp_path, affine) + src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) + dst = (np.c_[src, np.ones(3)] @ affine.T)[:, :2] + kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) + kp.to_csv(tmp_path / "keypoints.csv", index=False) + assert _check_keypoints(tmp_path) == pytest.approx(0.0, abs=1e-9) + assert "misplaced" not in caplog.text + kp.assign(fixedX=kp["fixedX"] + 50).to_csv(tmp_path / "keypoints.csv", index=False) + assert _check_keypoints(tmp_path) == pytest.approx(50.0) + assert "misplaced" in caplog.text + pd.DataFrame({"a": [1]}).to_csv(tmp_path / "keypoints.csv", index=False) # unknown columns: skipped + assert _check_keypoints(tmp_path) is None From 4eaefc9c0a3c8e4c9bf84f2e76bf526a1d50a904 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Tue, 10 Nov 2026 17:30:23 +0100 Subject: [PATCH 02/29] fix(xenium,atera): fit the keypoint fallback as a full affine On the Atera preview bundles a similarity fit to 10x's keypoints leaves 21.8 and 22.7 px residuals (about 0.2% anisotropy in the scans); a least-squares affine brings every keypoint under 7.2 px. On the v1 bundles, where 10x's alignment CSV is the similarity fit, the affine stays within about 3 px of it. Folds f712d90 (the similarity-fit fallback this commit replaces), which is not ported on its own. --- src/spatialdata_io/readers/atera.py | 25 +++++++++------- src/spatialdata_io/readers/xenium.py | 19 ++++++++++++ tests/readers/test_atera.py | 43 ++++++++++++++++++++++++---- 3 files changed, 72 insertions(+), 15 deletions(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 7f9a617a..15eec0eb 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -120,7 +120,7 @@ def atera( aligned_images Whether to read the post-run images aligned to the morphology images (``*he_image.ome.tif`` / ``*if_image.ome.tif`` in the bundle root), placed with their ``*imagealignment.csv`` affine. When a - ``keypoints.csv`` is present, a warning is logged if the affine does not map its keypoints onto each other. + ``*keypoints.csv`` is present, a warning is logged if the affine does not map its keypoints onto each other. var_columns Which columns of `var` to keep, out of the full table (which includes ~135 per-cluster differential-expression columns). Defaults to a small subset (`{xx.FEATURE_NAME!r}`, @@ -213,11 +213,7 @@ def atera( ) if aligned_images: - # same file conventions (and 3 x 3 image px -> morphology px affine) as the Xenium bundles - images |= _add_aligned_images( - path, image_models_kwargs=image_models_kwargs - ) # imread_kwargs are TiffFile kwargs here - _check_keypoints(path) + images |= _get_aligned_images(path, image_models_kwargs) tables = {"table": table} if table is not None else {} sdata = SpatialData(images=images, labels=labels, points=points, tables=tables, shapes=shapes) @@ -226,20 +222,29 @@ def atera( return sdata -KEYPOINTS_FILE = "keypoints.csv" +KEYPOINTS_SUFFIX = "keypoints.csv" KEYPOINTS_MAX_RESIDUAL_PX = 10.0 +def _get_aligned_images(path: Path, image_models_kwargs: Mapping[str, Any]) -> dict[str, DataTree]: + """Aligned post-run images (H&E, IF) in the global (morphology px) coordinate system, as in the Xenium bundles.""" + aligned = _add_aligned_images(path, image_models_kwargs=image_models_kwargs) # atera's imread_kwargs are TiffFile's + _check_keypoints(path) + return aligned + + def _check_keypoints(path: Path) -> float | None: """Warn when an alignment affine does not map ``keypoints.csv`` onto itself; return the median residual (px). Xenium Explorer writes keypoints as ``fixedX, fixedY`` (morphology image, px) and ``alignmentX, alignmentY`` (aligned image, px). Files with other columns are skipped. """ - kp_file = path / KEYPOINTS_FILE - alignments = sorted(path.glob("*imagealignment.csv")) - if not kp_file.exists() or len(alignments) != 1: + kp_files, alignments = sorted(path.glob(f"*{KEYPOINTS_SUFFIX}")), sorted(path.glob("*imagealignment.csv")) + if ( + len(kp_files) != 1 or len(alignments) != 1 + ): # 10x names them _keypoints.csv, _he_imagealignment.csv return None + kp_file = kp_files[0] kp = pd.read_csv(kp_file) cols = ["fixedX", "fixedY", "alignmentX", "alignmentY"] if not set(cols) <= set(kp.columns) or kp.empty: diff --git a/src/spatialdata_io/readers/xenium.py b/src/spatialdata_io/readers/xenium.py index cd6700dd..fd13a33b 100644 --- a/src/spatialdata_io/readers/xenium.py +++ b/src/spatialdata_io/readers/xenium.py @@ -38,6 +38,7 @@ ShapesModel, TableModel, ) +from spatialdata.transformations import set_transformation from spatialdata.transformations.transformations import Affine, Identity, Scale, Translation from xarray import DataArray, DataTree @@ -1102,10 +1103,28 @@ def _add_aligned_images( # parse the image image = xenium_aligned_image(file, alignment_file, imread_kwargs, image_models_kwargs) + keypoints = [f for f in csv_files if f.name.endswith("keypoints.csv")] + if alignment_file is None and element_name == "he_image" and len(keypoints) == 1: + # bundles with keypoints but no alignment CSV (the Atera previews): 10x's alignment CSV is the + # least-squares fit to the keypoints, so an affine fit to them places the image + set_transformation(image, _affine_from_keypoints(keypoints[0]), "global") + logging.info(f"{file.name} placed by an affine fit to {keypoints[0].name} (no alignment CSV).") images[element_name] = image return images +def _affine_from_keypoints(kp_file: Path) -> Affine: + """Fit the least-squares affine from the keypoints' aligned-image px onto their morphology px. + + ``alignmentX/Y`` -> ``fixedX/Y``. A full affine rather than a similarity: the Atera preview scans carry about 0.2% anisotropy, which leaves 22 px + residuals under a similarity fit and under 8 px under an affine (the Atera preview bundles). + """ + kp = pd.read_csv(kp_file) + src = np.c_[kp[["alignmentX", "alignmentY"]].to_numpy(dtype=float), np.ones(len(kp))] + coef, *_ = np.linalg.lstsq(src, kp[["fixedX", "fixedY"]].to_numpy(dtype=float), rcond=None) + return Affine(np.r_[coef.T, [[0.0, 0.0, 1.0]]], input_axes=("x", "y"), output_axes=("x", "y")) + + def xenium_aligned_image( image_path: str | Path, alignment_file: str | Path | None, diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index c5e05815..1d03d1c3 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -169,9 +169,9 @@ def _write_he(path: Path, affine: np.ndarray) -> None: import tifffile tifffile.imwrite( - path / "he_image.ome.tif", np.zeros((3, 40, 60), dtype=np.uint8), ome=True, metadata={"axes": "CYX"} + path / "S1_he_image.ome.tif", np.zeros((3, 40, 60), dtype=np.uint8), ome=True, metadata={"axes": "CYX"} ) - pd.DataFrame(affine).to_csv(path / "he_imagealignment.csv", header=False, index=False) + pd.DataFrame(affine).to_csv(path / "S1_he_imagealignment.csv", header=False, index=False) def test_aligned_he_image_is_placed_by_its_affine(tmp_path: Path) -> None: @@ -197,11 +197,44 @@ def test_keypoints_check(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> No src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) dst = (np.c_[src, np.ones(3)] @ affine.T)[:, :2] kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) - kp.to_csv(tmp_path / "keypoints.csv", index=False) + kp.to_csv(tmp_path / "S1_keypoints.csv", index=False) assert _check_keypoints(tmp_path) == pytest.approx(0.0, abs=1e-9) assert "misplaced" not in caplog.text - kp.assign(fixedX=kp["fixedX"] + 50).to_csv(tmp_path / "keypoints.csv", index=False) + kp.assign(fixedX=kp["fixedX"] + 50).to_csv(tmp_path / "S1_keypoints.csv", index=False) assert _check_keypoints(tmp_path) == pytest.approx(50.0) assert "misplaced" in caplog.text - pd.DataFrame({"a": [1]}).to_csv(tmp_path / "keypoints.csv", index=False) # unknown columns: skipped + pd.DataFrame({"a": [1]}).to_csv(tmp_path / "S1_keypoints.csv", index=False) # unknown columns: skipped assert _check_keypoints(tmp_path) is None + + +def test_affine_from_keypoints_reproduces_alignment(tmp_path: Path) -> None: + from spatialdata_io.readers.xenium import _affine_from_keypoints + + # about the Atera v1 kidney alignment, with 0.2% anisotropy as in the preview scans + affine = np.array([[0.0086, 1.2898, 914.1], [-1.2871, 0.0086, 32040.6], [0.0, 0.0, 1.0]]) + src = np.array([[16690.0, 14065.0], [22114.0, 25667.0], [13029.0, 25846.0], [16798.0, 35944.0]]) + dst = (np.c_[src, np.ones(len(src))] @ affine.T)[:, :2] + kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) + kp.to_csv(tmp_path / "S1_keypoints.csv", index=False) + m = _affine_from_keypoints(tmp_path / "S1_keypoints.csv").to_affine_matrix( + input_axes=("x", "y"), output_axes=("x", "y") + ) + np.testing.assert_allclose(m, affine, atol=1e-6) + + +def test_he_without_alignment_csv_is_placed_by_keypoints(tmp_path: Path) -> None: + from spatialdata.transformations import get_transformation + + from spatialdata_io.readers.atera import _get_aligned_images + + affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) + _write_he(tmp_path, affine) + (tmp_path / "S1_he_imagealignment.csv").unlink() + src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) + dst = (np.c_[src, np.ones(3)] @ affine.T)[:, :2] + pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}).to_csv( + tmp_path / "S1_keypoints.csv", index=False + ) + he = _get_aligned_images(tmp_path, {})["he_image"] + m = get_transformation(he, "global").to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) + np.testing.assert_allclose(m, affine, atol=1e-9) From 8fb5310d98b1d149a6b3541ef559d953b49b87dc Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Tue, 10 Nov 2026 17:08:36 +0100 Subject: [PATCH 03/29] fix(xenium,atera): read planar RGB aligned images (OME axes SYX) dask_image reads a planar RGB OME-TIFF as (1, 3, y, x), which the 4-D branch of xenium_aligned_image rejected (it expected (1, y, x, 3)). The Atera v1 Onboard 1.1.1 H&E images are planar (breast, CRC, brain); the 1.0.4 kidney is interleaved. --- src/spatialdata_io/readers/xenium.py | 8 ++++++-- tests/readers/test_atera.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/spatialdata_io/readers/xenium.py b/src/spatialdata_io/readers/xenium.py index fd13a33b..419dea2f 100644 --- a/src/spatialdata_io/readers/xenium.py +++ b/src/spatialdata_io/readers/xenium.py @@ -1177,9 +1177,13 @@ def xenium_aligned_image( if dims is None: if len(image.shape) == 4: assert image.shape[0] == 1 - assert image.shape[-1] == 3 image = image.squeeze(0) - dims = ("y", "x", "c") + # interleaved RGB reads as (1, y, x, 3); planar RGB (OME axes SYX, Atera v1 Onboard 1.1.1) as (1, 3, y, x) + if image.shape[-1] in [3, 4]: + dims = ("y", "x", "c") + else: + assert image.shape[0] in [3, 4] + dims = ("c", "y", "x") else: assert len(image.shape) == 3 assert image.shape[0] in [3, 4] diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 1d03d1c3..b873d1e5 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -238,3 +238,19 @@ def test_he_without_alignment_csv_is_placed_by_keypoints(tmp_path: Path) -> None he = _get_aligned_images(tmp_path, {})["he_image"] m = get_transformation(he, "global").to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) np.testing.assert_allclose(m, affine, atol=1e-9) + + +@pytest.mark.parametrize("planar", [False, True]) +def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool) -> None: + import tifffile + + from spatialdata_io.readers.xenium import _add_aligned_images + + rgb = np.arange(3 * 40 * 60, dtype=np.uint32).reshape(3, 40, 60).astype(np.uint8) + if planar: # OME axes SYX, as in the Atera v1 Onboard 1.1.1 H&E + tifffile.imwrite(tmp_path / "S1_he_image.ome.tif", rgb, ome=True, photometric="rgb", planarconfig="separate") + else: # YXS, as in the 1.0.4 kidney H&E + tifffile.imwrite(tmp_path / "S1_he_image.ome.tif", np.moveaxis(rgb, 0, -1), ome=True, photometric="rgb") + he = _add_aligned_images(tmp_path)["he_image"] + assert list(he.coords["c"].values) == ["r", "g", "b"] + np.testing.assert_array_equal(np.asarray(he.transpose("c", "y", "x").data), rgb) From a6a00870b06a64b0474b0e644ba4f59fc6d015fd Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Thu, 12 Nov 2026 07:07:28 +0100 Subject: [PATCH 04/29] fix(atera): leave the obs index unnamed where it duplicates the barcode column spatialdata's table joins call obs.reset_index() (spatialdata 0.8.0 _core/query/relational_query.py:259), which fails with "cannot insert barcode, already exists" when the index carries the name of an existing column. The column keeps the values. --- src/spatialdata_io/readers/atera.py | 5 ++++- tests/readers/test_atera.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 15eec0eb..3e7ef505 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -292,7 +292,7 @@ def _lazy_x( def _align_index_column(df: pd.DataFrame) -> pd.DataFrame: - """Make a column that shares the index's name have the index's dtype. + """Make a column that shares the index's name have the index's dtype, and leave the index unnamed. The bundles' `obs`/`var` list the index (`barcode`/`feature_id`) as a column too. Under pandas 3, `read_elem` returns that index as `str` but the column as `object`, and `anndata` refuses to write @@ -302,6 +302,9 @@ def _align_index_column(df: pd.DataFrame) -> pd.DataFrame: name = df.index.name if name is not None and name in df.columns: df[name] = df[name].astype(df.index.dtype) + # the column keeps the values; an index of the same name breaks `obs.reset_index()`, which spatialdata's + # table joins call (spatialdata 0.8.0 _core/query/relational_query.py:259): "cannot insert barcode" + df.index.name = None return df diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index b873d1e5..26e68c74 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -108,6 +108,22 @@ def test_table_obs_and_var_are_writable(atera_path: Path, table: str) -> None: write_elem(zarr.open_group(zarr.storage.MemoryStore(), mode="w", zarr_format=2), name, df) +@pytest.mark.parametrize("table", ["_get_table", "read_table_for_cells", "read_table_for_genes"]) +def test_table_obs_index_does_not_shadow_the_barcode_column(atera_path: Path, table: str) -> None: + # spatialdata's table joins call `obs.reset_index()` (spatialdata 0.8.0 _core/query/relational_query.py:259), + # which fails with "cannot insert barcode, already exists" when the index carries the column's name + if table == "_get_table": + adata = _get_table(atera_path)[0] + elif table == "read_table_for_cells": + adata = read_table_for_cells(atera_path, cell_ids=CELL_IDS[:2]) + else: + adata = read_table_for_genes(atera_path, "GENE_B") + + assert adata.obs.index.name is None + assert adata.obs["barcode"].tolist() == adata.obs_names.tolist() + adata.obs.reset_index() + + def test_read_var_is_writable(atera_path: Path) -> None: write_elem(zarr.open_group(zarr.storage.MemoryStore(), mode="w", zarr_format=2), "var", read_var(atera_path)) From 8cb6c5b892778dd005f4aaf665883560bfa11eff Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 21:40:40 +0100 Subject: [PATCH 05/29] test(atera): read a whole synthetic bundle with atera(), write it and query it --- tests/readers/test_atera.py | 96 +++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 26e68c74..18969c0f 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -56,9 +56,86 @@ def atera_path(tmp_path: Path) -> Path: elem.attrs["column-order"] = [str(df.index.name), *cast("list[str]", elem.attrs["column-order"])] finally: store.close() + _write_cells_transcripts_and_images(tmp_path, n) return tmp_path +PIXEL_SIZE = 0.5 + + +def _write_cells_transcripts_and_images(path: Path, n: int) -> None: + """The rest of a bundle: cell `i` is the 6 x 6 px square at ``10 i + 2``, with one 4-vertex polygon per mask.""" + import json + + import tifffile + + (path / AteraKeys.SPECS_FILE).write_text(json.dumps({"pixel_size": PIXEL_SIZE})) + store = zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, mode="w") + try: + root = zarr.open_group(store, mode="w", zarr_format=2) + mask = np.zeros((64, 96), dtype=np.uint32) + for c in range(n): + mask[10 * c + 2 : 10 * c + 8, 10 * c + 2 : 10 * c + 8] = c + 1 + bboxes = np.array([[10 * c + 2, 10 * c + 2, 10 * c + 8, 10 * c + 8] for c in range(n)], np.float32) * PIXEL_SIZE + root.create_array("bboxes", data=bboxes) + for i in ("0", "1"): + root.create_array(f"masks/{i}", data=mask) + square = bboxes[:, [0, 1, 2, 1, 2, 3, 0, 3]] + root.create_array(f"polygon_sets/{i}/vertices", data=np.c_[square, np.zeros((n, 2), np.float32)]) + root.create_array(f"polygon_sets/{i}/num_vertices", data=np.full(n, 4, np.int32)) + root.create_array(f"polygon_sets/{i}/cell_index", data=np.arange(n, dtype=np.uint32)) + gridded = root.create_group(f"gridded_polygon_sets/{i}") + gridded.attrs["grid_size"] = [100.0, 100.0] + for level in range(5): + gridded.create_array(f"{level}/0,0/num_vertices", data=np.full(n, 4, np.int32)) + gridded.create_array(f"{level}/0,0/cell_index", data=np.arange(n, dtype=np.uint32)) + corners = np.array([0, 0, 255, 0, 255, 255, 0, 255], np.uint8) + gridded.create_array(f"{level}/0,0/relative_vertices", data=np.tile(corners, n)) + finally: + store.close() + + # one tile; GENE_A owns rows 0-2 and GENE_B rows 3-5, the first transcript is unassigned + store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, mode="w") + try: + grid = zarr.open_group(store, mode="w", zarr_format=2).create_group(str(AteraKeys.GRID_GROUP)) + grid.attrs[str(AteraKeys.GRID_KEYS)] = ["0,0"] + cell_id = np.zeros((6, 2), np.uint32) + cell_id[0] = np.iinfo(np.uint32).max + arrays = { + "location": np.c_[np.arange(6), np.arange(6), np.zeros(6)].astype(np.float32), + "gene_offset": np.array([[0, 3], [3, 6]], np.uint32), + "cell_id": cell_id, + "overlaps_nucleus": np.zeros((6, 1), np.uint8), + "quality_score": np.full((6, 1), 30, np.float32), + "codeword_identity": np.full((6, 1), 7, np.uint16), + } + for key, data in arrays.items(): + grid.create_array(f"0,0/{key}", data=data) + finally: + store.close() + + # each 2D file holds every channel, the "chNNNN" one carries the data; one 3D file per channel + rng = np.random.default_rng(0) + planes = rng.integers(0, 255, (2, 64, 96), dtype=np.uint8) + (path / AteraKeys.MORPHOLOGY_2D_DIR).mkdir() + (path / AteraKeys.MORPHOLOGY_3D_DIR).mkdir() + for i, name in enumerate(["DAPI", "VIM"]): + tifffile.imwrite( + path / AteraKeys.MORPHOLOGY_2D_DIR / f"ch{i:04d}_{name.lower()}.ome.tif", + planes, + ome=True, + tile=(32, 32), + metadata={"axes": "CYX", "Channel": {"Name": ["DAPI", "VIM"]}}, + ) + tifffile.imwrite( + path / AteraKeys.MORPHOLOGY_3D_DIR / f"ch{i:04d}_{name.lower()}_3d.ome.tif", + np.stack([planes[i]] * 3), + ome=True, + tile=(32, 32), + metadata={"axes": "ZYX", "Channel": {"Name": [name]}}, + ) + + def test_cell_labels_table_uses_label_id(atera_path: Path) -> None: table, _ = _get_table(atera_path, region="cell_labels") @@ -270,3 +347,22 @@ def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool) -> he = _add_aligned_images(tmp_path)["he_image"] assert list(he.coords["c"].values) == ["r", "g", "b"] np.testing.assert_array_equal(np.asarray(he.transpose("c", "y", "x").data), rgb) + + +def test_atera_reads_writes_and_queries(atera_path: Path, tmp_path: Path) -> None: + from spatialdata import bounding_box_query, read_zarr + + from spatialdata_io import atera + + sdata = atera(atera_path, transcripts=True) + assert set(sdata.images) == {"morphology"} + assert set(sdata.labels) == {"cell_labels", "nucleus_labels"} + assert set(sdata.shapes) == {"cell_boundaries", "nucleus_boundaries"} + assert len(sdata.points["transcripts"]) == 6 + sdata.write(tmp_path / "out.zarr") + back = read_zarr(tmp_path / "out.zarr") + np.testing.assert_array_equal(back.tables["table"].X.toarray(), sdata.tables["table"].X.compute().toarray()) + crop = bounding_box_query( + back, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[15, 15], target_coordinate_system="global" + ) + assert crop.tables["table"].obs["cell_id"].tolist() == CELL_IDS[:2] From 5d87a6fefb2c0772fed54b71524516628aa3ac55 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 21:49:44 +0100 Subject: [PATCH 06/29] refactor(atera): one zip context manager, one transcript decoder, one transcripts store shared by all tasks Every reader opened its zip in a try/finally; _open_zip replaces 11 of them. read_transcripts_for_genes copied the tile decode of _read_transcript_tile; both now call _transcript_rows. _get_points and atera_to_parquet open transcripts.zarr.zip once instead of once per partition (the review measured 70.8 s against 12.4 s for 200 partitions); the ZipStore pickles and reopens, so the graph still runs under the processes scheduler. Adapted to ec169cf helpers: zarr lookups go through _get_array/_get_group (zarr < 3.3). --- .../converters/atera_to_parquet.py | 20 +-- src/spatialdata_io/readers/_atera_common.py | 169 +++++++----------- src/spatialdata_io/readers/atera.py | 37 +--- tests/readers/test_atera.py | 7 +- 4 files changed, 86 insertions(+), 147 deletions(-) diff --git a/src/spatialdata_io/converters/atera_to_parquet.py b/src/spatialdata_io/converters/atera_to_parquet.py index cafcbcce..ff55e36d 100644 --- a/src/spatialdata_io/converters/atera_to_parquet.py +++ b/src/spatialdata_io/converters/atera_to_parquet.py @@ -6,10 +6,15 @@ import numpy as np import pyarrow as pa import pyarrow.parquet as pq -import zarr from spatialdata_io._constants._constants import AteraKeys -from spatialdata_io.readers._atera_common import _get_array, _get_group, _read_transcript_tile, _tile_row_ranges +from spatialdata_io.readers._atera_common import ( + _get_array, + _get_group, + _open_zip, + _read_transcript_tile, + _tile_row_ranges, +) from spatialdata_io.readers.atera import read_var __all__ = ["atera_to_parquet"] @@ -77,10 +82,8 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path: feature_names = read_var(path)[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() - store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True) - writer = pq.ParquetWriter(output, _PROSEG_SCHEMA) - try: - grid = _get_group(zarr.open_group(store, mode="r"), str(AteraKeys.GRID_GROUP)) + with _open_zip(path / AteraKeys.TRANSCRIPTS_FILE) as group, pq.ParquetWriter(output, _PROSEG_SCHEMA) as writer: + grid = _get_group(group, str(AteraKeys.GRID_GROUP)) tile_keys = [str(key) for key in cast("list[str]", grid.attrs[str(AteraKeys.GRID_KEYS)])] # Stored IDs are used only if every tile has them; otherwise every transcript is numbered # contiguously in `grid_keys` order, so stored and generated IDs never mix. @@ -92,7 +95,7 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path: n_tile = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] # A tile can hold 100M+ transcripts, so it is read and written in row ranges (one parquet row group each). for start, stop in _tile_row_ranges(n_tile): - tile_df = _read_transcript_tile(path, tile_key, feature_names, start, stop) + tile_df = _read_transcript_tile(grid, tile_key, feature_names, start, stop) n = len(tile_df) if n == 0: continue @@ -122,8 +125,5 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path: ) writer.write_table(table) offset += n - finally: - writer.close() - store.close() return output diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 849d933b..0ee48751 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -78,6 +78,12 @@ def _get_group(group: zarr.Group, path: str) -> zarr.Group: _UINT32_SENTINEL = np.iinfo(np.uint32).max +@contextlib.contextmanager +def _open_zip(path: Path) -> Iterator[zarr.Group]: + with zarr.storage.ZipStore(path, read_only=True) as store: + yield zarr.open_group(store, mode="r") + + @contextlib.contextmanager def _patched_ragged_vlen_chunk_decode() -> Iterator[None]: """Work around ``zarr``'s `V2Codec` failing to decode a vlen (string) array's ragged trailing chunk. @@ -243,82 +249,79 @@ def _tile_row_ranges(n_rows: int, chunk_rows: int = DEFAULT_TRANSCRIPT_CHUNK_ROW return [(start, min(start + chunk_rows, n_rows)) for start in range(0, n_rows, chunk_rows)] or [(0, 0)] -def _read_transcript_tile( - path: Path, tile_key: str, feature_names: list[str], start: int = 0, stop: int | None = None -) -> pd.DataFrame: - """Read and decode rows ``[start, stop)`` of a single spatial tile of transcripts.zarr.zip (default: the whole tile). +def _transcript_rows(tile: zarr.Group, start: int, stop: int, feature_name: pd.Categorical) -> pd.DataFrame: + """Decode rows ``[start, stop)`` of one tile of transcripts.zarr.zip, labelled with `feature_name`.""" - Opens its own zip store (rather than sharing one across tiles) so this function can safely - be called from independent `dask` tasks. - """ - store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - tile = _get_group(group, f"{AteraKeys.GRID_GROUP}/{tile_key}") - location_array = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)) - stop = location_array.shape[0] if stop is None else min(stop, location_array.shape[0]) - location = np.asarray(location_array[start:stop]) - gene_offset = np.asarray(_get_array(tile, str(AteraKeys.TRANSCRIPTS_GENE_OFFSET))[...]) - cell_id_raw = np.asarray(_get_array(tile, str(AteraKeys.CELL_ID))[start:stop]) - overlaps_nucleus = np.asarray( - _get_array(tile, str(AteraKeys.TRANSCRIPTS_OVERLAPS_NUCLEUS))[start:stop] - ).reshape(-1) - quality_score = np.asarray(_get_array(tile, str(AteraKeys.TRANSCRIPTS_QUALITY_SCORE))[start:stop]).reshape(-1) - finally: - store.close() - - # Rows within a tile are sorted by gene; gene_offset[g] = [start, end) gives the row range for - # gene g, so the per-row gene index must be reconstructed rather than read directly: row r belongs to - # the first gene whose range ends after r. - gene_index = np.searchsorted(gene_offset[:, 1], np.arange(start, stop), side="right") - feature_name = pd.Categorical.from_codes(gene_index, categories=pd.Index(feature_names)) + def rows(key: AteraKeys) -> np.ndarray: + return np.asarray(_get_array(tile, str(key))[start:stop]) + location = rows(AteraKeys.TRANSCRIPTS_LOCATION) return pd.DataFrame( { str(AteraKeys.TRANSCRIPTS_X): location[:, 0].astype(np.float32), str(AteraKeys.TRANSCRIPTS_Y): location[:, 1].astype(np.float32), str(AteraKeys.TRANSCRIPTS_Z): location[:, 2].astype(np.float32), str(AteraKeys.FEATURE_NAME): feature_name, - str(AteraKeys.CELL_ID): _decode_packed_cell_id(cell_id_raw), - "overlaps_nucleus": overlaps_nucleus.astype(bool), - "quality_score": quality_score.astype(np.float32), + str(AteraKeys.CELL_ID): _decode_packed_cell_id(rows(AteraKeys.CELL_ID)), + "overlaps_nucleus": rows(AteraKeys.TRANSCRIPTS_OVERLAPS_NUCLEUS).reshape(-1).astype(bool), + "quality_score": rows(AteraKeys.TRANSCRIPTS_QUALITY_SCORE).reshape(-1).astype(np.float32), } ) -def _get_points(path: Path, pixel_size: float, feature_names: list[str]) -> dd.DataFrame: - """Lazily read transcripts.zarr.zip, one `dask` task per spatial tile.""" - store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - grid = _get_group(group, str(AteraKeys.GRID_GROUP)) - tile_keys = sorted(grid.group_keys()) - tile_sizes = { - key: _get_array(_get_group(grid, key), str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] for key in tile_keys - } - finally: - store.close() +def _read_transcript_tile( + grid: zarr.Group, tile_key: str, feature_names: list[str], start: int = 0, stop: int | None = None +) -> pd.DataFrame: + """Read and decode rows ``[start, stop)`` of a single spatial tile of transcripts.zarr.zip (default: the whole tile). - cat_dtype = pd.CategoricalDtype(categories=feature_names) - meta = pd.DataFrame( + `grid` is the store's ``grid`` group. All `dask` tasks share it, so the zip is opened once per process, not once + per task. + """ + tile = _get_group(grid, tile_key) + n_rows = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] + stop = n_rows if stop is None else min(stop, n_rows) + gene_offset = np.asarray(_get_array(tile, str(AteraKeys.TRANSCRIPTS_GENE_OFFSET))[...]) + # Rows within a tile are sorted by gene; gene_offset[g] = [start, end) gives the row range for + # gene g, so the per-row gene index must be reconstructed rather than read directly: row r belongs to + # the first gene whose range ends after r. + gene_index = np.searchsorted(gene_offset[:, 1], np.arange(start, stop), side="right") + feature_name = pd.Categorical.from_codes(gene_index, categories=pd.Index(feature_names)) + return _transcript_rows(tile, start, stop, feature_name) + + +def _transcripts_meta(categories: list[str]) -> pd.DataFrame: + """An empty transcripts table with the column dtypes of `_transcript_rows`.""" + return pd.DataFrame( { str(AteraKeys.TRANSCRIPTS_X): pd.array([], dtype="float32"), str(AteraKeys.TRANSCRIPTS_Y): pd.array([], dtype="float32"), str(AteraKeys.TRANSCRIPTS_Z): pd.array([], dtype="float32"), - str(AteraKeys.FEATURE_NAME): pd.array([], dtype=cat_dtype), + str(AteraKeys.FEATURE_NAME): pd.Categorical([], categories=categories), str(AteraKeys.CELL_ID): pd.array([], dtype="int64"), "overlaps_nucleus": pd.array([], dtype="bool"), "quality_score": pd.array([], dtype="float32"), } ) + + +def _get_points(path: Path, pixel_size: float, feature_names: list[str]) -> dd.DataFrame: + """Lazily read transcripts.zarr.zip, one `dask` task per spatial tile.""" + # the store stays referenced by the graph; it reopens on first use, also after pickling + grid = zarr.open_group(zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True), mode="r") + grid = _get_group(grid, str(AteraKeys.GRID_GROUP)) + tile_keys = sorted(grid.group_keys()) + tile_sizes = { + key: _get_array(_get_group(grid, key), str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] for key in tile_keys + } + # One partition per `chunk_rows`-row slice of a tile (not per whole tile), so peak memory is bounded by # `dask`'s number of concurrent tasks times the chunk size, however large a tile is. delayed_tiles = [ - delayed(_read_transcript_tile)(path, key, feature_names, start, stop) + delayed(_read_transcript_tile)(grid, key, feature_names, start, stop) for key in tile_keys for start, stop in _tile_row_ranges(tile_sizes[key]) ] - table = dd.from_delayed(delayed_tiles, meta=meta) + table = dd.from_delayed(delayed_tiles, meta=_transcripts_meta(feature_names)) transform = Scale([1.0 / pixel_size, 1.0 / pixel_size], axes=("x", "y")) return PointsModel.parse( @@ -413,60 +416,24 @@ def read_transcripts_for_genes( else: categories = genes - columns = [ - str(AteraKeys.TRANSCRIPTS_X), - str(AteraKeys.TRANSCRIPTS_Y), - str(AteraKeys.TRANSCRIPTS_Z), - str(AteraKeys.FEATURE_NAME), - str(AteraKeys.CELL_ID), - "overlaps_nucleus", - "quality_score", - ] - store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - tile_keys = sorted(_get_group(group, str(AteraKeys.GRID_GROUP)).group_keys()) - - frames = [] - for tile_key in tile_keys: - tile = _get_group(group, f"{AteraKeys.GRID_GROUP}/{tile_key}") + frames = [] + with _open_zip(path / AteraKeys.TRANSCRIPTS_FILE) as group: + grid = _get_group(group, str(AteraKeys.GRID_GROUP)) + for tile_key in sorted(grid.group_keys()): + tile = _get_group(grid, tile_key) gene_offset = _get_array(tile, str(AteraKeys.TRANSCRIPTS_GENE_OFFSET)) for gene, gene_idx in gene_indices.items(): - start, end = np.asarray(gene_offset[gene_idx]) + start, end = (int(v) for v in np.asarray(gene_offset[gene_idx])) if end <= start: continue - location = np.asarray(_get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION))[start:end]) - cell_id_raw = np.asarray(_get_array(tile, str(AteraKeys.CELL_ID))[start:end]) - overlaps_nucleus = np.asarray( - _get_array(tile, str(AteraKeys.TRANSCRIPTS_OVERLAPS_NUCLEUS))[start:end] - ).reshape(-1) - quality_score = np.asarray( - _get_array(tile, str(AteraKeys.TRANSCRIPTS_QUALITY_SCORE))[start:end] - ).reshape(-1) if by_codeword: - codeword_identity = np.asarray( - _get_array(tile, str(AteraKeys.CODEWORD_IDENTITY))[start:end] - ).reshape(-1) - feature_name = pd.Categorical([f"{gene}_cw{cw}" for cw in codeword_identity], categories=categories) + codeword = np.asarray(_get_array(tile, str(AteraKeys.CODEWORD_IDENTITY))[start:end]).reshape(-1) + labels = [f"{gene}_cw{cw}" for cw in codeword] else: - feature_name = pd.Categorical([gene] * (end - start), categories=categories) - frames.append( - pd.DataFrame( - { - str(AteraKeys.TRANSCRIPTS_X): location[:, 0].astype(np.float32), - str(AteraKeys.TRANSCRIPTS_Y): location[:, 1].astype(np.float32), - str(AteraKeys.TRANSCRIPTS_Z): location[:, 2].astype(np.float32), - str(AteraKeys.FEATURE_NAME): feature_name, - str(AteraKeys.CELL_ID): _decode_packed_cell_id(cell_id_raw), - "overlaps_nucleus": overlaps_nucleus.astype(bool), - "quality_score": quality_score.astype(np.float32), - } - ) - ) - finally: - store.close() + labels = [gene] * (end - start) + frames.append(_transcript_rows(tile, start, end, pd.Categorical(labels, categories=categories))) - df = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame(columns=columns) + df = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame(columns=list(_transcripts_meta([]).columns)) transform = Scale([1.0 / pixel_size, 1.0 / pixel_size], axes=("x", "y")) sdata.points[points_key] = PointsModel.parse( @@ -504,12 +471,8 @@ def _cell_row_positions_in_bbox(path: Path, bbox: tuple[float, float, float, flo pixel_size = _get_pixel_size(path) xmin, ymin, xmax, ymax = (v * pixel_size for v in bbox) - store = zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") + with _open_zip(path / AteraKeys.CELLS_FILE) as group: cell_bboxes = np.asarray(_get_array(group, str(AteraKeys.BBOXES))[...]).reshape(-1, 4) - finally: - store.close() keep = ( (cell_bboxes[:, 0] < xmax) @@ -588,9 +551,7 @@ def read_cell_boundaries( xmin, ymin, xmax, ymax = bbox raw_bbox = (xmin * pixel_size, ymin * pixel_size, xmax * pixel_size, ymax * pixel_size) - store = zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") + with _open_zip(path / AteraKeys.CELLS_FILE) as group: cell_bboxes = np.asarray(_get_array(group, str(AteraKeys.BBOXES))[...]).reshape(-1, 4) mask_group = _get_group(group, f"{AteraKeys.GRIDDED_POLYGON_SETS_GROUP}/{mask_index}") # Each pyramid level merges 2x2 blocks of tiles from the level below, so a tile's spatial @@ -634,8 +595,6 @@ def read_cell_boundaries( coords_parts.append((vertex_mins + relative_vertices / 255 * vertex_spans).astype(np.float32)) num_vertices_parts.append(num_vertices) cell_index_parts.append(cell_index) - finally: - store.close() coords = np.concatenate(coords_parts) if coords_parts else np.empty((0, 2), dtype=np.float32) num_vertices = np.concatenate(num_vertices_parts) if num_vertices_parts else np.empty((0,), dtype=np.int32) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 3e7ef505..d2d50ba4 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -27,6 +27,7 @@ _get_pixel_size, _get_points, _get_polygons, + _open_zip, _patched_ragged_vlen_chunk_decode, read_cell_boundaries, read_transcripts_for_genes, @@ -180,9 +181,7 @@ def atera( images: dict[str, DataArray | DataTree] = {} if needs_cells_zarr: - cells_store = zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, read_only=True) - cells_group = zarr.open_group(cells_store, mode="r") - + cells_group = zarr.open_group(zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, read_only=True), mode="r") cell_ids = table.obs[str(AteraKeys.CELL_ID)].to_numpy() if table is not None else None if nucleus_labels: @@ -270,12 +269,8 @@ def _read_x_chunk(zarr_path: Path, zarr_key: AteraKeys, start: int, stop: int) - """ from anndata.io import sparse_dataset - store = zarr.storage.ZipStore(zarr_path / zarr_key, read_only=True) - try: - group = zarr.open_group(store, mode="r") + with _open_zip(zarr_path / zarr_key) as group: return cast(csr_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[start:stop]) - finally: - store.close() def _lazy_x( @@ -388,10 +383,7 @@ def _get_table( from anndata import AnnData from anndata.io import read_elem, sparse_dataset - store = zarr.storage.ZipStore(path / AteraKeys.CELL_FEATURE_MATRIX_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - + with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: with _patched_ragged_vlen_chunk_decode(): var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() @@ -416,8 +408,6 @@ def _get_table( # Genes detected per cell is the number of stored entries per row of the CSR matrix, which `indptr` # already encodes; this avoids a full pass over the (lazy) counts. n_genes = np.diff(_get_array(group, f"{AteraKeys.X_GROUP}/indptr")[:]) if x_ds.format == "csr" else None - finally: - store.close() x = _lazy_x(path, AteraKeys.CELL_FEATURE_MATRIX_FILE, n_obs, n_var, dtype, row_chunk_size) @@ -453,13 +443,9 @@ def read_var(path: str | Path) -> pd.DataFrame: The complete `var` `DataFrame`, indexed by `feature_id`. """ path = Path(path) - store = zarr.storage.ZipStore(path / AteraKeys.CELL_FEATURE_MATRIX_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") + with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: with _patched_ragged_vlen_chunk_decode(): var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) - finally: - store.close() return var_df @@ -501,9 +487,7 @@ def read_table_for_cells( from anndata.io import sparse_dataset path = Path(path) - store = zarr.storage.ZipStore(path / AteraKeys.CELL_FEATURE_MATRIX_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") + with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: with _patched_ragged_vlen_chunk_decode(): var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) if var_columns != "all": @@ -524,8 +508,6 @@ def read_table_for_cells( x_ds = sparse_dataset(group[str(AteraKeys.X_GROUP)]) x = cast(csr_matrix, x_ds[row_positions]) obs_subset = obs_df.iloc[row_positions] - finally: - store.close() adata = AnnData(X=x, obs=obs_subset, var=var_df) adata.obsm["spatial"] = cast(pd.DataFrame, adata.obs)[ @@ -561,10 +543,7 @@ def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnDat genes = [genes] if isinstance(genes, str) else list(genes) path = Path(path) - store = zarr.storage.ZipStore(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE, read_only=True) - try: - group = zarr.open_group(store, mode="r") - + with _open_zip(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE) as group: with _patched_ragged_vlen_chunk_decode(): var_df = _read_dataframe(group, AteraKeys.VAR_GROUP)[ [str(AteraKeys.FEATURE_ID), str(AteraKeys.FEATURE_NAME)] @@ -587,8 +566,6 @@ def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnDat x_ds = sparse_dataset(group[str(AteraKeys.X_GROUP)]) x: csc_matrix = cast(csc_matrix, x_ds[:, col_positions]) - finally: - store.close() adata = AnnData(X=x, obs=obs_df, var=var_subset) return _parse_table(adata, "cell_labels") diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 18969c0f..3ff214ad 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -214,9 +214,12 @@ def test_tile_row_ranges() -> None: def test_chunked_tile_reads_match_whole_tile(tmp_path: Path) -> None: _write_atera_dataset(tmp_path, {"0,0": 7}, ["0,0"], ids={}) feature_names = ["GENE_A", "GENE_B"] - whole = _read_transcript_tile(tmp_path, "0,0", feature_names) + grid = zarr.open_group(zarr.storage.ZipStore(tmp_path / AteraKeys.TRANSCRIPTS_FILE, read_only=True), mode="r")[ + "grid" + ] + whole = _read_transcript_tile(grid, "0,0", feature_names) # GENE_A owns rows 0-2 and GENE_B rows 3-6, so the 2-row chunks straddle the gene boundary. - parts = [_read_transcript_tile(tmp_path, "0,0", feature_names, a, b) for a, b in _tile_row_ranges(7, 2)] + parts = [_read_transcript_tile(grid, "0,0", feature_names, a, b) for a, b in _tile_row_ranges(7, 2)] assert pd.concat(parts, ignore_index=True).equals(whole) assert whole["feature_name"].tolist() == ["GENE_A"] * 3 + ["GENE_B"] * 4 From f0526ecb8910b757b4f7b57fad3896986467309b Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 21:51:02 +0100 Subject: [PATCH 07/29] refactor(atera): read X with anndata's read_elem_lazy Replaces the hand-built _read_x_chunk/_lazy_x. Each chunk now reads only its slice of indptr (anndata opens the CSR with should_cache_indptr=False), where every task read the whole indptr before, and the unused zarr_key argument goes away. read_elem_lazy needs anndata 0.12. Adapted to ec169cf helpers: X and indptr are looked up with _get_group/_get_array (zarr < 3.3). --- pyproject.toml | 2 +- src/spatialdata_io/readers/atera.py | 44 ++++++----------------------- tests/readers/test_atera.py | 9 ++++++ 3 files changed, 18 insertions(+), 37 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 23853bac..7542f173 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dynamic = [ "version", # allow version to be set by git tags ] dependencies = [ - "anndata", + "anndata>=0.12", "click", "dask-image", "h5py", diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index d2d50ba4..ca633380 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -6,11 +6,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, cast -import dask.array as da import numpy as np import pandas as pd import zarr -from dask import delayed from scipy.sparse import csc_matrix, csr_matrix from spatialdata import SpatialData from spatialdata.models import TableModel @@ -22,6 +20,7 @@ DEFAULT_VAR_COLUMNS, _cell_row_positions_in_bbox, _get_array, + _get_group, _get_labels, _get_morphology_images, _get_pixel_size, @@ -261,31 +260,6 @@ def _check_keypoints(path: Path) -> float | None: return median -def _read_x_chunk(zarr_path: Path, zarr_key: AteraKeys, start: int, stop: int) -> csr_matrix: - """Read one contiguous row-range of `X` via `anndata.io.sparse_dataset`'s own row slicing. - - Opens its own zip store (rather than sharing one across chunks) so this function can safely be - called from independent `dask` tasks. - """ - from anndata.io import sparse_dataset - - with _open_zip(zarr_path / zarr_key) as group: - return cast(csr_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[start:stop]) - - -def _lazy_x( - zarr_path: Path, zarr_key: AteraKeys, n_obs: int, n_var: int, dtype: np.dtype, row_chunk_size: int -) -> da.Array: - """Build a `dask`-backed `n_obs x n_var` CSR array, delegating each chunk's read to `sparse_dataset`.""" - meta = csr_matrix((0, n_var), dtype=dtype) - blocks = [] - for start in range(0, n_obs, row_chunk_size): - stop = min(start + row_chunk_size, n_obs) - block = delayed(_read_x_chunk)(zarr_path, AteraKeys.CELL_FEATURE_MATRIX_FILE, start, stop) - blocks.append(da.from_delayed(block, shape=(stop - start, n_var), dtype=dtype, meta=meta)) - return da.concatenate(blocks, axis=0) - - def _align_index_column(df: pd.DataFrame) -> pd.DataFrame: """Make a column that shares the index's name have the index's dtype, and leave the index unnamed. @@ -373,15 +347,15 @@ def _get_table( `obs`/`var`/`obsm` are read with `anndata.io.read_elem` (which also means `obs`/`var` come back indexed by whichever column the store's ``_index`` attr names -- e.g. `barcode`/`feature_id`). - `X` is built as a lazily `dask`-backed array in row chunks (see `_lazy_x`), each chunk read via - `anndata.io.sparse_dataset` rather than manual `indptr` arithmetic. + `X` is a lazy `dask` array of `row_chunk_size`-row CSR chunks (`anndata.experimental.read_elem_lazy`). `region` should be the name of a `SpatialElement` that will actually be present in the `SpatialData` object the table is assembled into (see the ``default_region`` logic in `atera`), so the table doesn't end up annotating an element that was never loaded. """ from anndata import AnnData - from anndata.io import read_elem, sparse_dataset + from anndata.experimental import read_elem_lazy + from anndata.io import read_elem with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: with _patched_ragged_vlen_chunk_decode(): @@ -402,14 +376,12 @@ def _get_table( ) manifest_extras = _read_manifest_extras(group) - x_ds = sparse_dataset(group[str(AteraKeys.X_GROUP)]) - n_obs, n_var = x_ds.shape - dtype = x_ds.dtype + # the graph keeps the store, which reopens on first use (also after pickling) + x_group = _get_group(group, str(AteraKeys.X_GROUP)) + x = read_elem_lazy(x_group, chunks=(row_chunk_size, -1)) # Genes detected per cell is the number of stored entries per row of the CSR matrix, which `indptr` # already encodes; this avoids a full pass over the (lazy) counts. - n_genes = np.diff(_get_array(group, f"{AteraKeys.X_GROUP}/indptr")[:]) if x_ds.format == "csr" else None - - x = _lazy_x(path, AteraKeys.CELL_FEATURE_MATRIX_FILE, n_obs, n_var, dtype, row_chunk_size) + n_genes = np.diff(_get_array(x_group, "indptr")[:]) if x_group.attrs["encoding-type"] == "csr_matrix" else None if n_genes is not None: obs_df["n_genes"] = n_genes diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 3ff214ad..b725b9d4 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -369,3 +369,12 @@ def test_atera_reads_writes_and_queries(atera_path: Path, tmp_path: Path) -> Non back, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[15, 15], target_coordinate_system="global" ) assert crop.tables["table"].obs["cell_id"].tolist() == CELL_IDS[:2] + + +def test_lazy_x_matches_the_store_and_pickles(atera_path: Path) -> None: + import pickle + + x = _get_table(atera_path, row_chunk_size=3)[0].X + assert x.chunks[0] == (3, 1) + expected = np.arange(len(CELL_IDS) * 2, dtype=np.float32).reshape(-1, 2) + np.testing.assert_array_equal(pickle.loads(pickle.dumps(x)).compute().toarray(), expected) From 4027b82d8f1acdabc0b06bcc9a1b2797e03c770f Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 21:52:40 +0100 Subject: [PATCH 08/29] refactor(atera): reuse xenium's _get_labels and parse_physical_size; one bbox test; drop unused keys - _get_labels was a copy of xenium._get_labels. - The OME pixel-size fallback now goes through parse_physical_size, so a size in nm is converted to um. - One vectorised _bbox_overlaps serves the three bbox filters. - _decode_packed_cell_id is the little-endian int64 view of the uint32 pair; the sentinel views as -1. - Six AteraKeys members were never read. Adapted to ec169cf helpers: the bbox lookup uses _get_array; keeps ec169cf's VARM_GROUP, COLOR and MANIFEST_* keys (the writer uses them) and drops only CATEGORIES/CODES. --- src/spatialdata_io/_constants/_constants.py | 6 -- src/spatialdata_io/readers/_atera_common.py | 61 +++++---------------- src/spatialdata_io/readers/atera.py | 3 +- tests/readers/test_atera.py | 22 +++++++- 4 files changed, 37 insertions(+), 55 deletions(-) diff --git a/src/spatialdata_io/_constants/_constants.py b/src/spatialdata_io/_constants/_constants.py index b4add6eb..479682d9 100644 --- a/src/spatialdata_io/_constants/_constants.py +++ b/src/spatialdata_io/_constants/_constants.py @@ -190,15 +190,10 @@ class AteraKeys(ModeEnum): # cell_feature_matrix.zarr.zip groups/keys X_GROUP = "X" - X_DATA = "data" - X_INDICES = "indices" - X_INDPTR = "indptr" OBS_GROUP = "obs" VAR_GROUP = "var" OBSM_GROUP = "obsm" VARM_GROUP = "varm" - CATEGORIES = "categories" - CODES = "codes" COLOR = "color" # root `.zattrs` manifest (RFC "cell feature anndata", schema anndata_zattrs/v0.1.0) @@ -217,7 +212,6 @@ class AteraKeys(ModeEnum): LABEL_ID = "label_id" CENTROID_X = "centroid_column" CENTROID_Y = "centroid_row" - CELL_AREA = "cell_area" # var / features FEATURE_ID = "feature_id" diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 0ee48751..1bb74a02 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -20,7 +20,6 @@ from spatialdata.models import ( Image2DModel, Image3DModel, - Labels2DModel, PointsModel, ShapesModel, ) @@ -29,6 +28,7 @@ from xarray import DataTree from spatialdata_io._constants._constants import AteraKeys +from spatialdata_io.readers._utils._utils import parse_physical_size if TYPE_CHECKING: from collections.abc import Iterator, Mapping @@ -75,8 +75,6 @@ def _get_group(group: zarr.Group, path: str) -> zarr.Group: # Max transcripts per `dask` partition / `atera_to_parquet` batch (~130 MB of decoded columns). DEFAULT_TRANSCRIPT_CHUNK_ROWS = 5_000_000 -_UINT32_SENTINEL = np.iinfo(np.uint32).max - @contextlib.contextmanager def _open_zip(path: Path) -> Iterator[zarr.Group]: @@ -154,25 +152,10 @@ def decode() -> Any: def _decode_packed_cell_id(raw: np.ndarray) -> np.ndarray: """Decode a (N, 2) uint32 packed cell id array into the int64 ids used in the cell-feature-matrix obs. - The two uint32 columns pack into an int64 as ``low + (high << 32)``. The sentinel - ``(uint32 max, uint32 max)`` marks "no cell assigned" and is decoded to ``-1``. + The two uint32 columns pack into an int64 as ``low + (high << 32)``, which is the little-endian int64 view of + the pair. The sentinel ``(uint32 max, uint32 max)`` marks "no cell assigned" and so views as ``-1``. """ - low = raw[:, 0].astype(np.int64) - high = raw[:, 1].astype(np.int64) - packed = low + (high << 32) - unassigned = (raw[:, 0] == _UINT32_SENTINEL) & (raw[:, 1] == _UINT32_SENTINEL) - packed[unassigned] = -1 - return packed - - -def _get_labels( - cells_group: zarr.Group, - mask_index: int, - labels_models_kwargs: Mapping[str, Any] = MappingProxyType({}), -) -> DataArray | DataTree: - """Read the labels raster from cells.zarr.zip masks/{mask_index} (0 = nucleus, 1 = cell).""" - masks = da.from_array(_get_array(cells_group, f"{AteraKeys.MASKS_GROUP}/{mask_index}")) - return Labels2DModel.parse(masks, dims=("y", "x"), transformations={"global": Identity()}, **labels_models_kwargs) + return np.ascontiguousarray(raw, dtype=" tuple[float, float, float, return tile_x * tile_size, tile_y * tile_size, (tile_x + 1) * tile_size, (tile_y + 1) * tile_size -def _bbox_overlaps(a: tuple[float, float, float, float], b: tuple[float, float, float, float]) -> bool: - return a[0] < b[2] and b[0] < a[2] and a[1] < b[3] and b[1] < a[3] +def _bbox_overlaps(boxes: np.ndarray, bbox: tuple[float, float, float, float]) -> np.ndarray: + """Which of the ``(..., 4)`` (xmin, ymin, xmax, ymax) `boxes` overlap `bbox`.""" + boxes = np.asarray(boxes) + return (boxes[..., 0] < bbox[2]) & (bbox[0] < boxes[..., 2]) & (boxes[..., 1] < bbox[3]) & (bbox[1] < boxes[..., 3]) def _cell_row_positions_in_bbox(path: Path, bbox: tuple[float, float, float, float]) -> np.ndarray: @@ -469,18 +454,13 @@ def _cell_row_positions_in_bbox(path: Path, bbox: tuple[float, float, float, flo fact `read_cell_boundaries` relies on for its own `bbox` filtering). """ pixel_size = _get_pixel_size(path) - xmin, ymin, xmax, ymax = (v * pixel_size for v in bbox) - with _open_zip(path / AteraKeys.CELLS_FILE) as group: cell_bboxes = np.asarray(_get_array(group, str(AteraKeys.BBOXES))[...]).reshape(-1, 4) - - keep = ( - (cell_bboxes[:, 0] < xmax) - & (xmin < cell_bboxes[:, 2]) - & (cell_bboxes[:, 1] < ymax) - & (ymin < cell_bboxes[:, 3]) - ) - return np.nonzero(keep)[0] + return np.nonzero( + _bbox_overlaps( + cell_bboxes, (bbox[0] * pixel_size, bbox[1] * pixel_size, bbox[2] * pixel_size, bbox[3] * pixel_size) + ) + )[0] def read_cell_boundaries( @@ -575,13 +555,7 @@ def read_cell_boundaries( relative_vertices = np.asarray(_get_array(tile, str(AteraKeys.RELATIVE_VERTICES))[...]).reshape(-1, 2) if raw_bbox is not None: - tile_cell_bboxes = cell_bboxes[cell_index] - keep = ( - (tile_cell_bboxes[:, 0] < raw_bbox[2]) - & (raw_bbox[0] < tile_cell_bboxes[:, 2]) - & (tile_cell_bboxes[:, 1] < raw_bbox[3]) - & (raw_bbox[1] < tile_cell_bboxes[:, 3]) - ) + keep = _bbox_overlaps(cell_bboxes[cell_index], raw_bbox) if not keep.any(): continue relative_vertices = relative_vertices[np.repeat(keep, num_vertices)] @@ -633,8 +607,6 @@ def _get_pixel_size(path: Path) -> float: specs = json.load(f) return specs[str(AteraKeys.PIXEL_SIZE)] - from ome_types import from_tiff - morphology_dir = path / AteraKeys.MORPHOLOGY_2D_DIR files = ( sorted(f for f in morphology_dir.iterdir() if f.name.endswith(".ome.tif") and not f.name.startswith("._")) @@ -646,10 +618,7 @@ def _get_pixel_size(path: Path) -> float: f"Cannot determine the pixel size: {path!s} contains neither {AteraKeys.SPECS_FILE!s} nor any " f"morphology OME-TIFF files in {morphology_dir.name!s}/." ) - physical_size_x = from_tiff(files[0]).images[0].pixels.physical_size_x - if physical_size_x is None: - raise ValueError(f"{files[0]!s} does not specify `PhysicalSizeX` in its OME-XML metadata.") - return physical_size_x + return parse_physical_size(files[0]) def _tiled_imread( diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index ca633380..a2c96010 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -21,7 +21,6 @@ _cell_row_positions_in_bbox, _get_array, _get_group, - _get_labels, _get_morphology_images, _get_pixel_size, _get_points, @@ -32,7 +31,7 @@ read_transcripts_for_genes, ) from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata -from spatialdata_io.readers.xenium import _add_aligned_images +from spatialdata_io.readers.xenium import _add_aligned_images, _get_labels if TYPE_CHECKING: from collections.abc import Mapping, Sequence diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index b725b9d4..7f32d2af 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -361,7 +361,8 @@ def test_atera_reads_writes_and_queries(atera_path: Path, tmp_path: Path) -> Non assert set(sdata.images) == {"morphology"} assert set(sdata.labels) == {"cell_labels", "nucleus_labels"} assert set(sdata.shapes) == {"cell_boundaries", "nucleus_boundaries"} - assert len(sdata.points["transcripts"]) == 6 + transcripts = sdata.points["transcripts"].compute() + assert transcripts["cell_id"].tolist() == [-1, 0, 0, 0, 0, 0] # the packed (uint32 max, uint32 max) is unassigned sdata.write(tmp_path / "out.zarr") back = read_zarr(tmp_path / "out.zarr") np.testing.assert_array_equal(back.tables["table"].X.toarray(), sdata.tables["table"].X.compute().toarray()) @@ -378,3 +379,22 @@ def test_lazy_x_matches_the_store_and_pickles(atera_path: Path) -> None: assert x.chunks[0] == (3, 1) expected = np.arange(len(CELL_IDS) * 2, dtype=np.float32).reshape(-1, 2) np.testing.assert_array_equal(pickle.loads(pickle.dumps(x)).compute().toarray(), expected) + + +def test_pixel_size_falls_back_to_the_ome_metadata_in_microns(tmp_path: Path) -> None: + import tifffile + + (tmp_path / AteraKeys.MORPHOLOGY_2D_DIR).mkdir() + tifffile.imwrite( + tmp_path / AteraKeys.MORPHOLOGY_2D_DIR / "ch0000_dapi.ome.tif", + np.zeros((1, 8, 8), np.uint8), + ome=True, + metadata={ + "axes": "CYX", + "PhysicalSizeX": 212.0, + "PhysicalSizeXUnit": "nm", + "PhysicalSizeY": 212.0, + "PhysicalSizeYUnit": "nm", + }, + ) + assert common._get_pixel_size(tmp_path) == pytest.approx(0.212) From 48e53fa1b23ab3aee882779007c763d84a0aabf3 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 21:58:55 +0100 Subject: [PATCH 09/29] fix(atera): pad 10x's short trailing vlen chunks in a store wrapper, not by patching zarr _patched_ragged_vlen_chunk_decode swapped V2Codec._decode_single for the whole process. Two threads reading at once left the patch installed or removed it mid-read, any short 1-D chunk was accepted and broadcast, and FusedCodecPipeline bypassed it. _PadRaggedVlenStore pads only the trailing chunk of a 1-D vlen-utf8 array, and only when it holds exactly the remaining items, at the store level, so it holds under both codec pipelines and needs no global state. --- src/spatialdata_io/readers/_atera_common.py | 109 +++++++------------- src/spatialdata_io/readers/atera.py | 77 +++++++------- tests/readers/test_atera.py | 38 +++++++ 3 files changed, 113 insertions(+), 111 deletions(-) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 1bb74a02..5382bb3b 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import contextlib import json import re @@ -34,6 +33,9 @@ from collections.abc import Iterator, Mapping from xarray import DataArray + from zarr.abc.buffer import Buffer + from zarr.abc.store import ByteRequest + from zarr.core.buffer import BufferPrototype # `zarr.Group.get_array` / `zarr.Group.get_group` (typed lookups that raise on a missing node or on the @@ -76,77 +78,46 @@ def _get_group(group: zarr.Group, path: str) -> zarr.Group: DEFAULT_TRANSCRIPT_CHUNK_ROWS = 5_000_000 -@contextlib.contextmanager -def _open_zip(path: Path) -> Iterator[zarr.Group]: - with zarr.storage.ZipStore(path, read_only=True) as store: - yield zarr.open_group(store, mode="r") +class _PadRaggedVlenStore(zarr.storage.WrapperStore[zarr.storage.ZipStore]): + """Pad the short trailing chunk of a 1-D ``vlen-utf8`` zarr v2 array to the nominal chunk size. + + 10x writes the last chunk of the `obs`/`var` string columns (e.g. `barcode`, chunks of 1,000,000) with only the + items it holds. zarr decodes every v2 chunk at the nominal size and raises "cannot reshape array of size 376951 + into shape (1000000,)"; zarr's own writer pads that chunk. Only a trailing chunk that holds exactly the remaining + items is padded; any other chunk is passed on unchanged, so zarr still raises on a corrupt one. + """ + + async def get(self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None) -> Buffer | None: + buf = await super().get(key, prototype, byte_range) + parent, _, name = key.rpartition("/") + if buf is None or byte_range is not None or not name.isdigit(): + return buf + meta_buf = await super().get(f"{parent}/.zarray" if parent else ".zarray", prototype) + meta = json.loads(meta_buf.to_bytes()) if meta_buf is not None else {} + if len(meta.get("shape", ())) != 1 or [f["id"] for f in meta.get("filters") or []] != ["vlen-utf8"]: + return buf + (size,), (chunk,) = meta["shape"], meta["chunks"] + n_items = size - int(name) * chunk + if not 0 < n_items < chunk: + return buf + from numcodecs import VLenUTF8, get_codec + + compressor = get_codec(meta["compressor"]) if meta.get("compressor") else None + raw = buf.to_bytes() + items = VLenUTF8().decode(compressor.decode(raw) if compressor else raw) + if len(items) != n_items: + return buf + padded = np.full(chunk, meta.get("fill_value") or "", dtype=object) + padded[:n_items] = items + encoded = VLenUTF8().encode(padded) + return prototype.buffer.from_bytes(compressor.encode(encoded) if compressor else encoded) @contextlib.contextmanager -def _patched_ragged_vlen_chunk_decode() -> Iterator[None]: - """Work around ``zarr``'s `V2Codec` failing to decode a vlen (string) array's ragged trailing chunk. - - `obs`/`var`'s string columns (e.g. `barcode`) are stored as 1-D zarr v2 arrays with a fixed - chunk size chosen independently of the row count (e.g. `1_000_000`), so the trailing chunk is - usually smaller than that nominal size. For fixed-width dtypes this is harmless: `zarr` pads an - incomplete edge chunk to the full chunk shape (with the fill value) before compression, so it - always decodes back to the full nominal size. Variable-length (`vlen-utf8`/`StringDType`) - chunks aren't padded this way -- only the elements actually written are stored -- but - `V2Codec._decode_single` unconditionally reshapes the decoded chunk to the nominal chunk shape - regardless of dtype, so reading across that trailing chunk raises e.g. ``ValueError: cannot - reshape array of size 376951 into shape (1000000,)``. Reproduced against `zarr` 3.1.6 and - 3.4.0 (the latest release as of writing); no matching issue found upstream yet. - - This patches `V2Codec._decode_single` (the abstract-method entry point present on every - `zarr` 3.x release, unlike the private `_decode_sync`/`_decode_single` split introduced partway - through the 3.x series) for the duration of the context, falling back to the chunk's actual - (smaller) size instead of raising, only for the 1-D case this affects; any other reshape failure - (e.g. a genuinely corrupt chunk, or a ragged chunk in more than one dimension) is still raised. - """ - try: - from numcodecs.compat import ensure_ndarray_like - from zarr.codecs._v2 import V2Codec - from zarr.registry import get_ndbuffer_class - except ImportError: - # `zarr`'s internal layout changed enough that this patch no longer applies; read without - # it rather than fail outright (bundles without a ragged vlen chunk are unaffected either way). - yield - return - - original_decode_single = V2Codec._decode_single - - async def patched_decode_single(self: V2Codec, chunk_bytes: Any, chunk_spec: Any) -> Any: - def decode() -> Any: - cdata = chunk_bytes.as_array_like() - chunk = self.compressor.decode(cdata) if self.compressor else cdata - if self.filters: - for f in reversed(self.filters): - chunk = f.decode(chunk) - chunk = ensure_ndarray_like(chunk) - if chunk_spec.dtype.dtype_cls is not np.dtypes.ObjectDType: - try: - chunk = chunk.view(chunk_spec.dtype.to_native_dtype()) - except TypeError: - chunk = np.array(chunk).astype(chunk_spec.dtype.to_native_dtype()) - elif chunk.dtype != object: - raise RuntimeError("cannot read object array without object codec") - - chunk = chunk.reshape(-1, order="A") - nominal_size = int(np.prod(chunk_spec.shape)) - if chunk.size == nominal_size: - chunk = chunk.reshape(chunk_spec.shape, order=chunk_spec.order) - elif len(chunk_spec.shape) != 1: - raise ValueError(f"cannot reshape array of size {chunk.size} into shape {chunk_spec.shape}") - # else: ragged trailing chunk of a 1-D vlen array -- keep its natural, smaller size. - return get_ndbuffer_class().from_ndarray_like(chunk) - - return await asyncio.to_thread(decode) - - V2Codec._decode_single = patched_decode_single # type: ignore[method-assign] - try: - yield - finally: - V2Codec._decode_single = original_decode_single # type: ignore[method-assign] +def _open_zip(path: Path) -> Iterator[zarr.Group]: + """Open a zipped zarr store read-only (see `_PadRaggedVlenStore`).""" + with zarr.storage.ZipStore(path, read_only=True) as store: + yield zarr.open_group(_PadRaggedVlenStore(store), mode="r") def _decode_packed_cell_id(raw: np.ndarray) -> np.ndarray: diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index a2c96010..f31deb35 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -26,7 +26,6 @@ _get_points, _get_polygons, _open_zip, - _patched_ragged_vlen_chunk_decode, read_cell_boundaries, read_transcripts_for_genes, ) @@ -357,22 +356,21 @@ def _get_table( from anndata.io import read_elem with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: - with _patched_ragged_vlen_chunk_decode(): - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) - feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() - if var_columns != "all": - var_df = var_df[list(var_columns)] - colors = _read_category_colors(group, AteraKeys.OBS_GROUP) - colors.update(_read_category_colors(group, AteraKeys.VAR_GROUP, list(var_df.columns))) - - obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) - _add_label_id(obs_df) - - obsm = ( - cast(dict[str, Any], read_elem(group[str(AteraKeys.OBSM_GROUP)])) - if str(AteraKeys.OBSM_GROUP) in group - else {} - ) + var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) + feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() + if var_columns != "all": + var_df = var_df[list(var_columns)] + colors = _read_category_colors(group, AteraKeys.OBS_GROUP) + colors.update(_read_category_colors(group, AteraKeys.VAR_GROUP, list(var_df.columns))) + + obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) + _add_label_id(obs_df) + + obsm = ( + cast(dict[str, Any], read_elem(group[str(AteraKeys.OBSM_GROUP)])) + if str(AteraKeys.OBSM_GROUP) in group + else {} + ) manifest_extras = _read_manifest_extras(group) # the graph keeps the store, which reopens on first use (also after pickling) @@ -415,8 +413,7 @@ def read_var(path: str | Path) -> pd.DataFrame: """ path = Path(path) with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: - with _patched_ragged_vlen_chunk_decode(): - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) + var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) return var_df @@ -459,12 +456,11 @@ def read_table_for_cells( path = Path(path) with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: - with _patched_ragged_vlen_chunk_decode(): - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) - if var_columns != "all": - var_df = var_df[list(var_columns)] - obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) - _add_label_id(obs_df) + var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) + if var_columns != "all": + var_df = var_df[list(var_columns)] + obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) + _add_label_id(obs_df) if cell_ids is not None: index = pd.Index(obs_df[str(AteraKeys.CELL_ID)].to_numpy()) @@ -515,25 +511,22 @@ def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnDat genes = [genes] if isinstance(genes, str) else list(genes) path = Path(path) with _open_zip(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE) as group: - with _patched_ragged_vlen_chunk_decode(): - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP)[ - [str(AteraKeys.FEATURE_ID), str(AteraKeys.FEATURE_NAME)] - ] - feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() - index = pd.Index(feature_names) - col_positions = index.get_indexer(pd.Index(np.asarray(genes))) - missing = np.asarray(genes)[col_positions == -1] - if missing.size: - raise ValueError( - f"gene(s) not found in {AteraKeys.VAR_GROUP!s}.{AteraKeys.FEATURE_NAME!s}: {missing.tolist()}" - ) + var_df = _read_dataframe(group, AteraKeys.VAR_GROUP)[[str(AteraKeys.FEATURE_ID), str(AteraKeys.FEATURE_NAME)]] + feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() + index = pd.Index(feature_names) + col_positions = index.get_indexer(pd.Index(np.asarray(genes))) + missing = np.asarray(genes)[col_positions == -1] + if missing.size: + raise ValueError( + f"gene(s) not found in {AteraKeys.VAR_GROUP!s}.{AteraKeys.FEATURE_NAME!s}: {missing.tolist()}" + ) - var_subset = var_df.iloc[col_positions] - var_subset.index = pd.Index(np.asarray(genes), name=str(AteraKeys.FEATURE_NAME)) - _align_index_column(var_subset) + var_subset = var_df.iloc[col_positions] + var_subset.index = pd.Index(np.asarray(genes), name=str(AteraKeys.FEATURE_NAME)) + _align_index_column(var_subset) - obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) - _add_label_id(obs_df) + obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) + _add_label_id(obs_df) x_ds = sparse_dataset(group[str(AteraKeys.X_GROUP)]) x: csc_matrix = cast(csc_matrix, x_ds[:, col_positions]) diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 7f32d2af..ed2e9b85 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -398,3 +398,41 @@ def test_pixel_size_falls_back_to_the_ome_metadata_in_microns(tmp_path: Path) -> }, ) assert common._get_pixel_size(tmp_path) == pytest.approx(0.212) + + +@pytest.mark.parametrize("pipeline", ["BatchedCodecPipeline", "FusedCodecPipeline"]) +def test_short_trailing_vlen_chunk_is_padded(tmp_path: Path, pipeline: str) -> None: + import json + import zipfile + + import zarr.core.codec_pipeline + from numcodecs import VLenUTF8, get_codec + + if not hasattr(zarr.core.codec_pipeline, pipeline): + pytest.skip(f"zarr {zarr.__version__} has no {pipeline}") + root = tmp_path / "s.zarr" + zarr.create_array(zarr.storage.LocalStore(root), shape=(5,), chunks=(3,), dtype=str, zarr_format=2)[:] = np.array( + ["a", "bb", "ccc", "dd", "e"], dtype=object + ) + meta = json.loads((root / ".zarray").read_text()) + compressor = get_codec(meta["compressor"]) if meta["compressor"] else None + + def zip_with_trailing_chunk(items: list[str]) -> Path: + encoded = VLenUTF8().encode(np.array(items, dtype=object)) + (root / "1").write_bytes(compressor.encode(encoded) if compressor else encoded) + with zipfile.ZipFile(tmp_path / "s.zip", "w") as zf: + zf.writestr(".zgroup", json.dumps({"zarr_format": 2})) + for f in root.iterdir(): + zf.write(f, f"s/{f.name}") + return tmp_path / "s.zip" + + with zarr.config.set({"codec_pipeline.path": f"zarr.core.codec_pipeline.{pipeline}"}): + # 10x's writer stores only the 2 items of the trailing chunk; zarr reads it at the nominal size of 3 + path = zip_with_trailing_chunk(["dd", "e"]) + with pytest.raises(ValueError, match="reshape"): + zarr.open_group(zarr.storage.ZipStore(path, read_only=True), mode="r")["s"][:] + with common._open_zip(path) as group: + assert group["s"][:].tolist() == ["a", "bb", "ccc", "dd", "e"] + path = zip_with_trailing_chunk(["dd"]) # one item short: corrupt, not padded + with common._open_zip(path) as group, pytest.raises(ValueError, match="reshape"): + group["s"][:] From fc5eabaac73adadaf0c8449ed1f8ff1c24a190b7 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:02:59 +0100 Subject: [PATCH 10/29] fix(xenium,atera): check the keypoints before the H&E fallback fit; skip macOS ._ files The fallback took any *keypoints.csv in the root and fitted it unchecked. Other columns raised a KeyError, a NaN row a LinAlgError, fewer than 3 rows gave a NaN affine, and S2_keypoints.csv placed S1's H&E. It now reads _keypoints.csv of the same sample as the image and needs the four Xenium Explorer columns, at least 3 finite rows and non-collinear points. Otherwise it warns and keeps the image at Identity, as main does. The aligned-image glob skips ._ sidecars, which crashed atera() (aligned_images=True by default). Real bundles name the files _he_image.ome.tif and _keypoints.csv (all six Atera v1 bundles checked). --- src/spatialdata_io/readers/atera.py | 42 +++++----- src/spatialdata_io/readers/xenium.py | 110 ++++++++++++++++++--------- tests/readers/test_atera.py | 63 ++++++++++++--- 3 files changed, 146 insertions(+), 69 deletions(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index f31deb35..b5da7473 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -30,7 +30,13 @@ read_transcripts_for_genes, ) from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata -from spatialdata_io.readers.xenium import _add_aligned_images, _get_labels +from spatialdata_io.readers.xenium import ( + _add_aligned_images, + _aligned_image_files, + _get_labels, + _keypoints_file, + _read_keypoints, +) if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -218,41 +224,29 @@ def atera( return sdata -KEYPOINTS_SUFFIX = "keypoints.csv" KEYPOINTS_MAX_RESIDUAL_PX = 10.0 def _get_aligned_images(path: Path, image_models_kwargs: Mapping[str, Any]) -> dict[str, DataTree]: """Aligned post-run images (H&E, IF) in the global (morphology px) coordinate system, as in the Xenium bundles.""" aligned = _add_aligned_images(path, image_models_kwargs=image_models_kwargs) # atera's imread_kwargs are TiffFile's - _check_keypoints(path) + for name, file, alignment_file in _aligned_image_files(path): + if name == "he_image" and alignment_file is not None: + _check_keypoints(alignment_file, _keypoints_file(file)) return aligned -def _check_keypoints(path: Path) -> float | None: - """Warn when an alignment affine does not map ``keypoints.csv`` onto itself; return the median residual (px). - - Xenium Explorer writes keypoints as ``fixedX, fixedY`` (morphology image, px) and ``alignmentX, alignmentY`` - (aligned image, px). Files with other columns are skipped. - """ - kp_files, alignments = sorted(path.glob(f"*{KEYPOINTS_SUFFIX}")), sorted(path.glob("*imagealignment.csv")) - if ( - len(kp_files) != 1 or len(alignments) != 1 - ): # 10x names them _keypoints.csv, _he_imagealignment.csv - return None - kp_file = kp_files[0] - kp = pd.read_csv(kp_file) - cols = ["fixedX", "fixedY", "alignmentX", "alignmentY"] - if not set(cols) <= set(kp.columns) or kp.empty: - logging.info(f"{kp_file.name}: columns {list(kp.columns)} not recognised, keypoints not checked.") +def _check_keypoints(alignment_file: Path, kp_file: Path) -> float | None: + """Warn when the alignment affine does not map the keypoints onto each other; return the median residual (px).""" + keypoints = _read_keypoints(kp_file) + if keypoints is None: return None - affine = pd.read_csv(alignments[0], header=None).to_numpy(dtype=float) - src = np.c_[kp[["alignmentX", "alignmentY"]].to_numpy(dtype=float), np.ones(len(kp))] - residual = np.linalg.norm((src @ affine.T)[:, :2] - kp[["fixedX", "fixedY"]].to_numpy(dtype=float), axis=1) - median = float(np.median(residual)) + src, dst = keypoints + affine = pd.read_csv(alignment_file, header=None).to_numpy(dtype=float) + median = float(np.median(np.linalg.norm((np.c_[src, np.ones(len(src))] @ affine.T)[:, :2] - dst, axis=1))) if median > KEYPOINTS_MAX_RESIDUAL_PX: logging.warning( - f"{alignments[0].name} maps {kp_file.name} with a median residual of {median:.1f} px " + f"{alignment_file.name} maps {kp_file.name} with a median residual of {median:.1f} px " f"(> {KEYPOINTS_MAX_RESIDUAL_PX}); the aligned image may be misplaced." ) return median diff --git a/src/spatialdata_io/readers/xenium.py b/src/spatialdata_io/readers/xenium.py index 419dea2f..19e74a19 100644 --- a/src/spatialdata_io/readers/xenium.py +++ b/src/spatialdata_io/readers/xenium.py @@ -1073,6 +1073,30 @@ def filter(self, record: logging.LogRecord) -> bool: return result +def _aligned_image_files(path: Path) -> list[tuple[str, Path, Path | None]]: + """The aligned images in `path` (``*he_image.ome.tif``, ``*if_image.ome.tif``), each with its alignment CSV or None. + + macOS ``._`` sidecars are skipped. + """ + found = [] + for file in sorted(path.glob("*.ome.tif")): + for suffix in [XeniumKeys.ALIGNED_HE_IMAGE_SUFFIX, XeniumKeys.ALIGNED_IF_IMAGE_SUFFIX]: + if file.name.endswith(suffix) and not file.name.startswith("._"): + csv = file.with_name( + file.name.replace( + XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_REMOVE, XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_ADD + ) + ) + found.append( + ( + suffix.replace(XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_REMOVE, ""), + file, + csv if csv.is_file() else None, + ) + ) + return found + + def _add_aligned_images( path: Path, imread_kwargs: Mapping[str, Any] = MappingProxyType({}), @@ -1080,48 +1104,62 @@ def _add_aligned_images( ) -> dict[str, DataTree]: """Discover and parse aligned images.""" images = {} - ome_tif_files = list(path.glob("*.ome.tif")) - csv_files = list(path.glob("*.csv")) - for file in ome_tif_files: - element_name = None - for suffix in [ - XeniumKeys.ALIGNED_HE_IMAGE_SUFFIX, - XeniumKeys.ALIGNED_IF_IMAGE_SUFFIX, - ]: - if file.name.endswith(suffix): - element_name = suffix.replace(XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_REMOVE, "") - break - if element_name is not None: - # check if an alignment file exists - expected_filename = file.name.replace( - XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_REMOVE, - XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_ADD, - ) - alignment_files = [f for f in csv_files if f.name == expected_filename] - assert len(alignment_files) <= 1, f"Found more than one alignment file for {file.name}." - alignment_file = alignment_files[0] if alignment_files else None - - # parse the image - image = xenium_aligned_image(file, alignment_file, imread_kwargs, image_models_kwargs) - keypoints = [f for f in csv_files if f.name.endswith("keypoints.csv")] - if alignment_file is None and element_name == "he_image" and len(keypoints) == 1: - # bundles with keypoints but no alignment CSV (the Atera previews): 10x's alignment CSV is the - # least-squares fit to the keypoints, so an affine fit to them places the image - set_transformation(image, _affine_from_keypoints(keypoints[0]), "global") - logging.info(f"{file.name} placed by an affine fit to {keypoints[0].name} (no alignment CSV).") - images[element_name] = image + for element_name, file, alignment_file in _aligned_image_files(path): + image = xenium_aligned_image(file, alignment_file, imread_kwargs, image_models_kwargs) + if alignment_file is None and element_name == "he_image": + # bundles with keypoints but no alignment CSV (the Atera previews): 10x's alignment CSV is the + # least-squares fit to the keypoints, so an affine fit to them places the image + affine = _affine_from_keypoints(_keypoints_file(file)) + if affine is not None: + set_transformation(image, affine, "global") + images[element_name] = image return images -def _affine_from_keypoints(kp_file: Path) -> Affine: - """Fit the least-squares affine from the keypoints' aligned-image px onto their morphology px. +KEYPOINT_COLUMNS = ["fixedX", "fixedY", "alignmentX", "alignmentY"] + + +def _keypoints_file(he_image: Path) -> Path: + """Xenium Explorer's ``_keypoints.csv`` for ``_he_image.ome.tif``.""" + return he_image.with_name(he_image.name.removesuffix(XeniumKeys.ALIGNED_HE_IMAGE_SUFFIX) + "keypoints.csv") + - ``alignmentX/Y`` -> ``fixedX/Y``. A full affine rather than a similarity: the Atera preview scans carry about 0.2% anisotropy, which leaves 22 px - residuals under a similarity fit and under 8 px under an affine (the Atera preview bundles). +def _read_keypoints(kp_file: Path) -> tuple[np.ndarray, np.ndarray] | None: + """The finite keypoint pairs (aligned image px, morphology px) in `kp_file`, or None if it is missing or unusable. + + Xenium Explorer writes ``fixedX, fixedY`` (morphology image) and ``alignmentX, alignmentY`` (aligned image). A file + without these columns or with fewer than 3 finite rows is reported with a warning. """ + if not kp_file.is_file(): + return None kp = pd.read_csv(kp_file) - src = np.c_[kp[["alignmentX", "alignmentY"]].to_numpy(dtype=float), np.ones(len(kp))] - coef, *_ = np.linalg.lstsq(src, kp[["fixedX", "fixedY"]].to_numpy(dtype=float), rcond=None) + if not set(KEYPOINT_COLUMNS) <= set(kp.columns): + warnings.warn(f"{kp_file.name}: expected columns {KEYPOINT_COLUMNS}, found {list(kp.columns)}.", stacklevel=2) + return None + xy = kp[KEYPOINT_COLUMNS].apply(pd.to_numeric, errors="coerce").to_numpy(dtype=float) + xy = xy[np.isfinite(xy).all(axis=1)] + if len(xy) < 3: + warnings.warn(f"{kp_file.name}: {len(xy)} finite keypoints, at least 3 are needed.", stacklevel=2) + return None + return xy[:, 2:], xy[:, :2] + + +def _affine_from_keypoints(kp_file: Path) -> Affine | None: + """Fit the least-squares affine from the keypoints' aligned-image px onto their morphology px. + + ``alignmentX/Y`` -> ``fixedX/Y``. A full affine rather than a similarity: the Atera preview scans carry about 0.2% + anisotropy, which leaves 22 px residuals under a similarity fit and under 8 px under an affine (the Atera preview + bundles). None, with a warning, if the keypoints are unusable or collinear; the image then stays unaligned. + """ + keypoints = _read_keypoints(kp_file) + if keypoints is None: + return None + src, dst = keypoints + coef, _, rank, _ = np.linalg.lstsq(np.c_[src, np.ones(len(src))], dst, rcond=None) + if rank < 3: + warnings.warn(f"{kp_file.name}: the keypoints are collinear, so they fix no affine.", stacklevel=2) + return None + logging.info(f"Placed the H&E image by an affine fit to {kp_file.name} (no alignment CSV).") return Affine(np.r_[coef.T, [[0.0, 0.0, 1.0]]], input_axes=("x", "y"), output_axes=("x", "y")) diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index ed2e9b85..9c6b77c3 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -285,22 +285,67 @@ def test_aligned_he_image_is_placed_by_its_affine(tmp_path: Path) -> None: np.testing.assert_allclose(m, affine) +def _write_keypoints(path: Path, affine: np.ndarray, name: str = "S1_keypoints.csv") -> pd.DataFrame: + src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) + dst = (np.c_[src, np.ones(3)] @ affine.T)[:, :2] + kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) + kp.to_csv(path / name, index=False) + return kp + + def test_keypoints_check(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: from spatialdata_io.readers.atera import _check_keypoints affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) _write_he(tmp_path, affine) - src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) - dst = (np.c_[src, np.ones(3)] @ affine.T)[:, :2] - kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) - kp.to_csv(tmp_path / "S1_keypoints.csv", index=False) - assert _check_keypoints(tmp_path) == pytest.approx(0.0, abs=1e-9) + kp = _write_keypoints(tmp_path, affine) + csv, kp_file = tmp_path / "S1_he_imagealignment.csv", tmp_path / "S1_keypoints.csv" + assert _check_keypoints(csv, kp_file) == pytest.approx(0.0, abs=1e-9) assert "misplaced" not in caplog.text - kp.assign(fixedX=kp["fixedX"] + 50).to_csv(tmp_path / "S1_keypoints.csv", index=False) - assert _check_keypoints(tmp_path) == pytest.approx(50.0) + kp.assign(fixedX=kp["fixedX"] + 50).to_csv(kp_file, index=False) + assert _check_keypoints(csv, kp_file) == pytest.approx(50.0) assert "misplaced" in caplog.text - pd.DataFrame({"a": [1]}).to_csv(tmp_path / "S1_keypoints.csv", index=False) # unknown columns: skipped - assert _check_keypoints(tmp_path) is None + pd.DataFrame({"a": [1]}).to_csv(kp_file, index=False) + with pytest.warns(UserWarning, match="expected columns"): + assert _check_keypoints(csv, kp_file) is None + + +@pytest.mark.parametrize( + "case", ["other_columns", "two_rows", "nan_rows", "collinear", "other_sample", "appledouble", "no_keypoints"] +) +def test_he_stays_unaligned_without_usable_keypoints(tmp_path: Path, case: str) -> None: + import warnings + + from spatialdata.transformations import Identity, get_transformation + + from spatialdata_io.readers.xenium import _add_aligned_images + + affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) + _write_he(tmp_path, affine) + (tmp_path / "S1_he_imagealignment.csv").unlink() + kp = _write_keypoints(tmp_path, affine) + kp_file = tmp_path / "S1_keypoints.csv" + if case == "other_columns": + kp.set_axis(["a", "b", "c", "d"], axis=1).to_csv(kp_file, index=False) + elif case == "two_rows": + kp.iloc[:2].to_csv(kp_file, index=False) + elif case == "nan_rows": + kp.assign(fixedX=[1.0, np.nan, "x"]).to_csv(kp_file, index=False) + elif case == "collinear": + kp.assign(alignmentX=[1.0, 2.0, 3.0], alignmentY=[1.0, 2.0, 3.0]).to_csv(kp_file, index=False) + elif case in ["other_sample", "appledouble"]: + kp_file.rename(tmp_path / ("S2_keypoints.csv" if case == "other_sample" else "._S1_keypoints.csv")) + else: + kp_file.unlink() + (tmp_path / "._S1_he_image.ome.tif").write_bytes(b"\x00" * 16) # a macOS tar sidecar, not a TIFF + + expect_warning = case in ["other_columns", "two_rows", "nan_rows", "collinear"] + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + images = _add_aligned_images(tmp_path) + assert list(images) == ["he_image"] + assert isinstance(get_transformation(images["he_image"], "global"), Identity) + assert any("keypoint" in str(w.message) for w in record) == expect_warning def test_affine_from_keypoints_reproduces_alignment(tmp_path: Path) -> None: From c1a9a242ef7755b62cd5d3499fe2138a9730c71d Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:05:56 +0100 Subject: [PATCH 11/29] fix(atera): one image read path, one tile per dask task, pyramid from level 0 The H&E (aligned_images=True by default) went through dask_image, so every chunk of every scale waited on one task that decoded the whole plane; the four public Atera v1 H&E files are 2.8 to 7.3 GB. The morphology images read tifffile's stored levels: scale1 and below kept coordinates in their own pixels (a bounding_box_query cropped 2^i too wide at scale i), scale_factors other than 2 were ignored, and the graph held tifffile's open file handle, so it failed under the processes scheduler or any dask.distributed client, sdata.write() included. H&E, IF and morphology images now read level 0 through _TiffTiles: each dask task opens the TIFF and reads one tile of the file's tile grid, so the graph pickles. Image2DModel/Image3DModel.parse then build the pyramid with the standard scale_factors. The aligned images keep their affine and the keypoint fallback. 2D and 3D channels are both named from the OME-XML (3D used the file name). --- src/spatialdata_io/readers/_atera_common.py | 161 +++++++++----------- src/spatialdata_io/readers/atera.py | 56 +++++-- tests/readers/test_atera.py | 79 ++++++---- 3 files changed, 160 insertions(+), 136 deletions(-) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 5382bb3b..ab7307d4 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -22,7 +22,6 @@ PointsModel, ShapesModel, ) -from spatialdata.transformations import set_transformation from spatialdata.transformations.transformations import Identity, Scale from xarray import DataTree @@ -549,22 +548,6 @@ def read_cell_boundaries( return sdata -def _parse_morphology_channel_name(filename: str) -> str: - """Parse the channel name out of a ``chNNNN_.ome.tif`` morphology image filename.""" - match = re.match(r"ch\d+_(.+)\.ome\.tif+$", filename) - if match is None: - raise ValueError(f"Expected a morphology image filename of the form 'chNNNN_.ome.tif', found {filename}") - return match.group(1) - - -def _parse_morphology_channel_index(filename: str) -> int: - """Parse the channel index out of a ``chNNNN_.ome.tif`` morphology image filename.""" - match = re.match(r"ch(\d+)_.+\.ome\.tif+$", filename) - if match is None: - raise ValueError(f"Expected a morphology image filename of the form 'chNNNN_.ome.tif', found {filename}") - return int(match.group(1)) - - def _get_pixel_size(path: Path) -> float: """Return the dataset's pixel size (microns/pixel). @@ -592,53 +575,57 @@ def _get_pixel_size(path: Path) -> float: return parse_physical_size(files[0]) -def _tiled_imread( - path: Path, imread_kwargs: Mapping[str, Any] = MappingProxyType({}), n_levels: int = 1 -) -> list[da.Array]: - """Read an OME-TIFF as `dask` arrays (one per pyramid level) chunked by the file's own on-disk tile grid. - - `dask_image.imread.imread` gives each page (e.g. each channel or z-plane) exactly one `dask` - chunk -- the whole plane, which for these whole-slide-image morphology files is tens of GB. - That single giant chunk then has to be fully decoded before `Image2DModel.parse`/`Image3DModel.parse` - can rechunk it into tiles or build a downsampled pyramid level, so even plotting one channel at - the lowest pyramid resolution requires materializing the entire full-resolution plane in memory. - - This instead reads through `tifffile.TiffPageSeries.aszarr(level=i)`, which exposes the file's - native tile grid (e.g. 1024x1024, matching `p.is_tiled`/`p.tilewidth`/`p.tilelength`) as `zarr` - chunks, so only the tiles actually touched downstream get decoded. - - These OME-TIFFs embed their own multi-level pyramid (each level half the size of the previous one). - Reading up to `n_levels` of those stored levels, rather than downsampling `level=0` lazily, means - a low-resolution level only costs reading that (small) level from disk, not every full-resolution - tile. If the file has fewer levels than requested, only the available ones are returned. +class _TiffTiles: + """Level 0 of a TIFF's first series as an array that `dask` slices one tile per task. + + Each slice opens the file itself, so a `dask` graph holds only the path and pickles (tifffile's zarr store keeps + an open file handle and does not), and a task decodes only the tiles it covers, not the whole plane as + `dask_image.imread` does. Chunks follow the file's tile grid (strips for an untiled file). """ - import tifffile - tf = tifffile.TiffFile(path, **imread_kwargs) - series = tf.series[0] - n_levels = min(n_levels, len(series.levels)) - return [da.from_zarr(zarr.open(series.aszarr(level=i), mode="r")) for i in range(n_levels)] + def __init__(self, path: Path, imread_kwargs: Mapping[str, Any]) -> None: + import tifffile + + self.path, self.imread_kwargs = path, dict(imread_kwargs) + with tifffile.TiffFile(path, **self.imread_kwargs) as tf: + series, page = tf.series[0], tf.series[0].keyframe + self.shape, self.dtype, self.axes = tuple(series.shape), series.dtype, series.axes + tile = (page.tilelength, page.tilewidth) if page.is_tiled else (page.rowsperstrip, page.imagewidth) + self.ndim = len(self.shape) + # one tile in y and x, every sample (S) of a pixel, one plane of any other axis + self.chunks = tuple( + {"Y": tile[0], "X": tile[1], "S": n}.get(axis, 1) for axis, n in zip(self.axes, self.shape, strict=True) + ) + def __getitem__(self, key: Any) -> np.ndarray: + import tifffile -def _parse_morphology_pyramid( - levels: list[da.Array], - dims: tuple[str, ...], - c_coords: list[str], - image_models_kwargs: Mapping[str, Any], - model: type[Image2DModel] | type[Image3DModel], -) -> DataArray | DataTree: - """Build a (multiscale) image model from pre-computed pyramid levels instead of downsampling `level=0`.""" - from xarray import Dataset + with tifffile.TiffFile(self.path, **self.imread_kwargs) as tf: + return cast(np.ndarray, zarr.open_array(tf.series[0].aszarr(level=0), mode="r")[key]) - kwargs = {k: v for k, v in image_models_kwargs.items() if k != "scale_factors"} - parsed = [model.parse(level, dims=dims, c_coords=c_coords, **kwargs) for level in levels] - if len(parsed) == 1: - tree: DataArray | DataTree = parsed[0] - else: - tree = DataTree.from_dict({f"scale{i}": Dataset({"image": p}) for i, p in enumerate(parsed)}) - set_transformation(tree, Identity(), "global") - model.validate(tree) - return tree + +def _tiled_imread( + path: Path, imread_kwargs: Mapping[str, Any] = MappingProxyType({}) +) -> tuple[da.Array, tuple[str, ...]]: + """Read level 0 of an OME-TIFF lazily, one `dask` task per tile; return it with its dims. + + Stored pyramid levels are not read: the image models build the pyramid from level 0 with `scale_factors`. + Axes of length 1 other than y and x are dropped, samples (``S``) and channels (``C``) both become ``c``. + """ + tiles = _TiffTiles(path, imread_kwargs) + image = da.from_array(tiles, chunks=tiles.chunks, meta=np.empty((0,) * tiles.ndim, tiles.dtype), lock=False) + keep = [i for i, (axis, n) in enumerate(zip(tiles.axes, tiles.shape, strict=True)) if n > 1 or axis in "YX"] + image = image.squeeze(axis=tuple(i for i in range(tiles.ndim) if i not in keep)) + return image, tuple("c" if tiles.axes[i] in "SC" else tiles.axes[i].lower() for i in keep) + + +def _ome_channel_name(path: Path, index: int) -> str: + from ome_types import from_tiff + + name = from_tiff(path).images[0].pixels.channels[index].name + if name is None: + raise ValueError(f"Channel {index} in {path!s} has no name in its OME-XML metadata.") + return name def _get_morphology_images( @@ -647,37 +634,31 @@ def _get_morphology_images( image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), three_d: bool = False, ) -> DataArray | DataTree: - """Read the morphology OME-TIFF files in a directory as a single multi-channel image.""" + """Read the ``chNNNN_.ome.tif`` morphology files in a directory as one multi-channel image. + + A 2D file holds a plane per channel of the panel, of which only plane ``NNNN`` carries this file's channel; a 3D + file holds the z-stack of its one channel. Channels are named from the OME-XML in both cases. + """ files = sorted(f for f in dir_path.iterdir() if f.name.endswith(".ome.tif") and not f.name.startswith("._")) if not files: raise FileNotFoundError(f"No morphology OME-TIFF files found in {dir_path}.") - # `scale_factors=[2, 2, 2, 2]` (the default) means 5 levels in total; use the file's stored levels for them. - n_levels = len(image_models_kwargs.get("scale_factors") or []) + 1 - - if three_d: - # one file per channel, each file has shape (z, y, x); stack along a new channel axis - channel_names = [_parse_morphology_channel_name(f.name) for f in files] - per_file_levels = [_tiled_imread(f, imread_kwargs, n_levels) for f in files] - n = min(len(lv) for lv in per_file_levels) - levels = [da.stack([lv[i] for lv in per_file_levels], axis=0) for i in range(n)] - return _parse_morphology_pyramid(levels, ("c", "z", "y", "x"), channel_names, image_models_kwargs, Image3DModel) - - # Each 2D file is itself a multi-page OME-TIFF containing every channel of the panel (shape - # (n_channels, y, x)), but only the plane at the index given by the file's own "chNNNN" prefix - # holds that channel's real image data; take just that plane from each file. - from ome_types import from_tiff - - channel_indices = [_parse_morphology_channel_index(f.name) for f in files] - channel_names = [] - for f, i in zip(files, channel_indices, strict=True): - name = from_tiff(f).images[0].pixels.channels[i].name - if name is None: - raise ValueError(f"Channel {i} in {f!s} has no name in its OME-XML metadata.") - channel_names.append(name) - per_file_levels = [ - [level[i] for level in _tiled_imread(f, imread_kwargs, n_levels)] - for f, i in zip(files, channel_indices, strict=True) - ] - n = min(len(lv) for lv in per_file_levels) - levels = [da.stack([lv[j] for lv in per_file_levels], axis=0) for j in range(n)] - return _parse_morphology_pyramid(levels, ("c", "y", "x"), channel_names, image_models_kwargs, Image2DModel) + indices = [] + for f in files: + match = re.match(r"ch(\d+)_.+\.ome\.tif$", f.name) + if match is None: + raise ValueError( + f"Expected a morphology image filename of the form 'chNNNN_.ome.tif', found {f.name}" + ) + indices.append(0 if three_d else int(match.group(1))) + planes = [] + for f, i in zip(files, indices, strict=True): + image, dims = _tiled_imread(f, imread_kwargs) + planes.append(image if three_d or "c" not in dims else image[i]) + model = Image3DModel if three_d else Image2DModel + return model.parse( + da.stack(planes), + dims=("c", "z", "y", "x") if three_d else ("c", "y", "x"), + c_coords=[_ome_channel_name(f, i) for f, i in zip(files, indices, strict=True)], + transformations={"global": Identity()}, + **image_models_kwargs, + ) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index b5da7473..02a32a91 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -11,7 +11,8 @@ import zarr from scipy.sparse import csc_matrix, csr_matrix from spatialdata import SpatialData -from spatialdata.models import TableModel +from spatialdata.models import Image2DModel, TableModel +from spatialdata.transformations import Affine, BaseTransformation, Identity from spatialdata_io._constants._constants import AteraKeys from spatialdata_io._docs import inject_docs @@ -26,12 +27,13 @@ _get_points, _get_polygons, _open_zip, + _tiled_imread, read_cell_boundaries, read_transcripts_for_genes, ) from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata from spatialdata_io.readers.xenium import ( - _add_aligned_images, + _affine_from_keypoints, _aligned_image_files, _get_labels, _keypoints_file, @@ -122,9 +124,11 @@ def atera( Whether to read the 3D morphology images. This is opt-in and defaults to `False` because these images can be very large. aligned_images - Whether to read the post-run images aligned to the morphology images (``*he_image.ome.tif`` / - ``*if_image.ome.tif`` in the bundle root), placed with their ``*imagealignment.csv`` affine. When a - ``*keypoints.csv`` is present, a warning is logged if the affine does not map its keypoints onto each other. + Whether to read the post-run images aligned to the morphology images (``_he_image.ome.tif`` / + ``_if_image.ome.tif`` in the bundle root), placed with their ``*imagealignment.csv`` affine. A + warning is logged if that affine does not map ``_keypoints.csv`` onto itself. An H&E without an + alignment CSV is placed by an affine fit to its keypoints, or left unaligned (with a warning) if they are + unusable. var_columns Which columns of `var` to keep, out of the full table (which includes ~135 per-cluster differential-expression columns). Defaults to a small subset (`{xx.FEATURE_NAME!r}`, @@ -136,9 +140,9 @@ def atera( Larger values mean fewer, larger `dask` tasks (faster, but more peak memory per chunk); smaller values mean more, smaller ones. imread_kwargs - Keyword arguments passed to `tifffile.TiffFile` when reading the morphology images (see - `_tiled_imread`); the images are read tile-by-tile off `tifffile`'s own `zarr` store rather - than via `dask_image.imread.imread`, so this only accepts `TiffFile` constructor kwargs. + Keyword arguments passed to `tifffile.TiffFile` when reading the morphology and aligned images (see + `_tiled_imread`). The images are read one tile per `dask` task rather than with `dask_image.imread.imread`, + so this only accepts `TiffFile` constructor kwargs. image_models_kwargs Keyword arguments to pass to the image models. labels_models_kwargs @@ -215,7 +219,7 @@ def atera( ) if aligned_images: - images |= _get_aligned_images(path, image_models_kwargs) + images |= _get_aligned_images(path, imread_kwargs, image_models_kwargs) tables = {"table": table} if table is not None else {} sdata = SpatialData(images=images, labels=labels, points=points, tables=tables, shapes=shapes) @@ -227,13 +231,35 @@ def atera( KEYPOINTS_MAX_RESIDUAL_PX = 10.0 -def _get_aligned_images(path: Path, image_models_kwargs: Mapping[str, Any]) -> dict[str, DataTree]: - """Aligned post-run images (H&E, IF) in the global (morphology px) coordinate system, as in the Xenium bundles.""" - aligned = _add_aligned_images(path, image_models_kwargs=image_models_kwargs) # atera's imread_kwargs are TiffFile's +def _get_aligned_images( + path: Path, imread_kwargs: Mapping[str, Any], image_models_kwargs: Mapping[str, Any] +) -> dict[str, DataArray | DataTree]: + """The post-run H&E and IF images, read like the morphology images, in the global (morphology px) system. + + Each is placed by its alignment CSV or, for an H&E without one, by an affine fit to its keypoints (as in `xenium`). + """ + images = {} for name, file, alignment_file in _aligned_image_files(path): - if name == "he_image" and alignment_file is not None: - _check_keypoints(alignment_file, _keypoints_file(file)) - return aligned + image, dims = _tiled_imread(file, imread_kwargs) + transformation: BaseTransformation = Identity() + if alignment_file is not None: + affine = pd.read_csv(alignment_file, header=None).to_numpy(dtype=float) + transformation = Affine(affine, input_axes=("x", "y"), output_axes=("x", "y")) + if name == "he_image": + _check_keypoints(alignment_file, _keypoints_file(file)) + elif name == "he_image": + transformation = _affine_from_keypoints(_keypoints_file(file)) or transformation + if "c" not in dims: + image, dims = image[None], ("c", *dims) + n_channels = image.shape[dims.index("c")] + images[name] = Image2DModel.parse( + image, + dims=dims, + c_coords=["r", "g", "b", "a"][:n_channels] if name == "he_image" and n_channels in [3, 4] else None, + transformations={"global": transformation}, + **image_models_kwargs, + ) + return images def _check_keypoints(alignment_file: Path, kp_file: Path) -> float | None: diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 9c6b77c3..dc365e6b 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -234,31 +234,43 @@ def test_points_partitions_are_bounded(tmp_path: Path, monkeypatch: pytest.Monke assert len(points) == 7 -def test_morphology_images_use_embedded_pyramid(tmp_path: Path) -> None: - import tifffile - - rng = np.random.default_rng(0) - full = rng.integers(0, 255, (2, 64, 96), dtype=np.uint8) - for idx, name in enumerate(["dapi", "vim"]): - with tifffile.TiffWriter(tmp_path / f"ch{idx:04d}_{name}.ome.tif", ome=True) as tif: - levels = [full[:, :: 2**i, :: 2**i] for i in range(3)] - tif.write( - levels[0], - tile=(16, 16), - subifds=len(levels) - 1, - metadata={"axes": "CYX", "Channel": {"Name": ["dapi", "vim"]}}, - ) - for level in levels[1:]: - tif.write(level, tile=(16, 16), subfiletype=1) +@pytest.mark.parametrize("scale_factors", [[2, 2], [4]]) +def test_morphology_pyramid_is_built_from_level_0(atera_path: Path, scale_factors: list[int]) -> None: + from spatialdata import bounding_box_query + kwargs = {"chunks": (1, 32, 32), "scale_factors": scale_factors} image = cast( - DataTree, - common._get_morphology_images(tmp_path, image_models_kwargs={"chunks": (1, 32, 32), "scale_factors": [2, 2]}), + DataTree, common._get_morphology_images(atera_path / AteraKeys.MORPHOLOGY_2D_DIR, image_models_kwargs=kwargs) ) - assert list(image.children) == ["scale0", "scale1", "scale2"] - assert [image[k]["image"].shape for k in image.children] == [(2, 64, 96), (2, 32, 48), (2, 16, 24)] - # The lowest level is read from the stored pyramid, not downsampled from `scale0`. - np.testing.assert_array_equal(np.asarray(image["scale2"]["image"])[0], full[0, ::4, ::4]) + factors = np.cumprod([1, *scale_factors]) + assert [image[f"scale{i}"]["image"].shape for i in range(len(factors))] == [(2, 64 // f, 96 // f) for f in factors] + assert image["scale0"]["image"].coords["c"].values.tolist() == ["DAPI", "VIM"] + # each scale has coordinates in scale0 px, so a query crops the same region at every scale + crop = bounding_box_query( + image, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[32, 32], target_coordinate_system="global" + ) + assert [crop[k]["image"].shape[1:] for k in crop.children] == [(32 // f, 32 // f) for f in factors] + + +def test_morphology_3d_names_channels_like_2d(atera_path: Path) -> None: + image = common._get_morphology_images(atera_path / AteraKeys.MORPHOLOGY_3D_DIR, three_d=True) + assert image.coords["c"].values.tolist() == ["DAPI", "VIM"] + assert image.dims == ("c", "z", "y", "x") + + +def test_tiled_imread_reads_one_tile_per_task_and_pickles(atera_path: Path) -> None: + import pickle + + import dask + import tifffile + + file = atera_path / AteraKeys.MORPHOLOGY_2D_DIR / "ch0001_vim.ome.tif" + image, dims = common._tiled_imread(file) + assert dims == ("c", "y", "x") + assert image.chunks == ((1, 1), (32, 32), (32, 32, 32)) + graph = pickle.loads(pickle.dumps(image)) + with dask.config.set(scheduler="processes", num_workers=2): + np.testing.assert_array_equal(graph.compute(), tifffile.imread(file)) def _write_he(path: Path, affine: np.ndarray) -> None: @@ -270,14 +282,13 @@ def _write_he(path: Path, affine: np.ndarray) -> None: pd.DataFrame(affine).to_csv(path / "S1_he_imagealignment.csv", header=False, index=False) -def test_aligned_he_image_is_placed_by_its_affine(tmp_path: Path) -> None: +@pytest.mark.parametrize("reader", ["atera", "xenium"]) +def test_aligned_he_image_is_placed_by_its_affine(tmp_path: Path, reader: str) -> None: from spatialdata.transformations import get_transformation - from spatialdata_io.readers.xenium import _add_aligned_images - affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) _write_he(tmp_path, affine) - images = _add_aligned_images(tmp_path) + images = _read_aligned(tmp_path, reader) assert list(images) == ["he_image"] he = images["he_image"] assert list(he.coords["c"].values) == ["r", "g", "b"] @@ -376,23 +387,29 @@ def test_he_without_alignment_csv_is_placed_by_keypoints(tmp_path: Path) -> None pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}).to_csv( tmp_path / "S1_keypoints.csv", index=False ) - he = _get_aligned_images(tmp_path, {})["he_image"] + he = _get_aligned_images(tmp_path, {}, {})["he_image"] m = get_transformation(he, "global").to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) np.testing.assert_allclose(m, affine, atol=1e-9) +def _read_aligned(path: Path, reader: str) -> dict: + from spatialdata_io.readers.atera import _get_aligned_images + from spatialdata_io.readers.xenium import _add_aligned_images + + return _get_aligned_images(path, {}, {}) if reader == "atera" else _add_aligned_images(path) + + +@pytest.mark.parametrize("reader", ["atera", "xenium"]) @pytest.mark.parametrize("planar", [False, True]) -def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool) -> None: +def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool, reader: str) -> None: import tifffile - from spatialdata_io.readers.xenium import _add_aligned_images - rgb = np.arange(3 * 40 * 60, dtype=np.uint32).reshape(3, 40, 60).astype(np.uint8) if planar: # OME axes SYX, as in the Atera v1 Onboard 1.1.1 H&E tifffile.imwrite(tmp_path / "S1_he_image.ome.tif", rgb, ome=True, photometric="rgb", planarconfig="separate") else: # YXS, as in the 1.0.4 kidney H&E tifffile.imwrite(tmp_path / "S1_he_image.ome.tif", np.moveaxis(rgb, 0, -1), ome=True, photometric="rgb") - he = _add_aligned_images(tmp_path)["he_image"] + he = _read_aligned(tmp_path, reader)["he_image"] assert list(he.coords["c"].values) == ["r", "g", "b"] np.testing.assert_array_equal(np.asarray(he.transpose("c", "y", "x").data), rgb) From 0662beb99c88be1a4f357e2b897bdb04d143bac4 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:06:42 +0100 Subject: [PATCH 12/29] fix(atera): give the 4-D 3D morphology image its own chunks and keep z morphology_3d_images=True always raised with the default arguments: the (1, 4096, 4096) chunks meant for (c, y, x) reached Image3DModel.parse for (c, z, y, x) ('zip() argument 2 is shorter than argument 1' on the 10x tiny sample). The 3D parse now gets one z plane per chunk, and the scale factors downsample y and x only, since a z-stack of a few planes cannot be halved four times. --- src/spatialdata_io/readers/atera.py | 9 ++++++++- tests/readers/test_atera.py | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 02a32a91..0eff937a 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -214,8 +214,15 @@ def atera( path / AteraKeys.MORPHOLOGY_2D_DIR, imread_kwargs, image_models_kwargs ) if morphology_3d_images: + # (c, z, y, x): the 3-tuple default chunks gain one z plane per chunk, and z is not downsampled + kwargs_3d = dict(image_models_kwargs) + if isinstance(chunks := kwargs_3d.get("chunks"), tuple | list) and len(chunks) == 3: + kwargs_3d["chunks"] = (chunks[0], 1, *chunks[1:]) + kwargs_3d["scale_factors"] = [ + f if isinstance(f, dict) else {"z": 1, "y": f, "x": f} for f in kwargs_3d.get("scale_factors") or [] + ] images["morphology_3d"] = _get_morphology_images( - path / AteraKeys.MORPHOLOGY_3D_DIR, imread_kwargs, image_models_kwargs, three_d=True + path / AteraKeys.MORPHOLOGY_3D_DIR, imread_kwargs, kwargs_3d, three_d=True ) if aligned_images: diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index dc365e6b..88156409 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -498,3 +498,12 @@ def zip_with_trailing_chunk(items: list[str]) -> Path: path = zip_with_trailing_chunk(["dd"]) # one item short: corrupt, not padded with common._open_zip(path) as group, pytest.raises(ValueError, match="reshape"): group["s"][:] + + +def test_atera_reads_3d_morphology_with_default_kwargs(atera_path: Path) -> None: + from spatialdata_io import atera + + image = atera(atera_path, morphology_3d_images=True).images["morphology_3d"] + assert image["scale0"]["image"].dims == ("c", "z", "y", "x") + assert image["scale0"]["image"].data.chunksize[:2] == (1, 1) + assert [image[k]["image"].shape for k in image.children] == [(2, 3, 64 // f, 96 // f) for f in (1, 2, 4, 8, 16)] From 2dd3c29f25a78257fdbb2c27af50057b26302a6f Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:16:28 +0100 Subject: [PATCH 13/29] fix(atera): map transcript and polygon positions through the bundle, not the in-memory table read_transcripts_for_genes took a gene's position in gene_offset from sdata.tables['table'].var, and read_cell_boundaries took cell ids for polygon positions from its obs. After an HVG subset, the first returned another gene's transcripts under the requested name; after a cell filter or reorder, the second attached other cells' ids or raised IndexError (also in the documented pairing with read_table_for_cells(bbox)). Both now read the positions from the bundle's own var and obs, so they work with a filtered table or none. Adapted to the ec169cf table layout (69723bd and b908dff not ported here): keeps ec169cf's cast(float, pixel_size), and the test picks GENE_B by var["feature_name"], since var is indexed by feature_id. --- src/spatialdata_io/readers/_atera_common.py | 32 ++++++++++++++++----- tests/readers/test_atera.py | 16 +++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index ab7307d4..7d7b8470 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -119,6 +119,22 @@ def _open_zip(path: Path) -> Iterator[zarr.Group]: yield zarr.open_group(_PadRaggedVlenStore(store), mode="r") +def _read_feature_names(path: Path) -> list[str]: + """The feature names of the bundle in store order, which is the order of each tile's ``gene_offset``.""" + from anndata.io import read_elem + + with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: + return np.asarray(read_elem(group[f"{AteraKeys.VAR_GROUP}/{AteraKeys.FEATURE_NAME}"]), dtype=str).tolist() + + +def _read_cell_ids(path: Path) -> np.ndarray: + """The cell ids of the bundle in store order, which is the row order of ``cells.zarr.zip``.""" + from anndata.io import read_elem + + with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: + return np.asarray(read_elem(group[f"{AteraKeys.OBS_GROUP}/{AteraKeys.CELL_ID}"])) + + def _decode_packed_cell_id(raw: np.ndarray) -> np.ndarray: """Decode a (N, 2) uint32 packed cell id array into the int64 ids used in the cell-feature-matrix obs. @@ -332,8 +348,8 @@ def read_transcripts_for_genes( plotting a handful of genes across the whole dataset much cheaper than loading and filtering the full transcripts table. - Requires ``sdata`` to have been read with ``cells_table=True``, to look up gene indices (via - the table's `var`) and `pixel_size` (via ``sdata.attrs``). + Gene positions come from the bundle's own `var`, so ``sdata`` can hold a filtered table or none; its + ``sdata.attrs["pixel_size"]`` is used. Parameters ---------- @@ -359,7 +375,10 @@ def read_transcripts_for_genes( if isinstance(genes, str): genes = [genes] path = Path(path) - feature_names = sdata.tables["table"].var[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() + feature_names = _read_feature_names(path) + missing = [gene for gene in genes if gene not in feature_names] + if missing: + raise ValueError(f"gene(s) not found in {AteraKeys.CELL_FEATURE_MATRIX_FILE!s} var: {missing}") gene_indices = {gene: feature_names.index(gene) for gene in genes} pixel_size = cast(float, sdata.attrs[str(AteraKeys.PIXEL_SIZE)]) @@ -465,9 +484,8 @@ def read_cell_boundaries( the top-level ``bboxes`` array); this un-quantizes them via ``absolute = cell_min + relative / 255 * (cell_max - cell_min)``. - Requires ``sdata`` to have been read with ``cells_table=True``, to map row positions in - ``gridded_polygon_sets`` to cell ids (via the table's `obs`), and `pixel_size` (via - ``sdata.attrs``). + Row positions in ``gridded_polygon_sets`` are mapped to cell ids through the bundle's own `obs`, so ``sdata`` + can hold a filtered table or none; its ``sdata.attrs["pixel_size"]`` is used. Parameters ---------- @@ -493,7 +511,7 @@ def read_cell_boundaries( """ path = Path(path) pixel_size = cast(float, sdata.attrs[str(AteraKeys.PIXEL_SIZE)]) - cell_ids = sdata.tables["table"].obs[str(AteraKeys.CELL_ID)].to_numpy() + cell_ids = _read_cell_ids(path) is_nucleus = mask_index == 0 raw_bbox = None diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 88156409..a2453239 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -507,3 +507,19 @@ def test_atera_reads_3d_morphology_with_default_kwargs(atera_path: Path) -> None assert image["scale0"]["image"].dims == ("c", "z", "y", "x") assert image["scale0"]["image"].data.chunksize[:2] == (1, 1) assert [image[k]["image"].shape for k in image.children] == [(2, 3, 64 // f, 96 // f) for f in (1, 2, 4, 8, 16)] + + +def test_helpers_map_positions_through_the_bundle_not_the_table(atera_path: Path) -> None: + from spatialdata_io import atera + from spatialdata_io.readers.atera import read_cell_boundaries, read_transcripts_for_genes + + sdata = atera(atera_path) + table = sdata.tables["table"] + # an HVG-style gene subset and a reordered cell subset of the table + sdata.tables["table"] = table[[2, 0], (table.var["feature_name"] == "GENE_B").to_numpy()].copy() + points = read_transcripts_for_genes(sdata, atera_path, "GENE_B").points["transcripts_subset"].compute() + assert points["x"].tolist() == [3.0, 4.0, 5.0] # GENE_B owns rows 3-5 of the tile + shapes = read_cell_boundaries(sdata, atera_path).shapes["cell_boundaries"] + assert shapes.index.tolist() == CELL_IDS + del sdata.tables["table"] # nor do they need a table + assert len(read_transcripts_for_genes(sdata, atera_path, ["GENE_A"]).points["transcripts_subset"]) == 3 From d87297469c9142fcfc81232569bd53c4d5649f1b Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Wed, 11 Nov 2026 00:02:03 +0100 Subject: [PATCH 14/29] fix(atera): lay out var as the xenium reader does (symbol index, gene_ids, feature_types) The Atera bundles index var by feature_id (Ensembl) and name the type column feature_type; the xenium reader, and downstream gene search and QC, read symbols from the var index, Ensembl ids from gene_ids and the type from feature_types. The type values are already the Xenium ones. Applies to all three table readers. --- src/spatialdata_io/readers/atera.py | 27 ++++++++++++++++++++++++++- tests/readers/test_atera.py | 16 ++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 0eff937a..1b1df195 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -352,6 +352,28 @@ def _add_label_id(obs_df: pd.DataFrame) -> None: obs_df[str(AteraKeys.LABEL_ID)] = np.arange(1, len(obs_df) + 1, dtype=np.uint32) +def _xenium_var(var: pd.DataFrame) -> pd.DataFrame: + """Lay out `var` as the `xenium` reader does. + + Gene symbols are the index, Ensembl ids go to ``gene_ids`` and the feature type to ``feature_types`` (the + bundles index by ``feature_id`` and name the type ``feature_type``). The type values are already the Xenium + ones (``Gene Expression``, ``Negative Control Probe``, ...). + """ + from anndata.utils import make_index_unique + + name, ftype = str(AteraKeys.FEATURE_NAME), str(AteraKeys.VAR_FEATURE_TYPE) + if name not in var.columns: + return var + var = var.copy() + fid = str(AteraKeys.FEATURE_ID) + var["gene_ids"] = (var[fid] if fid in var.columns else var.index.to_series()).astype(str).to_numpy() + var.index = pd.Index(var[name].astype(str), name=None) + if not var.index.is_unique: + logging.warning(f"{int(var.index.duplicated().sum())} duplicate feature names; made unique with a suffix.") + var.index = make_index_unique(var.index) + return var.rename(columns={ftype: "feature_types"}) + + def _parse_table(adata: AnnData, region: str) -> AnnData: """Annotate `region` with `adata`, using whichever `obs` column holds that element's instance ids. @@ -359,6 +381,7 @@ def _parse_table(adata: AnnData, region: str) -> AnnData: """ instance_key = AteraKeys.LABEL_ID if region == "cell_labels" else AteraKeys.CELL_ID adata.obs["region"] = pd.Categorical([region] * adata.n_obs) + adata.var = _xenium_var(adata.var) return TableModel.parse(adata, region=region, region_key="region", instance_key=str(instance_key)) @@ -538,7 +561,9 @@ def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnDat genes = [genes] if isinstance(genes, str) else list(genes) path = Path(path) with _open_zip(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE) as group: - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP)[[str(AteraKeys.FEATURE_ID), str(AteraKeys.FEATURE_NAME)]] + var_df = _read_dataframe(group, AteraKeys.VAR_GROUP)[ + [str(AteraKeys.FEATURE_ID), str(AteraKeys.FEATURE_NAME), str(AteraKeys.VAR_FEATURE_TYPE)] + ] feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() index = pd.Index(feature_names) col_positions = index.get_indexer(pd.Index(np.asarray(genes))) diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index a2453239..0cfa683c 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -523,3 +523,19 @@ def test_helpers_map_positions_through_the_bundle_not_the_table(atera_path: Path assert shapes.index.tolist() == CELL_IDS del sdata.tables["table"] # nor do they need a table assert len(read_transcripts_for_genes(sdata, atera_path, ["GENE_A"]).points["transcripts_subset"]) == 3 + + +@pytest.mark.parametrize("table", ["_get_table", "read_table_for_cells", "read_table_for_genes"]) +def test_var_follows_the_xenium_layout(atera_path: Path, table: str) -> None: + # The xenium() reader indexes var by gene symbol, with `gene_ids` (Ensembl) and `feature_types`; downstream + # (gene search, QC) reads symbols from the index, ids from `gene_ids` and controls from `feature_types`. + if table == "_get_table": + adata = _get_table(atera_path)[0] + elif table == "read_table_for_cells": + adata = read_table_for_cells(atera_path, cell_ids=CELL_IDS[:2]) + else: + adata = read_table_for_genes(atera_path, ["GENE_A", "GENE_B"]) + assert adata.var.index.tolist() == ["GENE_A", "GENE_B"] + assert adata.var["gene_ids"].tolist() == ["ID_A", "ID_B"] + assert adata.var["feature_types"].tolist() == ["Gene Expression", "Gene Expression"] + assert "feature_type" not in adata.var.columns From b98400dd135bca062e8f62ab44c360c131e4c913 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Wed, 11 Nov 2026 02:36:57 +0100 Subject: [PATCH 15/29] fix(atera): leave the var index unnamed pd.Index(series, name=None) keeps the Series' name, so var was indexed by a 'feature_name' index beside the 'feature_name' column. Once a duplicate symbol is suffixed the two differ and anndata refuses to write (hub conversions hit it). --- src/spatialdata_io/readers/atera.py | 2 +- tests/readers/test_atera.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 1b1df195..bbf202a2 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -367,7 +367,7 @@ def _xenium_var(var: pd.DataFrame) -> pd.DataFrame: var = var.copy() fid = str(AteraKeys.FEATURE_ID) var["gene_ids"] = (var[fid] if fid in var.columns else var.index.to_series()).astype(str).to_numpy() - var.index = pd.Index(var[name].astype(str), name=None) + var.index = pd.Index(var[name].astype(str).to_numpy()) # from a Series, pd.Index would keep its name if not var.index.is_unique: logging.warning(f"{int(var.index.duplicated().sum())} duplicate feature names; made unique with a suffix.") var.index = make_index_unique(var.index) diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 0cfa683c..55fad27b 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -536,6 +536,7 @@ def test_var_follows_the_xenium_layout(atera_path: Path, table: str) -> None: else: adata = read_table_for_genes(atera_path, ["GENE_A", "GENE_B"]) assert adata.var.index.tolist() == ["GENE_A", "GENE_B"] + assert adata.var.index.name is None # a name shared with the `feature_name` column breaks writes once suffixed assert adata.var["gene_ids"].tolist() == ["ID_A", "ID_B"] assert adata.var["feature_types"].tolist() == ["Gene Expression", "Gene Expression"] assert "feature_type" not in adata.var.columns From b92f8bbe9f662cbcd5814a04a973b8d3a6126d3b Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 21:39:20 +0100 Subject: [PATCH 16/29] fix(atera): cast var to DataFrame for mypy --- src/spatialdata_io/readers/atera.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index bbf202a2..ffaa0474 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -381,7 +381,7 @@ def _parse_table(adata: AnnData, region: str) -> AnnData: """ instance_key = AteraKeys.LABEL_ID if region == "cell_labels" else AteraKeys.CELL_ID adata.obs["region"] = pd.Categorical([region] * adata.n_obs) - adata.var = _xenium_var(adata.var) + adata.var = _xenium_var(cast(pd.DataFrame, adata.var)) return TableModel.parse(adata, region=region, region_key="region", instance_key=str(instance_key)) From 022a6a2ea581ecbcb191505884bdd9f8e03249ef Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:15:23 +0100 Subject: [PATCH 17/29] fix(atera): X holds the Gene Expression features, the controls go to obsm n_genes counted every stored feature, the negative control probes and codewords and the unassigned codewords included: on the 10x tiny sample 79 of 132 features are not genes, and n_genes exceeded the gene count in 6 of 13 cells. As in xenium() (gex_only=True), X and var now hold the Gene Expression features. The counts of all other features go to obsm['control_counts'], a DataFrame with one column per feature (read from the CSC copy, where columns are cheap), and n_genes counts genes only. On the tiny sample: X 13 x 53, control_counts 13 x 79, n_genes equal to the nonzeros of X, and the DataFrame survives sdata.write() and read_zarr(). The three table readers now share _read_obs_var and _assemble_table, so read_table_for_cells has n_genes, control_counts and obsm['spatial'] too, and read_var returns var in the same layout as the table (indexed by symbol), so its documented recipe adds a column by name. Adapted to ec169cf helpers (indptr via _get_array); keeps ec169cf's category colors and manifest extras in the table's uns. Ported only because the commits after it build on it; c3f7e70 replaces obsm["control_counts"] with table_controls. --- src/spatialdata_io/readers/atera.py | 181 ++++++++++++++-------------- tests/readers/test_atera.py | 41 +++++-- 2 files changed, 122 insertions(+), 100 deletions(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index ffaa0474..0ed34204 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -106,7 +106,9 @@ def atera( path Path to the dataset. cells_table - Whether to read the cell annotations in the `AnnData` table. + Whether to read the cell annotations in the `AnnData` table. As in `xenium`, `X` and `var` hold the Gene + Expression features; the counts of the other features (negative control probes and codewords, unassigned + codewords) are in ``obsm["control_counts"]``, a `DataFrame` with one column per feature. cells_boundaries Whether to read cell boundaries (polygons). nucleus_boundaries @@ -353,25 +355,65 @@ def _add_label_id(obs_df: pd.DataFrame) -> None: def _xenium_var(var: pd.DataFrame) -> pd.DataFrame: - """Lay out `var` as the `xenium` reader does. + """Lay out the store's `var` as the `xenium` reader does. - Gene symbols are the index, Ensembl ids go to ``gene_ids`` and the feature type to ``feature_types`` (the - bundles index by ``feature_id`` and name the type ``feature_type``). The type values are already the Xenium - ones (``Gene Expression``, ``Negative Control Probe``, ...). + Gene symbols are the index (made unique with a suffix), Ensembl ids go to ``gene_ids`` and the feature type to + ``feature_types`` (the bundles index by ``feature_id`` and name the type ``feature_type``). The type values are + already the Xenium ones (``Gene Expression``, ``Negative Control Probe``, ...). """ from anndata.utils import make_index_unique - name, ftype = str(AteraKeys.FEATURE_NAME), str(AteraKeys.VAR_FEATURE_TYPE) - if name not in var.columns: - return var - var = var.copy() - fid = str(AteraKeys.FEATURE_ID) - var["gene_ids"] = (var[fid] if fid in var.columns else var.index.to_series()).astype(str).to_numpy() - var.index = pd.Index(var[name].astype(str).to_numpy()) # from a Series, pd.Index would keep its name - if not var.index.is_unique: - logging.warning(f"{int(var.index.duplicated().sum())} duplicate feature names; made unique with a suffix.") - var.index = make_index_unique(var.index) - return var.rename(columns={ftype: "feature_types"}) + out = var.rename(columns={str(AteraKeys.VAR_FEATURE_TYPE): "feature_types"}) + out["gene_ids"] = var.index.astype(str).to_numpy() + names = pd.Index(var[str(AteraKeys.FEATURE_NAME)].astype(str).to_numpy()) # from a Series, it would keep its name + if not names.is_unique: + logging.warning(f"{int(names.duplicated().sum())} duplicate feature names; made unique with a suffix.") + out.index = make_index_unique(names) + return out + + +def _is_gene(var: pd.DataFrame) -> np.ndarray: + """Which features are ``Gene Expression``, the only ones `xenium` keeps in `X` by default (``gex_only=True``).""" + if "feature_types" not in var.columns: + return np.ones(len(var), dtype=bool) + return (var["feature_types"] == "Gene Expression").to_numpy() + + +def _read_obs_var(group: zarr.Group) -> tuple[pd.DataFrame, pd.DataFrame]: + """`obs` with ``label_id`` and the full `var` in the `xenium` layout, in store order.""" + obs = _read_dataframe(group, AteraKeys.OBS_GROUP) + _add_label_id(obs) + return obs, _xenium_var(_read_dataframe(group, AteraKeys.VAR_GROUP)) + + +CONTROL_COUNTS_KEY = "control_counts" + + +def _assemble_table( + x: Any, + obs: pd.DataFrame, + var: pd.DataFrame, + controls: csr_matrix | csc_matrix, + var_columns: list[str] | Literal["all"], + region: str, + obsm: Mapping[str, Any] = MappingProxyType({}), +) -> AnnData: + """X and var hold the Gene Expression features, ``obsm["control_counts"]`` the counts of all other features. + + The other features are the negative control probes and codewords and the unassigned codewords, one named + column each, so that `X` and ``n_genes`` count genes only. `var_columns` name the store's columns. + """ + from anndata import AnnData + + is_gene = _is_gene(var) + renamed = {str(AteraKeys.VAR_FEATURE_TYPE): "feature_types"} + columns = var.columns if var_columns == "all" else [renamed.get(c, c) for c in var_columns] + ["gene_ids"] + obsm = { + **obsm, + CONTROL_COUNTS_KEY: pd.DataFrame(controls.toarray(), index=obs.index, columns=var.index[~is_gene]), + "spatial": obs[[str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)]].to_numpy(), + } + return _parse_table(AnnData(X=x, obs=obs, var=var.loc[is_gene, list(columns)], obsm=obsm), region) def _parse_table(adata: AnnData, region: str) -> AnnData: @@ -381,7 +423,6 @@ def _parse_table(adata: AnnData, region: str) -> AnnData: """ instance_key = AteraKeys.LABEL_ID if region == "cell_labels" else AteraKeys.CELL_ID adata.obs["region"] = pd.Categorical([region] * adata.n_obs) - adata.var = _xenium_var(cast(pd.DataFrame, adata.var)) return TableModel.parse(adata, region=region, region_key="region", instance_key=str(instance_key)) @@ -401,46 +442,38 @@ def _get_table( `SpatialData` object the table is assembled into (see the ``default_region`` logic in `atera`), so the table doesn't end up annotating an element that was never loaded. """ - from anndata import AnnData from anndata.experimental import read_elem_lazy - from anndata.io import read_elem + from anndata.io import read_elem, sparse_dataset with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) - feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() - if var_columns != "all": - var_df = var_df[list(var_columns)] + obs, var = _read_obs_var(group) colors = _read_category_colors(group, AteraKeys.OBS_GROUP) - colors.update(_read_category_colors(group, AteraKeys.VAR_GROUP, list(var_df.columns))) - - obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) - _add_label_id(obs_df) - + colors.update( + _read_category_colors(group, AteraKeys.VAR_GROUP, None if var_columns == "all" else list(var_columns)) + ) obsm = ( cast(dict[str, Any], read_elem(group[str(AteraKeys.OBSM_GROUP)])) if str(AteraKeys.OBSM_GROUP) in group else {} ) - manifest_extras = _read_manifest_extras(group) # the graph keeps the store, which reopens on first use (also after pickling) x_group = _get_group(group, str(AteraKeys.X_GROUP)) x = read_elem_lazy(x_group, chunks=(row_chunk_size, -1)) - # Genes detected per cell is the number of stored entries per row of the CSR matrix, which `indptr` - # already encodes; this avoids a full pass over the (lazy) counts. - n_genes = np.diff(_get_array(x_group, "indptr")[:]) if x_group.attrs["encoding-type"] == "csr_matrix" else None - - if n_genes is not None: - obs_df["n_genes"] = n_genes - - adata = AnnData(X=x, obs=obs_df, var=var_df, obsm=obsm) - adata.obsm["spatial"] = cast(pd.DataFrame, adata.obs)[ - [str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)] - ].to_numpy() - adata.uns.update(colors) + stored_per_row = np.diff(_get_array(x_group, "indptr")[:]) + is_gene = _is_gene(var) + if (path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE).is_file(): # column reads are cheap in the CSC copy + with _open_zip(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE) as group: + controls = sparse_dataset(group[str(AteraKeys.X_GROUP)])[:, np.flatnonzero(~is_gene)] + else: + controls = x[:, ~is_gene].compute() + # genes detected per cell: the stored entries of each CSR row (its `indptr` step) minus those of the controls + obs["n_genes"] = stored_per_row - controls.getnnz(axis=1) + table = _assemble_table(x[:, is_gene], obs, var, controls, var_columns, region, obsm) + table.uns.update(colors) if manifest_extras is not None: - adata.uns[str(AteraKeys.UNS_MANIFEST)] = manifest_extras - return _parse_table(adata, region), feature_names + table.uns[str(AteraKeys.UNS_MANIFEST)] = manifest_extras + return table, var[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() def read_var(path: str | Path) -> pd.DataFrame: @@ -449,8 +482,8 @@ def read_var(path: str | Path) -> pd.DataFrame: Includes the ~135 per-cluster differential-expression columns that `atera` drops by default (see `var_columns` on `atera`) to avoid holding that wider table in memory. - Use this to build your own `var`/`AnnData` with whichever columns you need, e.g. - ``sdata.tables["table"].var = read_var(path)[["feature_name", "de_leiden_res_1.0_c0_score"]]``. + It has the same index as the table's `var` (gene symbols), plus the non-gene features, so columns can be added + by name, e.g. ``table.var["score"] = read_var(path)["de_leiden_res_1.0_c0_score"]``. Parameters ---------- @@ -459,12 +492,10 @@ def read_var(path: str | Path) -> pd.DataFrame: Returns ------- - The complete `var` `DataFrame`, indexed by `feature_id`. + The complete `var` `DataFrame` of every feature, in the `xenium` layout (see `atera`). """ - path = Path(path) - with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) - return var_df + with _open_zip(Path(path) / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: + return _xenium_var(_read_dataframe(group, AteraKeys.VAR_GROUP)) def read_table_for_cells( @@ -501,36 +532,24 @@ def read_table_for_cells( """ if (cell_ids is None) == (bbox is None): raise ValueError("Exactly one of `cell_ids`/`bbox` must be given.") - from anndata import AnnData from anndata.io import sparse_dataset path = Path(path) with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP) - if var_columns != "all": - var_df = var_df[list(var_columns)] - obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) - _add_label_id(obs_df) - + obs, var = _read_obs_var(group) if cell_ids is not None: - index = pd.Index(obs_df[str(AteraKeys.CELL_ID)].to_numpy()) - row_positions = index.get_indexer(pd.Index(np.asarray(cell_ids))) + row_positions = pd.Index(obs[str(AteraKeys.CELL_ID)].to_numpy()).get_indexer(np.asarray(cell_ids)) missing = np.asarray(cell_ids)[row_positions == -1] if missing.size: raise ValueError(f"cell_id(s) not found in {AteraKeys.CELL_FEATURE_MATRIX_FILE!s}: {missing.tolist()}") else: assert bbox is not None row_positions = _cell_row_positions_in_bbox(path, bbox) - - x_ds = sparse_dataset(group[str(AteraKeys.X_GROUP)]) - x = cast(csr_matrix, x_ds[row_positions]) - obs_subset = obs_df.iloc[row_positions] - - adata = AnnData(X=x, obs=obs_subset, var=var_df) - adata.obsm["spatial"] = cast(pd.DataFrame, adata.obs)[ - [str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)] - ].to_numpy() - return _parse_table(adata, "cell_labels") + x = cast(csr_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[row_positions]) + obs = obs.iloc[row_positions].copy() + is_gene = _is_gene(var) + obs["n_genes"] = x[:, is_gene].getnnz(axis=1) + return _assemble_table(x[:, is_gene], obs, var, x[:, ~is_gene], var_columns, "cell_labels") def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnData: @@ -561,27 +580,13 @@ def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnDat genes = [genes] if isinstance(genes, str) else list(genes) path = Path(path) with _open_zip(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE) as group: - var_df = _read_dataframe(group, AteraKeys.VAR_GROUP)[ - [str(AteraKeys.FEATURE_ID), str(AteraKeys.FEATURE_NAME), str(AteraKeys.VAR_FEATURE_TYPE)] - ] - feature_names = var_df[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() - index = pd.Index(feature_names) - col_positions = index.get_indexer(pd.Index(np.asarray(genes))) + obs, var = _read_obs_var(group) + col_positions = var.index.get_indexer(np.asarray(genes)) missing = np.asarray(genes)[col_positions == -1] if missing.size: raise ValueError( f"gene(s) not found in {AteraKeys.VAR_GROUP!s}.{AteraKeys.FEATURE_NAME!s}: {missing.tolist()}" ) - - var_subset = var_df.iloc[col_positions] - var_subset.index = pd.Index(np.asarray(genes), name=str(AteraKeys.FEATURE_NAME)) - _align_index_column(var_subset) - - obs_df = _read_dataframe(group, AteraKeys.OBS_GROUP) - _add_label_id(obs_df) - - x_ds = sparse_dataset(group[str(AteraKeys.X_GROUP)]) - x: csc_matrix = cast(csc_matrix, x_ds[:, col_positions]) - - adata = AnnData(X=x, obs=obs_df, var=var_subset) - return _parse_table(adata, "cell_labels") + x = cast(csc_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[:, col_positions]) + obsm = {"spatial": obs[[str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)]].to_numpy()} + return _parse_table(AnnData(X=x, obs=obs, var=var.iloc[col_positions], obsm=obsm), "cell_labels") diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 55fad27b..4899ed85 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -18,6 +18,7 @@ # Atera `cell_id`s are large packed integers, unrelated to the `1..n_obs` values in the `cell_labels` raster. CELL_IDS = [4_320_252_178, 4_975_876_403, 5_016_677_209, 5_217_424_078] +NEG_COUNTS = [0.0, 1.0, 0.0, 2.0] # the negative control probe NEG_1, after the genes GENE_A and GENE_B @pytest.fixture @@ -33,20 +34,20 @@ def atera_path(tmp_path: Path) -> Path: ) var = pd.DataFrame( { - str(AteraKeys.FEATURE_NAME): ["GENE_A", "GENE_B"], - str(AteraKeys.VAR_FILTERED): [False, False], - str(AteraKeys.VAR_HIGHLY_VARIABLE): [False, False], - str(AteraKeys.VAR_FEATURE_TYPE): ["Gene Expression", "Gene Expression"], - str(AteraKeys.VAR_GENOME): ["g", "g"], + str(AteraKeys.FEATURE_NAME): ["GENE_A", "GENE_B", "NEG_1"], + str(AteraKeys.VAR_FILTERED): [False] * 3, + str(AteraKeys.VAR_HIGHLY_VARIABLE): [False] * 3, + str(AteraKeys.VAR_FEATURE_TYPE): ["Gene Expression", "Gene Expression", "Negative Control Probe"], + str(AteraKeys.VAR_GENOME): ["g"] * 3, }, - index=pd.Index(["ID_A", "ID_B"], dtype=object, name=str(AteraKeys.FEATURE_ID)), + index=pd.Index(["ID_A", "ID_B", "ID_NEG"], dtype=object, name=str(AteraKeys.FEATURE_ID)), ) - x = csr_matrix(np.arange(n * 2, dtype=np.float32).reshape(n, 2)) + x = csr_matrix(np.c_[np.arange(n * 2, dtype=np.float32).reshape(n, 2), NEG_COUNTS]) for file in (AteraKeys.CELL_FEATURE_MATRIX_FILE, AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE): store = zarr.storage.ZipStore(tmp_path / file, mode="w") try: group = zarr.open_group(store, mode="w", zarr_format=2) - write_elem(group, str(AteraKeys.X_GROUP), x) + write_elem(group, str(AteraKeys.X_GROUP), x if file == AteraKeys.CELL_FEATURE_MATRIX_FILE else x.tocsc()) write_elem(group, str(AteraKeys.OBS_GROUP), obs) write_elem(group, str(AteraKeys.VAR_GROUP), var) # Real bundles also list the index (`barcode`/`feature_id`) in `column-order`, so it comes back from @@ -103,7 +104,7 @@ def _write_cells_transcripts_and_images(path: Path, n: int) -> None: cell_id[0] = np.iinfo(np.uint32).max arrays = { "location": np.c_[np.arange(6), np.arange(6), np.zeros(6)].astype(np.float32), - "gene_offset": np.array([[0, 3], [3, 6]], np.uint32), + "gene_offset": np.array([[0, 3], [3, 6], [6, 6]], np.uint32), "cell_id": cell_id, "overlaps_nucleus": np.zeros((6, 1), np.uint8), "quality_score": np.full((6, 1), 30, np.float32), @@ -145,10 +146,21 @@ def test_cell_labels_table_uses_label_id(atera_path: Path) -> None: # the `cell_labels` raster is numbered 1..n_obs by row position assert table.obs["label_id"].tolist() == [1, 2, 3, 4] assert table.obs["cell_id"].tolist() == CELL_IDS - # genes detected per cell, read from the CSR `indptr` (cell 0 only has a zero in its first column) + # genes detected per cell (cell 0 only has a zero in its first column); NEG_1 is not a gene assert table.obs["n_genes"].tolist() == [1, 2, 2, 2] +@pytest.mark.parametrize("table", ["_get_table", "read_table_for_cells"]) +def test_x_holds_genes_and_obsm_the_controls(atera_path: Path, table: str) -> None: + adata = _get_table(atera_path)[0] if table == "_get_table" else read_table_for_cells(atera_path, cell_ids=CELL_IDS) + assert adata.var_names.tolist() == ["GENE_A", "GENE_B"] + assert adata.obs["n_genes"].tolist() == [1, 2, 2, 2] + controls = adata.obsm["control_counts"] + assert isinstance(controls, pd.DataFrame) + assert controls.columns.tolist() == ["NEG_1"] + assert controls["NEG_1"].tolist() == NEG_COUNTS + + def test_cell_boundaries_table_uses_cell_id(atera_path: Path) -> None: table, _ = _get_table(atera_path, region="cell_boundaries") @@ -201,8 +213,10 @@ def test_table_obs_index_does_not_shadow_the_barcode_column(atera_path: Path, ta adata.obs.reset_index() -def test_read_var_is_writable(atera_path: Path) -> None: - write_elem(zarr.open_group(zarr.storage.MemoryStore(), mode="w", zarr_format=2), "var", read_var(atera_path)) +def test_read_var_is_writable_and_indexed_like_the_table(atera_path: Path) -> None: + var = read_var(atera_path) + assert var.index.tolist() == ["GENE_A", "GENE_B", "NEG_1"] + write_elem(zarr.open_group(zarr.storage.MemoryStore(), mode="w", zarr_format=2), "var", var) def test_tile_row_ranges() -> None: @@ -428,6 +442,9 @@ def test_atera_reads_writes_and_queries(atera_path: Path, tmp_path: Path) -> Non sdata.write(tmp_path / "out.zarr") back = read_zarr(tmp_path / "out.zarr") np.testing.assert_array_equal(back.tables["table"].X.toarray(), sdata.tables["table"].X.compute().toarray()) + pd.testing.assert_frame_equal( + back.tables["table"].obsm["control_counts"], sdata.tables["table"].obsm["control_counts"] + ) crop = bounding_box_query( back, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[15, 15], target_coordinate_system="global" ) From f37e1a093c7579b782429fcf29513a43b5b55ab9 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:17:44 +0100 Subject: [PATCH 18/29] fix(atera): make duplicate feature names unique once and use them everywhere Only _xenium_var made a repeated feature_name unique. Every other path used the raw names as categorical categories or as a lookup: atera(transcripts=True) and atera_to_parquet raised 'Categorical categories must be unique', read_table_for_genes raised InvalidIndexError for every gene, and read_transcripts_for_genes returned only the first duplicate's rows. All of them now use the same anndata-style unique names (G, G-1), so G-1 addresses the second feature. Adapted to ec169cf helpers: atera_to_parquet keeps _get_array/_get_group; _get_table keeps the uns colors and manifest extras. --- .../converters/atera_to_parquet.py | 4 +-- src/spatialdata_io/readers/_atera_common.py | 15 +++++++-- src/spatialdata_io/readers/atera.py | 10 ++---- tests/readers/test_atera.py | 33 ++++++++++++++++--- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/spatialdata_io/converters/atera_to_parquet.py b/src/spatialdata_io/converters/atera_to_parquet.py index ff55e36d..8eeffcc7 100644 --- a/src/spatialdata_io/converters/atera_to_parquet.py +++ b/src/spatialdata_io/converters/atera_to_parquet.py @@ -12,10 +12,10 @@ _get_array, _get_group, _open_zip, + _read_feature_names, _read_transcript_tile, _tile_row_ranges, ) -from spatialdata_io.readers.atera import read_var __all__ = ["atera_to_parquet"] @@ -80,7 +80,7 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path: path = Path(path) output = Path(output) - feature_names = read_var(path)[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() + feature_names = _read_feature_names(path) with _open_zip(path / AteraKeys.TRANSCRIPTS_FILE) as group, pq.ParquetWriter(output, _PROSEG_SCHEMA) as writer: grid = _get_group(group, str(AteraKeys.GRID_GROUP)) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 7d7b8470..25136fa4 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -2,6 +2,7 @@ import contextlib import json +import logging import re from pathlib import Path from types import MappingProxyType @@ -119,12 +120,22 @@ def _open_zip(path: Path) -> Iterator[zarr.Group]: yield zarr.open_group(_PadRaggedVlenStore(store), mode="r") +def _unique_feature_names(names: Any) -> pd.Index: + """Feature names made unique as anndata does (``G``, ``G-1``), so they can index `var` and label categories.""" + from anndata.utils import make_index_unique + + names = pd.Index(np.asarray(names, dtype=str)) + if not names.is_unique: + logging.warning(f"{int(names.duplicated().sum())} duplicate feature names; made unique with a suffix.") + return make_index_unique(names) + + def _read_feature_names(path: Path) -> list[str]: - """The feature names of the bundle in store order, which is the order of each tile's ``gene_offset``.""" + """The unique feature names of the bundle in store order, which is the order of each tile's ``gene_offset``.""" from anndata.io import read_elem with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: - return np.asarray(read_elem(group[f"{AteraKeys.VAR_GROUP}/{AteraKeys.FEATURE_NAME}"]), dtype=str).tolist() + return _unique_feature_names(read_elem(group[f"{AteraKeys.VAR_GROUP}/{AteraKeys.FEATURE_NAME}"])).tolist() def _read_cell_ids(path: Path) -> np.ndarray: diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 0ed34204..6980b9b4 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -28,6 +28,7 @@ _get_polygons, _open_zip, _tiled_imread, + _unique_feature_names, read_cell_boundaries, read_transcripts_for_genes, ) @@ -361,14 +362,9 @@ def _xenium_var(var: pd.DataFrame) -> pd.DataFrame: ``feature_types`` (the bundles index by ``feature_id`` and name the type ``feature_type``). The type values are already the Xenium ones (``Gene Expression``, ``Negative Control Probe``, ...). """ - from anndata.utils import make_index_unique - out = var.rename(columns={str(AteraKeys.VAR_FEATURE_TYPE): "feature_types"}) out["gene_ids"] = var.index.astype(str).to_numpy() - names = pd.Index(var[str(AteraKeys.FEATURE_NAME)].astype(str).to_numpy()) # from a Series, it would keep its name - if not names.is_unique: - logging.warning(f"{int(names.duplicated().sum())} duplicate feature names; made unique with a suffix.") - out.index = make_index_unique(names) + out.index = _unique_feature_names(var[str(AteraKeys.FEATURE_NAME)]) return out @@ -473,7 +469,7 @@ def _get_table( table.uns.update(colors) if manifest_extras is not None: table.uns[str(AteraKeys.UNS_MANIFEST)] = manifest_extras - return table, var[str(AteraKeys.FEATURE_NAME)].astype(str).tolist() + return table, var.index.tolist() def read_var(path: str | Path) -> pd.DataFrame: diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 4899ed85..ff0b8d7b 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -23,6 +23,12 @@ @pytest.fixture def atera_path(tmp_path: Path) -> Path: + _write_matrices(tmp_path, ["GENE_A", "GENE_B", "NEG_1"]) + _write_cells_transcripts_and_images(tmp_path, len(CELL_IDS)) + return tmp_path + + +def _write_matrices(path: Path, feature_names: list[str]) -> None: n = len(CELL_IDS) obs = pd.DataFrame( { @@ -34,7 +40,7 @@ def atera_path(tmp_path: Path) -> Path: ) var = pd.DataFrame( { - str(AteraKeys.FEATURE_NAME): ["GENE_A", "GENE_B", "NEG_1"], + str(AteraKeys.FEATURE_NAME): feature_names, str(AteraKeys.VAR_FILTERED): [False] * 3, str(AteraKeys.VAR_HIGHLY_VARIABLE): [False] * 3, str(AteraKeys.VAR_FEATURE_TYPE): ["Gene Expression", "Gene Expression", "Negative Control Probe"], @@ -44,7 +50,7 @@ def atera_path(tmp_path: Path) -> Path: ) x = csr_matrix(np.c_[np.arange(n * 2, dtype=np.float32).reshape(n, 2), NEG_COUNTS]) for file in (AteraKeys.CELL_FEATURE_MATRIX_FILE, AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE): - store = zarr.storage.ZipStore(tmp_path / file, mode="w") + store = zarr.storage.ZipStore(path / file, mode="w") try: group = zarr.open_group(store, mode="w", zarr_format=2) write_elem(group, str(AteraKeys.X_GROUP), x if file == AteraKeys.CELL_FEATURE_MATRIX_FILE else x.tocsc()) @@ -57,8 +63,6 @@ def atera_path(tmp_path: Path) -> Path: elem.attrs["column-order"] = [str(df.index.name), *cast("list[str]", elem.attrs["column-order"])] finally: store.close() - _write_cells_transcripts_and_images(tmp_path, n) - return tmp_path PIXEL_SIZE = 0.5 @@ -557,3 +561,24 @@ def test_var_follows_the_xenium_layout(atera_path: Path, table: str) -> None: assert adata.var["gene_ids"].tolist() == ["ID_A", "ID_B"] assert adata.var["feature_types"].tolist() == ["Gene Expression", "Gene Expression"] assert "feature_type" not in adata.var.columns + + +def test_duplicate_feature_names_are_made_unique_everywhere(atera_path: Path, tmp_path: Path) -> None: + import pyarrow.parquet as pq + + from spatialdata_io import atera + from spatialdata_io.converters.atera_to_parquet import atera_to_parquet + from spatialdata_io.readers.atera import read_transcripts_for_genes + + _write_matrices(atera_path, ["GENE_A", "GENE_A", "NEG_1"]) + sdata = atera(atera_path, transcripts=True) + assert sdata.tables["table"].var_names.tolist() == ["GENE_A", "GENE_A-1"] + transcripts = sdata.points["transcripts"].compute() + assert transcripts["feature_name"].tolist() == ["GENE_A"] * 3 + ["GENE_A-1"] * 3 + sdata.write(tmp_path / "out.zarr") + genes = read_table_for_genes(atera_path, ["GENE_A-1", "GENE_A"]) + np.testing.assert_array_equal(genes.X.toarray(), np.arange(8).reshape(4, 2)[:, ::-1]) + second = read_transcripts_for_genes(sdata, atera_path, "GENE_A-1").points["transcripts_subset"].compute() + assert second["x"].tolist() == [3.0, 4.0, 5.0] + table = pq.read_table(atera_to_parquet(atera_path, tmp_path / "t.parquet")) + assert table["feature_name"].to_pylist() == ["GENE_A"] * 3 + ["GENE_A-1"] * 3 From 74c81fa755f11b50e01c29c49e75819badb6b62f Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:18:33 +0100 Subject: [PATCH 19/29] fix(atera): empty selections give empty elements read_transcripts_for_genes for a gene without transcripts (NegControlProbe_00009_2 on the 10x tiny sample) fell back to an object-dtype frame that PointsModel rejects; it now returns an empty points element with the usual dtypes. read_table_for_cells with no cells (cell_ids=[] or an empty bbox) built a 0-row table that TableModel rejects; it returns the empty table without a region, since spatialdata cannot validate an empty annotating table. spatialdata has no empty shapes element (ShapesModel rejects an empty GeoDataFrame), so read_cell_boundaries over an empty bbox raises a ValueError that says so instead of an IndexError. --- src/spatialdata_io/readers/_atera_common.py | 10 +++++----- src/spatialdata_io/readers/atera.py | 3 +++ tests/readers/test_atera.py | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 25136fa4..35536d77 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -416,7 +416,7 @@ def read_transcripts_for_genes( labels = [gene] * (end - start) frames.append(_transcript_rows(tile, start, end, pd.Categorical(labels, categories=categories))) - df = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame(columns=list(_transcripts_meta([]).columns)) + df = pd.concat(frames, ignore_index=True) if frames else _transcripts_meta(categories) transform = Scale([1.0 / pixel_size, 1.0 / pixel_size], axes=("x", "y")) sdata.points[points_key] = PointsModel.parse( @@ -569,10 +569,10 @@ def read_cell_boundaries( num_vertices_parts.append(num_vertices) cell_index_parts.append(cell_index) - coords = np.concatenate(coords_parts) if coords_parts else np.empty((0, 2), dtype=np.float32) - num_vertices = np.concatenate(num_vertices_parts) if num_vertices_parts else np.empty((0,), dtype=np.int32) - cell_index = np.concatenate(cell_index_parts) if cell_index_parts else np.empty((0,), dtype=np.uint32) - + if not coords_parts: # spatialdata has no empty shapes element: ShapesModel rejects an empty GeoDataFrame + raise ValueError(f"No polygons of {AteraKeys.CELLS_FILE!s} overlap bbox={bbox}.") + coords, num_vertices = np.concatenate(coords_parts), np.concatenate(num_vertices_parts) + cell_index = np.concatenate(cell_index_parts) sdata.shapes[shapes_key] = _polygons_to_shapes(coords, num_vertices, cell_index, cell_ids, pixel_size, is_nucleus) return sdata diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 6980b9b4..75ca1c5b 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -417,6 +417,9 @@ def _parse_table(adata: AnnData, region: str) -> AnnData: ``cell_labels`` raster values are ``label_id``; ``cell_boundaries`` shapes are indexed by ``cell_id``. """ + if adata.n_obs == 0: # spatialdata rejects an annotating table without rows ("Regions ... do not match") + logging.info(f"No cells selected; the empty table annotates no element instead of {region!r}.") + return TableModel.parse(adata) instance_key = AteraKeys.LABEL_ID if region == "cell_labels" else AteraKeys.CELL_ID adata.obs["region"] = pd.Categorical([region] * adata.n_obs) return TableModel.parse(adata, region=region, region_key="region", instance_key=str(instance_key)) diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index ff0b8d7b..aa49c3a4 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -582,3 +582,20 @@ def test_duplicate_feature_names_are_made_unique_everywhere(atera_path: Path, tm assert second["x"].tolist() == [3.0, 4.0, 5.0] table = pq.read_table(atera_to_parquet(atera_path, tmp_path / "t.parquet")) assert table["feature_name"].to_pylist() == ["GENE_A"] * 3 + ["GENE_A-1"] * 3 + + +def test_empty_selections(atera_path: Path) -> None: + from spatialdata import SpatialData + + from spatialdata_io import atera + from spatialdata_io.readers.atera import read_cell_boundaries, read_transcripts_for_genes + + sdata = atera(atera_path) + points = read_transcripts_for_genes(sdata, atera_path, "NEG_1").points["transcripts_subset"] # no transcripts + assert len(points) == 0 + assert points["feature_name"].dtype == "category" + for table in [read_table_for_cells(atera_path, cell_ids=[]), read_table_for_cells(atera_path, bbox=(80, 0, 90, 5))]: + assert table.shape == (0, 2) + SpatialData(tables={"table": table}) # a valid element + with pytest.raises(ValueError, match="No polygons"): + read_cell_boundaries(sdata, atera_path, bbox=(80, 0, 90, 5)) From 76f0d83fc701548cee1d61110fd76cf146bb12ab Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:19:23 +0100 Subject: [PATCH 20/29] fix(atera): keep a polygon with 1 or 2 vertices as an empty polygon shapely needs 3 vertices for a ring, so a single such polygon in polygon_sets made from_ragged_array raise ('A linearring requires at least 4 coordinates') and aborted the default atera() read (cell and nucleus boundaries are on by default). Such polygons are now kept as empty polygons, in place, with a warning that counts them. XOA does emit 2- and 3-vertex polygons; whether Atera bundles do is not known. --- src/spatialdata_io/readers/_atera_common.py | 5 +++++ tests/readers/test_atera.py | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 35536d77..efb52a0c 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -174,6 +174,11 @@ def _polygons_to_shapes( `gridded_polygon_sets`), which differ only in how they produce these ragged arrays. """ n_polygons = len(num_vertices) + degenerate = (num_vertices > 0) & (num_vertices < 3) # shapely needs 3 vertices for a ring + if degenerate.any(): + logging.warning(f"{int(degenerate.sum())} polygon(s) with 1 or 2 vertices kept as empty polygons.") + coords = coords[np.repeat(~degenerate, num_vertices)] + num_vertices = np.where(degenerate, 0, num_vertices) ring_offsets = np.concatenate([[0], np.cumsum(num_vertices)]) geom_offsets = np.arange(n_polygons + 1) geoms = from_ragged_array(GeometryType.POLYGON, coords, offsets=(ring_offsets, geom_offsets)) diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index aa49c3a4..853a2445 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -599,3 +599,15 @@ def test_empty_selections(atera_path: Path) -> None: SpatialData(tables={"table": table}) # a valid element with pytest.raises(ValueError, match="No polygons"): read_cell_boundaries(sdata, atera_path, bbox=(80, 0, 90, 5)) + + +def test_polygons_with_one_or_two_vertices_are_kept_empty(caplog: pytest.LogCaptureFixture) -> None: + import logging + + num_vertices = np.array([3, 2, 1, 0, 4]) + coords = np.arange(2 * num_vertices.sum(), dtype=np.float32).reshape(-1, 2) + with caplog.at_level(logging.WARNING): + shapes = common._polygons_to_shapes(coords, num_vertices, np.arange(5), None, 1.0, is_nucleus=False) + assert shapes.geometry.is_empty.tolist() == [False, True, True, True, False] + np.testing.assert_array_equal(np.asarray(shapes.geometry.iloc[4].exterior.coords)[:4], coords[6:10]) + assert "2 polygon(s) with 1 or 2 vertices" in caplog.text From 05535c5513e8844bdeadd2fe84236e75dffad4ef Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:24:19 +0100 Subject: [PATCH 21/29] fix(atera): honour cells_table=False; annotate only an element that is read cells_table=False was overridden whenever labels, boundaries or transcripts were read, because the polygons took cell ids from the table and the transcripts took feature names from it. Both now read them from the bundle, so cells_table=False gives no table. With neither cell_labels nor cell_boundaries read (only nucleus elements, or none), the table annotated the absent cell_labels and spatialdata warned; it now annotates no element, since nucleus labels and polygons are numbered per nucleus, not per cell. --- src/spatialdata_io/readers/atera.py | 46 ++++++++++------------------- tests/readers/test_atera.py | 29 ++++++++++++++++++ 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 75ca1c5b..aa2bce83 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -27,6 +27,8 @@ _get_points, _get_polygons, _open_zip, + _read_cell_ids, + _read_feature_names, _tiled_imread, _unique_feature_names, read_cell_boundaries, @@ -167,32 +169,19 @@ def atera( pixel_size = _get_pixel_size(path) - needs_cells_zarr = cells_boundaries or nucleus_boundaries or cells_labels or nucleus_labels - if not cells_table and (needs_cells_zarr or transcripts): - logging.info("Reading the table is required for the requested elements; setting cells_table=True.") - cells_table = True - - # Prefer `cell_labels` as the table's annotation target (matching the `xenium` reader's - # convention; instance ids are `obs["label_id"]`), but fall back to `cell_boundaries` (instance ids - # are `obs["cell_id"]`) when labels aren't being loaded, so the table doesn't end up annotating an - # element that isn't actually present in `sdata`. - default_region = "cell_labels" if cells_labels else "cell_boundaries" if cells_boundaries else "cell_labels" - - table: AnnData | None = None - feature_names: list[str] | None = None - if cells_table: - table, feature_names = _get_table( - path, var_columns=var_columns, row_chunk_size=table_row_chunk_size, region=default_region - ) + # The table annotates `cell_labels` (instance ids `obs["label_id"]`, as in `xenium`), else `cell_boundaries` + # (`obs["cell_id"]`), else nothing: the nucleus elements are numbered per nucleus, not per cell. + region = "cell_labels" if cells_labels else "cell_boundaries" if cells_boundaries else None + table = _get_table(path, var_columns, table_row_chunk_size, region)[0] if cells_table else None shapes: dict[str, Any] = {} labels: dict[str, DataArray | DataTree] = {} points: dict[str, Any] = {} images: dict[str, DataArray | DataTree] = {} - if needs_cells_zarr: + if cells_boundaries or nucleus_boundaries or cells_labels or nucleus_labels: cells_group = zarr.open_group(zarr.storage.ZipStore(path / AteraKeys.CELLS_FILE, read_only=True), mode="r") - cell_ids = table.obs[str(AteraKeys.CELL_ID)].to_numpy() if table is not None else None + cell_ids = _read_cell_ids(path) if nucleus_labels: labels["nucleus_labels"] = _get_labels(cells_group, mask_index=0, labels_models_kwargs=labels_models_kwargs) @@ -208,9 +197,7 @@ def atera( ) if transcripts: - if feature_names is None: - raise ValueError("Reading transcripts requires `cells_table=True` (to map gene indices to names).") - points["transcripts"] = _get_points(path, pixel_size, feature_names) + points["transcripts"] = _get_points(path, pixel_size, _read_feature_names(path)) if morphology_images: images["morphology"] = _get_morphology_images( @@ -391,7 +378,7 @@ def _assemble_table( var: pd.DataFrame, controls: csr_matrix | csc_matrix, var_columns: list[str] | Literal["all"], - region: str, + region: str | None, obsm: Mapping[str, Any] = MappingProxyType({}), ) -> AnnData: """X and var hold the Gene Expression features, ``obsm["control_counts"]`` the counts of all other features. @@ -412,13 +399,12 @@ def _assemble_table( return _parse_table(AnnData(X=x, obs=obs, var=var.loc[is_gene, list(columns)], obsm=obsm), region) -def _parse_table(adata: AnnData, region: str) -> AnnData: - """Annotate `region` with `adata`, using whichever `obs` column holds that element's instance ids. +def _parse_table(adata: AnnData, region: str | None) -> AnnData: + """Annotate `region` (if any) with `adata`, using whichever `obs` column holds that element's instance ids. ``cell_labels`` raster values are ``label_id``; ``cell_boundaries`` shapes are indexed by ``cell_id``. """ - if adata.n_obs == 0: # spatialdata rejects an annotating table without rows ("Regions ... do not match") - logging.info(f"No cells selected; the empty table annotates no element instead of {region!r}.") + if region is None or adata.n_obs == 0: # spatialdata rejects an empty annotating table ("Regions ... do not match") return TableModel.parse(adata) instance_key = AteraKeys.LABEL_ID if region == "cell_labels" else AteraKeys.CELL_ID adata.obs["region"] = pd.Categorical([region] * adata.n_obs) @@ -429,7 +415,7 @@ def _get_table( path: Path, var_columns: list[str] | Literal["all"] = DEFAULT_VAR_COLUMNS, row_chunk_size: int = DEFAULT_TABLE_ROW_CHUNK_SIZE, - region: str = "cell_labels", + region: str | None = "cell_labels", ) -> tuple[AnnData, list[str]]: """Read ``cell_feature_matrix.zarr.zip`` into an AnnData table using native `anndata` zarr IO. @@ -437,9 +423,7 @@ def _get_table( indexed by whichever column the store's ``_index`` attr names -- e.g. `barcode`/`feature_id`). `X` is a lazy `dask` array of `row_chunk_size`-row CSR chunks (`anndata.experimental.read_elem_lazy`). - `region` should be the name of a `SpatialElement` that will actually be present in the - `SpatialData` object the table is assembled into (see the ``default_region`` logic in - `atera`), so the table doesn't end up annotating an element that was never loaded. + `region` is the element the table annotates; it must be read too (see `atera`). None annotates nothing. """ from anndata.experimental import read_elem_lazy from anndata.io import read_elem, sparse_dataset diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 853a2445..90c9121a 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -611,3 +611,32 @@ def test_polygons_with_one_or_two_vertices_are_kept_empty(caplog: pytest.LogCapt assert shapes.geometry.is_empty.tolist() == [False, True, True, True, False] np.testing.assert_array_equal(np.asarray(shapes.geometry.iloc[4].exterior.coords)[:4], coords[6:10]) assert "2 polygon(s) with 1 or 2 vertices" in caplog.text + + +@pytest.mark.parametrize( + ("kwargs", "region"), + [ + ({"cells_table": False, "transcripts": True}, None), + ({"cells_labels": False, "cells_boundaries": False}, None), + ({"cells_labels": False}, "cell_boundaries"), + ({}, "cell_labels"), + ], +) +def test_table_and_its_region_follow_the_flags(atera_path: Path, kwargs: dict, region: str | None) -> None: + import warnings + + from spatialdata.models import TableModel + + from spatialdata_io import atera + + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + sdata = atera(atera_path, morphology_images=False, aligned_images=False, **kwargs) + assert not [w for w in record if "which is not present in the SpatialData object" in str(w.message)] + if kwargs.get("cells_table") is False: + assert not sdata.tables + assert len(sdata.points["transcripts"]) == 6 + assert sdata.shapes["cell_boundaries"].index.tolist() == CELL_IDS + else: + attrs = sdata.tables["table"].uns.get(TableModel.ATTRS_KEY, {}) + assert attrs.get("region") == region From 8aa0d48c6215384f83fcdc8d1a68ea4e387589c7 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:24:55 +0100 Subject: [PATCH 22/29] fix(atera_to_parquet): write to a .partial file and move it into place when complete ParquetWriter closed in a finally, so any failure left a truncated but valid parquet file, and a failed rerun had already truncated the earlier complete one at open. Adapted to ec169cf helpers: the zarr lookups use _get_array/_get_group. --- .../converters/atera_to_parquet.py | 95 ++++++++++--------- tests/converters/test_atera_to_parquet.py | 20 ++++ 2 files changed, 71 insertions(+), 44 deletions(-) diff --git a/src/spatialdata_io/converters/atera_to_parquet.py b/src/spatialdata_io/converters/atera_to_parquet.py index 8eeffcc7..2ae4ba9d 100644 --- a/src/spatialdata_io/converters/atera_to_parquet.py +++ b/src/spatialdata_io/converters/atera_to_parquet.py @@ -82,48 +82,55 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path: feature_names = _read_feature_names(path) - with _open_zip(path / AteraKeys.TRANSCRIPTS_FILE) as group, pq.ParquetWriter(output, _PROSEG_SCHEMA) as writer: - grid = _get_group(group, str(AteraKeys.GRID_GROUP)) - tile_keys = [str(key) for key in cast("list[str]", grid.attrs[str(AteraKeys.GRID_KEYS)])] - # Stored IDs are used only if every tile has them; otherwise every transcript is numbered - # contiguously in `grid_keys` order, so stored and generated IDs never mix. - use_stored_ids = all(str(AteraKeys.TRANSCRIPTS_ID) in _get_group(grid, key) for key in tile_keys) - - offset = 0 - for tile_key in tile_keys: - tile = _get_group(grid, tile_key) - n_tile = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] - # A tile can hold 100M+ transcripts, so it is read and written in row ranges (one parquet row group each). - for start, stop in _tile_row_ranges(n_tile): - tile_df = _read_transcript_tile(grid, tile_key, feature_names, start, stop) - n = len(tile_df) - if n == 0: - continue - - if use_stored_ids: - stored_ids = _get_array(tile, str(AteraKeys.TRANSCRIPTS_ID))[start:stop] - transcript_id = np.asarray(stored_ids, dtype=np.uint64).reshape(-1) - else: - transcript_id = np.arange(offset, offset + n, dtype=np.uint64) - - cell_id = tile_df[str(AteraKeys.CELL_ID)].to_numpy() - cell_id_str = np.where(cell_id == -1, _CELL_ID_UNASSIGNED, cell_id.astype(str)) - - table = pa.table( - { - "transcript_id": transcript_id, - "cell_id": cell_id_str, - "overlaps_nucleus": tile_df["overlaps_nucleus"].to_numpy().astype(np.uint8), - "feature_name": tile_df[str(AteraKeys.FEATURE_NAME)].astype(str).to_numpy(), - "x_location": tile_df[str(AteraKeys.TRANSCRIPTS_X)].to_numpy().astype(np.float32), - "y_location": tile_df[str(AteraKeys.TRANSCRIPTS_Y)].to_numpy().astype(np.float32), - "z_location": tile_df[str(AteraKeys.TRANSCRIPTS_Z)].to_numpy().astype(np.float32), - "qv": tile_df["quality_score"].to_numpy().astype(np.float32), - "fov_name": np.full(n, tile_key), - }, - schema=_PROSEG_SCHEMA, - ) - writer.write_table(table) - offset += n - + # written next to `output` and moved there when complete, so a failed run leaves no truncated (but valid) + # parquet file and keeps an earlier complete one + partial = output.with_name(output.name + ".partial") + try: + with _open_zip(path / AteraKeys.TRANSCRIPTS_FILE) as group, pq.ParquetWriter(partial, _PROSEG_SCHEMA) as writer: + grid = _get_group(group, str(AteraKeys.GRID_GROUP)) + tile_keys = [str(key) for key in cast("list[str]", grid.attrs[str(AteraKeys.GRID_KEYS)])] + # Stored IDs are used only if every tile has them; otherwise every transcript is numbered + # contiguously in `grid_keys` order, so stored and generated IDs never mix. + use_stored_ids = all(str(AteraKeys.TRANSCRIPTS_ID) in _get_group(grid, key) for key in tile_keys) + + offset = 0 + for tile_key in tile_keys: + tile = _get_group(grid, tile_key) + n_tile = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0] + # A tile can hold 100M+ transcripts, so it is read and written in row ranges (one parquet row group each). + for start, stop in _tile_row_ranges(n_tile): + tile_df = _read_transcript_tile(grid, tile_key, feature_names, start, stop) + n = len(tile_df) + if n == 0: + continue + + if use_stored_ids: + stored_ids = _get_array(tile, str(AteraKeys.TRANSCRIPTS_ID))[start:stop] + transcript_id = np.asarray(stored_ids, dtype=np.uint64).reshape(-1) + else: + transcript_id = np.arange(offset, offset + n, dtype=np.uint64) + + cell_id = tile_df[str(AteraKeys.CELL_ID)].to_numpy() + cell_id_str = np.where(cell_id == -1, _CELL_ID_UNASSIGNED, cell_id.astype(str)) + + table = pa.table( + { + "transcript_id": transcript_id, + "cell_id": cell_id_str, + "overlaps_nucleus": tile_df["overlaps_nucleus"].to_numpy().astype(np.uint8), + "feature_name": tile_df[str(AteraKeys.FEATURE_NAME)].astype(str).to_numpy(), + "x_location": tile_df[str(AteraKeys.TRANSCRIPTS_X)].to_numpy().astype(np.float32), + "y_location": tile_df[str(AteraKeys.TRANSCRIPTS_Y)].to_numpy().astype(np.float32), + "z_location": tile_df[str(AteraKeys.TRANSCRIPTS_Z)].to_numpy().astype(np.float32), + "qv": tile_df["quality_score"].to_numpy().astype(np.float32), + "fov_name": np.full(n, tile_key), + }, + schema=_PROSEG_SCHEMA, + ) + writer.write_table(table) + offset += n + except BaseException: + partial.unlink(missing_ok=True) + raise + partial.replace(output) return output diff --git a/tests/converters/test_atera_to_parquet.py b/tests/converters/test_atera_to_parquet.py index e3bade15..250805e2 100644 --- a/tests/converters/test_atera_to_parquet.py +++ b/tests/converters/test_atera_to_parquet.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Any import numpy as np import pandas as pd @@ -92,3 +93,22 @@ def test_chunked_conversion_matches_unchunked( assert chunked.equals(whole) assert chunked["feature_name"].to_pylist() == ["GENE_A"] * 3 + ["GENE_B"] * 4 + ["GENE_A"] * 2 + ["GENE_B"] * 3 + + +def test_failed_conversion_keeps_the_earlier_output(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _write_atera_dataset(tmp_path, {"0,0": 4, "1,0": 4}, ["0,0", "1,0"], ids={}) + output = atera_to_parquet(tmp_path, tmp_path / "transcripts.parquet") + before = output.read_bytes() + + read_tile = atera_module._read_transcript_tile + + def fail_on_second_tile(grid: zarr.Group, tile_key: str, *args: Any) -> pd.DataFrame: + if tile_key == "1,0": + raise OSError("disk full") + return read_tile(grid, tile_key, *args) + + monkeypatch.setattr(atera_module, "_read_transcript_tile", fail_on_second_tile) + with pytest.raises(OSError, match="disk full"): + atera_to_parquet(tmp_path, output) + assert output.read_bytes() == before + assert not (tmp_path / "transcripts.parquet.partial").exists() From 04a58fe9acae549eb268fdbc6bd2dbd51bf7afaf Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:25:49 +0100 Subject: [PATCH 23/29] fix(atera): log through spatialdata's logger, as the other readers do The module-level logging.warning/info calls went to the root logger, and the first of them ran logging.basicConfig as a side effect in the user's process. --- src/spatialdata_io/readers/_atera_common.py | 6 +++--- src/spatialdata_io/readers/atera.py | 4 ++-- tests/readers/test_atera.py | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index efb52a0c..44ca1070 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -2,7 +2,6 @@ import contextlib import json -import logging import re from pathlib import Path from types import MappingProxyType @@ -17,6 +16,7 @@ from geopandas import GeoDataFrame from shapely import GeometryType, from_ragged_array from spatialdata import SpatialData +from spatialdata._logging import logger from spatialdata.models import ( Image2DModel, Image3DModel, @@ -126,7 +126,7 @@ def _unique_feature_names(names: Any) -> pd.Index: names = pd.Index(np.asarray(names, dtype=str)) if not names.is_unique: - logging.warning(f"{int(names.duplicated().sum())} duplicate feature names; made unique with a suffix.") + logger.warning(f"{int(names.duplicated().sum())} duplicate feature names; made unique with a suffix.") return make_index_unique(names) @@ -176,7 +176,7 @@ def _polygons_to_shapes( n_polygons = len(num_vertices) degenerate = (num_vertices > 0) & (num_vertices < 3) # shapely needs 3 vertices for a ring if degenerate.any(): - logging.warning(f"{int(degenerate.sum())} polygon(s) with 1 or 2 vertices kept as empty polygons.") + logger.warning(f"{int(degenerate.sum())} polygon(s) with 1 or 2 vertices kept as empty polygons.") coords = coords[np.repeat(~degenerate, num_vertices)] num_vertices = np.where(degenerate, 0, num_vertices) ring_offsets = np.concatenate([[0], np.cumsum(num_vertices)]) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index aa2bce83..264f5dfa 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import logging from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, cast @@ -11,6 +10,7 @@ import zarr from scipy.sparse import csc_matrix, csr_matrix from spatialdata import SpatialData +from spatialdata._logging import logger from spatialdata.models import Image2DModel, TableModel from spatialdata.transformations import Affine, BaseTransformation, Identity @@ -268,7 +268,7 @@ def _check_keypoints(alignment_file: Path, kp_file: Path) -> float | None: affine = pd.read_csv(alignment_file, header=None).to_numpy(dtype=float) median = float(np.median(np.linalg.norm((np.c_[src, np.ones(len(src))] @ affine.T)[:, :2] - dst, axis=1))) if median > KEYPOINTS_MAX_RESIDUAL_PX: - logging.warning( + logger.warning( f"{alignment_file.name} maps {kp_file.name} with a median residual of {median:.1f} px " f"(> {KEYPOINTS_MAX_RESIDUAL_PX}); the aligned image may be misplaced." ) diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 90c9121a..e853976f 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -322,9 +322,13 @@ def _write_keypoints(path: Path, affine: np.ndarray, name: str = "S1_keypoints.c return kp -def test_keypoints_check(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: +def test_keypoints_check(tmp_path: Path, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: + from spatialdata._logging import logger + from spatialdata_io.readers.atera import _check_keypoints + monkeypatch.setattr(logger, "propagate", True) + affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) _write_he(tmp_path, affine) kp = _write_keypoints(tmp_path, affine) @@ -601,9 +605,15 @@ def test_empty_selections(atera_path: Path) -> None: read_cell_boundaries(sdata, atera_path, bbox=(80, 0, 90, 5)) -def test_polygons_with_one_or_two_vertices_are_kept_empty(caplog: pytest.LogCaptureFixture) -> None: +def test_polygons_with_one_or_two_vertices_are_kept_empty( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: import logging + from spatialdata._logging import logger + + monkeypatch.setattr(logger, "propagate", True) + num_vertices = np.array([3, 2, 1, 0, 4]) coords = np.arange(2 * num_vertices.sum(), dtype=np.float32).reshape(-1, 2) with caplog.at_level(logging.WARNING): From 33bf1025e77e7f972833ff03848530574a3e66b9 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:26:15 +0100 Subject: [PATCH 24/29] docs(atera): fix the stale docstrings The transcripts are partitioned by row range of a tile, not by tile; atera()'s X is lazy, not eager; the module note listed what no longer lives where it said. (The aligned_images entry was updated with the keypoint fallback.) --- src/spatialdata_io/converters/atera_to_parquet.py | 6 +++--- src/spatialdata_io/readers/_atera_common.py | 2 +- src/spatialdata_io/readers/atera.py | 7 +++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/spatialdata_io/converters/atera_to_parquet.py b/src/spatialdata_io/converters/atera_to_parquet.py index 2ae4ba9d..ed83d1ed 100644 --- a/src/spatialdata_io/converters/atera_to_parquet.py +++ b/src/spatialdata_io/converters/atera_to_parquet.py @@ -62,9 +62,9 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path: ``transcript_id`` is taken from each tile's stored ``id`` array when every tile has one, and is otherwise assigned contiguously in the order of the grid group's ``grid_keys`` attribute. - The input is read and written one spatial tile at a time (mirroring how `atera`'s own transcripts - reader is built), so converting even a dataset with tens of millions of transcripts does not require - holding the whole table in memory at once. + The input is read and written one row range of a spatial tile at a time (one parquet row group each, + as `atera`'s own transcripts reader partitions it), so converting even a dataset with hundreds of millions + of transcripts does not require holding the whole table in memory at once. Parameters ---------- diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 44ca1070..7a347244 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -290,7 +290,7 @@ def _transcripts_meta(categories: list[str]) -> pd.DataFrame: def _get_points(path: Path, pixel_size: float, feature_names: list[str]) -> dd.DataFrame: - """Lazily read transcripts.zarr.zip, one `dask` task per spatial tile.""" + """Lazily read transcripts.zarr.zip, one `dask` task per row range of a spatial tile (see `_tile_row_ranges`).""" # the store stays referenced by the graph; it reopens on first use, also after pickling grid = zarr.open_group(zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True), mode="r") grid = _get_group(grid, str(AteraKeys.GRID_GROUP)) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 264f5dfa..62748cf8 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -63,8 +63,7 @@ # only works on bundles whose zarr stores carry the standard AnnData `encoding-type`/ # `encoding-version`/`shape` attrs. # -# Every non-table element (labels/shapes/points/morphology images) lives in `_atera_common.py` -# rather than duplicated here. +# The tables are built here; the labels, shapes, points and morphology images in `_atera_common.py`. @inject_docs(xx=AteraKeys) @@ -489,8 +488,8 @@ def read_table_for_cells( ) -> AnnData: """Read the cell-by-gene table for only a subset of cells, without loading the full `X`. - Unlike ``atera(path, cells_table=True)``, which eagerly builds the whole `X`, this reads only - the requested rows directly off `anndata.io.sparse_dataset`'s own fancy-indexing support + Unlike ``atera(path, cells_table=True)``, whose `X` is lazy over every cell, this reads only + the requested rows, in memory, directly off `anndata.io.sparse_dataset`'s own fancy-indexing support (``x_ds[row_positions]``), which already handles arbitrary/unsorted row selections internally. Exactly one of `cell_ids`/`bbox` must be given. From 889ef2f1bb82c530577ce0539807fd97a37ef619 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 22:50:18 +0100 Subject: [PATCH 25/29] fix(atera): type the new code for mypy No new mypy errors in the changed files against 5827d7b plus our 7 commits, with the repo's mypy hook. Adapted to ec169cf helpers: X and indptr stay on _get_group/_get_array, keys stay wrapped in str() (7fc9d3c is not ported). --- src/spatialdata_io/readers/_atera_common.py | 5 +- src/spatialdata_io/readers/atera.py | 14 ++--- tests/readers/test_atera.py | 59 ++++++++++++--------- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 7a347244..d643399d 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -32,6 +32,7 @@ if TYPE_CHECKING: from collections.abc import Iterator, Mapping + from numpy.typing import ArrayLike from xarray import DataArray from zarr.abc.buffer import Buffer from zarr.abc.store import ByteRequest @@ -234,7 +235,7 @@ def _tile_row_ranges(n_rows: int, chunk_rows: int = DEFAULT_TRANSCRIPT_CHUNK_ROW return [(start, min(start + chunk_rows, n_rows)) for start in range(0, n_rows, chunk_rows)] or [(0, 0)] -def _transcript_rows(tile: zarr.Group, start: int, stop: int, feature_name: pd.Categorical) -> pd.DataFrame: +def _transcript_rows(tile: zarr.Group, start: int, stop: int, feature_name: pd.Categorical[Any]) -> pd.DataFrame: """Decode rows ``[start, stop)`` of one tile of transcripts.zarr.zip, labelled with `feature_name`.""" def rows(key: AteraKeys) -> np.ndarray: @@ -444,7 +445,7 @@ def _tile_bounds(tile_key: str, tile_size: float) -> tuple[float, float, float, return tile_x * tile_size, tile_y * tile_size, (tile_x + 1) * tile_size, (tile_y + 1) * tile_size -def _bbox_overlaps(boxes: np.ndarray, bbox: tuple[float, float, float, float]) -> np.ndarray: +def _bbox_overlaps(boxes: ArrayLike, bbox: tuple[float, float, float, float]) -> np.ndarray: """Which of the ``(..., 4)`` (xmin, ymin, xmax, ymax) `boxes` overlap `bbox`.""" boxes = np.asarray(boxes) return (boxes[..., 0] < bbox[2]) & (bbox[0] < boxes[..., 2]) & (boxes[..., 1] < bbox[3]) & (bbox[1] < boxes[..., 3]) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 62748cf8..df9671a4 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -46,6 +46,7 @@ if TYPE_CHECKING: from collections.abc import Mapping, Sequence + import dask.array as da from anndata import AnnData from xarray import DataArray, DataTree @@ -441,14 +442,15 @@ def _get_table( manifest_extras = _read_manifest_extras(group) # the graph keeps the store, which reopens on first use (also after pickling) x_group = _get_group(group, str(AteraKeys.X_GROUP)) - x = read_elem_lazy(x_group, chunks=(row_chunk_size, -1)) + x = cast("da.Array", read_elem_lazy(x_group, chunks=(row_chunk_size, -1))) stored_per_row = np.diff(_get_array(x_group, "indptr")[:]) is_gene = _is_gene(var) + controls: csr_matrix | csc_matrix if (path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE).is_file(): # column reads are cheap in the CSC copy with _open_zip(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE) as group: - controls = sparse_dataset(group[str(AteraKeys.X_GROUP)])[:, np.flatnonzero(~is_gene)] + controls = cast(csc_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[:, np.flatnonzero(~is_gene)]) else: - controls = x[:, ~is_gene].compute() + controls = cast(csr_matrix, x[:, ~is_gene].compute()) # genes detected per cell: the stored entries of each CSR row (its `indptr` step) minus those of the controls obs["n_genes"] = stored_per_row - controls.getnnz(axis=1) table = _assemble_table(x[:, is_gene], obs, var, controls, var_columns, region, obsm) @@ -520,7 +522,7 @@ def read_table_for_cells( with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: obs, var = _read_obs_var(group) if cell_ids is not None: - row_positions = pd.Index(obs[str(AteraKeys.CELL_ID)].to_numpy()).get_indexer(np.asarray(cell_ids)) + row_positions = pd.Index(obs[str(AteraKeys.CELL_ID)].to_numpy()).get_indexer(pd.Index(np.asarray(cell_ids))) missing = np.asarray(cell_ids)[row_positions == -1] if missing.size: raise ValueError(f"cell_id(s) not found in {AteraKeys.CELL_FEATURE_MATRIX_FILE!s}: {missing.tolist()}") @@ -563,12 +565,12 @@ def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnDat path = Path(path) with _open_zip(path / AteraKeys.CSC_CELL_FEATURE_MATRIX_FILE) as group: obs, var = _read_obs_var(group) - col_positions = var.index.get_indexer(np.asarray(genes)) + col_positions = var.index.get_indexer(pd.Index(genes)) missing = np.asarray(genes)[col_positions == -1] if missing.size: raise ValueError( f"gene(s) not found in {AteraKeys.VAR_GROUP!s}.{AteraKeys.FEATURE_NAME!s}: {missing.tolist()}" ) x = cast(csc_matrix, sparse_dataset(group[str(AteraKeys.X_GROUP)])[:, col_positions]) - obsm = {"spatial": obs[[str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)]].to_numpy()} + obsm: dict[str, Any] = {"spatial": obs[[str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)]].to_numpy()} return _parse_table(AnnData(X=x, obs=obs, var=var.iloc[col_positions], obsm=obsm), "cell_labels") diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index e853976f..7d13c596 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -1,13 +1,16 @@ from pathlib import Path -from typing import cast +from typing import Any, cast import numpy as np import pandas as pd import pytest +import shapely import zarr from anndata.io import write_elem from scipy.sparse import csr_matrix +from spatialdata import SpatialData from spatialdata.models import get_table_keys +from spatialdata.transformations import Affine from tests.converters.test_atera_to_parquet import _write_atera_dataset from xarray import DataTree @@ -124,17 +127,17 @@ def _write_cells_transcripts_and_images(path: Path, n: int) -> None: planes = rng.integers(0, 255, (2, 64, 96), dtype=np.uint8) (path / AteraKeys.MORPHOLOGY_2D_DIR).mkdir() (path / AteraKeys.MORPHOLOGY_3D_DIR).mkdir() - for i, name in enumerate(["DAPI", "VIM"]): + for c, name in enumerate(["DAPI", "VIM"]): tifffile.imwrite( - path / AteraKeys.MORPHOLOGY_2D_DIR / f"ch{i:04d}_{name.lower()}.ome.tif", + path / AteraKeys.MORPHOLOGY_2D_DIR / f"ch{c:04d}_{name.lower()}.ome.tif", planes, ome=True, tile=(32, 32), metadata={"axes": "CYX", "Channel": {"Name": ["DAPI", "VIM"]}}, ) tifffile.imwrite( - path / AteraKeys.MORPHOLOGY_3D_DIR / f"ch{i:04d}_{name.lower()}_3d.ome.tif", - np.stack([planes[i]] * 3), + path / AteraKeys.MORPHOLOGY_3D_DIR / f"ch{c:04d}_{name.lower()}_3d.ome.tif", + np.stack([planes[c]] * 3), ome=True, tile=(32, 32), metadata={"axes": "ZYX", "Channel": {"Name": [name]}}, @@ -214,7 +217,7 @@ def test_table_obs_index_does_not_shadow_the_barcode_column(atera_path: Path, ta assert adata.obs.index.name is None assert adata.obs["barcode"].tolist() == adata.obs_names.tolist() - adata.obs.reset_index() + cast(pd.DataFrame, adata.obs).reset_index() def test_read_var_is_writable_and_indexed_like_the_table(atera_path: Path) -> None: @@ -232,9 +235,8 @@ def test_tile_row_ranges() -> None: def test_chunked_tile_reads_match_whole_tile(tmp_path: Path) -> None: _write_atera_dataset(tmp_path, {"0,0": 7}, ["0,0"], ids={}) feature_names = ["GENE_A", "GENE_B"] - grid = zarr.open_group(zarr.storage.ZipStore(tmp_path / AteraKeys.TRANSCRIPTS_FILE, read_only=True), mode="r")[ - "grid" - ] + root = zarr.open_group(zarr.storage.ZipStore(tmp_path / AteraKeys.TRANSCRIPTS_FILE, read_only=True), mode="r") + grid = root.get_group("grid") whole = _read_transcript_tile(grid, "0,0", feature_names) # GENE_A owns rows 0-2 and GENE_B rows 3-6, so the 2-row chunks straddle the gene boundary. parts = [_read_transcript_tile(grid, "0,0", feature_names, a, b) for a, b in _tile_row_ranges(7, 2)] @@ -264,8 +266,11 @@ def test_morphology_pyramid_is_built_from_level_0(atera_path: Path, scale_factor assert [image[f"scale{i}"]["image"].shape for i in range(len(factors))] == [(2, 64 // f, 96 // f) for f in factors] assert image["scale0"]["image"].coords["c"].values.tolist() == ["DAPI", "VIM"] # each scale has coordinates in scale0 px, so a query crops the same region at every scale - crop = bounding_box_query( - image, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[32, 32], target_coordinate_system="global" + crop = cast( + DataTree, + bounding_box_query( + image, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[32, 32], target_coordinate_system="global" + ), ) assert [crop[k]["image"].shape[1:] for k in crop.children] == [(32 // f, 32 // f) for f in factors] @@ -310,7 +315,7 @@ def test_aligned_he_image_is_placed_by_its_affine(tmp_path: Path, reader: str) - assert list(images) == ["he_image"] he = images["he_image"] assert list(he.coords["c"].values) == ["r", "g", "b"] - m = get_transformation(he, "global").to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) + m = cast(Affine, get_transformation(he, "global")).to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) np.testing.assert_allclose(m, affine) @@ -390,7 +395,7 @@ def test_affine_from_keypoints_reproduces_alignment(tmp_path: Path) -> None: dst = (np.c_[src, np.ones(len(src))] @ affine.T)[:, :2] kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) kp.to_csv(tmp_path / "S1_keypoints.csv", index=False) - m = _affine_from_keypoints(tmp_path / "S1_keypoints.csv").to_affine_matrix( + m = cast(Affine, _affine_from_keypoints(tmp_path / "S1_keypoints.csv")).to_affine_matrix( input_axes=("x", "y"), output_axes=("x", "y") ) np.testing.assert_allclose(m, affine, atol=1e-6) @@ -410,7 +415,7 @@ def test_he_without_alignment_csv_is_placed_by_keypoints(tmp_path: Path) -> None tmp_path / "S1_keypoints.csv", index=False ) he = _get_aligned_images(tmp_path, {}, {})["he_image"] - m = get_transformation(he, "global").to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) + m = cast(Affine, get_transformation(he, "global")).to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) np.testing.assert_allclose(m, affine, atol=1e-9) @@ -449,12 +454,16 @@ def test_atera_reads_writes_and_queries(atera_path: Path, tmp_path: Path) -> Non assert transcripts["cell_id"].tolist() == [-1, 0, 0, 0, 0, 0] # the packed (uint32 max, uint32 max) is unassigned sdata.write(tmp_path / "out.zarr") back = read_zarr(tmp_path / "out.zarr") - np.testing.assert_array_equal(back.tables["table"].X.toarray(), sdata.tables["table"].X.compute().toarray()) + table, table_back = sdata.tables["table"], back.tables["table"] + np.testing.assert_array_equal(cast(csr_matrix, table_back.X).toarray(), cast(Any, table.X).compute().toarray()) pd.testing.assert_frame_equal( - back.tables["table"].obsm["control_counts"], sdata.tables["table"].obsm["control_counts"] + cast(pd.DataFrame, table_back.obsm["control_counts"]), cast(pd.DataFrame, table.obsm["control_counts"]) ) - crop = bounding_box_query( - back, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[15, 15], target_coordinate_system="global" + crop = cast( + SpatialData, + bounding_box_query( + back, axes=("x", "y"), min_coordinate=[0, 0], max_coordinate=[15, 15], target_coordinate_system="global" + ), ) assert crop.tables["table"].obs["cell_id"].tolist() == CELL_IDS[:2] @@ -462,7 +471,7 @@ def test_atera_reads_writes_and_queries(atera_path: Path, tmp_path: Path) -> Non def test_lazy_x_matches_the_store_and_pickles(atera_path: Path) -> None: import pickle - x = _get_table(atera_path, row_chunk_size=3)[0].X + x = cast(Any, _get_table(atera_path, row_chunk_size=3)[0].X) assert x.chunks[0] == (3, 1) expected = np.arange(len(CELL_IDS) * 2, dtype=np.float32).reshape(-1, 2) np.testing.assert_array_equal(pickle.loads(pickle.dumps(x)).compute().toarray(), expected) @@ -517,12 +526,12 @@ def zip_with_trailing_chunk(items: list[str]) -> Path: # 10x's writer stores only the 2 items of the trailing chunk; zarr reads it at the nominal size of 3 path = zip_with_trailing_chunk(["dd", "e"]) with pytest.raises(ValueError, match="reshape"): - zarr.open_group(zarr.storage.ZipStore(path, read_only=True), mode="r")["s"][:] + zarr.open_group(zarr.storage.ZipStore(path, read_only=True), mode="r").get_array("s")[:] with common._open_zip(path) as group: - assert group["s"][:].tolist() == ["a", "bb", "ccc", "dd", "e"] + assert np.asarray(group.get_array("s")[:]).tolist() == ["a", "bb", "ccc", "dd", "e"] path = zip_with_trailing_chunk(["dd"]) # one item short: corrupt, not padded with common._open_zip(path) as group, pytest.raises(ValueError, match="reshape"): - group["s"][:] + group.get_array("s")[:] def test_atera_reads_3d_morphology_with_default_kwargs(atera_path: Path) -> None: @@ -581,7 +590,7 @@ def test_duplicate_feature_names_are_made_unique_everywhere(atera_path: Path, tm assert transcripts["feature_name"].tolist() == ["GENE_A"] * 3 + ["GENE_A-1"] * 3 sdata.write(tmp_path / "out.zarr") genes = read_table_for_genes(atera_path, ["GENE_A-1", "GENE_A"]) - np.testing.assert_array_equal(genes.X.toarray(), np.arange(8).reshape(4, 2)[:, ::-1]) + np.testing.assert_array_equal(cast(csr_matrix, genes.X).toarray(), np.arange(8).reshape(4, 2)[:, ::-1]) second = read_transcripts_for_genes(sdata, atera_path, "GENE_A-1").points["transcripts_subset"].compute() assert second["x"].tolist() == [3.0, 4.0, 5.0] table = pq.read_table(atera_to_parquet(atera_path, tmp_path / "t.parquet")) @@ -589,8 +598,6 @@ def test_duplicate_feature_names_are_made_unique_everywhere(atera_path: Path, tm def test_empty_selections(atera_path: Path) -> None: - from spatialdata import SpatialData - from spatialdata_io import atera from spatialdata_io.readers.atera import read_cell_boundaries, read_transcripts_for_genes @@ -619,7 +626,7 @@ def test_polygons_with_one_or_two_vertices_are_kept_empty( with caplog.at_level(logging.WARNING): shapes = common._polygons_to_shapes(coords, num_vertices, np.arange(5), None, 1.0, is_nucleus=False) assert shapes.geometry.is_empty.tolist() == [False, True, True, True, False] - np.testing.assert_array_equal(np.asarray(shapes.geometry.iloc[4].exterior.coords)[:4], coords[6:10]) + np.testing.assert_array_equal(shapely.get_coordinates(shapes.geometry.iloc[4])[:4], coords[6:10]) assert "2 polygon(s) with 1 or 2 vertices" in caplog.text From 50b95cf0f2bc6d47d57b727c763684c236247b62 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 23:00:11 +0100 Subject: [PATCH 26/29] fix(atera): build unique feature names from a numpy StringDType array On the 10x tiny sample, read_elem returns var/feature_name as a numpy StringDType array, which np.asarray(..., dtype=str) cannot cast, so atera() failed in _read_feature_names. --- src/spatialdata_io/readers/_atera_common.py | 2 +- tests/readers/test_atera.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index d643399d..795956dd 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -125,7 +125,7 @@ def _unique_feature_names(names: Any) -> pd.Index: """Feature names made unique as anndata does (``G``, ``G-1``), so they can index `var` and label categories.""" from anndata.utils import make_index_unique - names = pd.Index(np.asarray(names, dtype=str)) + names = pd.Index([str(name) for name in names]) # numpy can't cast its StringDType, which zarr may return if not names.is_unique: logger.warning(f"{int(names.duplicated().sum())} duplicate feature names; made unique with a suffix.") return make_index_unique(names) diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index 7d13c596..daeb8a5d 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -657,3 +657,8 @@ def test_table_and_its_region_follow_the_flags(atera_path: Path, kwargs: dict, r else: attrs = sdata.tables["table"].uns.get(TableModel.ATTRS_KEY, {}) assert attrs.get("region") == region + + +def test_unique_feature_names_from_a_numpy_string_array() -> None: + names = np.array(["G", "G", "NEG_1"], dtype=np.dtypes.StringDType()) # what read_elem gives on real bundles + assert common._unique_feature_names(names).tolist() == ["G", "G-1", "NEG_1"] From f1dc16ed9ff7681008ee6250ae9c0eaa17456e74 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Mon, 16 Nov 2026 23:34:34 +0100 Subject: [PATCH 27/29] fix(atera): put the non-gene features in a second table, table_controls The dense obsm['control_counts'] DataFrame took 10.58 GB on the Atera v1 WTA kidney (399,120 cells x 6,625 non-gene features), and anndata cannot write a pandas sparse DataFrame. atera() now returns table_controls next to table: a sparse CSR X with the stored dtype, the negative control probes and codewords and the unassigned codewords in var with their feature_types, and the same obs index, instance ids and annotated element as table. table keeps the Gene Expression features only, and read_table_for_cells returns those. Adapted to ec169cf: the gene table keeps the category colors and manifest extras in uns. --- src/spatialdata_io/readers/atera.py | 57 ++++++++++++++++------------- tests/readers/test_atera.py | 32 ++++++++++------ 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index df9671a4..8d8782d2 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -110,8 +110,9 @@ def atera( Path to the dataset. cells_table Whether to read the cell annotations in the `AnnData` table. As in `xenium`, `X` and `var` hold the Gene - Expression features; the counts of the other features (negative control probes and codewords, unassigned - codewords) are in ``obsm["control_counts"]``, a `DataFrame` with one column per feature. + Expression features. The counts of the other features (negative control probes and codewords, unassigned + codewords) go to a second table, ``table_controls``, with the same cells, `obs` index and annotation, + a sparse `X` and those features in `var`. cells_boundaries Whether to read cell boundaries (polygons). nucleus_boundaries @@ -172,7 +173,17 @@ def atera( # The table annotates `cell_labels` (instance ids `obs["label_id"]`, as in `xenium`), else `cell_boundaries` # (`obs["cell_id"]`), else nothing: the nucleus elements are numbered per nucleus, not per cell. region = "cell_labels" if cells_labels else "cell_boundaries" if cells_boundaries else None - table = _get_table(path, var_columns, table_row_chunk_size, region)[0] if cells_table else None + tables = ( + dict( + zip( + ["table", "table_controls"], + _get_table(path, var_columns, table_row_chunk_size, region)[:2], + strict=True, + ) + ) + if cells_table + else {} + ) shapes: dict[str, Any] = {} labels: dict[str, DataArray | DataTree] = {} @@ -218,7 +229,6 @@ def atera( if aligned_images: images |= _get_aligned_images(path, imread_kwargs, image_models_kwargs) - tables = {"table": table} if table is not None else {} sdata = SpatialData(images=images, labels=labels, points=points, tables=tables, shapes=shapes) sdata = _set_reader_metadata(sdata, "atera") sdata.attrs[str(AteraKeys.PIXEL_SIZE)] = pixel_size @@ -369,10 +379,7 @@ def _read_obs_var(group: zarr.Group) -> tuple[pd.DataFrame, pd.DataFrame]: return obs, _xenium_var(_read_dataframe(group, AteraKeys.VAR_GROUP)) -CONTROL_COUNTS_KEY = "control_counts" - - -def _assemble_table( +def _assemble_tables( x: Any, obs: pd.DataFrame, var: pd.DataFrame, @@ -380,23 +387,23 @@ def _assemble_table( var_columns: list[str] | Literal["all"], region: str | None, obsm: Mapping[str, Any] = MappingProxyType({}), -) -> AnnData: - """X and var hold the Gene Expression features, ``obsm["control_counts"]`` the counts of all other features. +) -> tuple[AnnData, AnnData]: + """The Gene Expression table and the ``table_controls`` table of all other features, for the same cells. - The other features are the negative control probes and codewords and the unassigned codewords, one named - column each, so that `X` and ``n_genes`` count genes only. `var_columns` name the store's columns. + The other features are the negative control probes and codewords and the unassigned codewords, so `X` and + ``n_genes`` of the first table count genes only. The second has a sparse `X` (CSR, as stored), the same `obs` + index and instance id columns, and annotates the same element. `var_columns` name the store's columns. """ from anndata import AnnData is_gene = _is_gene(var) renamed = {str(AteraKeys.VAR_FEATURE_TYPE): "feature_types"} - columns = var.columns if var_columns == "all" else [renamed.get(c, c) for c in var_columns] + ["gene_ids"] - obsm = { - **obsm, - CONTROL_COUNTS_KEY: pd.DataFrame(controls.toarray(), index=obs.index, columns=var.index[~is_gene]), - "spatial": obs[[str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)]].to_numpy(), - } - return _parse_table(AnnData(X=x, obs=obs, var=var.loc[is_gene, list(columns)], obsm=obsm), region) + columns = list(var.columns if var_columns == "all" else [renamed.get(c, c) for c in var_columns] + ["gene_ids"]) + obsm = {**obsm, "spatial": obs[[str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)]].to_numpy()} + table = _parse_table(AnnData(X=x, obs=obs, var=var.loc[is_gene, columns], obsm=obsm), region) + ids = obs[[str(AteraKeys.CELL_ID), str(AteraKeys.LABEL_ID)]].copy() + controls_table = AnnData(X=csr_matrix(controls), obs=ids, var=var.loc[~is_gene, columns]) + return table, _parse_table(controls_table, region) def _parse_table(adata: AnnData, region: str | None) -> AnnData: @@ -416,8 +423,8 @@ def _get_table( var_columns: list[str] | Literal["all"] = DEFAULT_VAR_COLUMNS, row_chunk_size: int = DEFAULT_TABLE_ROW_CHUNK_SIZE, region: str | None = "cell_labels", -) -> tuple[AnnData, list[str]]: - """Read ``cell_feature_matrix.zarr.zip`` into an AnnData table using native `anndata` zarr IO. +) -> tuple[AnnData, AnnData, list[str]]: + """Read ``cell_feature_matrix.zarr.zip`` into the gene table and the ``table_controls`` table using native `anndata` zarr IO. `obs`/`var`/`obsm` are read with `anndata.io.read_elem` (which also means `obs`/`var` come back indexed by whichever column the store's ``_index`` attr names -- e.g. `barcode`/`feature_id`). @@ -453,11 +460,11 @@ def _get_table( controls = cast(csr_matrix, x[:, ~is_gene].compute()) # genes detected per cell: the stored entries of each CSR row (its `indptr` step) minus those of the controls obs["n_genes"] = stored_per_row - controls.getnnz(axis=1) - table = _assemble_table(x[:, is_gene], obs, var, controls, var_columns, region, obsm) + table, controls_table = _assemble_tables(x[:, is_gene], obs, var, controls, var_columns, region, obsm) table.uns.update(colors) if manifest_extras is not None: table.uns[str(AteraKeys.UNS_MANIFEST)] = manifest_extras - return table, var.index.tolist() + return table, controls_table, var.index.tolist() def read_var(path: str | Path) -> pd.DataFrame: @@ -512,7 +519,7 @@ def read_table_for_cells( Returns ------- - An `AnnData` `TableModel` with only the requested cells. + An `AnnData` `TableModel` with only the requested cells and the Gene Expression features. """ if (cell_ids is None) == (bbox is None): raise ValueError("Exactly one of `cell_ids`/`bbox` must be given.") @@ -533,7 +540,7 @@ def read_table_for_cells( obs = obs.iloc[row_positions].copy() is_gene = _is_gene(var) obs["n_genes"] = x[:, is_gene].getnnz(axis=1) - return _assemble_table(x[:, is_gene], obs, var, x[:, ~is_gene], var_columns, "cell_labels") + return _assemble_tables(x[:, is_gene], obs, var, x[:, ~is_gene], var_columns, "cell_labels")[0] def read_table_for_genes(path: str | Path, genes: str | Sequence[str]) -> AnnData: diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index daeb8a5d..d685763e 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -145,7 +145,7 @@ def _write_cells_transcripts_and_images(path: Path, n: int) -> None: def test_cell_labels_table_uses_label_id(atera_path: Path) -> None: - table, _ = _get_table(atera_path, region="cell_labels") + table = _get_table(atera_path, region="cell_labels")[0] region, _, instance_key = get_table_keys(table) assert region == "cell_labels" @@ -157,19 +157,27 @@ def test_cell_labels_table_uses_label_id(atera_path: Path) -> None: assert table.obs["n_genes"].tolist() == [1, 2, 2, 2] -@pytest.mark.parametrize("table", ["_get_table", "read_table_for_cells"]) -def test_x_holds_genes_and_obsm_the_controls(atera_path: Path, table: str) -> None: - adata = _get_table(atera_path)[0] if table == "_get_table" else read_table_for_cells(atera_path, cell_ids=CELL_IDS) +@pytest.mark.parametrize("region", ["cell_labels", "cell_boundaries"]) +def test_x_holds_genes_and_table_controls_the_controls(atera_path: Path, region: str) -> None: + from scipy.sparse import issparse + + adata, controls, _ = _get_table(atera_path, region=region) assert adata.var_names.tolist() == ["GENE_A", "GENE_B"] assert adata.obs["n_genes"].tolist() == [1, 2, 2, 2] - controls = adata.obsm["control_counts"] - assert isinstance(controls, pd.DataFrame) - assert controls.columns.tolist() == ["NEG_1"] - assert controls["NEG_1"].tolist() == NEG_COUNTS + assert read_table_for_cells(atera_path, cell_ids=CELL_IDS).var_names.tolist() == ["GENE_A", "GENE_B"] + assert issparse(controls.X) + assert controls.X.dtype == adata.X.dtype # as stored + assert cast(csr_matrix, controls.X).toarray()[:, 0].tolist() == NEG_COUNTS + assert controls.var_names.tolist() == ["NEG_1"] + assert controls.var["feature_types"].tolist() == ["Negative Control Probe"] + assert controls.obs_names.equals(adata.obs_names) + assert get_table_keys(controls) == get_table_keys(adata) + _, _, instance_key = get_table_keys(adata) + assert controls.obs[instance_key].tolist() == adata.obs[instance_key].tolist() def test_cell_boundaries_table_uses_cell_id(atera_path: Path) -> None: - table, _ = _get_table(atera_path, region="cell_boundaries") + table = _get_table(atera_path, region="cell_boundaries")[0] assert get_table_keys(table) == ("cell_boundaries", "region", "cell_id") @@ -456,9 +464,9 @@ def test_atera_reads_writes_and_queries(atera_path: Path, tmp_path: Path) -> Non back = read_zarr(tmp_path / "out.zarr") table, table_back = sdata.tables["table"], back.tables["table"] np.testing.assert_array_equal(cast(csr_matrix, table_back.X).toarray(), cast(Any, table.X).compute().toarray()) - pd.testing.assert_frame_equal( - cast(pd.DataFrame, table_back.obsm["control_counts"]), cast(pd.DataFrame, table.obsm["control_counts"]) - ) + controls, controls_back = sdata.tables["table_controls"], back.tables["table_controls"] + np.testing.assert_array_equal(cast(csr_matrix, controls_back.X).toarray(), cast(csr_matrix, controls.X).toarray()) + assert controls_back.var_names.tolist() == ["NEG_1"] crop = cast( SpatialData, bounding_box_query( From a11e05303aa23e2190667db95e04f329759805dc Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 20 Nov 2026 18:01:45 +0100 Subject: [PATCH 28/29] feat(atera writer): write table + table_controls back as one cell_feature_matrix atera() splits the features into table (Gene Expression) and table_controls, with var in the xenium layout. The writer now appends table_controls (same cells in the same order, checked) and maps feature_types/gene_ids/the symbol index back to feature_type/feature_id/feature_name. Features are written genes first, then controls, which is the 10x tiny Atera bundle's own order (53 Gene Expression features first). A single table in the bundle's var layout is still written as it is. The reader keys the var category colors by the column it names (feature_types_colors) and puts them on both tables. --- src/spatialdata_io/readers/atera.py | 24 +++++-- src/spatialdata_io/writers/atera.py | 100 ++++++++++++++++++++++++---- tests/readers/test_atera_writer.py | 65 ++++++++++++++++-- 3 files changed, 164 insertions(+), 25 deletions(-) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 8d8782d2..9650389a 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -352,6 +352,10 @@ def _add_label_id(obs_df: pd.DataFrame) -> None: obs_df[str(AteraKeys.LABEL_ID)] = np.arange(1, len(obs_df) + 1, dtype=np.uint32) +# store `var` columns that the `xenium` layout renames +_XENIUM_VAR_NAMES = {str(AteraKeys.VAR_FEATURE_TYPE): "feature_types"} + + def _xenium_var(var: pd.DataFrame) -> pd.DataFrame: """Lay out the store's `var` as the `xenium` reader does. @@ -359,7 +363,7 @@ def _xenium_var(var: pd.DataFrame) -> pd.DataFrame: ``feature_types`` (the bundles index by ``feature_id`` and name the type ``feature_type``). The type values are already the Xenium ones (``Gene Expression``, ``Negative Control Probe``, ...). """ - out = var.rename(columns={str(AteraKeys.VAR_FEATURE_TYPE): "feature_types"}) + out = var.rename(columns=_XENIUM_VAR_NAMES) out["gene_ids"] = var.index.astype(str).to_numpy() out.index = _unique_feature_names(var[str(AteraKeys.FEATURE_NAME)]) return out @@ -397,8 +401,9 @@ def _assemble_tables( from anndata import AnnData is_gene = _is_gene(var) - renamed = {str(AteraKeys.VAR_FEATURE_TYPE): "feature_types"} - columns = list(var.columns if var_columns == "all" else [renamed.get(c, c) for c in var_columns] + ["gene_ids"]) + columns = list( + var.columns if var_columns == "all" else [_XENIUM_VAR_NAMES.get(c, c) for c in var_columns] + ["gene_ids"] + ) obsm = {**obsm, "spatial": obs[[str(AteraKeys.CENTROID_X), str(AteraKeys.CENTROID_Y)]].to_numpy()} table = _parse_table(AnnData(X=x, obs=obs, var=var.loc[is_gene, columns], obsm=obsm), region) ids = obs[[str(AteraKeys.CELL_ID), str(AteraKeys.LABEL_ID)]].copy() @@ -438,9 +443,13 @@ def _get_table( with _open_zip(path / AteraKeys.CELL_FEATURE_MATRIX_FILE) as group: obs, var = _read_obs_var(group) colors = _read_category_colors(group, AteraKeys.OBS_GROUP) - colors.update( - _read_category_colors(group, AteraKeys.VAR_GROUP, None if var_columns == "all" else list(var_columns)) - ) + # keyed by the column's name in `var` (`feature_types_colors`), as scanpy looks them up + var_colors = { + f"{_XENIUM_VAR_NAMES.get(k.removesuffix('_colors'), k.removesuffix('_colors'))}_colors": v + for k, v in _read_category_colors( + group, AteraKeys.VAR_GROUP, None if var_columns == "all" else list(var_columns) + ).items() + } obsm = ( cast(dict[str, Any], read_elem(group[str(AteraKeys.OBSM_GROUP)])) if str(AteraKeys.OBSM_GROUP) in group @@ -461,7 +470,8 @@ def _get_table( # genes detected per cell: the stored entries of each CSR row (its `indptr` step) minus those of the controls obs["n_genes"] = stored_per_row - controls.getnnz(axis=1) table, controls_table = _assemble_tables(x[:, is_gene], obs, var, controls, var_columns, region, obsm) - table.uns.update(colors) + table.uns.update({**colors, **var_colors}) + controls_table.uns.update(var_colors) if manifest_extras is not None: table.uns[str(AteraKeys.UNS_MANIFEST)] = manifest_extras return table, controls_table, var.index.tolist() diff --git a/src/spatialdata_io/writers/atera.py b/src/spatialdata_io/writers/atera.py index cc23c4d6..bad8c85d 100644 --- a/src/spatialdata_io/writers/atera.py +++ b/src/spatialdata_io/writers/atera.py @@ -52,6 +52,7 @@ def write_atera_cell_feature_matrix( path: str | Path, *, table_name: str = "table", + controls_table: str | AnnData | None = "table_controls", x_layer: str | None = None, major_axis: Literal["cell", "feature"] = "cell", chunk_size: int = 1_000_000, @@ -63,6 +64,14 @@ def write_atera_cell_feature_matrix( The table's `obs`/`var` columns are written as they are (names, order, dtypes), except for the columns `atera` adds on read (``label_id``, ``n_genes``, ``region``) and ``obsm["spatial"]``. + `atera` reads the Gene Expression features into ``table`` and the other features (negative control probes + and codewords, unassigned codewords) into ``table_controls``, with `var` in the `xenium` layout (symbols as + the index, ``gene_ids``, ``feature_types``). Both are written back into one matrix with the bundle's `var` + columns (``feature_name``, ``feature_id``, ``feature_type``): the genes first, then the controls, each in + its table's order. That is the bundles' own order where we checked it (the 10x tiny Atera bundle lists its + 53 Gene Expression features first); a bundle that interleaves the two is written in a different feature order. + A single table with a ``feature_type`` column (the bundle's own `var` layout) is written as it is. + Parameters ---------- sdata @@ -74,6 +83,10 @@ def write_atera_cell_feature_matrix( ``cell_feature_matrix.zarr.zip``). table_name Which table of `sdata` to write. + controls_table + The table of the non-gene features to append to `table_name`'s (same cells, in the same order): its name + in `sdata` (skipped if `sdata` has no such table, or is an `AnnData`), or the `AnnData` itself. `None` + writes `table_name` alone. x_layer Write ``table.layers[x_layer]`` as ``X`` instead of ``table.X``. The bundle's ``X`` is raw integer counts, so use this (e.g. ``x_layer="counts"``) when ``table.X`` has been normalised in place. @@ -96,6 +109,7 @@ def write_atera_cell_feature_matrix( if major_axis not in ("cell", "feature"): raise ValueError(f"`major_axis` must be 'cell' or 'feature', got {major_axis!r}.") table = sdata.tables[table_name] if isinstance(sdata, SpatialData) else sdata + controls = _controls_table(sdata, controls_table) path = Path(path) if path.suffix != ".zip": path = path / str(AteraKeys.CELL_FEATURE_MATRIX_FILE) @@ -104,9 +118,11 @@ def write_atera_cell_feature_matrix( if x_layer is not None and x_layer not in table.layers: raise KeyError(f"`x_layer={x_layer!r}` is not in the table's layers ({list(table.layers)}).") x: Any = table.X if x_layer is None else table.layers[x_layer] + if controls is not None: + x = _append_controls(table, controls, x) obs, obs_colors = _prepare_obs(table) - var, var_colors = _prepare_var(table) - n_obs, n_var = table.shape + var, var_colors = _prepare_var(table, controls) + n_obs, n_var = table.n_obs, len(var) if tuple(x.shape) != (n_obs, n_var): # pragma: no cover - AnnData enforces this raise ValueError("`X` shape does not match `obs`/`var`.") @@ -130,12 +146,14 @@ def write_atera_cell_feature_matrix( ) obsm = _embeddings(table.obsm, n_obs, "obsm") obsm_manifest = _write_embeddings(root, str(AteraKeys.OBSM_GROUP), obsm, chunk_size, compressor) - varm = _embeddings(table.varm, n_var, "varm") + varm = _embeddings(_varm(table, controls), n_var, "varm") varm_manifest = _write_embeddings(root, str(AteraKeys.VARM_GROUP), varm, chunk_size, compressor) _write_x(root, zf, x, (n_obs, n_var), major_axis, chunk_size, row_chunk_size, n_threads, compressor) - manifest = _build_manifest(table, obs_manifest, var_manifest, obsm_manifest, varm_manifest, var, major_axis) + manifest = _build_manifest( + table, n_var, obs_manifest, var_manifest, obsm_manifest, varm_manifest, var, major_axis + ) root.attrs.update(manifest) for key, buffer in entries.items(): zf.writestr(key, buffer.to_bytes()) @@ -196,11 +214,15 @@ def _clean_columns(df: pd.DataFrame) -> pd.DataFrame: return pd.DataFrame(columns) -def _colors(table: AnnData, df: pd.DataFrame) -> dict[str, np.ndarray]: +def _colors(table: AnnData, df: pd.DataFrame, sources: dict[str, str] | None = None) -> dict[str, np.ndarray]: colors = {} for column in df.columns: if isinstance(df[column].dtype, pd.CategoricalDtype): - found = _colors_for(table, str(column), len(df[column].cat.categories)) + k = len(df[column].cat.categories) + source = (sources or {}).get(str(column)) + found = _colors_for(table, source, k) if source is not None else None + if found is None: + found = _colors_for(table, str(column), k) if found is not None: colors[str(column)] = found return colors @@ -218,18 +240,71 @@ def _prepare_obs(table: AnnData) -> tuple[pd.DataFrame, dict[str, np.ndarray]]: return df, _colors(table, df) -def _prepare_var(table: AnnData) -> tuple[pd.DataFrame, dict[str, np.ndarray]]: - var = cast(pd.DataFrame, table.var).reset_index(drop=True) +def _controls_table(sdata: SpatialData | AnnData, controls_table: str | AnnData | None) -> AnnData | None: + if controls_table is None or isinstance(controls_table, str): + if isinstance(sdata, SpatialData) and controls_table is not None and controls_table in sdata.tables: + return sdata.tables[controls_table] + return None + return controls_table + + +def _append_controls(table: AnnData, controls: AnnData, x: Any) -> Any: + """`x` with the controls' `X` appended as columns, after checking both tables hold the same cells in order.""" + if controls.n_obs != table.n_obs or not controls.obs_names.equals(table.obs_names): + raise ValueError("`table_controls` must hold the same cells as the table, in the same order (`obs_names`).") + cid = str(AteraKeys.CELL_ID) + if cid in controls.obs.columns and cid in table.obs.columns: + if not np.array_equal(controls.obs[cid].to_numpy(), table.obs[cid].to_numpy()): + raise ValueError(f"`table_controls` and the table disagree on obs[{cid!r}].") + c: Any = controls.X + if isinstance(x, da.Array): + c = ( + c + if isinstance(c, da.Array) + else da.from_array(sparse.csr_matrix(c), chunks=(x.chunks[0], -1), asarray=False) + ) + return da.concatenate([x, c.rechunk({0: x.chunks[0]})], axis=1) + if isinstance(c, da.Array): + c = c.compute() + return sparse.hstack([sparse.csr_matrix(x), sparse.csr_matrix(c)], format="csr") + + +# `var` columns of the `xenium` layout that `atera` reads (symbol index) -> the bundle's own column +_BUNDLE_VAR_NAMES = {"feature_types": str(AteraKeys.VAR_FEATURE_TYPE), "gene_ids": str(AteraKeys.FEATURE_ID)} + + +def _varm(table: AnnData, controls: AnnData | None) -> dict[str, Any]: + if controls is None: + return dict(table.varm) + if set(table.varm) != set(controls.varm): + raise ValueError("The table and `table_controls` must have the same `varm` keys.") + return {k: np.concatenate([np.asarray(table.varm[k]), np.asarray(controls.varm[k])]) for k in table.varm} + + +def _prepare_var(table: AnnData, controls: AnnData | None = None) -> tuple[pd.DataFrame, dict[str, np.ndarray]]: + frames = [cast(pd.DataFrame, t.var) for t in (table, controls) if t is not None] + var = pd.concat(frames) if len(frames) > 1 else frames[0] + index = var.index.astype(str) + var = var.reset_index(drop=True) + sources = {} # written column -> the table's column, whose `uns[f"{source}_colors"]` holds its colors + for column, bundle in _BUNDLE_VAR_NAMES.items(): + if column in var.columns: + var = var.drop(columns=[bundle], errors="ignore").rename(columns={column: bundle}) + sources[bundle] = column + if str(AteraKeys.FEATURE_NAME) not in var.columns and "feature_types" in sources.values(): + var.insert(0, str(AteraKeys.FEATURE_NAME), index) # the `xenium` layout's symbols if "feature_id" not in var.columns: - var.insert(0, "feature_id", table.var.index.astype(str)) + var.insert(0, "feature_id", index) for required in (str(AteraKeys.FEATURE_NAME), str(AteraKeys.VAR_FEATURE_TYPE)): if required not in var.columns: - raise ValueError(f"The table's var must have a {required!r} column.") + raise ValueError( + f"The table's var must have a {required!r} column (or `feature_types`, as `atera` reads it)." + ) df = _clean_columns(var) for name in (str(AteraKeys.VAR_FEATURE_TYPE), str(AteraKeys.VAR_GENOME)): if name in df.columns and not isinstance(df[name].dtype, pd.CategoricalDtype): df[name] = df[name].astype("category") - return df, _colors(table, df) + return df, _colors(table, df, sources) # -------------------------------------------------------------------------------------------------- @@ -537,6 +612,7 @@ def _drain_one(self) -> None: def _build_manifest( table: AnnData, + n_var: int, obs: list[dict[str, Any]], var: list[dict[str, Any]], obsm: list[dict[str, Any]], @@ -551,7 +627,7 @@ def _build_manifest( str(AteraKeys.MANIFEST_VERSION): dict(_SCHEMA_VERSION), str(AteraKeys.MANIFEST_MATRIX_PROPERTIES): { "num_cells": int(table.n_obs), - "num_features": int(table.n_vars), + "num_features": int(n_var), "major_axis": major_axis, "feature_types": feature_types, }, diff --git a/tests/readers/test_atera_writer.py b/tests/readers/test_atera_writer.py index f2b035ef..8327e27c 100644 --- a/tests/readers/test_atera_writer.py +++ b/tests/readers/test_atera_writer.py @@ -107,8 +107,10 @@ def test_roundtrip(tmp_path: Path, x_kind: str, chunk_size: int) -> None: path = write_atera_cell_feature_matrix(table, tmp_path, chunk_size=chunk_size, row_chunk_size=4) assert path == tmp_path / "cell_feature_matrix.zarr.zip" - back, feature_names = _get_table(tmp_path, var_columns="all") - assert np.array_equal(back.X.compute().toarray(), expected_x) + back, back_controls, feature_names = _get_table(tmp_path, var_columns="all") + # `atera` splits the features: Gene Expression in the table, the Negative Control Probe in `table_controls` + assert np.array_equal(back.X.compute().toarray(), expected_x[:, :-1]) + assert np.array_equal(back_controls.X.toarray(), expected_x[:, -1:]) assert feature_names == [f"G{i}" for i in range(N_VAR)] assert back.obs["cell_id"].tolist() == CELL_IDS.tolist() assert back.obs["label_id"].tolist() == list(range(1, N_OBS + 1)) @@ -118,7 +120,8 @@ def test_roundtrip(tmp_path: Path, x_kind: str, chunk_size: int) -> None: assert np.allclose(back.obs["cell_area"], table.obs["cell_area"]) assert np.allclose(back.obsm["spatial"], table.obsm["spatial"]) assert np.allclose(back.obsm["X_umap"], table.obsm["X_umap"]) - assert back.var["feature_type"].astype(str).tolist() == table.var["feature_type"].tolist() + feature_types = [*back.var["feature_types"].astype(str), *back_controls.var["feature_types"].astype(str)] + assert feature_types == table.var["feature_type"].tolist() assert "de_leiden_res_1.0_c0_logfc" in read_var(tmp_path).columns subset = read_table_for_cells(tmp_path, cell_ids=CELL_IDS[[3, 1]]) assert subset.obs["cell_id"].tolist() == CELL_IDS[[3, 1]].tolist() @@ -234,13 +237,13 @@ def test_x_layer_writes_raw_counts_when_x_is_normalised(tmp_path: Path) -> None: write_atera_cell_feature_matrix(table, tmp_path, x_layer="nope") write_atera_cell_feature_matrix(table, tmp_path, x_layer="counts") - back, _ = _get_table(tmp_path, var_columns="all") - assert np.array_equal(back.X.compute().toarray(), counts.toarray()) + back, _, _ = _get_table(tmp_path, var_columns="all") + assert np.array_equal(back.X.compute().toarray(), counts.toarray()[:, :-1]) def test_colors_survive_a_roundtrip(tmp_path: Path) -> None: write_atera_cell_feature_matrix(_make_table(), tmp_path) - back, _ = _get_table(tmp_path, var_columns="all") + back, _, _ = _get_table(tmp_path, var_columns="all") assert back.uns["leiden_res_1.0_colors"] == ["#ff0000ff", "#00ff00ff", "#0000ffff"] assert "many_colors" not in back.uns @@ -250,3 +253,53 @@ def test_non_integer_x_is_rejected(tmp_path: Path) -> None: table.X = table.X.astype(np.float32) * 0.5 with pytest.raises(ValueError, match="integer counts"): write_atera_cell_feature_matrix(table, tmp_path) + + +def _interleaved_table() -> ad.AnnData: + """`_make_table` with the features interleaved, as a bundle could store them: G0, NEG, G1, CW, G2, G3.""" + table = _make_table() + types = ["Gene Expression", "Negative Control Probe", "Gene Expression", "Negative Control Codeword"] + table.var["feature_type"] = pd.Categorical(types + ["Gene Expression"] * 2) + table.uns["feature_type_colors"] = ["#111111", "#222222", "#333333"] + return table + + +@pytest.mark.parametrize("lazy", [True, False]) +def test_atera_tables_round_trip_genes_first_then_controls(tmp_path: Path, lazy: bool) -> None: + source = _interleaved_table() + write_atera_cell_feature_matrix(source, tmp_path / "a") + table, controls, _ = _get_table(tmp_path / "a", var_columns="all") + if not lazy: + table.X = table.X.compute() + write_atera_cell_feature_matrix(SpatialData(tables={"table": table, "table_controls": controls}), tmp_path / "b") + + counts = source.X.toarray() + genes, others = [0, 2, 4, 5], [1, 3] + root = _open(tmp_path / "b" / "cell_feature_matrix.zarr.zip") + assert dict(root.attrs)["matrix_properties"]["num_features"] == N_VAR + assert np.array_equal(sparse_dataset(root["X"])[:].toarray(), counts[:, genes + others]) + var = ad.io.read_elem(root["var"]) + assert var.index.tolist() == [f"ENS{i}" for i in genes + others] + assert var["feature_name"].tolist() == [f"G{i}" for i in genes + others] + assert ( + var["feature_type"].astype(str).tolist() == source.var["feature_type"].astype(str).iloc[genes + others].tolist() + ) + assert "gene_ids" not in var.columns and "feature_types" not in var.columns + + table_back, controls_back, _ = _get_table(tmp_path / "b", var_columns="all") + assert table_back.var_names.tolist() == table.var_names.tolist() + assert controls_back.var_names.tolist() == controls.var_names.tolist() + assert np.array_equal(table_back.X.compute().toarray(), counts[:, genes]) + assert np.array_equal(controls_back.X.toarray(), counts[:, others]) + assert table_back.obs["cell_id"].tolist() == CELL_IDS.tolist() + # the var colors are keyed by the column `atera` names `feature_types`, and survive + assert table.uns["feature_types_colors"] == ["#111111ff", "#222222ff", "#333333ff"] + assert table_back.uns["feature_types_colors"] == table.uns["feature_types_colors"] + + +def test_table_controls_with_other_cells_is_rejected(tmp_path: Path) -> None: + write_atera_cell_feature_matrix(_interleaved_table(), tmp_path / "a") + table, controls, _ = _get_table(tmp_path / "a", var_columns="all") + with pytest.raises(ValueError, match="same cells"): + write_atera_cell_feature_matrix(table, tmp_path / "b", controls_table=controls[::-1].copy()) + assert not (tmp_path / "b").exists() or not list((tmp_path / "b").iterdir()) From 71a2f7560a02ea0a7a69c65d1817b0b9f2dc4e05 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 20 Nov 2026 21:13:45 +0100 Subject: [PATCH 29/29] simplify --- src/spatialdata_io/readers/_atera_common.py | 15 +-- src/spatialdata_io/readers/atera.py | 53 ++++---- src/spatialdata_io/readers/xenium.py | 104 ++++++--------- src/spatialdata_io/writers/atera.py | 32 ++--- tests/readers/test_atera.py | 141 ++++++-------------- tests/test_xenium.py | 109 +++++++++++++++ 6 files changed, 231 insertions(+), 223 deletions(-) diff --git a/src/spatialdata_io/readers/_atera_common.py b/src/spatialdata_io/readers/_atera_common.py index 795956dd..9c17f044 100644 --- a/src/spatialdata_io/readers/_atera_common.py +++ b/src/spatialdata_io/readers/_atera_common.py @@ -30,7 +30,7 @@ from spatialdata_io.readers._utils._utils import parse_physical_size if TYPE_CHECKING: - from collections.abc import Iterator, Mapping + from collections.abc import Iterator, Mapping, Sequence from numpy.typing import ArrayLike from xarray import DataArray @@ -445,7 +445,7 @@ def _tile_bounds(tile_key: str, tile_size: float) -> tuple[float, float, float, return tile_x * tile_size, tile_y * tile_size, (tile_x + 1) * tile_size, (tile_y + 1) * tile_size -def _bbox_overlaps(boxes: ArrayLike, bbox: tuple[float, float, float, float]) -> np.ndarray: +def _bbox_overlaps(boxes: ArrayLike, bbox: Sequence[float]) -> np.ndarray: """Which of the ``(..., 4)`` (xmin, ymin, xmax, ymax) `boxes` overlap `bbox`.""" boxes = np.asarray(boxes) return (boxes[..., 0] < bbox[2]) & (bbox[0] < boxes[..., 2]) & (boxes[..., 1] < bbox[3]) & (bbox[1] < boxes[..., 3]) @@ -462,11 +462,7 @@ def _cell_row_positions_in_bbox(path: Path, bbox: tuple[float, float, float, flo pixel_size = _get_pixel_size(path) with _open_zip(path / AteraKeys.CELLS_FILE) as group: cell_bboxes = np.asarray(_get_array(group, str(AteraKeys.BBOXES))[...]).reshape(-1, 4) - return np.nonzero( - _bbox_overlaps( - cell_bboxes, (bbox[0] * pixel_size, bbox[1] * pixel_size, bbox[2] * pixel_size, bbox[3] * pixel_size) - ) - )[0] + return np.flatnonzero(_bbox_overlaps(cell_bboxes, [v * pixel_size for v in bbox])) def read_cell_boundaries( @@ -531,10 +527,7 @@ def read_cell_boundaries( cell_ids = _read_cell_ids(path) is_nucleus = mask_index == 0 - raw_bbox = None - if bbox is not None: - xmin, ymin, xmax, ymax = bbox - raw_bbox = (xmin * pixel_size, ymin * pixel_size, xmax * pixel_size, ymax * pixel_size) + raw_bbox = None if bbox is None else [v * pixel_size for v in bbox] with _open_zip(path / AteraKeys.CELLS_FILE) as group: cell_bboxes = np.asarray(_get_array(group, str(AteraKeys.BBOXES))[...]).reshape(-1, 4) diff --git a/src/spatialdata_io/readers/atera.py b/src/spatialdata_io/readers/atera.py index 9650389a..f8045313 100644 --- a/src/spatialdata_io/readers/atera.py +++ b/src/spatialdata_io/readers/atera.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import warnings from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Literal, cast @@ -35,13 +36,7 @@ read_transcripts_for_genes, ) from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata -from spatialdata_io.readers.xenium import ( - _affine_from_keypoints, - _aligned_image_files, - _get_labels, - _keypoints_file, - _read_keypoints, -) +from spatialdata_io.readers.xenium import _affine_from_keypoints, _get_labels if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -63,8 +58,6 @@ # `anndata`'s own zarr reading functions (`anndata.io.read_elem`/`anndata.io.sparse_dataset`). This # only works on bundles whose zarr stores carry the standard AnnData `encoding-type`/ # `encoding-version`/`shape` attrs. -# -# The tables are built here; the labels, shapes, points and morphology images in `_atera_common.py`. @inject_docs(xx=AteraKeys) @@ -173,17 +166,9 @@ def atera( # The table annotates `cell_labels` (instance ids `obs["label_id"]`, as in `xenium`), else `cell_boundaries` # (`obs["cell_id"]`), else nothing: the nucleus elements are numbered per nucleus, not per cell. region = "cell_labels" if cells_labels else "cell_boundaries" if cells_boundaries else None - tables = ( - dict( - zip( - ["table", "table_controls"], - _get_table(path, var_columns, table_row_chunk_size, region)[:2], - strict=True, - ) - ) - if cells_table - else {} - ) + tables: dict[str, AnnData] = {} + if cells_table: + tables["table"], tables["table_controls"], _ = _get_table(path, var_columns, table_row_chunk_size, region) shapes: dict[str, Any] = {} labels: dict[str, DataArray | DataTree] = {} @@ -246,16 +231,21 @@ def _get_aligned_images( Each is placed by its alignment CSV or, for an H&E without one, by an affine fit to its keypoints (as in `xenium`). """ images = {} - for name, file, alignment_file in _aligned_image_files(path): + for file in sorted(path.glob("*.ome.tif")): + name = next((n for n in ["he_image", "if_image"] if file.name.endswith(f"{n}.ome.tif")), None) + if name is None or file.name.startswith("._"): # macOS sidecars + continue + alignment_file = file.with_name(file.name.replace(".ome.tif", "alignment.csv")) + kp_file = file.with_name(file.name.removesuffix("he_image.ome.tif") + "keypoints.csv") image, dims = _tiled_imread(file, imread_kwargs) transformation: BaseTransformation = Identity() - if alignment_file is not None: + if alignment_file.is_file(): affine = pd.read_csv(alignment_file, header=None).to_numpy(dtype=float) transformation = Affine(affine, input_axes=("x", "y"), output_axes=("x", "y")) if name == "he_image": - _check_keypoints(alignment_file, _keypoints_file(file)) + _check_keypoints(alignment_file, kp_file) elif name == "he_image": - transformation = _affine_from_keypoints(_keypoints_file(file)) or transformation + transformation = _affine_from_keypoints(kp_file) or transformation if "c" not in dims: image, dims = image[None], ("c", *dims) n_channels = image.shape[dims.index("c")] @@ -271,11 +261,20 @@ def _get_aligned_images( def _check_keypoints(alignment_file: Path, kp_file: Path) -> float | None: """Warn when the alignment affine does not map the keypoints onto each other; return the median residual (px).""" - keypoints = _read_keypoints(kp_file) - if keypoints is None: + if not kp_file.is_file(): + return None + columns = ["fixedX", "fixedY", "alignmentX", "alignmentY"] + kp = pd.read_csv(kp_file) + if not set(columns) <= set(kp.columns): + warnings.warn(f"{kp_file.name}: expected columns {columns}, found {list(kp.columns)}.", stacklevel=2) + return None + xy = kp[columns].apply(pd.to_numeric, errors="coerce").to_numpy(dtype=float) + xy = xy[np.isfinite(xy).all(axis=1)] + if len(xy) < 3: + warnings.warn(f"{kp_file.name}: {len(xy)} finite keypoints, at least 3 are needed.", stacklevel=2) return None - src, dst = keypoints affine = pd.read_csv(alignment_file, header=None).to_numpy(dtype=float) + src, dst = xy[:, 2:], xy[:, :2] median = float(np.median(np.linalg.norm((np.c_[src, np.ones(len(src))] @ affine.T)[:, :2] - dst, axis=1))) if median > KEYPOINTS_MAX_RESIDUAL_PX: logger.warning( diff --git a/src/spatialdata_io/readers/xenium.py b/src/spatialdata_io/readers/xenium.py index 19e74a19..c0daf024 100644 --- a/src/spatialdata_io/readers/xenium.py +++ b/src/spatialdata_io/readers/xenium.py @@ -1073,30 +1073,6 @@ def filter(self, record: logging.LogRecord) -> bool: return result -def _aligned_image_files(path: Path) -> list[tuple[str, Path, Path | None]]: - """The aligned images in `path` (``*he_image.ome.tif``, ``*if_image.ome.tif``), each with its alignment CSV or None. - - macOS ``._`` sidecars are skipped. - """ - found = [] - for file in sorted(path.glob("*.ome.tif")): - for suffix in [XeniumKeys.ALIGNED_HE_IMAGE_SUFFIX, XeniumKeys.ALIGNED_IF_IMAGE_SUFFIX]: - if file.name.endswith(suffix) and not file.name.startswith("._"): - csv = file.with_name( - file.name.replace( - XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_REMOVE, XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_ADD - ) - ) - found.append( - ( - suffix.replace(XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_REMOVE, ""), - file, - csv if csv.is_file() else None, - ) - ) - return found - - def _add_aligned_images( path: Path, imread_kwargs: Mapping[str, Any] = MappingProxyType({}), @@ -1104,58 +1080,59 @@ def _add_aligned_images( ) -> dict[str, DataTree]: """Discover and parse aligned images.""" images = {} - for element_name, file, alignment_file in _aligned_image_files(path): - image = xenium_aligned_image(file, alignment_file, imread_kwargs, image_models_kwargs) - if alignment_file is None and element_name == "he_image": - # bundles with keypoints but no alignment CSV (the Atera previews): 10x's alignment CSV is the - # least-squares fit to the keypoints, so an affine fit to them places the image - affine = _affine_from_keypoints(_keypoints_file(file)) - if affine is not None: - set_transformation(image, affine, "global") - images[element_name] = image + ome_tif_files = list(path.glob("*.ome.tif")) + csv_files = list(path.glob("*.csv")) + for file in ome_tif_files: + if file.name.startswith("._"): # macOS sidecar files + continue + element_name = None + for suffix in [ + XeniumKeys.ALIGNED_HE_IMAGE_SUFFIX, + XeniumKeys.ALIGNED_IF_IMAGE_SUFFIX, + ]: + if file.name.endswith(suffix): + element_name = suffix.replace(XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_REMOVE, "") + break + if element_name is not None: + # check if an alignment file exists + expected_filename = file.name.replace( + XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_REMOVE, + XeniumKeys.ALIGNMENT_FILE_SUFFIX_TO_ADD, + ) + alignment_files = [f for f in csv_files if f.name == expected_filename] + assert len(alignment_files) <= 1, f"Found more than one alignment file for {file.name}." + alignment_file = alignment_files[0] if alignment_files else None + + # parse the image + image = xenium_aligned_image(file, alignment_file, imread_kwargs, image_models_kwargs) + if alignment_file is None and element_name == "he_image": + keypoints = file.with_name(file.name.removesuffix(XeniumKeys.ALIGNED_HE_IMAGE_SUFFIX) + "keypoints.csv") + affine = _affine_from_keypoints(keypoints) + if affine is not None: + set_transformation(image, affine, "global") + images[element_name] = image return images -KEYPOINT_COLUMNS = ["fixedX", "fixedY", "alignmentX", "alignmentY"] - - -def _keypoints_file(he_image: Path) -> Path: - """Xenium Explorer's ``_keypoints.csv`` for ``_he_image.ome.tif``.""" - return he_image.with_name(he_image.name.removesuffix(XeniumKeys.ALIGNED_HE_IMAGE_SUFFIX) + "keypoints.csv") - - -def _read_keypoints(kp_file: Path) -> tuple[np.ndarray, np.ndarray] | None: - """The finite keypoint pairs (aligned image px, morphology px) in `kp_file`, or None if it is missing or unusable. +def _affine_from_keypoints(kp_file: Path) -> Affine | None: + """Least-squares affine from Xenium Explorer keypoints (``alignmentX/Y`` -> ``fixedX/Y``), or None with a warning. - Xenium Explorer writes ``fixedX, fixedY`` (morphology image) and ``alignmentX, alignmentY`` (aligned image). A file - without these columns or with fewer than 3 finite rows is reported with a warning. + The alignment CSV that Xenium Explorer writes is this fit. A full affine, because a similarity fit leaves about 22 px + of error on scans with 0.2% anisotropy. """ if not kp_file.is_file(): return None + columns = ["fixedX", "fixedY", "alignmentX", "alignmentY"] kp = pd.read_csv(kp_file) - if not set(KEYPOINT_COLUMNS) <= set(kp.columns): - warnings.warn(f"{kp_file.name}: expected columns {KEYPOINT_COLUMNS}, found {list(kp.columns)}.", stacklevel=2) + if not set(columns) <= set(kp.columns): + warnings.warn(f"{kp_file.name}: expected columns {columns}, found {list(kp.columns)}.", stacklevel=2) return None - xy = kp[KEYPOINT_COLUMNS].apply(pd.to_numeric, errors="coerce").to_numpy(dtype=float) + xy = kp[columns].apply(pd.to_numeric, errors="coerce").to_numpy(dtype=float) xy = xy[np.isfinite(xy).all(axis=1)] if len(xy) < 3: warnings.warn(f"{kp_file.name}: {len(xy)} finite keypoints, at least 3 are needed.", stacklevel=2) return None - return xy[:, 2:], xy[:, :2] - - -def _affine_from_keypoints(kp_file: Path) -> Affine | None: - """Fit the least-squares affine from the keypoints' aligned-image px onto their morphology px. - - ``alignmentX/Y`` -> ``fixedX/Y``. A full affine rather than a similarity: the Atera preview scans carry about 0.2% - anisotropy, which leaves 22 px residuals under a similarity fit and under 8 px under an affine (the Atera preview - bundles). None, with a warning, if the keypoints are unusable or collinear; the image then stays unaligned. - """ - keypoints = _read_keypoints(kp_file) - if keypoints is None: - return None - src, dst = keypoints - coef, _, rank, _ = np.linalg.lstsq(np.c_[src, np.ones(len(src))], dst, rcond=None) + coef, _, rank, _ = np.linalg.lstsq(np.c_[xy[:, 2:], np.ones(len(xy))], xy[:, :2], rcond=None) if rank < 3: warnings.warn(f"{kp_file.name}: the keypoints are collinear, so they fix no affine.", stacklevel=2) return None @@ -1216,7 +1193,6 @@ def xenium_aligned_image( if len(image.shape) == 4: assert image.shape[0] == 1 image = image.squeeze(0) - # interleaved RGB reads as (1, y, x, 3); planar RGB (OME axes SYX, Atera v1 Onboard 1.1.1) as (1, 3, y, x) if image.shape[-1] in [3, 4]: dims = ("y", "x", "c") else: diff --git a/src/spatialdata_io/writers/atera.py b/src/spatialdata_io/writers/atera.py index bad8c85d..7733845e 100644 --- a/src/spatialdata_io/writers/atera.py +++ b/src/spatialdata_io/writers/atera.py @@ -151,9 +151,7 @@ def write_atera_cell_feature_matrix( _write_x(root, zf, x, (n_obs, n_var), major_axis, chunk_size, row_chunk_size, n_threads, compressor) - manifest = _build_manifest( - table, n_var, obs_manifest, var_manifest, obsm_manifest, varm_manifest, var, major_axis - ) + manifest = _build_manifest(table, obs_manifest, var_manifest, obsm_manifest, varm_manifest, var, major_axis) root.attrs.update(manifest) for key, buffer in entries.items(): zf.writestr(key, buffer.to_bytes()) @@ -241,28 +239,24 @@ def _prepare_obs(table: AnnData) -> tuple[pd.DataFrame, dict[str, np.ndarray]]: def _controls_table(sdata: SpatialData | AnnData, controls_table: str | AnnData | None) -> AnnData | None: - if controls_table is None or isinstance(controls_table, str): - if isinstance(sdata, SpatialData) and controls_table is not None and controls_table in sdata.tables: - return sdata.tables[controls_table] - return None + if isinstance(controls_table, str): + return ( + sdata.tables[controls_table] if isinstance(sdata, SpatialData) and controls_table in sdata.tables else None + ) return controls_table def _append_controls(table: AnnData, controls: AnnData, x: Any) -> Any: """`x` with the controls' `X` appended as columns, after checking both tables hold the same cells in order.""" - if controls.n_obs != table.n_obs or not controls.obs_names.equals(table.obs_names): + if not controls.obs_names.equals(table.obs_names): raise ValueError("`table_controls` must hold the same cells as the table, in the same order (`obs_names`).") cid = str(AteraKeys.CELL_ID) - if cid in controls.obs.columns and cid in table.obs.columns: - if not np.array_equal(controls.obs[cid].to_numpy(), table.obs[cid].to_numpy()): - raise ValueError(f"`table_controls` and the table disagree on obs[{cid!r}].") + if cid in controls.obs and cid in table.obs and not np.array_equal(controls.obs[cid], table.obs[cid]): + raise ValueError(f"`table_controls` and the table disagree on obs[{cid!r}].") c: Any = controls.X if isinstance(x, da.Array): - c = ( - c - if isinstance(c, da.Array) - else da.from_array(sparse.csr_matrix(c), chunks=(x.chunks[0], -1), asarray=False) - ) + if not isinstance(c, da.Array): + c = da.from_array(sparse.csr_matrix(c), chunks=(x.chunks[0], -1), asarray=False) return da.concatenate([x, c.rechunk({0: x.chunks[0]})], axis=1) if isinstance(c, da.Array): c = c.compute() @@ -282,8 +276,7 @@ def _varm(table: AnnData, controls: AnnData | None) -> dict[str, Any]: def _prepare_var(table: AnnData, controls: AnnData | None = None) -> tuple[pd.DataFrame, dict[str, np.ndarray]]: - frames = [cast(pd.DataFrame, t.var) for t in (table, controls) if t is not None] - var = pd.concat(frames) if len(frames) > 1 else frames[0] + var = pd.concat([cast(pd.DataFrame, t.var) for t in (table, controls) if t is not None]) index = var.index.astype(str) var = var.reset_index(drop=True) sources = {} # written column -> the table's column, whose `uns[f"{source}_colors"]` holds its colors @@ -612,7 +605,6 @@ def _drain_one(self) -> None: def _build_manifest( table: AnnData, - n_var: int, obs: list[dict[str, Any]], var: list[dict[str, Any]], obsm: list[dict[str, Any]], @@ -627,7 +619,7 @@ def _build_manifest( str(AteraKeys.MANIFEST_VERSION): dict(_SCHEMA_VERSION), str(AteraKeys.MANIFEST_MATRIX_PROPERTIES): { "num_cells": int(table.n_obs), - "num_features": int(n_var), + "num_features": len(var_df), "major_axis": major_axis, "feature_types": feature_types, }, diff --git a/tests/readers/test_atera.py b/tests/readers/test_atera.py index d685763e..f132f4a8 100644 --- a/tests/readers/test_atera.py +++ b/tests/readers/test_atera.py @@ -17,7 +17,14 @@ from spatialdata_io._constants._constants import AteraKeys from spatialdata_io.readers import _atera_common as common from spatialdata_io.readers._atera_common import _read_transcript_tile, _tile_row_ranges -from spatialdata_io.readers.atera import _get_table, read_table_for_cells, read_table_for_genes, read_var +from spatialdata_io.readers.atera import ( + _check_keypoints, + _get_aligned_images, + _get_table, + read_table_for_cells, + read_table_for_genes, + read_var, +) # Atera `cell_id`s are large packed integers, unrelated to the `1..n_obs` values in the `cell_labels` raster. CELL_IDS = [4_320_252_178, 4_975_876_403, 5_016_677_209, 5_217_424_078] @@ -304,47 +311,51 @@ def test_tiled_imread_reads_one_tile_per_task_and_pickles(atera_path: Path) -> N np.testing.assert_array_equal(graph.compute(), tifffile.imread(file)) -def _write_he(path: Path, affine: np.ndarray) -> None: +# an H&E aligned by a rotation by 90 degrees, a scale of 2 and a translation +HE_AFFINE = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) + + +def _write_he(path: Path, alignment_csv: bool = True) -> None: import tifffile tifffile.imwrite( path / "S1_he_image.ome.tif", np.zeros((3, 40, 60), dtype=np.uint8), ome=True, metadata={"axes": "CYX"} ) - pd.DataFrame(affine).to_csv(path / "S1_he_imagealignment.csv", header=False, index=False) + if alignment_csv: + pd.DataFrame(HE_AFFINE).to_csv(path / "S1_he_imagealignment.csv", header=False, index=False) + + +def _write_keypoints(path: Path) -> pd.DataFrame: + src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) + dst = (np.c_[src, np.ones(3)] @ HE_AFFINE.T)[:, :2] + kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) + kp.to_csv(path / "S1_keypoints.csv", index=False) + return kp -@pytest.mark.parametrize("reader", ["atera", "xenium"]) -def test_aligned_he_image_is_placed_by_its_affine(tmp_path: Path, reader: str) -> None: +def _global_affine(image: Any) -> np.ndarray: from spatialdata.transformations import get_transformation - affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) - _write_he(tmp_path, affine) - images = _read_aligned(tmp_path, reader) - assert list(images) == ["he_image"] - he = images["he_image"] - assert list(he.coords["c"].values) == ["r", "g", "b"] - m = cast(Affine, get_transformation(he, "global")).to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) - np.testing.assert_allclose(m, affine) + t = cast(Affine, get_transformation(image, "global")) + return t.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) -def _write_keypoints(path: Path, affine: np.ndarray, name: str = "S1_keypoints.csv") -> pd.DataFrame: - src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) - dst = (np.c_[src, np.ones(3)] @ affine.T)[:, :2] - kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) - kp.to_csv(path / name, index=False) - return kp +def test_aligned_he_image_is_placed_by_its_affine(tmp_path: Path) -> None: + _write_he(tmp_path) + (tmp_path / "._S1_he_image.ome.tif").write_bytes(b"\x00" * 16) # a macOS sidecar, not a TIFF + images = _get_aligned_images(tmp_path, {}, {}) + assert list(images) == ["he_image"] + assert list(images["he_image"].coords["c"].values) == ["r", "g", "b"] + np.testing.assert_allclose(_global_affine(images["he_image"]), HE_AFFINE) def test_keypoints_check(tmp_path: Path, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: from spatialdata._logging import logger - from spatialdata_io.readers.atera import _check_keypoints - monkeypatch.setattr(logger, "propagate", True) - affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) - _write_he(tmp_path, affine) - kp = _write_keypoints(tmp_path, affine) + _write_he(tmp_path) + kp = _write_keypoints(tmp_path) csv, kp_file = tmp_path / "S1_he_imagealignment.csv", tmp_path / "S1_keypoints.csv" assert _check_keypoints(csv, kp_file) == pytest.approx(0.0, abs=1e-9) assert "misplaced" not in caplog.text @@ -356,87 +367,15 @@ def test_keypoints_check(tmp_path: Path, caplog: pytest.LogCaptureFixture, monke assert _check_keypoints(csv, kp_file) is None -@pytest.mark.parametrize( - "case", ["other_columns", "two_rows", "nan_rows", "collinear", "other_sample", "appledouble", "no_keypoints"] -) -def test_he_stays_unaligned_without_usable_keypoints(tmp_path: Path, case: str) -> None: - import warnings - - from spatialdata.transformations import Identity, get_transformation - - from spatialdata_io.readers.xenium import _add_aligned_images - - affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) - _write_he(tmp_path, affine) - (tmp_path / "S1_he_imagealignment.csv").unlink() - kp = _write_keypoints(tmp_path, affine) - kp_file = tmp_path / "S1_keypoints.csv" - if case == "other_columns": - kp.set_axis(["a", "b", "c", "d"], axis=1).to_csv(kp_file, index=False) - elif case == "two_rows": - kp.iloc[:2].to_csv(kp_file, index=False) - elif case == "nan_rows": - kp.assign(fixedX=[1.0, np.nan, "x"]).to_csv(kp_file, index=False) - elif case == "collinear": - kp.assign(alignmentX=[1.0, 2.0, 3.0], alignmentY=[1.0, 2.0, 3.0]).to_csv(kp_file, index=False) - elif case in ["other_sample", "appledouble"]: - kp_file.rename(tmp_path / ("S2_keypoints.csv" if case == "other_sample" else "._S1_keypoints.csv")) - else: - kp_file.unlink() - (tmp_path / "._S1_he_image.ome.tif").write_bytes(b"\x00" * 16) # a macOS tar sidecar, not a TIFF - - expect_warning = case in ["other_columns", "two_rows", "nan_rows", "collinear"] - with warnings.catch_warnings(record=True) as record: - warnings.simplefilter("always") - images = _add_aligned_images(tmp_path) - assert list(images) == ["he_image"] - assert isinstance(get_transformation(images["he_image"], "global"), Identity) - assert any("keypoint" in str(w.message) for w in record) == expect_warning - - -def test_affine_from_keypoints_reproduces_alignment(tmp_path: Path) -> None: - from spatialdata_io.readers.xenium import _affine_from_keypoints - - # about the Atera v1 kidney alignment, with 0.2% anisotropy as in the preview scans - affine = np.array([[0.0086, 1.2898, 914.1], [-1.2871, 0.0086, 32040.6], [0.0, 0.0, 1.0]]) - src = np.array([[16690.0, 14065.0], [22114.0, 25667.0], [13029.0, 25846.0], [16798.0, 35944.0]]) - dst = (np.c_[src, np.ones(len(src))] @ affine.T)[:, :2] - kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) - kp.to_csv(tmp_path / "S1_keypoints.csv", index=False) - m = cast(Affine, _affine_from_keypoints(tmp_path / "S1_keypoints.csv")).to_affine_matrix( - input_axes=("x", "y"), output_axes=("x", "y") - ) - np.testing.assert_allclose(m, affine, atol=1e-6) - - def test_he_without_alignment_csv_is_placed_by_keypoints(tmp_path: Path) -> None: - from spatialdata.transformations import get_transformation - - from spatialdata_io.readers.atera import _get_aligned_images - - affine = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) - _write_he(tmp_path, affine) - (tmp_path / "S1_he_imagealignment.csv").unlink() - src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) - dst = (np.c_[src, np.ones(3)] @ affine.T)[:, :2] - pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}).to_csv( - tmp_path / "S1_keypoints.csv", index=False - ) + _write_he(tmp_path, alignment_csv=False) + _write_keypoints(tmp_path) he = _get_aligned_images(tmp_path, {}, {})["he_image"] - m = cast(Affine, get_transformation(he, "global")).to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) - np.testing.assert_allclose(m, affine, atol=1e-9) - - -def _read_aligned(path: Path, reader: str) -> dict: - from spatialdata_io.readers.atera import _get_aligned_images - from spatialdata_io.readers.xenium import _add_aligned_images + np.testing.assert_allclose(_global_affine(he), HE_AFFINE, atol=1e-9) - return _get_aligned_images(path, {}, {}) if reader == "atera" else _add_aligned_images(path) - -@pytest.mark.parametrize("reader", ["atera", "xenium"]) @pytest.mark.parametrize("planar", [False, True]) -def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool, reader: str) -> None: +def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool) -> None: import tifffile rgb = np.arange(3 * 40 * 60, dtype=np.uint32).reshape(3, 40, 60).astype(np.uint8) @@ -444,7 +383,7 @@ def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool, re tifffile.imwrite(tmp_path / "S1_he_image.ome.tif", rgb, ome=True, photometric="rgb", planarconfig="separate") else: # YXS, as in the 1.0.4 kidney H&E tifffile.imwrite(tmp_path / "S1_he_image.ome.tif", np.moveaxis(rgb, 0, -1), ome=True, photometric="rgb") - he = _read_aligned(tmp_path, reader)["he_image"] + he = _get_aligned_images(tmp_path, {}, {})["he_image"] assert list(he.coords["c"].values) == ["r", "g", "b"] np.testing.assert_array_equal(np.asarray(he.transpose("c", "y", "x").data), rgb) diff --git a/tests/test_xenium.py b/tests/test_xenium.py index 1c116e57..e514bac4 100644 --- a/tests/test_xenium.py +++ b/tests/test_xenium.py @@ -2,6 +2,7 @@ import warnings from pathlib import Path from tempfile import TemporaryDirectory +from typing import Any import numpy as np import pandas as pd @@ -10,14 +11,18 @@ from pytest_mock import MockerFixture from spatialdata import match_table_to_element, read_zarr from spatialdata.models import get_table_keys +from spatialdata.transformations import Affine, Identity, get_transformation from spatialdata_io.__main__ import xenium_wrapper from spatialdata_io.readers.xenium import ( + _add_aligned_images, + _affine_from_keypoints, _cell_id_str_from_prefix_suffix_uint32_reference, _warn_if_scaled_protein, cell_id_str_from_prefix_suffix_uint32, prefix_suffix_uint32_from_cell_id_str, xenium, + xenium_aligned_image, ) from tests._utils import skip_if_below_python_version @@ -453,3 +458,107 @@ def test_cli_xenium_valid_json_forwarded( assert result.exit_code == 0, result.output call_kwargs = mock_xenium.call_args.kwargs assert call_kwargs[kwarg_param] == {"chunks": 512} + + +@pytest.mark.parametrize("planar", [False, True]) +def test_aligned_he_image_planar_or_interleaved(tmp_path: Path, planar: bool) -> None: + import tifffile + + rgb = np.arange(3 * 40 * 60, dtype=np.uint32).reshape(3, 40, 60).astype(np.uint8) + path = tmp_path / "S1_he_image.ome.tif" + if planar: # OME axes SYX + tifffile.imwrite(path, rgb, ome=True, photometric="rgb", planarconfig="separate") + else: # YXS + tifffile.imwrite(path, np.moveaxis(rgb, 0, -1), ome=True, photometric="rgb") + he = xenium_aligned_image(path, None) + assert list(he.coords["c"].values) == ["r", "g", "b"] + np.testing.assert_array_equal(np.asarray(he.transpose("c", "y", "x").data), rgb) + + +# an H&E aligned by a rotation by 90 degrees, a scale of 2 and a translation +HE_AFFINE = np.array([[0.0, 2.0, 10.0], [-2.0, 0.0, 300.0], [0.0, 0.0, 1.0]]) + + +def _write_he(path: Path, affine: np.ndarray, alignment_csv: bool = True) -> None: + import tifffile + + tifffile.imwrite( + path / "S1_he_image.ome.tif", np.zeros((3, 40, 60), dtype=np.uint8), ome=True, metadata={"axes": "CYX"} + ) + if alignment_csv: + pd.DataFrame(affine).to_csv(path / "S1_he_imagealignment.csv", header=False, index=False) + + +def _write_keypoints(path: Path, affine: np.ndarray) -> pd.DataFrame: + src = np.array([[5.0, 5.0], [30.0, 10.0], [50.0, 35.0]]) + dst = (np.c_[src, np.ones(3)] @ affine.T)[:, :2] + kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) + kp.to_csv(path / "S1_keypoints.csv", index=False) + return kp + + +def _global_affine(image: Any) -> np.ndarray: + t = get_transformation(image, "global") + assert isinstance(t, Affine) + return t.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")) + + +def test_aligned_he_image_is_placed_by_its_alignment_csv(tmp_path: Path) -> None: + _write_he(tmp_path, HE_AFFINE) + shifted = HE_AFFINE.copy() + shifted[0, 2] += 100.0 + _write_keypoints(tmp_path, shifted) # the alignment CSV wins over the keypoints + images = _add_aligned_images(tmp_path) + assert list(images) == ["he_image"] + np.testing.assert_allclose(_global_affine(images["he_image"]), HE_AFFINE) + + +def test_he_without_alignment_csv_is_placed_by_keypoints(tmp_path: Path) -> None: + _write_he(tmp_path, HE_AFFINE, alignment_csv=False) + _write_keypoints(tmp_path, HE_AFFINE) + he = _add_aligned_images(tmp_path)["he_image"] + np.testing.assert_allclose(_global_affine(he), HE_AFFINE, atol=1e-9) + + +def test_affine_from_keypoints_reproduces_an_anisotropic_alignment(tmp_path: Path) -> None: + # x and y scales differ by 0.2%, which a similarity fit can't reproduce + affine = np.array([[0.0086, 1.2898, 914.1], [-1.2871, 0.0086, 32040.6], [0.0, 0.0, 1.0]]) + src = np.array([[16690.0, 14065.0], [22114.0, 25667.0], [13029.0, 25846.0], [16798.0, 35944.0]]) + dst = (np.c_[src, np.ones(len(src))] @ affine.T)[:, :2] + kp = pd.DataFrame({"fixedX": dst[:, 0], "fixedY": dst[:, 1], "alignmentX": src[:, 0], "alignmentY": src[:, 1]}) + kp.to_csv(tmp_path / "S1_keypoints.csv", index=False) + fitted = _affine_from_keypoints(tmp_path / "S1_keypoints.csv") + assert fitted is not None + np.testing.assert_allclose( + fitted.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")), affine, atol=1e-6 + ) + + +@pytest.mark.parametrize( + "case", ["other_columns", "two_rows", "nan_rows", "collinear", "other_sample", "appledouble", "no_keypoints"] +) +def test_he_stays_unaligned_without_usable_keypoints(tmp_path: Path, case: str) -> None: + _write_he(tmp_path, HE_AFFINE, alignment_csv=False) + kp = _write_keypoints(tmp_path, HE_AFFINE) + kp_file = tmp_path / "S1_keypoints.csv" + if case == "other_columns": + kp.set_axis(["a", "b", "c", "d"], axis=1).to_csv(kp_file, index=False) + elif case == "two_rows": + kp.iloc[:2].to_csv(kp_file, index=False) + elif case == "nan_rows": + kp.assign(fixedX=[1.0, np.nan, "x"]).to_csv(kp_file, index=False) + elif case == "collinear": + kp.assign(alignmentX=[1.0, 2.0, 3.0], alignmentY=[1.0, 2.0, 3.0]).to_csv(kp_file, index=False) + elif case in ["other_sample", "appledouble"]: + kp_file.rename(tmp_path / ("S2_keypoints.csv" if case == "other_sample" else "._S1_keypoints.csv")) + else: + kp_file.unlink() + (tmp_path / "._S1_he_image.ome.tif").write_bytes(b"\x00" * 16) # a macOS sidecar, not a TIFF + + expect_warning = case in ["other_columns", "two_rows", "nan_rows", "collinear"] + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + images = _add_aligned_images(tmp_path) + assert list(images) == ["he_image"] + assert isinstance(get_transformation(images["he_image"], "global"), Identity) + assert any("keypoint" in str(w.message) for w in record) == expect_warning