Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
3fbf4c6
feat(atera): read the aligned post-run H&E/IF images and check their …
Nov 10, 2026
4eaefc9
fix(xenium,atera): fit the keypoint fallback as a full affine
Nov 10, 2026
8fb5310
fix(xenium,atera): read planar RGB aligned images (OME axes SYX)
Nov 10, 2026
a6a0087
fix(atera): leave the obs index unnamed where it duplicates the barco…
Nov 12, 2026
8cb6c5b
test(atera): read a whole synthetic bundle with atera(), write it and…
Nov 16, 2026
5d87a6f
refactor(atera): one zip context manager, one transcript decoder, one…
Nov 16, 2026
f0526ec
refactor(atera): read X with anndata's read_elem_lazy
Nov 16, 2026
4027b82
refactor(atera): reuse xenium's _get_labels and parse_physical_size; …
Nov 16, 2026
48e53fa
fix(atera): pad 10x's short trailing vlen chunks in a store wrapper, …
Nov 16, 2026
fc5eaba
fix(xenium,atera): check the keypoints before the H&E fallback fit; s…
Nov 16, 2026
c1a9a24
fix(atera): one image read path, one tile per dask task, pyramid from…
Nov 16, 2026
0662beb
fix(atera): give the 4-D 3D morphology image its own chunks and keep z
Nov 16, 2026
2dd3c29
fix(atera): map transcript and polygon positions through the bundle, …
Nov 16, 2026
d872974
fix(atera): lay out var as the xenium reader does (symbol index, gene…
Nov 10, 2026
b98400d
fix(atera): leave the var index unnamed
Nov 11, 2026
b92f8bb
fix(atera): cast var to DataFrame for mypy
Nov 16, 2026
022a6a2
fix(atera): X holds the Gene Expression features, the controls go to …
Nov 16, 2026
f37e1a0
fix(atera): make duplicate feature names unique once and use them eve…
Nov 16, 2026
74c81fa
fix(atera): empty selections give empty elements
Nov 16, 2026
76f0d83
fix(atera): keep a polygon with 1 or 2 vertices as an empty polygon
Nov 16, 2026
05535c5
fix(atera): honour cells_table=False; annotate only an element that i…
Nov 16, 2026
8aa0d48
fix(atera_to_parquet): write to a .partial file and move it into plac…
Nov 16, 2026
04a58fe
fix(atera): log through spatialdata's logger, as the other readers do
Nov 16, 2026
33bf102
docs(atera): fix the stale docstrings
Nov 16, 2026
889ef2f
fix(atera): type the new code for mypy
Nov 16, 2026
50b95cf
fix(atera): build unique feature names from a numpy StringDType array
Nov 16, 2026
f1dc16e
fix(atera): put the non-gene features in a second table, table_controls
Nov 16, 2026
a11e053
feat(atera writer): write table + table_controls back as one cell_fea…
Nov 20, 2026
71a2f75
simplify
Nov 20, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ dynamic = [
"version", # allow version to be set by git tags
]
dependencies = [
"anndata",
"anndata>=0.12",
"click",
"dask-image",
"h5py",
Expand Down
6 changes: 0 additions & 6 deletions src/spatialdata_io/_constants/_constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,15 +190,10 @@ class AteraKeys(ModeEnum):

# cell_feature_matrix.zarr.zip groups/keys
X_GROUP = "X"
X_DATA = "data"
X_INDICES = "indices"
X_INDPTR = "indptr"
OBS_GROUP = "obs"
VAR_GROUP = "var"
OBSM_GROUP = "obsm"
VARM_GROUP = "varm"
CATEGORIES = "categories"
CODES = "codes"
COLOR = "color"

# root `.zattrs` manifest (RFC "cell feature anndata", schema anndata_zattrs/v0.1.0)
Expand All @@ -217,7 +212,6 @@ class AteraKeys(ModeEnum):
LABEL_ID = "label_id"
CENTROID_X = "centroid_column"
CENTROID_Y = "centroid_row"
CELL_AREA = "cell_area"

# var / features
FEATURE_ID = "feature_id"
Expand Down
117 changes: 62 additions & 55 deletions src/spatialdata_io/converters/atera_to_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,16 @@
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import zarr

from spatialdata_io._constants._constants import AteraKeys
from spatialdata_io.readers._atera_common import _get_array, _get_group, _read_transcript_tile, _tile_row_ranges
from spatialdata_io.readers.atera import read_var
from spatialdata_io.readers._atera_common import (
_get_array,
_get_group,
_open_zip,
_read_feature_names,
_read_transcript_tile,
_tile_row_ranges,
)

__all__ = ["atera_to_parquet"]

Expand Down Expand Up @@ -57,9 +62,9 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path:
``transcript_id`` is taken from each tile's stored ``id`` array when every tile has one, and is
otherwise assigned contiguously in the order of the grid group's ``grid_keys`` attribute.

The input is read and written one spatial tile at a time (mirroring how `atera`'s own transcripts
reader is built), so converting even a dataset with tens of millions of transcripts does not require
holding the whole table in memory at once.
The input is read and written one row range of a spatial tile at a time (one parquet row group each,
as `atera`'s own transcripts reader partitions it), so converting even a dataset with hundreds of millions
of transcripts does not require holding the whole table in memory at once.

Parameters
----------
Expand All @@ -75,55 +80,57 @@ def atera_to_parquet(path: str | Path, output: str | Path) -> Path:
path = Path(path)
output = Path(output)

feature_names = read_var(path)[str(AteraKeys.FEATURE_NAME)].astype(str).tolist()
feature_names = _read_feature_names(path)

store = zarr.storage.ZipStore(path / AteraKeys.TRANSCRIPTS_FILE, read_only=True)
writer = pq.ParquetWriter(output, _PROSEG_SCHEMA)
# written next to `output` and moved there when complete, so a failed run leaves no truncated (but valid)
# parquet file and keeps an earlier complete one
partial = output.with_name(output.name + ".partial")
try:
grid = _get_group(zarr.open_group(store, mode="r"), str(AteraKeys.GRID_GROUP))
tile_keys = [str(key) for key in cast("list[str]", grid.attrs[str(AteraKeys.GRID_KEYS)])]
# Stored IDs are used only if every tile has them; otherwise every transcript is numbered
# contiguously in `grid_keys` order, so stored and generated IDs never mix.
use_stored_ids = all(str(AteraKeys.TRANSCRIPTS_ID) in _get_group(grid, key) for key in tile_keys)

offset = 0
for tile_key in tile_keys:
tile = _get_group(grid, tile_key)
n_tile = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0]
# A tile can hold 100M+ transcripts, so it is read and written in row ranges (one parquet row group each).
for start, stop in _tile_row_ranges(n_tile):
tile_df = _read_transcript_tile(path, tile_key, feature_names, start, stop)
n = len(tile_df)
if n == 0:
continue

if use_stored_ids:
stored_ids = _get_array(tile, str(AteraKeys.TRANSCRIPTS_ID))[start:stop]
transcript_id = np.asarray(stored_ids, dtype=np.uint64).reshape(-1)
else:
transcript_id = np.arange(offset, offset + n, dtype=np.uint64)

cell_id = tile_df[str(AteraKeys.CELL_ID)].to_numpy()
cell_id_str = np.where(cell_id == -1, _CELL_ID_UNASSIGNED, cell_id.astype(str))

table = pa.table(
{
"transcript_id": transcript_id,
"cell_id": cell_id_str,
"overlaps_nucleus": tile_df["overlaps_nucleus"].to_numpy().astype(np.uint8),
"feature_name": tile_df[str(AteraKeys.FEATURE_NAME)].astype(str).to_numpy(),
"x_location": tile_df[str(AteraKeys.TRANSCRIPTS_X)].to_numpy().astype(np.float32),
"y_location": tile_df[str(AteraKeys.TRANSCRIPTS_Y)].to_numpy().astype(np.float32),
"z_location": tile_df[str(AteraKeys.TRANSCRIPTS_Z)].to_numpy().astype(np.float32),
"qv": tile_df["quality_score"].to_numpy().astype(np.float32),
"fov_name": np.full(n, tile_key),
},
schema=_PROSEG_SCHEMA,
)
writer.write_table(table)
offset += n
finally:
writer.close()
store.close()

with _open_zip(path / AteraKeys.TRANSCRIPTS_FILE) as group, pq.ParquetWriter(partial, _PROSEG_SCHEMA) as writer:
grid = _get_group(group, str(AteraKeys.GRID_GROUP))
tile_keys = [str(key) for key in cast("list[str]", grid.attrs[str(AteraKeys.GRID_KEYS)])]
# Stored IDs are used only if every tile has them; otherwise every transcript is numbered
# contiguously in `grid_keys` order, so stored and generated IDs never mix.
use_stored_ids = all(str(AteraKeys.TRANSCRIPTS_ID) in _get_group(grid, key) for key in tile_keys)

offset = 0
for tile_key in tile_keys:
tile = _get_group(grid, tile_key)
n_tile = _get_array(tile, str(AteraKeys.TRANSCRIPTS_LOCATION)).shape[0]
# A tile can hold 100M+ transcripts, so it is read and written in row ranges (one parquet row group each).
for start, stop in _tile_row_ranges(n_tile):
tile_df = _read_transcript_tile(grid, tile_key, feature_names, start, stop)
n = len(tile_df)
if n == 0:
continue

if use_stored_ids:
stored_ids = _get_array(tile, str(AteraKeys.TRANSCRIPTS_ID))[start:stop]
transcript_id = np.asarray(stored_ids, dtype=np.uint64).reshape(-1)
else:
transcript_id = np.arange(offset, offset + n, dtype=np.uint64)

cell_id = tile_df[str(AteraKeys.CELL_ID)].to_numpy()
cell_id_str = np.where(cell_id == -1, _CELL_ID_UNASSIGNED, cell_id.astype(str))

table = pa.table(
{
"transcript_id": transcript_id,
"cell_id": cell_id_str,
"overlaps_nucleus": tile_df["overlaps_nucleus"].to_numpy().astype(np.uint8),
"feature_name": tile_df[str(AteraKeys.FEATURE_NAME)].astype(str).to_numpy(),
"x_location": tile_df[str(AteraKeys.TRANSCRIPTS_X)].to_numpy().astype(np.float32),
"y_location": tile_df[str(AteraKeys.TRANSCRIPTS_Y)].to_numpy().astype(np.float32),
"z_location": tile_df[str(AteraKeys.TRANSCRIPTS_Z)].to_numpy().astype(np.float32),
"qv": tile_df["quality_score"].to_numpy().astype(np.float32),
"fov_name": np.full(n, tile_key),
},
schema=_PROSEG_SCHEMA,
)
writer.write_table(table)
offset += n
except BaseException:
partial.unlink(missing_ok=True)
raise
partial.replace(output)
return output
Loading
Loading