diff --git a/.flake8 b/.flake8 index abe8e86..765c418 100644 --- a/.flake8 +++ b/.flake8 @@ -7,3 +7,5 @@ exclude = venv, .conda max-complexity = 10 +# Black's PEP 8-compliant slice formatting conflicts with pycodestyle E203. +extend-ignore = E203 diff --git a/.github/workflows/test_and_lint.yml b/.github/workflows/lint.yml similarity index 72% rename from .github/workflows/test_and_lint.yml rename to .github/workflows/lint.yml index ef1c276..d1033e0 100644 --- a/.github/workflows/test_and_lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: Lint and run tests +name: Lint on: pull_request: @@ -18,9 +18,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: | + run: | python -m pip install -e . --group dev --no-cache-dir - name: Run linter checks - run: flake8 . && interrogate --verbose . - - name: Run tests and coverage - run: coverage run -m unittest discover && coverage report + run: flake8 . && interrogate --verbose . \ No newline at end of file diff --git a/.github/workflows/tag_and_publish.yml b/.github/workflows/tag_and_publish.yml index 3f8c804..b729b2e 100644 --- a/.github/workflows/tag_and_publish.yml +++ b/.github/workflows/tag_and_publish.yml @@ -3,8 +3,7 @@ on: push: branches: - main -# Remove line 65 to enable automated semantic version bumps. -# Change line 71 from "if: false" to "if: true" to enable PyPI publishing. +# Set the publish job's "if" condition to true to enable PyPI publishing. # Requires that svc-aindscicomp be added as an admin to repo. jobs: update_badges: @@ -41,19 +40,6 @@ jobs: new_interrogate_badge="interrogate-$interrogate_val%25-yellow" fi sed -i "s/$interrogate_badge/$new_interrogate_badge/g" README.md - - name: Get Coverage values and Update README.md - run: | - coverage run -m unittest discover - coverage_val=$(coverage report | grep "^TOTAL" | grep -o '[0-9]\+%' | grep -o '[0-9]\+') - coverage_badge=$(grep '^\!\[Coverage\]' README.md | sed -n 's/.*\(coverage-[^)]*\).*/\1/p') - if (( $(echo "$coverage_val >= 90.00" | bc -l) )); then - new_coverage_badge="coverage-$coverage_val%25-brightgreen" - elif (( $(echo "$coverage_val < 80.00" | bc -l) )); then - new_coverage_badge="coverage-$coverage_val%25-red" - else - new_coverage_badge="coverage-$coverage_val%25-yellow" - fi - sed -i "s/$coverage_badge/$new_coverage_badge/g" README.md - name: Commit changes uses: EndBug/add-and-commit@v10 with: diff --git a/.gitignore b/.gitignore index 06a56dd..405b401 100644 --- a/.gitignore +++ b/.gitignore @@ -37,19 +37,8 @@ MANIFEST pip-log.txt pip-delete-this-directory.txt -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* +# Tool caches .cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ # Translations *.mo @@ -135,5 +124,8 @@ dmypy.json # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ +# VS Code +.vscode/ + # MacOs **/.DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3c23d24..1f16d22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,15 +1,19 @@ # Contributing -### Linters and testing +### Development setup -There are several libraries used to run linters, check documentation, and run tests. - -- Please test your changes using the **coverage** library, which will run the tests and log a coverage report: +From the repository root, install the package and development tools with pip 25.1 or later: ```bash -coverage run -m unittest discover && coverage report +pip install -e . --group dev ``` +Alternatively, use `uv sync`. Interactive ROI work also requires tkinter in the Python installation. + +### Code quality + +Use the following tools to check documentation and maintain code style. + - Use **interrogate** to check that modules, methods, etc. have been documented thoroughly: ```bash @@ -47,7 +51,6 @@ where scope (optional) describes the packages affected by the code changes and t - **fix**: A bugfix - **perf**: A code change that improves performance - **refactor**: A code change that neither fixes a bug nor adds a feature -- **test**: Adding missing tests or correcting existing tests ### Semantic Release @@ -60,15 +63,17 @@ The table below, from [semantic release](https://github.com/semantic-release/sem | `perf(pencil): remove graphiteWidth option`

`BREAKING CHANGE: The graphiteWidth option has been removed.`
`The default graphite width of 10mm is always used for performance reasons.` | ~~Major~~ Breaking Release
(Note that the `BREAKING CHANGE: ` token must be in the footer of the commit and need to be all caps as shown) | ### Documentation -To generate the rst files source files for documentation, run -```bash -sphinx-apidoc -o docs/source/ src -``` -Then to create the documentation HTML files, run + +The maintained Sphinx pages include a curated public API reference; do not regenerate them with `sphinx-apidoc`. +Build the documentation and treat warnings as errors: + ```bash -sphinx-build -b html docs/source/ docs/build/html +sphinx-build -b html -n -W --keep-going docs/source/ docs/build/html ``` -More info on sphinx installation can be found [here](https://www.sphinx-doc.org/en/master/usage/installation.html). + +External API links use Python, NumPy, and h5py inventories downloaded by Sphinx; a clean build requires network access to those documentation sites. + +Keep the README focused on running BandSILo. Put option and staging details in [API and data conventions](docs/source/migration.rst), scientific details in [trace estimation](docs/source/traces.rst), schemas in [file formats](docs/source/file_formats.rst), and ownership decisions in [architecture](docs/source/architecture.rst). ### Read the Docs Deployment Note: Private repositories require **Read the Docs for Business** account. The following instructions are for a public repo. diff --git a/README.md b/README.md index f4169dd..b7e640a 100644 --- a/README.md +++ b/README.md @@ -1,405 +1,75 @@ -# GIAnT-python +# GIAnT-Python [![License](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE) -![Code Style](https://img.shields.io/badge/code%20style-black-black) -[![semantic-release: angular](https://img.shields.io/badge/semantic--release-angular-e10079?logo=semantic-release)](https://github.com/semantic-release/semantic-release) -![Interrogate](https://img.shields.io/badge/interrogate-97.5%25-brightgreen) -![Coverage](https://img.shields.io/badge/coverage-98%25-brightgreen) ![Python](https://img.shields.io/badge/python->=3.10-blue?logo=python) -Python translation of the [GIAnT-MATLAB](https://github.com/AllenNeuralDynamics/GIAnT-MATLAB) analysis package. +GIAnT-Python is a Python package for processing glutamate imaging data, with advanced capabilities for SLAP2 recordings. -## Level of Support - - [ ] Supported: We are releasing this code to the public as a tool we expect others to use. Issues are welcomed, and we expect to address them promptly; pull requests will be vetted by our staff before inclusion. - - [ ] Occasional updates: We are planning on occasional updating this tool with no fixed schedule. Community involvement is encouraged through both issues and pull requests. - - [ ] Unsupported: We are not currently supporting this code, but simply releasing it to the community AS IS but are not able to provide any guarantees of support. The community is welcome to submit issues, but you should not expect an active response. +Currently, only **SLAP2 Band Scan processing** is implemented: **BandSILo source extraction and Band Scan ROI annotation** from pre-registered recordings. Additional functionality is available in [GIAnT-MATLAB](https://github.com/AllenNeuralDynamics/GIAnT-MATLAB), and efforts are underway to port it into GIAnT-Python. -## Installation -To use the software, in the root directory, run -```bash -pip install -e . -``` - -To develop the code, run -```bash -pip install -e . --group dev -``` -Note: --group flag is available only in pip versions >=25.1 - -Alternatively, if using `uv`, run -```bash -uv sync -``` - -## Epoch and Analysis Trial - -For each experiment we run through the pipeline, we break down the data into epochs and analysis trials. - -Epochs are full experimental sessions that can be aligned with each other (i.e. the same field of view and regions of interest are being imaged). Analysis trials are generally contiguous subsets (in time) of an epoch. These analysis trials may not align exactly with experimental trials. - -For SLAP2, analysis trials are the experimental trials if the data was collected using the multi-trial functions of the SLAP2 and each trial is saved off the microscope in a different file. If data was continuously collected on SLAP2, the experiment will be split up into analysis trials of length 200000 lines (~20 sec) to help parallelize processing. - -For data not collected on SLAP2, the current GIAnT pipeline sets Epochs to be 1 and each file that is selected to be processed is an analysis trial. These analysis trials must be able to be aligned to one another. - -## Pipeline Outputs - -GIAnT-python writes the **same set of HDF5 output files with the same field names, shapes, dtypes, and conventions** as [GIAnT-MATLAB](https://github.com/AllenNeuralDynamics/GIAnT-MATLAB), so files produced by either toolbox are interchangeable. The schemas below are the on-disk contract; the typed data models in `giant_python.models` (`TrialTable`, `AlignmentData`, `ExperimentSummary`, ...) mirror these structures and own the `from_h5` / `to_h5` (de)serialization. - -### Reading and writing H5 - -GIAnT-python reads and writes these files with [`h5py`](https://www.h5py.org/), which stores arrays in **row-major (C) order**. Files written by GIAnT-python therefore set the `row_major` flag to `1`, and **every dimension tuple listed below matches the h5py `shape` directly** (i.e. read the tuples left-to-right as the axis order returned by `h5py`). - -Files produced by GIAnT-**MATLAB** instead set `row_major` to `0` (or omit it), because MATLAB stores arrays column-major and its dimension tuples follow MATLAB `size()`. GIAnT-python's loaders inspect the `row_major` flag and **permute axes as needed**, so a MATLAB-written file and a Python-written file deserialize to the same logical arrays. When in doubt: - -| `row_major` value | Layout | Axis order of the tuples below | -| --- | --- | --- | -| `1` | row-major (h5py / NumPy C order) | matches h5py `shape` as written here | -| `0` or absent | column-major (MATLAB `size()`) | reverse of the tuples here | - -**Vector shapes are preserved on disk.** Fields documented as `1 x N` (row vectors) or `N x 1` (column vectors) are written as rank-2 datasets with those exact dimensions โ€” e.g. `motionDSc` is shape `(1, nDSframes)`, not a squeezed `(nDSframes,)`. GIAnT-python does **not** collapse singleton dimensions when saving. Downstream code can therefore use the same size checks and indexing on MATLAB- or Python-generated files after handling the `row_major` flag (and any required axis permutation for multi-D arrays). - -### Index and coordinate conventions - -These conventions are shared with GIAnT-MATLAB and preserved on both read and write: - -- **Line/frame indices** (`first_line`, `last_line`, `DSframes`, `frame_line_idxs`) are **1-indexed** to retain the SLAP2 line-indexing convention. -- **Spatial coordinate fields** in the summaries (`sources/spatial/coords`, `act_im_peaks`, `per_trial_*_coords`, etc.) are **0-indexed** `[z_loc, y_loc, x_loc]`, matching image axis order (`fastz`, rows, cols). `z_loc` is the 0-based index into the `fastz` axis. -- **Annotation coordinate fields** (`position`, `center`) are **0-indexed** `[y_loc, x_loc]` when `coords_zero_indexed = 1`; legacy files without the flag use MATLAB `images.roi` convention (1-indexed `[x, y]`). -- **Images** are stored `rows x cols` (row = Y, col = X). Motion shifts: `motionDSr` / `motionR` are **row** shifts, `motionDSc` / `motionC` are **column** shifts. -- `params/activityChannel` is **1-indexed** into the recording's `numChannels` channels. +## Requirements and scope -Legend for the trees below (๐Ÿ—„๏ธ file ยท ๐Ÿ“ group ยท ๐Ÿ”ค string ยท ๐Ÿ”ข integer ยท ๐Ÿ“ˆ numeric ยท ๐Ÿ–ผ๏ธ image ยท โ˜‘๏ธ bool). +Provide an existing trial table, raw recordings with sidecars, reference TIFFs, alignment files, and the band registration lookup table. Python consumes these inputs; it does not organize or register recordings. Standard SILo, standalone deconvolution, trial-table writing, config-file loading, and checkpoint/resume are not supported. -### Trial Table +Analysis trials come from the supplied table; Python does not split or merge them. -Each experiment processed with GIAnT first gets a `trial_table.h5` file that summarizes relevant file locations and analysis trial structures. The `slap2_info` group is only populated for SLAP2 experiments. The `motion_correction` and `source_extraction` groups are populated by downstream pipeline stages and will only be present once those stages have run. (Model: `giant_python.models.TrialTable` / `Slap2Info`.) - -``` -๐Ÿ—„๏ธ trial_table.h5 - โ”œ ๐Ÿ”ค datadr - โ”œ ๐Ÿ”ค savedr - โ”œ ๐Ÿ”ค filename - โ”œ ๐Ÿ”ข true_trial_ix - โ”œ ๐Ÿ”ข epoch - โ”œ โ˜‘๏ธ row_major - โ”œ ๐Ÿ“ slap2_info - | โ”œ ๐Ÿ“ ref_stack - | | โ”” ๐Ÿ“ Path{1,2} - | | โ”œ ๐Ÿ–ผ๏ธ IM - | | โ”œ ๐Ÿ”ข channels - | | โ”œ ๐Ÿ“ˆ Zs - | | โ”” ๐Ÿ“ˆ dmdPixel2SampleTransform - | โ”œ ๐Ÿ”ข first_line - | โ”œ ๐Ÿ”ข last_line - | โ”œ ๐Ÿ”ข trial_start_time_inferred - | โ”” ๐Ÿ”ข trial_end_time_from_pc - โ”œ ๐Ÿ“ motion_correction - | โ”œ ๐Ÿ”ค fn_reg_ds - | โ”œ ๐Ÿ”ค fn_adata - | โ”œ ๐Ÿ”ค fn_raw - | โ”œ โ˜‘๏ธ registration_failed - | โ”œ ๐Ÿ”ข first_line_original - | โ”” ๐Ÿ“ align_params - โ”” ๐Ÿ“ source_extraction - โ”œ ๐Ÿ“ analysis_params - โ”” ๐Ÿ”ค fn_raw -``` - -### Alignment Data - -The motion correction stage saves a H5 file ending in `_ALIGNMENTDATA.h5` that contains the alignment data for each trial. (Model: `giant_python.models.AlignmentData`.) - -``` -๐Ÿ—„๏ธ _ALIGNMENTDATA.h5 - โ”œ โ˜‘๏ธ row_major - โ”œ ๐Ÿ“ˆ numChannels - โ”œ ๐Ÿ“ˆ frametime - โ”œ ๐Ÿ“ˆ alignHz - โ”œ ๐Ÿ“ˆ motionDSc - โ”œ ๐Ÿ“ˆ motionDSr - โ”œ ๐Ÿ“ˆ motionDSz (BandRegistration always; MultiRoiRegistration when refStackTemplate is enabled) - โ”œ ๐Ÿ–ผ๏ธ meanIM (StripRegistration and MultiRoiRegistration only; not written by BandRegistration) - โ”œ ๐Ÿ“ˆ recNegErr (StripRegistration and MultiRoiRegistration only; not written by BandRegistration) - โ”œ ๐Ÿ“ˆ motionC (StripRegistration / Bergamo only) - โ”œ ๐Ÿ“ˆ motionR (StripRegistration / Bergamo only) - โ”œ ๐Ÿ“ˆ motionZ (reserved; not written by any current script) - โ”œ ๐Ÿ“ˆ brightnessDS (BandRegistration only) - โ”œ ๐Ÿ“ˆ logLikelihoodDS (BandRegistration only) - โ”œ ๐Ÿ”ข DSframes (SLAP2 only: MultiRoiRegistration and BandRegistration) - โ”œ โ˜‘๏ธ registrationFailed (SLAP2 only: MultiRoiRegistration and BandRegistration) - โ”” ๐Ÿ“ slap2 (SLAP2 only) - โ”œ ๐Ÿ“ˆ onlineMotionXshift - โ”œ ๐Ÿ“ˆ onlineMotionYshift - โ”œ ๐Ÿ“ˆ onlineMotionZshift - โ”œ ๐Ÿ–ผ๏ธ varFacDS (MultiRoiRegistration only) - โ”œ ๐Ÿ“ˆ Z_depths (MultiRoiRegistration only) - โ”œ ๐Ÿ”ข cropRow (MultiRoiRegistration only) - โ”œ ๐Ÿ”ข cropCol (MultiRoiRegistration only) - โ”œ ๐Ÿ–ผ๏ธ viewC (MultiRoiRegistration only) - โ”œ ๐Ÿ–ผ๏ธ viewR (MultiRoiRegistration only) - โ”œ ๐Ÿ”ข trimRows (MultiRoiRegistration only) - โ”” ๐Ÿ”ข trimCols (MultiRoiRegistration only) -``` - -### Manual Annotations +## Installation -Users can manually annotate pixels to exclude from analysis, or pixels that correspond to soma whose signals should be extracted. When ROIs are annotated, information about the ROIs is saved in the `annotations.h5` file. String fields are stored as UTF-16 code units (`uint16`) for robust MATLAB/Python compatibility. +From the repository root, with Python 3.10 or later: +```bash +pip install -e . ``` -๐Ÿ—„๏ธ annotations.h5 - โ”œ โ˜‘๏ธ row_major - โ”œ โ˜‘๏ธ coords_zero_indexed - โ”” ๐Ÿ“ Path{1,2} - โ”œ ๐Ÿ”ค dr - โ”œ ๐Ÿ”ค fn - โ”œ ๐Ÿ”ข n_rois - โ”” ๐Ÿ“ roi_### - โ”œ ๐Ÿ”ค type - โ”œ ๐Ÿ”ค label - โ”œ ๐Ÿ–ผ๏ธ mask - โ”œ ๐Ÿ“ˆ position (polygon only; nVertices x 2 [y_loc, x_loc] when flagged) - โ”œ ๐Ÿ“ˆ center (circle/ellipse; 1 x 2 [y_loc, x_loc] when flagged) - โ”œ ๐Ÿ“ˆ semi_axes (ellipse) - โ”œ ๐Ÿ“ˆ rotation_angle (ellipse) - โ”” ๐Ÿ“ˆ radius (circle) -``` - -### Experiment Summary - -The final step of the pipeline, source extraction (Source Identification by Activity Localization; SILo), outputs an `experiment_summary.h5` file which contains the extracted sources as well as other useful data about the experiment. Dimensions use one `total frames` axis for all trials from that path stitched in time. (Model: `giant_python.models.ExperimentSummary` / `Source` / `UserRoi` / `Visualizations`.) -``` -๐Ÿ—„๏ธ experiment_summary.h5 - โ”œ โ˜‘๏ธ row_major - โ”œ ๐Ÿ“ params - โ”” ๐Ÿ“ Path{1,2} - โ”œ ๐Ÿ“ˆ Z_depths (fastz x 1) - โ”œ ๐Ÿ“ sources - | โ”œ ๐Ÿ“ temporal - | | โ”œ ๐Ÿ“ˆ dF_ls (sources x channels x total frames) - | | โ”œ ๐Ÿ“ˆ dF_denoised (sources x channels x total frames) - | | โ”œ ๐Ÿ“ˆ events (sources x channels x total frames) - | | โ”œ ๐Ÿ“ˆ F0 (sources x channels x total frames) - | | โ”” ๐Ÿ“ˆ SNR (sources x 1) - | โ”” ๐Ÿ“ spatial - | โ”œ ๐Ÿ–ผ๏ธ profiles (sources x fastz x rows x cols) - | โ”” ๐Ÿ“ˆ coords (sources x 3 [z_loc, y_loc, x_loc]) - โ”œ ๐Ÿ“ user_rois - | โ”œ ๐Ÿ”ค labels (rois x 1) - | โ”œ ๐Ÿ–ผ๏ธ mask (rois x fastz x rows x cols) - | โ”œ ๐Ÿ“ˆ Fsvd (rois x channels x total frames) - | โ”” ๐Ÿ“ˆ F (rois x channels x total frames) - โ”œ ๐Ÿ“ visualizations - | โ”œ ๐Ÿ–ผ๏ธ mean_im (channels x fastz x rows x cols) - | โ”œ ๐Ÿ–ผ๏ธ act_im (fastz x rows x cols) - | โ”” ๐Ÿ–ผ๏ธ act_im_peaks (sources x 3 [z_loc, y_loc, x_loc]) - โ”œ ๐Ÿ“ global - | โ”” ๐Ÿ“ˆ F (channels x total frames) - โ”” ๐Ÿ“ frame_info - โ”œ ๐Ÿ“ˆ offlineXshifts (total frames x 1) - โ”œ ๐Ÿ“ˆ offlineYshifts (total frames x 1) - โ”œ ๐Ÿ“ˆ offlineZshifts (total frames x 1) - โ”œ ๐Ÿ“ˆ onlineXshifts (total frames x 1) - โ”œ ๐Ÿ“ˆ onlineYshifts (total frames x 1) - โ”œ ๐Ÿ“ˆ onlineZshifts (total frames x 1) - โ”œ ๐Ÿ”ข trial_num_frames (trials x 1) - โ”œ ๐Ÿ”ข frame_line_idxs (total frames x 1) - โ”” โ˜‘๏ธ discard_frames (total frames x 1) -``` +Installation requires Git/network access for the `slap2_utils` dependency. Interactive ROI drawing uses OpenCV and tkinter; tkinter must be available in the Python installation. Headless extraction does not need a display. -A summary file of per-trial data is also saved as `per_trial_summary.h5` for any fields that may vary across analysis trials. The trial axis matches `trial_table.h5` (all analysis trials); trials without alignment or source-extraction data are left as NaN in the corresponding slices. +For development setup and documentation builds, see [CONTRIBUTING.md](CONTRIBUTING.md). -``` -๐Ÿ—„๏ธ per_trial_summary.h5 - โ”œ โ˜‘๏ธ row_major - โ”” ๐Ÿ“ Path{1,2} - โ”œ ๐Ÿ“ sources - | โ”œ ๐Ÿ“ temporal - | | โ”” ๐Ÿ“ˆ per_trial_SNR (trials x sources) - | โ”” ๐Ÿ“ spatial - | โ”œ ๐Ÿ–ผ๏ธ per_trial_profiles (trials x sources x fastz x rows x cols) - | โ”” ๐Ÿ“ˆ per_trial_coords (trials x sources x 3 [z_loc, y_loc, x_loc]) - โ”” ๐Ÿ“ visualizations - โ”œ ๐Ÿ–ผ๏ธ per_trial_mean_im (trials x channels x fastz x rows x cols) - โ”œ ๐Ÿ–ผ๏ธ per_trial_act_im (trials x fastz x rows x cols) - โ”œ ๐Ÿ–ผ๏ธ per_trial_act_im_peaks (trials x max_peaks x 3 [z_loc, y_loc, x_loc]) - โ”” ๐Ÿ”ข per_trial_num_peaks (trials x 1) -``` +## Run BandSILo -### Band Registration Lookup Table (intermediate file) +Use the actual trial-table filename, or pass a loaded `TrialTable`: -`BandRegistration` builds this file once per experiment under `motion_correction/bandRegLookupTable.h5` and reuses it on subsequent runs. XY search limits (`xPre`, `yPre`, etc.) are shared across paths; per-path superpixel and reference-stack data live under `Path{n}`. +```python +from giant_python import ExecutionOptions, extract_band_sources +summary = extract_band_sources( + "path/to/trial_table.h5", + execution=ExecutionOptions(max_workers=6, verbose=True), +) ``` -๐Ÿ—„๏ธ bandRegLookupTable.h5 - โ”œ ๐Ÿ”ข xPre - โ”œ ๐Ÿ”ข xPost - โ”œ ๐Ÿ”ข yPre - โ”œ ๐Ÿ”ข yPost - โ”œ โ˜‘๏ธ row_major - โ”” ๐Ÿ“ Path{1,2} - โ”œ ๐Ÿ“ˆ likelihood_means (Y x X x Z x C x nSP) - โ”œ ๐Ÿ”ข allSuperPixelIDs (nSP x 1) - โ”œ ๐Ÿ”ข sparseMaskInds (N x 2) - โ”œ ๐Ÿ”ข zPre - โ”œ ๐Ÿ”ข zPost - โ”” ๐Ÿ“ˆ fastZ2RefZ -``` - -## File Field Descriptions - -### `trial_table.h5` - -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag: `1` = row-major (sizes match h5py `shape`; written by GIAnT-python); `0` = column-major (MATLAB `size()`). **If absent, assume column-major (`0`).** | -| `datadr` | 1 x 1 | string | Data directory location | -| `savedr` | 1 x 1 | string | Results directory location | -| `filename` | nPaths x total trials | string (ragged) | Relative file name from `datadr` | -| `true_trial_ix` | nPaths x total trials | integer | Trial indices unraveled by epochs | -| `epoch` | nPaths x total trials | integer | Epoch numbers | -| `slap2_info` | โ€” | group | Only saved for SLAP2 experiments | -| `slap2_info/ref_stack/Path{1,2}/IM` | image dims | numeric | Reference stack image | -| `slap2_info/ref_stack/Path{1,2}/channels` | 1 x nChannels | numeric | Color channels | -| `slap2_info/ref_stack/Path{1,2}/Zs` | 1 x nZ | numeric | Z positions | -| `slap2_info/ref_stack/Path{1,2}/dmdPixel2SampleTransform` | 3 x 3 | numeric | Transformation matrix | -| `slap2_info/first_line` | nPaths x total trials | integer | First line of each trial (1-indexed) | -| `slap2_info/last_line` | nPaths x total trials | integer | Last line of each trial (1-indexed) | -| `slap2_info/trial_start_time_inferred` | 1 x total trials | integer | Inferred trial start times | -| `slap2_info/trial_end_time_from_pc` | 1 x total trials | integer | Trial end times from PC | -| `motion_correction` | โ€” | group | Written by motion correction stage | -| `motion_correction/fn_reg_ds` | nPaths x total trials | string | Registered + downsampled tif filename | -| `motion_correction/fn_adata` | nPaths x total trials | string | Alignment metadata `_ALIGNMENTDATA.h5` filename | -| `motion_correction/fn_raw` | nPaths x total trials | string | Registered raw-resolution file (Bergamo only) | -| `motion_correction/registration_failed` | nPaths x total trials | bool | Whether registration failed | -| `motion_correction/first_line_original` | nPaths x total trials | integer | Original `slap2_info/first_line` before reVolt adjustment | -| `motion_correction/align_params` | โ€” | group/struct | Alignment parameters used | -| `source_extraction` | โ€” | group | Written by source extraction stage | -| `source_extraction/analysis_params` | โ€” | group/struct | Analysis parameters used | -| `source_extraction/fn_raw` | nPaths x total trials | string | Raw file source extraction reads from per trial | - -### `_ALIGNMENTDATA.h5` -Top-level fields written by **all three** motion correction backends: `numChannels`, `frametime`, `alignHz`, `motionDSc`, `motionDSr`. `meanIM` and `recNegErr` are written by StripRegistration and MultiRoiRegistration but **not** by BandRegistration. `motionC`/`motionR` are written only by StripRegistration (Bergamo); `DSframes`/`registrationFailed` by both SLAP2 backends (MultiRoiRegistration and BandRegistration); `brightnessDS`/`logLikelihoodDS` by BandRegistration only. The `slap2` group is only populated for SLAP2 experiments. +Annotations are disabled by default. Use `BandSiloParams` for scientific settings, `ExecutionOptions` for workers/progress/trial limits, and `AnnotationOptions` for ROI inclusion and GUI policy. A chainable `Pipeline` is also available; see [API and data conventions](docs/source/migration.rst). -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `numChannels` | 1 x 1 | integer | Number of channels in the recording | -| `meanIM` | channels x rows x cols | single | Per-channel mean of motion-corrected frames (not written by BandRegistration) | -| `frametime` | 1 x 1 | numeric | Seconds per downsampled frame | -| `alignHz` | 1 x 1 | numeric | Frame rate (Hz) at which alignment was performed | -| `motionDSc` | 1 x nDSframes | numeric | Inferred column shift per downsampled frame | -| `motionDSr` | 1 x nDSframes | numeric | Inferred row shift per downsampled frame | -| `motionDSz` | 1 x nDSframes | numeric | Inferred Z shift per downsampled frame; always written by BandRegistration; written by MultiRoiRegistration only when `refStackTemplate` is enabled; never written by StripRegistration | -| `recNegErr` | 1 x nDSframes | numeric | Per-frame reconstruction error; alignment QC metric and used for motion censoring (not written by BandRegistration) | -| `brightnessDS` | nDSframes x channels | numeric | (BandRegistration only) Per-channel brightness/scaling factor at the selected motion shift | -| `logLikelihoodDS` | nDSframes x 1 | numeric | (BandRegistration only) Peak log-likelihood of the motion match per downsampled frame | -| `motionC` | 1 x nFrames | numeric | Column shift upsampled to raw frame rate (Bergamo only) | -| `motionR` | 1 x nFrames | numeric | Row shift upsampled to raw frame rate (Bergamo only) | -| `motionZ` | 1 x nFrames | numeric | (reserved; not written by any current script) Z shift upsampled to raw frame rate | -| `DSframes` | 1 x nDSframes | integer | Line indices of each downsampled frame (SLAP2 only: MultiRoiRegistration and BandRegistration; 1-indexed) | -| `registrationFailed` | 1 x 1 | bool | Whether registration failed for this trial (SLAP2 only: MultiRoiRegistration and BandRegistration) | -| `slap2` | โ€” | group | Only saved for SLAP2 experiments | -| `slap2/varFacDS` | rows x cols x nDSframes | numeric | (MultiRoiRegistration only) Variance factor; multiply pixel intensity to get a value proportional to its variance | -| `slap2/Z_depths` | fastz x 1 | numeric | (MultiRoiRegistration only) Imaged Z depths from microscope metadata | -| `slap2/cropRow` | 1 x 1 | integer | (MultiRoiRegistration only) Row offset to add to ROIs to index into original recording | -| `slap2/cropCol` | 1 x 1 | integer | (MultiRoiRegistration only) Column offset to add to ROIs to index into original recording | -| `slap2/viewC` | (rows+2ยทmaxshift) x (cols+2ยทmaxshift) | numeric | (MultiRoiRegistration only) Column interpolation grid for remapping into saved tiff space | -| `slap2/viewR` | (rows+2ยทmaxshift) x (cols+2ยทmaxshift) | numeric | (MultiRoiRegistration only) Row interpolation grid for remapping into saved tiff space | -| `slap2/trimRows` | 1 x nTrimRows | integer | (MultiRoiRegistration only) Row indices used to remap images from the datafile into saved tiff space | -| `slap2/trimCols` | 1 x nTrimCols | integer | (MultiRoiRegistration only) Column indices used to remap images from the datafile into saved tiff space | -| `slap2/onlineMotionXshift` | 1 x nDSframes | numeric | Online motion-correction X shift from the microscope | -| `slap2/onlineMotionYshift` | 1 x nDSframes | numeric | Online motion-correction Y shift from the microscope | -| `slap2/onlineMotionZshift` | 1 x nDSframes | numeric | Online motion-correction Z shift from the microscope | +**Paths matter:** raw inputs come from the table's `datadr`; results are written under its `savedr`, in `source_extraction/experiment_summary.h5`. Copying the table does not change either directory. For explicit directory overrides and a fresh output location, use [the staging example](examples/extract_band.py). -### `bandRegLookupTable.h5` +### Manual ROIs -`BandRegistration` writes this cached lookup table to `motion_correction/` on the first run and loads it on later runs. XY search limits (`xPre`, `yPre`, etc.) are shared across paths; per-path superpixel and reference-stack data live under `Path{n}` (one group per DMD, in trial-table path order). `Y`, `X`, and `Z` are the row, column, and reference-stack Z dimensions of the motion search cube (`yPre + yPost + 1`, etc.). +- Draw or load annotations with `annotate_rois(table_or_filename)` or `Pipeline.annotate()`. +- Extract using existing ROIs with `annotations=AnnotationOptions(enabled=True, interactive=False)`. Missing required annotations raise an error. +- The CLI provides `giant annotate` and `giant extract`, each taking a trial-table filename. For extraction with existing ROIs and no GUI, select `--draw-user-rois --headless`. -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `xPre` | 1 x 1 | numeric | Maximum column shift searched **before** the reference position (pixels); equals `align_params.maxshiftXY` | -| `xPost` | 1 x 1 | numeric | Maximum column shift searched **after** the reference position (pixels); equals `align_params.maxshiftXY` | -| `yPre` | 1 x 1 | numeric | Maximum row shift searched **before** the reference position (pixels); equals `align_params.maxshiftXY` | -| `yPost` | 1 x 1 | numeric | Maximum row shift searched **after** the reference position (pixels); equals `align_params.maxshiftXY` | -| `Path{n}` | โ€” | group | One group per imaging path (DMD) | -| `Path{n}/likelihood_means` | Y x X x Z x C x nSP | single | Precomputed expected superpixel mean intensity in the padded reference stack at each displacement in the search cube, per channel and superpixel; used as the template for Poisson or correlation motion inference | -| `Path{n}/allSuperPixelIDs` | nSP x 1 | numeric | Unique superpixel keys for this path: `superPixIdx * 100 + zIdx` (integration-mode pixels only when `integrationOnly` is true) | -| `Path{n}/sparseMaskInds` | N x 2 | numeric | Sparse ROI definition: column 1 = linear DMD pixel index (`rows x cols x numFastZs` layout); column 2 = superpixel index (1 โ€ฆ nSP) | -| `Path{n}/zPre` | 1 x 1 | numeric | Maximum reference-stack Z shift searched **before** the matched plane (planes); capped by `align_params.maxshiftZ` and available reference Z planes | -| `Path{n}/zPost` | 1 x 1 | numeric | Maximum reference-stack Z shift searched **after** the matched plane (planes); capped similarly to `zPre` | -| `Path{n}/fastZ2RefZ` | numFastZs x 1 | numeric | Maps each imaged fast-Z index to the nearest reference-stack Z plane index (used when sampling `likelihood_means`) | +Annotations are stored beneath the table's `savedr` in `annotations/annotations.h5`. -### `annotations.h5` +## Results and cautions -**Indexing conventions.** New files set `coords_zero_indexed` to `1` and store `position` / `center` as **0-indexed `[y_loc, x_loc]`** (row, column). Legacy files without this flag use MATLAB `images.roi` convention: **1-indexed `[x, y]`** (column, row). +The returned `ExperimentSummary` is also saved to disk. Access imaging paths through `summary.paths`; each path contains sources, ROIs, visualizations, frame information, depths, and global fluorescence. Arrays are eager and may share storage. `path_index` is zero-based; disk groups start at `Path1`. -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `coords_zero_indexed` | 1 x 1 | uint8 | When `1`, `position` / `center` are 0-indexed `[y_loc, x_loc]`; when absent or `0`, legacy 1-indexed `[x, y]` | -| `Path{n}` | โ€” | group | One group per imaging path in trial-table order | -| `Path{n}/dr` | 1 x nChars | uint16 | Motion-correction directory used while drawing these ROIs | -| `Path{n}/fn` | 1 x nChars | uint16 | Trial stem used when displaying ROI GUI | -| `Path{n}/n_rois` | 1 x 1 | uint32 | Number of saved ROI entries for this path | -| `Path{n}/roi_###/type` | 1 x nChars | uint16 | ROI geometry type: `polygon`, `circle`, or `ellipse` | -| `Path{n}/roi_###/label` | 1 x nChars | uint16 | User label (e.g., `SOMA`) | -| `Path{n}/roi_###/mask` | rows x cols | uint8 | Binary ROI mask in image coordinates (1 = included pixel) | -| `Path{n}/roi_###/position` | nVertices x 2 | double | Polygon vertices `[y_loc, x_loc]` when `coords_zero_indexed=1`, else legacy `[x, y]` | -| `Path{n}/roi_###/center` | 1 x 2 | double | Circle/ellipse center `[y_loc, x_loc]` when `coords_zero_indexed=1`, else legacy `[x, y]` | -| `Path{n}/roi_###/semi_axes` | 1 x 2 | double | Ellipse semi-axes lengths (ellipse only) | -| `Path{n}/roi_###/rotation_angle` | 1 x 1 | double | Ellipse rotation angle in degrees (ellipse only) | -| `Path{n}/roi_###/radius` | 1 x 1 | double | Circle radius (circle only) | +- Extraction saves least-squares `dF_ls`, convolution-constrained `dF_denoised`, and fitted `events`. Source activity uses channel zero; other channels and missing observations are NaN. Events are not automatically neuronal spikes, and smoother traces do not establish improved SNR. +- ROI `Fsvd` is written as NaN unless supplied explicitly; BandSILo does not compute it. Disabled or empty annotations produce zero-ROI datasets. +- Acquisition line indices are one-based; spatial coordinates are zero-based `[z, y, x]`. Saved offline shifts follow the MATLAB SILo alignment-displacement sign. +- **Use a fresh output location for each analysis.** Updating an existing summary preserves unrelated paths and existing root parameters/layout flags; writes are not atomic replacement or resume. -### `experiment_summary.h5` +## Reference -**Indexing conventions.** Pixel/plane coordinates in `sources/spatial/coords` and related peak/coordinate fields are written **0-indexed** as `[z_loc, y_loc, x_loc]`, matching image axis order (`fastz`, rows, cols). `frame_line_idxs` is kept **1-indexed** to retain the SLAP2 line-indexing convention. +Documentation provenance.[^documentation] -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `params` | โ€” | group/struct | Analysis parameters. `params/activityChannel` is **1-indexed** into the recording's `numChannels` channels โ€” use it to pick the glutamate channel from any `channels x โ€ฆ` dataset (e.g., `global/F`, `sources/temporal/dF_ls`) | -| `Path{n}` | โ€” | group | One group per imaging path | -| `Path{n}/Z_depths` | fastz x 1 | numeric | Z depths per imaging plane (SLAP2 only) | -| `Path{n}/frame_info/offlineXshifts` | total frames x 1 | numeric | Offline registration X shift per frame | -| `Path{n}/frame_info/offlineYshifts` | total frames x 1 | numeric | Offline registration Y shift per frame | -| `Path{n}/frame_info/offlineZshifts` | total frames x 1 | numeric | (optional) Offline registration Z shift per frame; written only when 3D alignment was performed | -| `Path{n}/frame_info/onlineXshifts` | total frames x 1 | numeric | (SLAP2 only) online X shift per frame | -| `Path{n}/frame_info/onlineYshifts` | total frames x 1 | numeric | (SLAP2 only) online Y shift per frame | -| `Path{n}/frame_info/onlineZshifts` | total frames x 1 | numeric | (SLAP2 only) online Z shift per frame | -| `Path{n}/frame_info/trial_num_frames` | trials x 1 | integer | Number of frames contributed by each analysis trial | -| `Path{n}/frame_info/frame_line_idxs` | total frames x 1 | integer | Raw line (SLAP2) or frame (other microscopes) index for each frame in the stitched series. **1-indexed** | -| `Path{n}/frame_info/discard_frames` | total frames x 1 | bool or uint8 | Frame excluded from analysis (e.g., motion censoring) | -| `Path{n}/visualizations/mean_im` | channels x fastz x rows x cols | numeric | Mean registered image per channel / Z slice | -| `Path{n}/visualizations/act_im` | fastz x rows x cols | numeric | Activity / localization summary image | -| `Path{n}/visualizations/act_im_peaks` | sources x 3 | numeric | Activity image peak locations `[z_loc, y_loc, x_loc]`, **0-indexed**; from source row/column coordinates, `z_loc` fixed at `0` | -| `Path{n}/global/F` | channels x total frames | numeric | Fluorescence traces over the whole field (one row per channel) | -| `Path{n}/user_rois/labels` | rois x 1 | string | User-defined ROI labels | -| `Path{n}/user_rois/mask` | rois x fastz x rows x cols | uint8 or bool | Stacked binary masks for each user ROI | -| `Path{n}/user_rois/Fsvd` | rois x channels x total frames | numeric | ROI signals after SVD / projection step (if used) | -| `Path{n}/user_rois/F` | rois x channels x total frames | numeric | Raw or baseline-corrected ROI fluorescence | -| `Path{n}/sources/spatial/profiles` | sources x fastz x rows x cols | numeric | Spatial component / pixel weights per source, averaged across trials with footprints | -| `Path{n}/sources/spatial/coords` | sources x 3 | numeric | Source centers per row: `[z_loc, y_loc, x_loc]`, **0-indexed** | -| `Path{n}/sources/temporal/dF_ls` | sources x channels x total frames | numeric | Least-squares ฮ”F (absolute or scaled) | -| `Path{n}/sources/temporal/dF_denoised` | sources x channels x total frames | numeric | Denoised ฮ”F | -| `Path{n}/sources/temporal/events` | sources x channels x total frames | numeric | Deconvolved source events | -| `Path{n}/sources/temporal/F0` | sources x channels x total frames | numeric | Baseline estimate used for normalization | -| `Path{n}/sources/temporal/SNR` | sources x 1 | numeric | (optional) Signal-to-noise ratio metric; only written when extraction emits per-source SNR | +- [API, configuration, and staging](docs/source/migration.rst) +- [Trace estimation, controls, and scientific caveats](docs/source/traces.rst) +- [File formats, coordinates, and MATLAB interoperability](docs/source/file_formats.rst) +- [Architecture and module ownership](docs/source/architecture.rst) -### `per_trial_summary.h5` +## Support -The trial axis matches `trial_table.h5` (all analysis trials); trials without alignment or source-extraction data are left as NaN in the corresponding slices. Coordinate fields use the same **0-indexed** `[z_loc, y_loc, x_loc]` convention as `experiment_summary.h5`; `z_loc` is currently always `0`. +**Occasional updates:** no fixed update schedule. Community involvement through issues and pull requests is welcome. -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `Path{n}` | โ€” | group | One group per imaging path | -| `Path{n}/visualizations/per_trial_mean_im` | trials x channels x fastz x rows x cols | numeric | Trial-aligned mean registered image per channel / Z slice | -| `Path{n}/visualizations/per_trial_act_im` | trials x fastz x rows x cols | numeric | Trial-aligned activity / localization summary image | -| `Path{n}/visualizations/per_trial_act_im_peaks` | trials x max_peaks x 3 | numeric | Per-trial detected peak locations `[z_loc, y_loc, x_loc]`, **0-indexed**, NaN-padded when a trial has fewer than `max_peaks` | -| `Path{n}/visualizations/per_trial_num_peaks` | trials x 1 | integer | Number of valid peaks per trial | -| `Path{n}/sources/spatial/per_trial_profiles` | trials x sources x fastz x rows x cols | numeric | Spatial component / pixel weights per source per trial | -| `Path{n}/sources/spatial/per_trial_coords` | trials x sources x 3 | numeric | Source centers per trial: `[z_loc, y_loc, x_loc]`, **0-indexed** | -| `Path{n}/sources/temporal/per_trial_SNR` | trials x sources | numeric | Per-source SNR for each analysis trial | +[^documentation]: Current documentation has been generated by large language models (LLMs) from human-verified code. \ No newline at end of file diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst new file mode 100644 index 0000000..5239b07 --- /dev/null +++ b/docs/source/architecture.rst @@ -0,0 +1,121 @@ +Architecture and ownership +========================== + +Scope +----- + +GIAnT-Python is a package for processing glutamate imaging data, with +advanced capabilities for SLAP2 recordings. The current implementation +supports only SLAP2 Band Scan processing: BandSILo source extraction and +Band Scan ROI annotation from pre-registered inputs. Additional functionality +is available in +`GIAnT-MATLAB `_, +with porting efforts underway. + +The architecture below describes the implemented functionality. Public usage, +configuration, results and staging are documented in :doc:`migration`; saved +schemas and coordinate conventions are in :doc:`file_formats`; trace +estimation is in :doc:`traces`. + +Module owners +------------- + +All module names below are relative to ``giant_python``. + +.. list-table:: Canonical responsibilities + :header-rows: 1 + :widths: 44 56 + + * - Owner + - Responsibility + * - ``pipeline.extract`` / ``pipeline.annotate`` / ``pipeline.pipeline`` + - Lazy public services, explicit capability routing and the chainable + ``Pipeline``. The CLI and standalone calls use the same services. + * - ``models.params`` / ``models.trial_table`` + - Split configuration and portable input metadata; construction does not + discover band assets or normalize motion. + * - ``models.experiment_summary`` + - Eager ``ExperimentSummary``, ``PathSummary``, ``FrameInfo``, ``Source``, + ``UserRoi`` and ``Visualizations`` models. + * - ``io.experiment_summary`` / ``io.hdf5`` / ``io.trial_table`` + - Summary codecs, shared HDF5 decoding and parameter-group writing, + and trial-table reading. Model convenience codecs delegate here. + * - ``io.slap2`` / ``io.tiff`` + - Raw acquisition/alignment reading and TIFF decoding, without private + band motion-sign normalization. + * - ``extraction.band.inputs`` + - Runtime paths, kept trials, resolved acquisition metadata, private + reference offsets, lookup/reference loading and PSF discovery. + * - ``extraction.band.workflow`` / ``extraction.band.result_assembly`` + - Scientific orchestration and construction of public path models from + numerical results; persistence is delegated to shared IO. + * - ``extraction.band.geometry`` / ``extraction.band.operators`` + - Pure pixel support, PSF padding/cropping and sparse projection operators. + * - ``extraction.band.motion_binning`` / ``extraction.band.background`` / + ``extraction.band.activity`` + - Band motion policies, background estimation and activity calculations. + * - ``extraction.band.localization`` + - Band-specific NMF/localization and spatial profile fitting. + * - ``extraction.band.traces`` / ``extraction.band.transient`` + - Trace extraction and fixed-profile temporal optimization. + * - ``extraction.band.summary_images`` / ``extraction.band.trial_data`` + - Image assembly and superpixel data reduction. + * - ``numerics.baseline`` / ``numerics.peaks`` / + ``numerics.interpolation`` / ``numerics.morphology`` + - Acquisition-independent numerical kernels and helpers. + * - ``io.annotations`` / ``extraction.band.rois`` / + ``extraction.band.annotation`` + - Annotation persistence, pure band ROI geometry and annotation policy. + * - ``gui.roi_editor`` / ``gui.parameters_gui`` + - Interactive presentation with lazy toolkit imports. + * - ``parallel`` / ``progress`` + - Ordered trial mapping and opt-in logging/progress; the declared + ``tqdm`` dependency is imported lazily for verbose progress. + * - ``assets/psfs/`` + - Bundled PSF templates. + +Backend and IO boundaries +------------------------- + +Runtime input preparation resolves raw and alignment locations from the +loaded table's explicit ``datadr`` and ``savedr``. ``source_path`` records +provenance, not a location reconstructed from ``savedr``. Copying a table +does not relocate its inputs; see :doc:`migration` for staging instructions. +``ResolvedAcquisition`` carries inferred channel metadata separately from +caller-owned scientific options. + +``PathResult`` collects numerical outputs; +``result_assembly.assemble_path_summary(PathResult, path_index=0)`` constructs +the public ``PathSummary``. Schema dictionaries inside IO are implementation +details, not a second public result API. The summary codec supports the +emitted band subset, not lossless round-tripping of every external field. +See :doc:`file_formats` for output coverage and non-atomic write behavior. + +``Slap2Reader`` is a structural typing protocol for the vendor reader's +attributes and batched ``getLineData`` method. It promises neither a context +manager nor a close method. OpenCV and tkinter are loaded for interactive +work, not model deserialization; the implemented ROI editor does not use +napari. + +Numerical boundaries +-------------------- + +Spatial localization/NMF and trace-only temporal optimization are separate +stages. Source activity uses channel zero. See :doc:`traces` for the trace +objective, validity masks, trial boundaries, convergence and reproducibility +controls rather than treating denoising as a selectable extraction mode. + +``numerics.peaks.gaussian_peaks_integrated`` accepts ``(N, 5)`` parameters +``[amp, mu_y, mu_x, sigma_y, sigma_x]``. Both widths must be explicit, even +for isotropic Gaussians. Fitting uses an isotropic warm start followed by +anisotropic refinement: the first stage ties width updates using their +combined derivative while retaining the five-column representation. + +Raw alignment reading preserves saved displacement. Band input preparation +negates it into private reference offsets; ``assemble_path_summary`` converts +those offsets back exactly once for public offline shifts. Generic IO does +not flip signs. Displacement maps reference/view-grid coordinates into raw +sampling coordinates; the inverse mapping uses its negative. No inter-trial +XY correction is invented where the workflow has not computed one. See +:doc:`file_formats` for saved motion signs, producer-provenance cautions and +the linear/clamped versus MATLAB PCHIP/extrapolation distinction. \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index e4ca118..9fd81d3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -29,13 +29,50 @@ extensions = [ "sphinx.ext.duration", - "sphinx.ext.doctest", "sphinx.ext.autodoc", "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", ] templates_path = ["_templates"] exclude_patterns = [] +intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "h5py": ("https://docs.h5py.org/en/stable/", None), +} + +napoleon_use_param = True +napoleon_use_rtype = True +napoleon_preprocess_types = True +napoleon_type_aliases = { + "Path": "pathlib.Path", + "ndarray": "numpy.ndarray", + "TrialTable": "giant_python.models.trial_table.TrialTable", + "Slap2Info": "giant_python.models.trial_table.Slap2Info", + "BandSiloParams": "giant_python.models.params.BandSiloParams", + "ExecutionOptions": "giant_python.models.params.ExecutionOptions", + "AnnotationOptions": "giant_python.models.params.AnnotationOptions", + "ExperimentSummary": ( + "giant_python.models.experiment_summary.ExperimentSummary" + ), +} + + +def _resolve_public_type_alias(app, env, node, contnode): + """Resolve h5py's runtime class name through its public documentation.""" + if ( + node.get("refdomain") == "py" + and node.get("reftarget") == "h5py._hl.group.Group" + ): + node["reftarget"] = "h5py.Group" + + +def setup(app): + """Normalize private type names before intersphinx resolves references.""" + app.connect("missing-reference", _resolve_public_type_alias, priority=400) + + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/docs/source/file_formats.rst b/docs/source/file_formats.rst new file mode 100644 index 0000000..347a463 --- /dev/null +++ b/docs/source/file_formats.rst @@ -0,0 +1,930 @@ +File formats +============ + +This reference describes Python BandSILo outputs and the external +`GIAnT-MATLAB `_ +schemas used to interpret pre-registered inputs. The full external field +reference is retained even where Python does not consume or emit a field. +Schema descriptions do not imply that every reference file round-trips +losslessly through ``ExperimentSummary``. + +Python outputs and write behavior +--------------------------------- + +BandSILo emits ``experiment_summary.h5`` with source ``profiles``, ``coords``, +``dF_ls``, ``dF_denoised``, ``events``, ``F0``, and ``SNR``; visualizations; +column-vector ``Z_depths``; global ``F``; frame bookkeeping and shifts; and +ROI ``labels``, ``mask``, ``Fsvd``, and ``F``. See :doc:`traces` for trace +estimation and interpretation. + +Disabled or enabled-but-empty annotations produce correctly shaped zero-ROI +datasets. The ``annotation_enabled`` attribute on each ``Path{n}/user_rois`` +group distinguishes these states. ROI ``Fsvd`` is **not calculated by +BandSILo**: the dataset contains NaN unless values are supplied explicitly +through the model. It is never copied from raw ``F``. Older summaries without +the optional temporal datasets remain readable; rewriting missing optional +data emits NaNs, not fabricated fits. + +``per_trial_summary.h5`` is an **unsupported, external-reference format**; +Python BandSILo does not emit it. Registration products and the band lookup +table are **external inputs**, not Python-generated outputs. Python loads +existing trial tables but does not write them. + +.. warning:: + + Summary writes replace supplied path groups but retain unrelated path + groups and existing root parameters and layout flags. This is neither an + atomic replacement nor a resume mechanism. Prefer a **fresh output + location** for a new analysis, and do not mix producers in place. + +HDF5 layout +----------- + +The implemented codecs use `h5py `_. New Python band +summary and annotation files set ``row_major`` to ``1``: the logical +dimensions below then match each dataset's h5py ``shape``. MATLAB files set +``row_major`` to ``0`` or omit it; their dimension tuples follow MATLAB +``size()``. The shared decoder inspects the flag and reverses multidimensional +axes as needed to recover logical orientation. + +.. list-table:: Layout flag and logical dimensions + :header-rows: 1 + :widths: 18 32 50 + + * - ``row_major`` + - Layout + - Relation to the dimensions listed below + * - ``1`` + - Row-major (h5py / NumPy C order) + - h5py ``shape`` matches the listed logical dimensions. + * - ``0`` or absent + - Column-major (MATLAB ``size()``) + - h5py axes are reversed relative to the listed logical dimensions. + +**Emitted vector shapes are preserved on disk.** Band summary frame vectors +retain rank two rather than being squeezed to ``(nFrames,)``. Reference-only +vector shapes describe the external schema, not a Python writer guarantee. +Layout conversion requires schema-specific interpretation and does not imply +full interchangeability. Updating an existing summary preserves its layout +flag; use a fresh destination when changing producers. + +Indices, coordinates, and motion signs +-------------------------------------- + +* Line/frame indices (``first_line``, ``last_line``, ``DSframes``, and + ``frame_line_idxs``) are **1-indexed**, retaining the SLAP2 convention. +* Summary spatial coordinates (``sources/spatial/coords``, ``act_im_peaks``, + and per-trial coordinate fields) are **0-indexed** + ``[z_loc, y_loc, x_loc]``, matching image axes ``(fastz, rows, cols)``. + ``z_loc`` indexes the fast-z axis; it is not a physical depth. +* Annotation ``position`` and ``center`` use **0-indexed** + ``[y_loc, x_loc]`` when ``coords_zero_indexed=1``. Absent or zero flags + denote legacy MATLAB ``images.roi`` coordinates: **1-indexed** ``[x, y]``. +* Images use ``rows x cols`` (row = Y, column = X). ``motionDSr`` and + ``motionR`` are row shifts; ``motionDSc`` and ``motionC`` are column shifts. +* Python source activity extraction always uses **channel zero**, with NaNs + in other source-activity channels. Channel count comes from acquisition + metadata, not ``BandSiloParams``. The external MATLAB parameter + ``params/activityChannel``, when present, is **1-indexed** into + ``numChannels``; it is not a Python activity-channel selector or a required + emitted parameter. + +.. warning:: + + Public ``FrameInfo.offlineXshifts``, ``offlineYshifts``, and + ``offlineZshifts`` contain alignment displacement with the standard MATLAB + SILo sign. Private band reference offsets have the opposite sign and are + converted once by ``assemble_path_summary()``, not by generic IO. + Establish producer/alignment provenance before interpreting or changing + shifts from experimental files. Never negate MATLAB-produced or + already-correct summaries again. Online shifts are unchanged. Matching + the sign does not make Python's linear/clamped interpolation equivalent + to MATLAB's PCHIP/extrapolation. + +Schema trees +------------ + +Trees show files, groups (names ending in ``/``), and datasets. Dimensions +are logical dimensions under the layout convention above. ``Path{n}`` means +one group per imaging path in trial-table order, with disk numbering starting +at one; ``PathSummary.path_index`` is zero-based. Typical two-path recordings +use ``Path1`` and ``Path2``. + +Trial table: external input +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A trial table summarizes file locations and analysis-trial structure. Python +loads an existing pre-registered table through ``TrialTable`` / ``Slap2Info``; +it does not create/save the model or append reference stage groups. +``slap2_info`` is SLAP2-specific. In the external pipeline, +``motion_correction`` and ``source_extraction`` are populated downstream:: + + trial_table.h5 + โ”œโ”€โ”€ datadr + โ”œโ”€โ”€ savedr + โ”œโ”€โ”€ filename + โ”œโ”€โ”€ true_trial_ix + โ”œโ”€โ”€ epoch + โ”œโ”€โ”€ row_major + โ”œโ”€โ”€ slap2_info/ + โ”‚ โ”œโ”€โ”€ ref_stack/ + โ”‚ โ”‚ โ””โ”€โ”€ Path{n}/ + โ”‚ โ”‚ โ”œโ”€โ”€ IM + โ”‚ โ”‚ โ”œโ”€โ”€ channels + โ”‚ โ”‚ โ”œโ”€โ”€ Zs + โ”‚ โ”‚ โ””โ”€โ”€ dmdPixel2SampleTransform + โ”‚ โ”œโ”€โ”€ first_line + โ”‚ โ”œโ”€โ”€ last_line + โ”‚ โ”œโ”€โ”€ trial_start_time_inferred + โ”‚ โ””โ”€โ”€ trial_end_time_from_pc + โ”œโ”€โ”€ motion_correction/ + โ”‚ โ”œโ”€โ”€ fn_reg_ds + โ”‚ โ”œโ”€โ”€ fn_adata + โ”‚ โ”œโ”€โ”€ fn_raw + โ”‚ โ”œโ”€โ”€ registration_failed + โ”‚ โ”œโ”€โ”€ first_line_original + โ”‚ โ””โ”€โ”€ align_params/ + โ””โ”€โ”€ source_extraction/ + โ”œโ”€โ”€ analysis_params/ + โ””โ”€โ”€ fn_raw + +Alignment data: external registration input +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +External motion correction saves one alignment HDF5 file per trial. Python +reads the band fields through +``giant_python.io.slap2.load_alignment_data_h5``; Python registration is not +implemented. The tree includes external fields not consumed by BandSILo:: + + _ALIGNMENTDATA.h5 + โ”œโ”€โ”€ row_major + โ”œโ”€โ”€ numChannels + โ”œโ”€โ”€ frametime + โ”œโ”€โ”€ alignHz + โ”œโ”€โ”€ motionDSc + โ”œโ”€โ”€ motionDSr + โ”œโ”€โ”€ motionDSz (BandRegistration; MultiRoiRegistration with refStackTemplate) + โ”œโ”€โ”€ meanIM (StripRegistration and MultiRoiRegistration) + โ”œโ”€โ”€ recNegErr (StripRegistration and MultiRoiRegistration) + โ”œโ”€โ”€ motionC (StripRegistration / Bergamo only) + โ”œโ”€โ”€ motionR (StripRegistration / Bergamo only) + โ”œโ”€โ”€ motionZ (reserved; not written by current external scripts) + โ”œโ”€โ”€ brightnessDS (BandRegistration only) + โ”œโ”€โ”€ logLikelihoodDS (BandRegistration only) + โ”œโ”€โ”€ DSframes (MultiRoiRegistration and BandRegistration) + โ”œโ”€โ”€ registrationFailed (MultiRoiRegistration and BandRegistration) + โ””โ”€โ”€ slap2/ (SLAP2 only) + โ”œโ”€โ”€ onlineMotionXshift + โ”œโ”€โ”€ onlineMotionYshift + โ”œโ”€โ”€ onlineMotionZshift + โ”œโ”€โ”€ varFacDS (MultiRoiRegistration only) + โ”œโ”€โ”€ Z_depths (MultiRoiRegistration only) + โ”œโ”€โ”€ cropRow (MultiRoiRegistration only) + โ”œโ”€โ”€ cropCol (MultiRoiRegistration only) + โ”œโ”€โ”€ viewC (MultiRoiRegistration only) + โ”œโ”€โ”€ viewR (MultiRoiRegistration only) + โ”œโ”€โ”€ trimRows (MultiRoiRegistration only) + โ””โ”€โ”€ trimCols (MultiRoiRegistration only) + +Manual annotations +~~~~~~~~~~~~~~~~~~ + +Manual annotations identify pixels to exclude from analysis or soma pixels +whose signals should be extracted. ROI information is saved in +``annotations.h5``. String fields use UTF-16 code units (``uint16``) for +MATLAB/Python compatibility:: + + annotations.h5 + โ”œโ”€โ”€ row_major + โ”œโ”€โ”€ coords_zero_indexed + โ””โ”€โ”€ Path{n}/ + โ”œโ”€โ”€ dr + โ”œโ”€โ”€ fn + โ”œโ”€โ”€ n_rois + โ””โ”€โ”€ roi_###/ + โ”œโ”€โ”€ type + โ”œโ”€โ”€ label + โ”œโ”€โ”€ mask + โ”œโ”€โ”€ position (polygon; nVertices x 2 [y_loc, x_loc] when flagged) + โ”œโ”€โ”€ center (circle/ellipse; 1 x 2 [y_loc, x_loc] when flagged) + โ”œโ”€โ”€ semi_axes (ellipse) + โ”œโ”€โ”€ rotation_angle (ellipse) + โ””โ”€โ”€ radius (circle) + +Experiment summary +~~~~~~~~~~~~~~~~~~ + +Source extraction (Source Identification by Activity Localization; SILo) +writes ``experiment_summary.h5`` with extracted sources and experiment data. +The ``total frames`` axis stitches trials from that path in time. The models +are ``ExperimentSummary``, ``Source``, ``UserRoi``, and ``Visualizations``:: + + experiment_summary.h5 + โ”œโ”€โ”€ row_major + โ”œโ”€โ”€ params/ + โ””โ”€โ”€ Path{n}/ + โ”œโ”€โ”€ Z_depths (fastz x 1) + โ”œโ”€โ”€ sources/ + โ”‚ โ”œโ”€โ”€ temporal/ + โ”‚ โ”‚ โ”œโ”€โ”€ dF_ls (sources x channels x total frames) + โ”‚ โ”‚ โ”œโ”€โ”€ dF_denoised (sources x channels x total frames) + โ”‚ โ”‚ โ”œโ”€โ”€ events (sources x channels x total frames) + โ”‚ โ”‚ โ”œโ”€โ”€ F0 (sources x channels x total frames) + โ”‚ โ”‚ โ””โ”€โ”€ SNR (sources x 1) + โ”‚ โ””โ”€โ”€ spatial/ + โ”‚ โ”œโ”€โ”€ profiles (sources x fastz x rows x cols) + โ”‚ โ””โ”€โ”€ coords (sources x 3 [z_loc, y_loc, x_loc]) + โ”œโ”€โ”€ user_rois/ [attribute: annotation_enabled] + โ”‚ โ”œโ”€โ”€ labels (rois x 1) + โ”‚ โ”œโ”€โ”€ mask (rois x fastz x rows x cols) + โ”‚ โ”œโ”€โ”€ Fsvd (rois x channels x total frames; NaN unless supplied) + โ”‚ โ””โ”€โ”€ F (rois x channels x total frames) + โ”œโ”€โ”€ visualizations/ + โ”‚ โ”œโ”€โ”€ mean_im (channels x fastz x rows x cols) + โ”‚ โ”œโ”€โ”€ act_im (fastz x rows x cols) + โ”‚ โ””โ”€โ”€ act_im_peaks (sources x 3 [z_loc, y_loc, x_loc]) + โ”œโ”€โ”€ global/ + โ”‚ โ””โ”€โ”€ F (channels x total frames) + โ””โ”€โ”€ frame_info/ + โ”œโ”€โ”€ offlineXshifts (total frames x 1) + โ”œโ”€โ”€ offlineYshifts (total frames x 1) + โ”œโ”€โ”€ offlineZshifts (total frames x 1) + โ”œโ”€โ”€ onlineXshifts (total frames x 1) + โ”œโ”€โ”€ onlineYshifts (total frames x 1) + โ”œโ”€โ”€ onlineZshifts (total frames x 1) + โ”œโ”€โ”€ trial_num_frames (trials x 1) + โ”œโ”€โ”€ frame_line_idxs (total frames x 1) + โ””โ”€โ”€ discard_frames (total frames x 1) + +Per-trial summary: unsupported external reference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Python BandSILo does not emit this file.** The external format holds fields +that vary across analysis trials. Its trial axis matches the reference trial +table (all analysis trials); trials without alignment or source-extraction +data are represented by NaN slices:: + + per_trial_summary.h5 + โ”œโ”€โ”€ row_major + โ””โ”€โ”€ Path{n}/ + โ”œโ”€โ”€ sources/ + โ”‚ โ”œโ”€โ”€ temporal/ + โ”‚ โ”‚ โ””โ”€โ”€ per_trial_SNR (trials x sources) + โ”‚ โ””โ”€โ”€ spatial/ + โ”‚ โ”œโ”€โ”€ per_trial_profiles (trials x sources x fastz x rows x cols) + โ”‚ โ””โ”€โ”€ per_trial_coords (trials x sources x 3 [z_loc, y_loc, x_loc]) + โ””โ”€โ”€ visualizations/ + โ”œโ”€โ”€ per_trial_mean_im (trials x channels x fastz x rows x cols) + โ”œโ”€โ”€ per_trial_act_im (trials x fastz x rows x cols) + โ”œโ”€โ”€ per_trial_act_im_peaks (trials x max_peaks x 3 [z_loc, y_loc, x_loc]) + โ””โ”€โ”€ per_trial_num_peaks (trials x 1) + +Band registration lookup table: external input +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +External MATLAB ``BandRegistration`` builds +``motion_correction/bandRegLookupTable.h5``. Python consumes this file; +registration and lookup-table generation are not implemented. XY search +limits are shared across paths; superpixel and reference-stack data are +per-path:: + + bandRegLookupTable.h5 + โ”œโ”€โ”€ xPre + โ”œโ”€โ”€ xPost + โ”œโ”€โ”€ yPre + โ”œโ”€โ”€ yPost + โ”œโ”€โ”€ row_major + โ””โ”€โ”€ Path{n}/ + โ”œโ”€โ”€ likelihood_means (Y x X x Z x C x nSP) + โ”œโ”€โ”€ allSuperPixelIDs (nSP x 1) + โ”œโ”€โ”€ sparseMaskInds (N x 2) + โ”œโ”€โ”€ zPre + โ”œโ”€โ”€ zPost + โ””โ”€โ”€ fastZ2RefZ + +Field descriptions +------------------ + +These tables retain the full external reference fields alongside Python +outputs. Registration outputs are inputs to Python; all per-trial-summary +fields are external-reference only. The layout flag has the same meaning in +each table: ``1`` means the listed dimensions match h5py ``shape``; ``0`` or +absence means MATLAB layout. + +Trial table fields +~~~~~~~~~~~~~~~~~~ + +.. list-table:: ``trial_table.h5`` (external input) + :header-rows: 1 + :widths: 30 20 12 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag; if absent, assume column-major (``0``). + * - ``datadr`` + - 1 x 1 + - string + - Data directory location. + * - ``savedr`` + - 1 x 1 + - string + - Results directory location. + * - ``filename`` + - nPaths x total trials + - string (ragged) + - Relative file name from ``datadr``. + * - ``true_trial_ix`` + - nPaths x total trials + - integer + - Trial indices unraveled by epochs. + * - ``epoch`` + - nPaths x total trials + - integer + - Epoch numbers. + * - ``slap2_info`` + - โ€” + - group + - Only saved for SLAP2 experiments. + * - ``slap2_info/ref_stack/Path{n}/IM`` + - image dims + - numeric + - Reference stack image. + * - ``slap2_info/ref_stack/Path{n}/channels`` + - 1 x nChannels + - numeric + - Color channels. + * - ``slap2_info/ref_stack/Path{n}/Zs`` + - 1 x nZ + - numeric + - Z positions. + * - ``slap2_info/ref_stack/Path{n}/dmdPixel2SampleTransform`` + - 3 x 3 + - numeric + - Transformation matrix. + * - ``slap2_info/first_line`` + - nPaths x total trials + - integer + - First line of each trial (1-indexed). + * - ``slap2_info/last_line`` + - nPaths x total trials + - integer + - Last line of each trial (1-indexed). + * - ``slap2_info/trial_start_time_inferred`` + - 1 x total trials + - integer + - Inferred trial start times. + * - ``slap2_info/trial_end_time_from_pc`` + - 1 x total trials + - integer + - Trial end times from PC. + * - ``motion_correction`` + - โ€” + - group + - Written by the external motion-correction stage. + * - ``motion_correction/fn_reg_ds`` + - nPaths x total trials + - string + - Registered and downsampled TIFF filename. + * - ``motion_correction/fn_adata`` + - nPaths x total trials + - string + - Alignment metadata ``_ALIGNMENTDATA.h5`` filename. + * - ``motion_correction/fn_raw`` + - nPaths x total trials + - string + - Registered raw-resolution file (Bergamo only). + * - ``motion_correction/registration_failed`` + - nPaths x total trials + - bool + - Whether registration failed. + * - ``motion_correction/first_line_original`` + - nPaths x total trials + - integer + - Original ``slap2_info/first_line`` before reVolt adjustment. + * - ``motion_correction/align_params`` + - โ€” + - group/struct + - Alignment parameters used. + * - ``source_extraction`` + - โ€” + - group + - Written by the external source-extraction stage. + * - ``source_extraction/analysis_params`` + - โ€” + - group/struct + - Analysis parameters used. + * - ``source_extraction/fn_raw`` + - nPaths x total trials + - string + - Raw file source extraction reads from per trial. + +Alignment data fields +~~~~~~~~~~~~~~~~~~~~~ + +All three external motion-correction backends write ``numChannels``, +``frametime``, ``alignHz``, ``motionDSc``, and ``motionDSr``. +``meanIM`` and ``recNegErr`` are written by StripRegistration and +MultiRoiRegistration, but not BandRegistration. ``motionC`` and ``motionR`` +are StripRegistration/Bergamo-only. Both SLAP2 backends write ``DSframes`` +and ``registrationFailed``. ``brightnessDS`` and ``logLikelihoodDS`` are +BandRegistration-only. The ``slap2`` group is populated only for SLAP2. + +.. list-table:: ``_ALIGNMENTDATA.h5`` (external registration input) + :header-rows: 1 + :widths: 28 22 12 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``numChannels`` + - 1 x 1 + - integer + - Number of channels in the recording. + * - ``meanIM`` + - channels x rows x cols + - single + - Per-channel mean of motion-corrected frames; not written by BandRegistration. + * - ``frametime`` + - 1 x 1 + - numeric + - Seconds per downsampled frame. + * - ``alignHz`` + - 1 x 1 + - numeric + - Frame rate (Hz) at which alignment was performed. + * - ``motionDSc`` + - 1 x nDSframes + - numeric + - Inferred column shift per downsampled frame. + * - ``motionDSr`` + - 1 x nDSframes + - numeric + - Inferred row shift per downsampled frame. + * - ``motionDSz`` + - 1 x nDSframes + - numeric + - Inferred Z shift per downsampled frame. Always written by BandRegistration; + written by MultiRoiRegistration only with ``refStackTemplate`` enabled; + never written by StripRegistration. + * - ``recNegErr`` + - 1 x nDSframes + - numeric + - Per-frame reconstruction error; alignment QC metric used for motion + censoring. Not written by BandRegistration. + * - ``brightnessDS`` + - nDSframes x channels + - numeric + - BandRegistration-only per-channel brightness/scaling factor at the + selected motion shift. + * - ``logLikelihoodDS`` + - nDSframes x 1 + - numeric + - BandRegistration-only peak log-likelihood of the motion match per + downsampled frame. + * - ``motionC`` + - 1 x nFrames + - numeric + - Column shift upsampled to raw frame rate (Bergamo only). + * - ``motionR`` + - 1 x nFrames + - numeric + - Row shift upsampled to raw frame rate (Bergamo only). + * - ``motionZ`` + - 1 x nFrames + - numeric + - Reserved Z shift upsampled to raw frame rate; not written by any + current external script. + * - ``DSframes`` + - 1 x nDSframes + - integer + - Line indices of each downsampled frame (1-indexed); SLAP2 only, + MultiRoiRegistration and BandRegistration. + * - ``registrationFailed`` + - 1 x 1 + - bool + - Whether registration failed for this trial; SLAP2 only, + MultiRoiRegistration and BandRegistration. + * - ``slap2`` + - โ€” + - group + - Only saved for SLAP2 experiments. + * - ``slap2/varFacDS`` + - rows x cols x nDSframes + - numeric + - MultiRoiRegistration-only variance factor; multiply pixel intensity + by it to obtain a value proportional to its variance. + * - ``slap2/Z_depths`` + - fastz x 1 + - numeric + - MultiRoiRegistration-only imaged Z depths from microscope metadata. + * - ``slap2/cropRow`` + - 1 x 1 + - integer + - MultiRoiRegistration-only row offset to add to ROIs to index the + original recording. + * - ``slap2/cropCol`` + - 1 x 1 + - integer + - MultiRoiRegistration-only column offset to add to ROIs to index the + original recording. + * - ``slap2/viewC`` + - (rows+2ยทmaxshift) x (cols+2ยทmaxshift) + - numeric + - MultiRoiRegistration-only column interpolation grid for remapping + into saved TIFF space. + * - ``slap2/viewR`` + - (rows+2ยทmaxshift) x (cols+2ยทmaxshift) + - numeric + - MultiRoiRegistration-only row interpolation grid for remapping + into saved TIFF space. + * - ``slap2/trimRows`` + - 1 x nTrimRows + - integer + - MultiRoiRegistration-only row indices used to remap images from + the datafile into saved TIFF space. + * - ``slap2/trimCols`` + - 1 x nTrimCols + - integer + - MultiRoiRegistration-only column indices used to remap images from + the datafile into saved TIFF space. + * - ``slap2/onlineMotionXshift`` + - 1 x nDSframes + - numeric + - Online motion-correction X shift from the microscope. + * - ``slap2/onlineMotionYshift`` + - 1 x nDSframes + - numeric + - Online motion-correction Y shift from the microscope. + * - ``slap2/onlineMotionZshift`` + - 1 x nDSframes + - numeric + - Online motion-correction Z shift from the microscope. + +Band registration lookup fields +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +External ``BandRegistration`` writes the cached lookup table into +``motion_correction/`` on its first run and loads it on later runs. XY search +limits (``xPre``, ``yPre``, etc.) are shared across paths; each ``Path{n}`` +holds superpixel and reference-stack data for one DMD in trial-table order. +``Y``, ``X``, and ``Z`` denote row, column, and reference-stack Z dimensions +of the search cube (``yPre + yPost + 1``, etc.). + +.. list-table:: ``bandRegLookupTable.h5`` (external registration input) + :header-rows: 1 + :widths: 28 22 12 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``xPre`` + - 1 x 1 + - numeric + - Maximum column shift searched before the reference position (pixels); + equals ``align_params.maxshiftXY``. + * - ``xPost`` + - 1 x 1 + - numeric + - Maximum column shift searched after the reference position (pixels); + equals ``align_params.maxshiftXY``. + * - ``yPre`` + - 1 x 1 + - numeric + - Maximum row shift searched before the reference position (pixels); + equals ``align_params.maxshiftXY``. + * - ``yPost`` + - 1 x 1 + - numeric + - Maximum row shift searched after the reference position (pixels); + equals ``align_params.maxshiftXY``. + * - ``Path{n}`` + - โ€” + - group + - One group per imaging path (DMD). + * - ``Path{n}/likelihood_means`` + - Y x X x Z x C x nSP + - single + - Precomputed expected superpixel mean intensity in the padded reference + stack at each search-cube displacement, per channel and superpixel; + template for Poisson or correlation motion inference. + * - ``Path{n}/allSuperPixelIDs`` + - nSP x 1 + - numeric + - Unique keys ``superPixIdx * 100 + zIdx`` for this path; band-scan pixels + only when ``bandOnly`` is true. + * - ``Path{n}/sparseMaskInds`` + - N x 2 + - numeric + - Sparse ROI definition. Column 1: linear DMD pixel index in + ``rows x cols x numFastZs`` layout. Column 2: superpixel index (1 โ€ฆ nSP). + * - ``Path{n}/zPre`` + - 1 x 1 + - numeric + - Maximum reference-stack Z shift searched before the matched plane + (planes); capped by ``align_params.maxshiftZ`` and available reference + Z planes. + * - ``Path{n}/zPost`` + - 1 x 1 + - numeric + - Maximum reference-stack Z shift searched after the matched plane + (planes); capped as for ``zPre``. + * - ``Path{n}/fastZ2RefZ`` + - numFastZs x 1 + - numeric + - Maps each imaged fast-Z index to its nearest reference-stack Z plane + index, used when sampling ``likelihood_means``. + +Annotation fields +~~~~~~~~~~~~~~~~~ + +New files set ``coords_zero_indexed=1`` and store ``position`` / ``center`` +as **0-indexed** ``[y_loc, x_loc]`` (row, column). Legacy files without the +flag use MATLAB ``images.roi`` convention: **1-indexed** ``[x, y]`` +(column, row). + +.. list-table:: ``annotations.h5`` + :header-rows: 1 + :widths: 28 22 12 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``coords_zero_indexed`` + - 1 x 1 + - uint8 + - When ``1``, ``position`` / ``center`` are 0-indexed + ``[y_loc, x_loc]``; when absent or ``0``, legacy 1-indexed ``[x, y]``. + * - ``Path{n}`` + - โ€” + - group + - One group per imaging path in trial-table order. + * - ``Path{n}/dr`` + - 1 x nChars + - uint16 + - Motion-correction directory used while drawing these ROIs. + * - ``Path{n}/fn`` + - 1 x nChars + - uint16 + - Trial stem used when displaying the ROI GUI. + * - ``Path{n}/n_rois`` + - 1 x 1 + - uint32 + - Number of saved ROI entries for this path. + * - ``Path{n}/roi_###/type`` + - 1 x nChars + - uint16 + - ROI geometry: ``polygon``, ``circle``, or ``ellipse``. + * - ``Path{n}/roi_###/label`` + - 1 x nChars + - uint16 + - User label, for example ``SOMA``. + * - ``Path{n}/roi_###/mask`` + - rows x cols + - uint8 + - Binary ROI mask in image coordinates (1 = included pixel). + * - ``Path{n}/roi_###/position`` + - nVertices x 2 + - double + - Polygon vertices ``[y_loc, x_loc]`` when ``coords_zero_indexed=1``; + otherwise legacy ``[x, y]``. + * - ``Path{n}/roi_###/center`` + - 1 x 2 + - double + - Circle/ellipse center ``[y_loc, x_loc]`` when + ``coords_zero_indexed=1``; otherwise legacy ``[x, y]``. + * - ``Path{n}/roi_###/semi_axes`` + - 1 x 2 + - double + - Ellipse semi-axis lengths (ellipse only). + * - ``Path{n}/roi_###/rotation_angle`` + - 1 x 1 + - double + - Ellipse rotation angle in degrees (ellipse only). + * - ``Path{n}/roi_###/radius`` + - 1 x 1 + - double + - Circle radius (circle only). + +Experiment summary fields +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Pixel/plane coordinates are **0-indexed** ``[z_loc, y_loc, x_loc]``; +``frame_line_idxs`` remains **1-indexed**. The Python writer emits every +dataset in the summary tree, including ``SNR`` and ``offlineZshifts``. +External summaries may omit SNR or Z shifts when unavailable. Missing model +SNR values are serialized as NaN. + +.. list-table:: ``experiment_summary.h5`` + :header-rows: 1 + :widths: 30 22 10 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``params`` + - โ€” + - group/struct + - Analysis parameters. External MATLAB ``activityChannel``, if present, + is 1-indexed into ``numChannels``. Python activity extraction uses + channel zero, not this reference selector. + * - ``Path{n}`` + - โ€” + - group + - One group per imaging path. + * - ``Path{n}/Z_depths`` + - fastz x 1 + - numeric + - Z depths per imaging plane (SLAP2). + * - ``Path{n}/frame_info/offlineXshifts`` + - total frames x 1 + - numeric + - Offline registration X displacement per frame, using the public + alignment sign convention above. + * - ``Path{n}/frame_info/offlineYshifts`` + - total frames x 1 + - numeric + - Offline registration Y displacement per frame. + * - ``Path{n}/frame_info/offlineZshifts`` + - total frames x 1 + - numeric + - Offline registration Z displacement per frame. Emitted by Python + BandSILo; optional in the external schema when 3D alignment is absent. + * - ``Path{n}/frame_info/onlineXshifts`` + - total frames x 1 + - numeric + - SLAP2 online X shift per frame. + * - ``Path{n}/frame_info/onlineYshifts`` + - total frames x 1 + - numeric + - SLAP2 online Y shift per frame. + * - ``Path{n}/frame_info/onlineZshifts`` + - total frames x 1 + - numeric + - SLAP2 online Z shift per frame. + * - ``Path{n}/frame_info/trial_num_frames`` + - trials x 1 + - integer + - Number of frames contributed by each analysis trial. + * - ``Path{n}/frame_info/frame_line_idxs`` + - total frames x 1 + - integer + - Raw line index (SLAP2) or frame index (other microscopes in the + external schema) for each stitched-series frame; 1-indexed. + * - ``Path{n}/frame_info/discard_frames`` + - total frames x 1 + - bool or uint8 + - Frame excluded from analysis, for example by motion censoring. + * - ``Path{n}/visualizations/mean_im`` + - channels x fastz x rows x cols + - numeric + - Mean registered image per channel / Z slice. + * - ``Path{n}/visualizations/act_im`` + - fastz x rows x cols + - numeric + - Activity / localization summary image. + * - ``Path{n}/visualizations/act_im_peaks`` + - sources x 3 + - numeric + - Activity-image peak locations ``[z_loc, y_loc, x_loc]``, 0-indexed; + Python uses source row/column coordinates with ``z_loc`` fixed at ``0``. + * - ``Path{n}/global/F`` + - channels x total frames + - numeric + - Whole-field fluorescence traces, one row per channel. + * - ``Path{n}/user_rois/labels`` + - rois x 1 + - string + - User-defined ROI labels (UTF-8 strings in Python summary output). + * - ``Path{n}/user_rois/mask`` + - rois x fastz x rows x cols + - uint8 or bool + - Stacked binary masks for each user ROI. + * - ``Path{n}/user_rois/Fsvd`` + - rois x channels x total frames + - numeric + - ROI SVD / projection signals. Emitted as NaN unless explicitly + supplied through the model; not computed by BandSILo or copied from ``F``. + * - ``Path{n}/user_rois/F`` + - rois x channels x total frames + - numeric + - ROI fluorescence (raw or baseline-corrected according to the producer). + * - ``Path{n}/user_rois`` attribute ``annotation_enabled`` + - scalar attribute + - bool + - Python annotation policy: distinguishes disabled from enabled-but-empty + annotations. Both produce zero-ROI ``labels``, ``mask``, ``Fsvd``, and + ``F`` datasets with the remaining dimensions intact. + * - ``Path{n}/sources/spatial/profiles`` + - sources x fastz x rows x cols + - numeric + - Spatial component / pixel weights per source, averaged across trials + with footprints. + * - ``Path{n}/sources/spatial/coords`` + - sources x 3 + - numeric + - Source centers, one ``[z_loc, y_loc, x_loc]`` per row, 0-indexed. + * - ``Path{n}/sources/temporal/dF_ls`` + - sources x channels x total frames + - numeric + - Least-squares fluorescence change (absolute or scaled); Python uses + baseline re-estimation and processes activity channel zero only. + * - ``Path{n}/sources/temporal/dF_denoised`` + - sources x channels x total frames + - numeric + - Fitted convolution fluorescence change, saved alongside least squares + without a second baseline subtraction. Missing optional model data + serialize as NaN. + * - ``Path{n}/sources/temporal/events`` + - sources x channels x total frames + - numeric + - Nonnegative fitted event amplitudes; excludes each trial's initial + state. Discarded frames and non-activity channels are NaN. + * - ``Path{n}/sources/temporal/F0`` + - sources x channels x total frames + - numeric + - Baseline estimate used for normalization; based on QR, not the + transient model. + * - ``Path{n}/sources/temporal/SNR`` + - sources x 1 + - numeric + - Per-source signal-to-noise ratio metric. Python emits this dataset + (NaN for missing model values); optional in external summaries. + +Per-trial summary fields: unsupported external reference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Python does not emit this file.** The reference trial axis includes all +analysis trials in the trial table. Trials without alignment or extraction +data have NaN slices. Coordinates use **0-indexed** +``[z_loc, y_loc, x_loc]``. + +.. list-table:: ``per_trial_summary.h5`` (external-reference only) + :header-rows: 1 + :widths: 30 22 10 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``Path{n}`` + - โ€” + - group + - One group per imaging path. + * - ``Path{n}/visualizations/per_trial_mean_im`` + - trials x channels x fastz x rows x cols + - numeric + - Trial-aligned mean registered image per channel / Z slice. + * - ``Path{n}/visualizations/per_trial_act_im`` + - trials x fastz x rows x cols + - numeric + - Trial-aligned activity / localization summary image. + * - ``Path{n}/visualizations/per_trial_act_im_peaks`` + - trials x max_peaks x 3 + - numeric + - Per-trial detected peak locations ``[z_loc, y_loc, x_loc]``, 0-indexed; + NaN-padded when a trial has fewer than ``max_peaks``. + * - ``Path{n}/visualizations/per_trial_num_peaks`` + - trials x 1 + - integer + - Number of valid peaks per trial. + * - ``Path{n}/sources/spatial/per_trial_profiles`` + - trials x sources x fastz x rows x cols + - numeric + - Spatial component / pixel weights per source per trial. + * - ``Path{n}/sources/spatial/per_trial_coords`` + - trials x sources x 3 + - numeric + - Source centers per trial, ``[z_loc, y_loc, x_loc]``, 0-indexed. + * - ``Path{n}/sources/temporal/per_trial_SNR`` + - trials x sources + - numeric + - Per-source SNR for each analysis trial. \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 07adcad..f9a210b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,17 +1,128 @@ -.. Doc Template documentation master file, created by - sphinx-quickstart on Wed Aug 17 15:36:32 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +GIAnT-Python +============ +GIAnT-Python is a Python package for processing glutamate imaging data, +with advanced capabilities for SLAP2 recordings. Currently, only **SLAP2 +Band Scan processing** is implemented: **BandSILo source extraction and +Band Scan ROI annotation** from a pre-registered trial table. -Welcome to this repository's documentation! -=========================================== +Additional functionality is available in +`GIAnT-MATLAB `_, +and efforts are underway to port it into GIAnT-Python. Organization, +registration, standard SILo and general-purpose deconvolution stages are +not yet implemented in Python. The current workflow requires existing raw +recordings, reference TIFFs, alignment files and the band registration +lookup table. .. toctree:: :maxdepth: 2 :caption: Contents: - modules + migration + traces + file_formats + architecture + +Getting started +--------------- + +Both an exact trial-table filename and a loaded ``TrialTable`` are accepted. +Given ``table_or_filename`` supplied by the caller: + +.. code-block:: python + + from giant_python import ( + AnnotationOptions, BandSiloParams, ExecutionOptions, + extract_band_sources, + ) + + summary = extract_band_sources( + table_or_filename, + BandSiloParams(), + execution=ExecutionOptions(max_workers=6, verbose=True), + annotations=AnnotationOptions(enabled=False, interactive=False), + ) + +For a chainable session, use ``Pipeline.from_trial_table(...).extract()`` and +read ``pipeline.summary``. The ``giant`` CLI offers ``annotate`` and ``extract``. +For headless extraction with existing ROIs, use +``AnnotationOptions(enabled=True, interactive=False)``; missing required +annotations fail explicitly. + +Output is written beneath the loaded table's ``savedr`` in +``source_extraction/experiment_summary.h5``. Copying a table does not rewrite +its stored directories. Use a **fresh output location**: summary updates are +not atomic replacement or resume. See :doc:`migration` for configuration, +the results API, parameter editing and safe staging with the repository's +``examples/extract_band.py``. + +Current outputs +--------------- + +``summary.paths`` holds eager per-path sources, ROIs, visualizations, frame +information, depths and global fluorescence. BandSILo emits ``dF_ls``, +``dF_denoised`` and ``events``, together with source profiles, coordinates, +``F0`` and ``SNR``. ROI ``Fsvd`` is written as NaN unless supplied through the +model. Disabled or enabled-but-empty annotations produce zero-ROI datasets; +the ``user_rois`` group's ``annotation_enabled`` attribute distinguishes them. + +See :doc:`traces` for estimation and interpretation, :doc:`file_formats` for +schemas and indexing, and :doc:`architecture` for module ownership and +numerical boundaries. External reference formats, including per-trial +summaries and registration products, do not imply Python output support. + +Public API reference +-------------------- + +These modules expose models, canonical IO and lazy workflow services. +Autodoc deliberately does not traverse the numerical backend or GUI toolkit +modules. + +.. automodule:: giant_python.pipeline.extract + :members: extract_band_sources, extract_sources + +.. automodule:: giant_python.pipeline.annotate + :members: annotate_rois + +.. autoclass:: giant_python.pipeline.pipeline.Pipeline + :members: from_trial_table, annotate, extract + +.. automodule:: giant_python.models.params + :members: BandSiloParams, ExecutionOptions, AnnotationOptions, resolve_band_options + +.. autodata:: giant_python.models.params.BandParamsInput + +.. autodata:: giant_python.models.params.ExecutionInput + +.. autodata:: giant_python.models.params.AnnotationInput + +.. autodata:: giant_python.pipeline.extract.TrialTableInput + +.. automodule:: giant_python.models.experiment_summary + :members: ExperimentSummary, PathSummary, FrameInfo, Source, UserRoi, Visualizations + +.. automodule:: giant_python.io.experiment_summary + :members: read_summary, write_summary + +.. autoclass:: giant_python.models.trial_table.TrialTable + :members: from_h5 + +.. autoclass:: giant_python.models.trial_table.Slap2Info + +.. automodule:: giant_python.io.slap2 + :members: Slap2Reader, open_slap2_file, load_alignment_data_h5 + +.. automodule:: giant_python.io.tiff + :members: read_tiff + +.. automodule:: giant_python.io.hdf5 + :members: load_struct_h5, write_dict_to_h5group + +Package exports: + +.. autodata:: giant_python.__all__ + +.. autodata:: giant_python.io.__all__ Indices and tables diff --git a/docs/source/migration.rst b/docs/source/migration.rst new file mode 100644 index 0000000..1f257f8 --- /dev/null +++ b/docs/source/migration.rst @@ -0,0 +1,168 @@ +API and data conventions +======================== + +Public workflow and options +--------------------------- + +The lazy package-root exports provide ``extract_band_sources``, +``extract_sources``, ``annotate_rois``, ``Pipeline`` and the input, option and +result models. Both extraction functions accept an exact trial-table filename +or a loaded ``TrialTable`` and return ``ExperimentSummary``. +``Pipeline.from_trial_table(...).extract()`` returns the same pipeline; +read its result through ``pipeline.summary``. ``Pipeline.annotate()`` and +``annotate_rois(...)`` load existing annotations or draw them when permitted. + +The band service signature is +``extract_band_sources(input, params=None, *, execution=None, annotations=None)``. +``Pipeline`` uses the same option scopes. Generic services and ``Pipeline`` +take explicit ``microscope`` / ``scan_mode`` routing arguments; only +``microscope='slap2'`` with ``scan_mode='band'`` (or ``None``) is supported. +Unsupported standalone routing raises ``ValueError`` before input/backend +loading. Pipeline operations require a loaded table. + +The ``giant`` CLI offers ``annotate`` and ``extract``, each with a trial-table +filename and ``slap2`` / ``band`` routing choices. Invalid commands/options +are argument errors (exit code 2). For existing headless ROIs use +``--draw-user-rois --headless``, or in Python +``AnnotationOptions(enabled=True, interactive=False)``. Missing required +annotations fail explicitly rather than silently disabling ROIs. + +Configuration +------------- + +.. list-table:: Independent configuration scopes + :header-rows: 1 + + * - Argument + - Dataclass + - Scope + * - ``params`` + - ``BandSiloParams`` + - Band science only + * - ``execution`` + - ``ExecutionOptions`` + - ``max_workers``, ``verbose``, ``max_trials`` + * - ``annotations`` + - ``AnnotationOptions`` + - ``enabled``, ``interactive`` + +``resolve_band_options(params=None, execution=None, annotations=None)`` returns +fresh validated objects in the order shown above. Each input accepts **only +its corresponding dataclass, a dictionary of that scope's fields, or None**. +Mixed dictionaries, cross-scope fields, unknown keys and unrelated dataclasses +raise ``TypeError``. Caller-owned inputs are not mutated, even when acquisition +channel counts or GUI policy are later resolved. + +Defaults include peak threshold 7, analysis rate 100 Hz, decay 0.15 seconds, +baseline window 4 seconds, denoise window 1 second, VIF 1.38 and PSF dilation 17. +Execution defaults to six workers, no trial limit and verbosity off. Annotation +defaults to disabled with automatic interactivity. See :doc:`traces` for +temporal-fit controls, motion-bin tolerance and random-seed semantics. + +Channel count is inferred from acquisition metadata and retained in output +metadata. Source activity extraction always uses **channel zero**; other +source-activity channels contain NaNs. Neither channel count nor activity +channel is a scientific option. If channel count cannot be inferred, +extraction raises ``ValueError``; supply acquisition metadata for a kept trial. + +``gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, +annotations=None)`` returns a validated ``(BandSiloParams, ExecutionOptions, +AnnotationOptions)`` triple, or ``None`` on cancellation, without mutating +inputs. Its sparse-factor control is in natural-log units. GUI toolkit imports +are lazy; see :doc:`architecture` for module ownership and numerical boundaries. + +Input identity and output location +---------------------------------- + +``Pipeline.from_trial_table`` loads an exact filename once and retains it as +``trial_table_path``; a loaded table is retained directly. ``source_path`` is +provenance, not a filename reconstructed from ``savedr``. Set ``datadr`` and +``savedr`` on the loaded ``TrialTable`` for explicit directory overrides and +pass that model to extraction. Copying the table file does not rewrite its +stored directories. + +Extraction writes beneath ``savedr`` to +``source_extraction/experiment_summary.h5``. Use a fresh output location: +updating an existing summary replaces supplied path groups but retains +unrelated paths and root parameters/layout flags. Array preparation before +each path replacement does not make writes file-wide atomic or provide +checkpoint/resume. Unknown external fields need not survive a model round-trip. +See :doc:`file_formats` for the persistence contract. + +Results API +----------- + +All imaging results belong to ``ExperimentSummary.paths``. Each ``PathSummary`` +owns its sources, ROIs, visualizations, ``FrameInfo``, depths and global +fluorescence. There are no flat or aggregate summary properties. ``path_index`` +is zero-based and maps to disk group ``Path{path_index + 1}``; a list position +is not necessarily that identity when reading selected paths. + +``FrameInfo`` exposes attributes and a shallow ``to_dict()`` only, not mapping +methods. Results are eager; per-source/ROI entries may share array storage. +They are neither lazy HDF5 proxies nor independent immutable snapshots. +Mutating shared storage can affect other views; serialization may stack +arrays and does not promise zero-copy writing. + +``io.experiment_summary.read_summary(path)`` returns ``ExperimentSummary`` and +``write_summary(summary, path)`` accepts that model and returns ``None``. +Both functions are also exported by ``giant_python.io``. +``ExperimentSummary.from_h5`` / ``to_h5`` delegate to these canonical functions. + +Production summaries emit ``dF_ls``, ``dF_denoised`` and ``events`` alongside +source profiles, ``F0``, ``SNR``, visualizations, depths, global fluorescence +and frame information. ROI ``Fsvd`` is emitted as NaN unless explicitly +supplied through the model; BandSILo does not calculate it. Disabled and +enabled-but-empty annotations both emit zero-ROI datasets, distinguished by +the ``annotation_enabled`` attribute on the ``user_rois`` group. + +New summaries set ``row_major=1`` and preserve rank-two frame vectors. +Acquisition line indexes are one-based; summary spatial coordinates are +zero-based ``[z, y, x]``. ``PathSummary.z_depths`` maps to ``Z_depths`` on disk. +See :doc:`file_formats` for complete shapes, MATLAB layout decoding, annotation +coordinate flags and motion-sign provenance; see :doc:`traces` for temporal +output interpretation. External per-trial summaries are not Python outputs. + +Staging example +--------------- + +The repository's ``examples/extract_band.py`` is safe to import: +there are no import-time filesystem operations, environment assignments, +matplotlib imports or heavy GIAnT imports. + +Provide ``--trial-table``, ``--data-dir``, ``--motion-correction`` and a **new** +``--results-dir``. The example copies the exact supplied table and copies +existing motion-correction assets by default. ``--stage-motion symlink`` +selects directory linking (Windows may require Developer Mode or link +privileges). ``--stage-data copy`` or ``symlink`` stages the complete raw +directory, including sidecars and reference TIFFs; ``none`` uses it in place. +This is staging of externally generated motion correction, not execution of +Python registration. + +Crucially, the example overrides ``datadr`` and ``savedr`` on the loaded +``TrialTable`` and passes that model directly to extraction. It does not +reload the copied file and lose those overrides. The copied file is an +unaltered provenance artifact: its stored directories are not rewritten, +and Python does not write trial tables. Input filename fields +must be relative without parent traversal; absolute names are rejected rather +than silently bypassing staging or being guessed from basenames. + +The example refuses any existing results file, directory or symlink, and +refuses output nested inside an input tree. It never deletes arbitrary +destinations or silently retries by replacing a link. Partial output after +failure is left for inspection; choose a new directory for another run. +Copied source symlinks are retained, so verify their targets when relocating +data between machines. Symlink mode is intentionally not a portable copy. + +``--joblib-temp`` optionally selects an existing directory with sufficient +space for worker memory maps. Paths, loaded metadata and scientific/run +options are validated before any staging or environment changes. Its +environment override is scoped to staging/extraction, before importing the +lazy extraction service, and restored even when extraction fails. Without +the flag, the existing environment and joblib defaults are left alone. +Dependencies must already be installed; the example does not install packages. + +Scientific and execution flags are explicit run choices, not changes to +library defaults. Use ``--use-rois --annotations-dir`` for existing headless +annotations, or add ``--interactive`` to permit drawing. Inspect ``--help`` +for all flags. \ No newline at end of file diff --git a/docs/source/traces.rst b/docs/source/traces.rst new file mode 100644 index 0000000..c7bb491 --- /dev/null +++ b/docs/source/traces.rst @@ -0,0 +1,118 @@ +Trace estimation +================ + +Production BandSILo extraction computes and saves least-squares +``sources/temporal/dF_ls``, convolution-constrained +``sources/temporal/dF_denoised``, and fitted ``sources/temporal/events``. +Each array has shape ``(sources, channels, total frames)``. Source activity +extraction uses **channel zero**; other channels and missing observations are +NaN. Both trace estimates are mandatory production outputs, not selectable +trace modes. Low-level loaded-array functions also support weighted averages +for diagnostics. See :doc:`file_formats` for the saved layout. + +Trial-wise objective +-------------------- + +The trace-only solver holds spatial profiles fixed and minimizes + +.. math:: + + \min_{s \geq 0}\; \frac{1}{2} + \left\|(d-b)-H_{\mathrm{motion}} A K s\right\|_2^2 + + \beta \sum_j \sum_{t=1}^{T-1} s_{j,t}. + +Here ``d`` is the observed data, ``b`` is the background, ``A`` contains the +fixed source profiles, ``H_motion`` is the motion-dependent observation +operator, and ``K`` applies a causal unit-peak exponential. For each source, +:math:`s_{j,0}` is an unpenalized nonnegative initial state; the remaining entries +are nonnegative event amplitudes. The solver operates chronologically over +each complete trial, including motion changes and missing frames. No state or +event inference crosses trial boundaries. Localization, NMF, and profile +fitting are separate from this trace-only optimization. + +Returned fluorescence changes preserve the QR complete-frame validity mask. +The standard least-squares trace and its background retain baseline +re-estimation for ``dF_ls`` and ``F0``; ``F0`` comes from QR, not the transient +model. ``dF_denoised`` is the fitted convolution trace itself, with no second +baseline subtraction. ``events`` excludes the initial state at each trial's +first frame and stores NaN for discarded frames. + +Controls and convergence +------------------------ + +``BandSiloParams.decay_tau_s`` controls both the trace kernel and localization +filtering. Its default is 0.15 seconds, and it is saved as ``decayTau_s``. +The GUI's Decay Tau field and CLI ``--decay-tau-s`` flag change this shared +setting; there is no separate trace decay parameter. + +.. list-table:: Trace-only controls + :header-rows: 1 + :widths: 28 12 60 + + * - Parameter + - Default + - Meaning + * - ``trace_beta_fraction`` + - ``0.01`` + - Fraction of each trial's positive adjoint scale; zero disables L1. + * - ``trace_max_iter`` + - ``500`` + - Maximum L-BFGS-B iterations per optimization stage. + * - ``trace_tolerance`` + - ``1e-7`` + - Objective/gradient tolerance. + * - ``trace_debias`` + - ``False`` + - Refit selected events without L1 shrinkage. + * - ``trace_event_threshold`` + - ``1e-6`` + - Absolute event-amplitude threshold for debias support. + +These controls are recorded in summary parameters and are available as +hyphenated CLI flags (for example, ``--trace-beta-fraction``). +A nonconverged production fit raises an error rather than saving an unfinished +estimate. Increase the iteration limit or inspect a low-level fit when this +occurs. Per-trial optimizer diagnostics and latent in-gap predictions are not +saved in the summary. + +Motion selection and reproducibility +------------------------------------ + +``BandSiloParams(z_tol=1.5)`` sets the low-resolution motion-bin selection +tolerance in microns. Bins must lie within this absolute distance of the +median frame z and satisfy the minimum frame count. The value must be positive +and finite. The CLI and staging example expose ``--z-tol``; the parameter GUI +labels it **Z Tolerance (ยตm)**. + +``BandSiloParams(random_seed=0)`` seeds the random motion-bin order used by +source localization (NMF and Adam). It must be a nonnegative integer. Each +path and outer iteration uses its own generator derived from this seed, +independently of the global torch RNG. The seed is saved in summary parameters +and exposed as ``--random-seed``. It fixes random draws, not floating-point +results across hardware, thread counts, or package versions. + +Loaded-array API +---------------- + +For loaded arrays, pass ``transient_options=TransientOptions(...)`` and +``analyze_hz=...`` to +``giant_python.extraction.band.traces.compute_high_res_traces``. +``TransientOptions`` and ``fit_transient_traces`` are defined in +``giant_python.extraction.band.transient``. The latter accepts an absolute +``beta`` or an explicit causal kernel and exposes events, the initial state, +in-gap model predictions, actual beta, and convergence diagnostics. +Retain the complete chronological output grid, including gaps, when supplying +arrays. Low-level nonconvergence warns and returns diagnostics; callers must +check ``metadata['converged']`` before saving a result. +``TrialTraceResult`` retains its original eight fields in order and appends +``d_f_denoised`` and ``events``. + +Scientific interpretation +------------------------- + +This is an output-grid estimator: overlapping acquisition-line averaging is +not modeled exactly. In-gap predictions are not observations, and glutamate +events are not automatically neuronal spikes. A smoother trace alone does not +establish improved SNR. Use a fresh output location when comparing estimates +or settings; summary updates are not an atomic replacement or a resume +mechanism. \ No newline at end of file diff --git a/examples/extract_band.py b/examples/extract_band.py new file mode 100644 index 0000000..ce7c805 --- /dev/null +++ b/examples/extract_band.py @@ -0,0 +1,325 @@ +"""Extract registered band data, optionally staging inputs in a fresh run. + +Use --help for arguments. No files, environment variables, GUI toolkits, or +GIAnT modules are touched on import. This example copies existing motion +correction; it does not run the unimplemented Python registration stage. + +The source table must use relative recording/alignment filenames. Include +raw acquisition sidecars and reference TIFFs in --data-dir. A copied table +retains its original on-disk metadata: effective directory overrides are +made on the loaded model, not saved with the unsupported TrialTable writer. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +from contextlib import contextmanager +from dataclasses import dataclass, replace +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import TYPE_CHECKING, Generator + +if TYPE_CHECKING: + from giant_python import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + TrialTable, + ) + + +@dataclass(frozen=True) +class InputPaths: + """Resolved input roots and a fresh destination, before any staging.""" + + table: Path + data: Path + motion: Path + results: Path + annotations: Path | None + joblib_temp: Path | None + + +@dataclass(frozen=True) +class PreparedRun: + """Validated paths, loaded metadata and independent run settings.""" + + paths: InputPaths + table: TrialTable + params: BandSiloParams + execution: ExecutionOptions + annotations: AnnotationOptions + + +def build_parser() -> argparse.ArgumentParser: + """Describe explicit input roots, safe staging, and split run options.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--trial-table", type=Path, required=True) + parser.add_argument( + "--data-dir", + type=Path, + required=True, + help="Raw recordings, sidecars, and reference TIFF directory", + ) + parser.add_argument( + "--motion-correction", + type=Path, + required=True, + help="Existing alignment and band registration lookup directory", + ) + parser.add_argument( + "--results-dir", + type=Path, + required=True, + help="New directory; an existing file, directory, or link is refused", + ) + parser.add_argument( + "--stage-data", + choices=("none", "copy", "symlink"), + default="none", + help="Keep raw inputs in place (default), copy, or link the full tree", + ) + parser.add_argument( + "--stage-motion", + choices=("copy", "symlink"), + default="copy", + help="Copy existing registration assets (default), or link them", + ) + parser.add_argument( + "--annotations-dir", + type=Path, + help="Optional existing annotation directory to copy into the run", + ) + parser.add_argument( + "--joblib-temp", + type=Path, + help="Existing temporary directory; overrides JOBLIB_TEMP_FOLDER", + ) + parser.add_argument("--max-workers", type=int, default=6) + parser.add_argument("--max-trials", type=int) + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--use-rois", action="store_true") + parser.add_argument("--interactive", action="store_true") + parser.add_argument("--analyze-hz", type=float, default=100.0) + parser.add_argument("--decay-tau-s", type=float, default=0.15) + parser.add_argument("--baseline-window-s", type=float, default=4.0) + parser.add_argument("--denoise-window-s", type=float, default=1.0) + parser.add_argument("--vif", type=float, default=1.38) + parser.add_argument("--d-xy", type=int, default=5) + parser.add_argument("--peak-buffer", type=int, default=3) + parser.add_argument("--psf-dilation", type=int, default=17) + parser.add_argument( + "--z-tol", + type=float, + default=1.5, + help="Z tolerance for motion-bin selection (microns)", + ) + return parser + + +def require_directory(path: Path) -> Path: + """Resolve a source directory and fail before staging if it is missing.""" + resolved = path.resolve(strict=True) + if not resolved.is_dir(): + raise NotADirectoryError(path) + return resolved + + +def validate_relative_names(values: object, field: str) -> None: + """Reject names that would bypass the explicitly selected input roots.""" + import numpy as np + + for entry in np.asarray(values, dtype=object).flat: + if entry is None or str(entry) == "": + continue + name = str(entry) + posix = PurePosixPath(name) + windows = PureWindowsPath(name) + if ( + posix.is_absolute() + or windows.anchor + or ".." in posix.parts + or ".." in windows.parts + ): + raise ValueError( + f"{field} must contain relative filenames without '..': " + f"{name!r}. Rebase the metadata explicitly before staging; " + "this example will not guess a replacement basename." + ) + + +def stage_directory(source: Path, destination: Path, mode: str) -> None: + """Copy or link a directory without replacing any existing destination.""" + if destination.exists() or destination.is_symlink(): + raise FileExistsError(destination) + if mode == "copy": + shutil.copytree(source, destination, symlinks=True) + elif mode == "symlink": + destination.symlink_to(source, target_is_directory=True) + else: + raise ValueError(f"Unknown staging mode: {mode}") + + +def validate_annotation_arguments( + args: argparse.Namespace, parser: argparse.ArgumentParser +) -> None: + """Reject inconsistent GUI policy before touching inputs.""" + if args.interactive and not args.use_rois: + parser.error("--interactive requires --use-rois") + if args.use_rois and not args.interactive and not args.annotations_dir: + parser.error("Headless --use-rois requires --annotations-dir") + + +def require_file(path: Path) -> Path: + """Resolve a required metadata file without creating anything.""" + resolved = path.resolve(strict=True) + if not resolved.is_file(): + raise FileNotFoundError(path) + return resolved + + +def validate_destination(requested: Path, sources: tuple[Path, ...]) -> Path: + """Refuse existing destinations and recursive staging into input trees.""" + if requested.exists() or requested.is_symlink(): + raise FileExistsError(f"Choose a fresh --results-dir: {requested}") + results = requested.resolve() + for source in sources: + if results.is_relative_to(source): + raise ValueError("Results must be outside all input directories") + return results + + +def validate_paths(args: argparse.Namespace) -> InputPaths: + """Check all paths without modifying files or the environment.""" + table = require_file(args.trial_table) + data = require_directory(args.data_dir) + motion = require_directory(args.motion_correction) + require_file(motion / "bandRegLookupTable.h5") + annotations = None + sources = (data, motion) + if args.annotations_dir is not None: + annotations = require_directory(args.annotations_dir) + require_file(annotations / "annotations.h5") + sources += (annotations,) + results = validate_destination(args.results_dir, sources) + joblib_temp = ( + require_directory(args.joblib_temp) + if args.joblib_temp is not None + else None + ) + return InputPaths(table, data, motion, results, annotations, joblib_temp) + + +def prepare_run(args: argparse.Namespace) -> PreparedRun: + """Validate metadata and options before staging or environment changes.""" + paths = validate_paths(args) + from giant_python import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + TrialTable, + ) + + table = TrialTable.from_h5(paths.table) + if table.slap2_info is None or not table.motion_correction: + raise ValueError("A pre-registered SLAP2 band trial table is required") + if "fn_adata" not in table.motion_correction: + raise ValueError("The table requires motion_correction/fn_adata") + validate_relative_names(table.filename, "filename") + validate_relative_names( + table.motion_correction["fn_adata"], "motion_correction/fn_adata" + ) + params = BandSiloParams( + analyze_hz=args.analyze_hz, + decay_tau_s=args.decay_tau_s, + baseline_window_s=args.baseline_window_s, + denoise_window_s=args.denoise_window_s, + vif=args.vif, + d_xy=args.d_xy, + peak_buffer=args.peak_buffer, + psf_dilation=args.psf_dilation, + z_tol=args.z_tol, + ) + execution = ExecutionOptions( + max_workers=args.max_workers, + max_trials=args.max_trials, + verbose=args.verbose, + ) + annotations = AnnotationOptions( + enabled=args.use_rois, + interactive=args.interactive, + ) + return PreparedRun(paths, table, params, execution, annotations) + + +@contextmanager +def joblib_environment( + directory: Path | None, +) -> Generator[None, None, None]: + """Temporarily override worker scratch space only for a validated run.""" + if directory is None: + yield + return + previous = os.environ.get("JOBLIB_TEMP_FOLDER") + os.environ["JOBLIB_TEMP_FOLDER"] = str(directory) + try: + yield + finally: + if previous is None: + os.environ.pop("JOBLIB_TEMP_FOLDER", None) + else: + os.environ["JOBLIB_TEMP_FOLDER"] = previous + + +def stage_and_extract(run: PreparedRun, args: argparse.Namespace) -> None: + """Stage validated inputs and pass the relocated model to extraction.""" + from giant_python import extract_band_sources + + paths = run.paths + results_dir = paths.results + data_dir = paths.data + + # No overwrite mode and no automatic cleanup: a failed run leaves its + # partial directory for inspection, never removes an existing tree/link. + results_dir.mkdir(parents=True, exist_ok=False) + staged_table = results_dir / "input_trial_table.h5" + shutil.copy2(paths.table, staged_table) + stage_directory( + paths.motion, results_dir / "motion_correction", args.stage_motion + ) + if args.stage_data != "none": + staged_data = results_dir / "input" + stage_directory(data_dir, staged_data, args.stage_data) + data_dir = staged_data + if paths.annotations is not None: + stage_directory(paths.annotations, results_dir / "annotations", "copy") + + # Copying the HDF5 file does not rewrite the paths stored inside it. + # Pass this overridden model, not staged_table, to make staging effective. + table = replace( + run.table, + datadr=data_dir, + savedr=results_dir, + source_path=staged_table, + ) + summary = extract_band_sources( + table, run.params, execution=run.execution, annotations=run.annotations + ) + print(f"Extracted {len(summary.paths)} imaging path(s)") + print(results_dir / "source_extraction" / "experiment_summary.h5") + + +def main(argv: list[str] | None = None) -> int: + """Validate first, then stage a fresh run and extract registered bands.""" + parser = build_parser() + args = parser.parse_args(argv) + validate_annotation_arguments(args, parser) + run = prepare_run(args) + with joblib_environment(run.paths.joblib_temp): + stage_and_extract(run, args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 29877a2..f23911b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,9 @@ classifiers = [ readme = "README.md" dynamic = ["version"] +# NOTE: The SLAP2 band-scan (BandSILo) source-extraction backend requires +# ``slap2_utils`` (raw SLAP2 data-file reader). It is not published on PyPI, so +# it is installed directly from source via the git URL below. dependencies = [ "numpy", "scipy", @@ -23,18 +26,20 @@ dependencies = [ "h5py", "tifffile", "joblib", + "tqdm", + "torch", + "opencv-python", + "pandas", + "bottleneck", + "slap2_utils @ git+https://github.com/m-xie/SLAP2_Utils.git@multidatafiles", ] -[project.optional-dependencies] -gui = ["napari"] - [project.scripts] giant = "giant_python.cli:main" [dependency-groups] dev = [ 'black', - 'coverage', 'flake8', 'interrogate', 'isort', @@ -45,6 +50,11 @@ dev = [ [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +# Bundle the default point-spread-function template used by the BandSILo +# (band-scan) source-extraction backend. +giant_python = ["assets/psfs/*.tif"] + [tool.setuptools.dynamic] version = {attr = "giant_python.__version__"} @@ -59,7 +69,6 @@ exclude = ''' | \.git # root of the project | \.hg | \.mypy_cache - | \.tox | \.venv | _build | build @@ -70,19 +79,6 @@ exclude = ''' ) ''' -[tool.coverage.run] -omit = ["*__init__*"] -source = ["giant_python", "tests"] - -[tool.coverage.report] -exclude_lines = [ - "if __name__ == .__main__.:", - "^from .* import .*", - "^import .*", - "pragma: no cover" -] -fail_under = 100 - [tool.isort] line_length = 79 profile = "black" diff --git a/src/giant_python/__init__.py b/src/giant_python/__init__.py index b613276..b9e94c7 100644 --- a/src/giant_python/__init__.py +++ b/src/giant_python/__init__.py @@ -1,45 +1,82 @@ -"""GIAnT-Python: Python translation of the GIAnT-MATLAB analysis package. - -Public API ----------- -The recommended entry point for the full pipeline is :class:`Pipeline`. Power -users can drive individual stages (``TrialTableBuilder``, ``MotionCorrector``, -``SourceExtractor``) or call the underlying functions directly. Typed data -models (``TrialTable``, ``AlignmentData``, ``ExperimentSummary``) own the -HDF5 serialization shared with GIAnT-MATLAB. - -The GUI lives in :mod:`giant_python.gui` and is imported lazily, so importing -this package never pulls in GUI dependencies. +"""GIAnT-Python: band extraction functions and a chainable ``Pipeline``. + +Use ``BandSiloParams`` for science, ``ExecutionOptions`` for run policy and +``AnnotationOptions`` for ROI/GUI policy. ``extract_sources`` defaults to the +implemented SLAP2 band workflow; explicitly requested unsupported modes raise. +Public exports are lazy: importing GIAnT does not load acquisition or GUI +toolkits. Summary arrays are currently eager, not disk-backed. """ -from .models import ( - AlignmentData, - AlignParams, - ExperimentSummary, - SiloParams, - TrialTable, -) -from .pipeline import ( - IntegrationSourceExtractor, - MotionCorrector, - Pipeline, - SourceExtractor, - StandardSourceExtractor, - TrialTableBuilder, -) +from importlib import import_module +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .models import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + ExperimentSummary, + FrameInfo, + PathSummary, + TrialTable, + resolve_band_options, + ) + from .pipeline import ( + Pipeline, + annotate_rois, + extract_band_sources, + extract_sources, + ) __version__ = "0.1.10" +_MODEL_EXPORTS = { + "AnnotationOptions", + "BandSiloParams", + "ExecutionOptions", + "ExperimentSummary", + "FrameInfo", + "PathSummary", + "TrialTable", + "resolve_band_options", +} +_PIPELINE_EXPORTS = { + "Pipeline", + "annotate_rois", + "extract_band_sources", + "extract_sources", +} + + +def __getattr__(name: str): + """Load a public model or workflow export on first access.""" + if name in _MODEL_EXPORTS: + module = import_module(".models", __name__) + elif name in _PIPELINE_EXPORTS: + module = import_module(".pipeline", __name__) + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(module, name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """Include lazy public exports in interactive completion.""" + return sorted(set(globals()) | set(__all__)) + + __all__ = [ - "AlignmentData", - "AlignParams", + "AnnotationOptions", + "BandSiloParams", + "ExecutionOptions", "ExperimentSummary", - "IntegrationSourceExtractor", - "MotionCorrector", + "FrameInfo", + "PathSummary", "Pipeline", - "SiloParams", - "SourceExtractor", - "StandardSourceExtractor", "TrialTable", - "TrialTableBuilder", + "annotate_rois", + "extract_band_sources", + "extract_sources", + "resolve_band_options", ] diff --git a/src/giant_python/assets/psfs/dil-05.tif b/src/giant_python/assets/psfs/dil-05.tif new file mode 100644 index 0000000..cfb47a1 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-05.tif differ diff --git a/src/giant_python/assets/psfs/dil-07.tif b/src/giant_python/assets/psfs/dil-07.tif new file mode 100644 index 0000000..5d4c8a2 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-07.tif differ diff --git a/src/giant_python/assets/psfs/dil-09.tif b/src/giant_python/assets/psfs/dil-09.tif new file mode 100644 index 0000000..ba8d2e9 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-09.tif differ diff --git a/src/giant_python/assets/psfs/dil-11.tif b/src/giant_python/assets/psfs/dil-11.tif new file mode 100644 index 0000000..76a1b0d Binary files /dev/null and b/src/giant_python/assets/psfs/dil-11.tif differ diff --git a/src/giant_python/assets/psfs/dil-13.tif b/src/giant_python/assets/psfs/dil-13.tif new file mode 100644 index 0000000..5f9923e Binary files /dev/null and b/src/giant_python/assets/psfs/dil-13.tif differ diff --git a/src/giant_python/assets/psfs/dil-15.tif b/src/giant_python/assets/psfs/dil-15.tif new file mode 100644 index 0000000..7803a7e Binary files /dev/null and b/src/giant_python/assets/psfs/dil-15.tif differ diff --git a/src/giant_python/assets/psfs/dil-17.tif b/src/giant_python/assets/psfs/dil-17.tif new file mode 100644 index 0000000..02d1d24 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-17.tif differ diff --git a/src/giant_python/assets/psfs/dil-19.tif b/src/giant_python/assets/psfs/dil-19.tif new file mode 100644 index 0000000..75630d6 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-19.tif differ diff --git a/src/giant_python/assets/psfs/dil-21.tif b/src/giant_python/assets/psfs/dil-21.tif new file mode 100644 index 0000000..fb17b79 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-21.tif differ diff --git a/src/giant_python/assets/psfs/dil-23.tif b/src/giant_python/assets/psfs/dil-23.tif new file mode 100644 index 0000000..ad38ee8 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-23.tif differ diff --git a/src/giant_python/assets/psfs/dil-25.tif b/src/giant_python/assets/psfs/dil-25.tif new file mode 100644 index 0000000..683e5a9 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-25.tif differ diff --git a/src/giant_python/assets/psfs/dil-27.tif b/src/giant_python/assets/psfs/dil-27.tif new file mode 100644 index 0000000..f95c50a Binary files /dev/null and b/src/giant_python/assets/psfs/dil-27.tif differ diff --git a/src/giant_python/assets/psfs/dil-29.tif b/src/giant_python/assets/psfs/dil-29.tif new file mode 100644 index 0000000..c40ebc4 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-29.tif differ diff --git a/src/giant_python/assets/psfs/dil-31.tif b/src/giant_python/assets/psfs/dil-31.tif new file mode 100644 index 0000000..bf617a3 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-31.tif differ diff --git a/src/giant_python/assets/psfs/dil-41.tif b/src/giant_python/assets/psfs/dil-41.tif new file mode 100644 index 0000000..2f42c4a Binary files /dev/null and b/src/giant_python/assets/psfs/dil-41.tif differ diff --git a/src/giant_python/assets/psfs/dil-51.tif b/src/giant_python/assets/psfs/dil-51.tif new file mode 100644 index 0000000..0052490 Binary files /dev/null and b/src/giant_python/assets/psfs/dil-51.tif differ diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index 69580c7..502e5f0 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -1,17 +1,207 @@ -"""Command-line interface: ``giant organize|register|annotate|extract``. +"""CLI for annotation and extraction from registered SLAP2 band inputs. -Thin wrapper over the pipeline stages for batch / cluster use. Wired as the -``giant`` console script in pyproject.toml. +``giant annotate`` loads or draws ROIs; ``giant extract`` runs BandSILo. +Both delegate to the public workflow services with separate scientific, +execution and annotation options. GUI imports occur only when needed. """ +from __future__ import annotations + +import argparse from typing import Optional, Sequence +from .models.params import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, +) + + +def _add_silo_options(parser: argparse.ArgumentParser) -> None: + """Attach the shared source-extraction options to a subparser.""" + for name, kind, help_text in ( + ( + "decay_tau_s", + float, + "Shared localization and trace decay time in seconds.", + ), + ( + "trace_beta_fraction", + float, + "Trial-relative nonnegative event penalty.", + ), + ("trace_max_iter", int, "Maximum iterations per transient fit stage."), + ("trace_tolerance", float, "Transient optimizer tolerance."), + ( + "trace_event_threshold", + float, + "Absolute event threshold for debias support.", + ), + ( + "random_seed", + int, + "Nonnegative seed for the localization motion order.", + ), + ): + parser.add_argument( + "--" + name.replace("_", "-"), + type=kind, + default=getattr(BandSiloParams, name), + help=help_text, + ) + parser.add_argument( + "--trace-debias", + action="store_true", + help="Refit selected transient events without the L1 penalty.", + ) + parser.add_argument( + "--z-tol", + type=float, + default=BandSiloParams.z_tol, + help="Z tolerance for motion-bin selection in microns (default: 1.5).", + ) + parser.add_argument( + "--microscope", + choices=("slap2",), + default="slap2", + help="Acquisition type (currently only SLAP2 band is implemented).", + ) + parser.add_argument( + "--scan-mode", + choices=("band",), + default="band", + help="Supported scan mode.", + ) + parser.add_argument( + "--draw-user-rois", + action="store_true", + help="Prompt for / use manual user ROIs.", + ) + interactive = parser.add_mutually_exclusive_group() + interactive.add_argument( + "--interactive", + dest="interactive", + action="store_true", + default=None, + help="Force the ROI-drawing GUI when annotations are missing.", + ) + interactive.add_argument( + "--headless", + dest="interactive", + action="store_false", + default=None, + help="Never open a GUI; fail fast if annotations are missing.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-stage status messages and progress bars.", + ) + parser.add_argument( + "--max-workers", + type=int, + default=ExecutionOptions.max_workers, + help="Positive worker count (default: 6).", + ) + parser.add_argument( + "--max-trials", + type=int, + default=None, + help="Positive debug trial limit per DMD (default: all trials).", + ) + + +def build_parser() -> argparse.ArgumentParser: + """Build the ``giant`` argument parser. + + Returns + ------- + argparse.ArgumentParser + Parser with ``annotate`` and ``extract`` + subcommands. Each subcommand stores its ``command`` name in the + parsed namespace. + """ + parser = argparse.ArgumentParser(prog="giant") + subparsers = parser.add_subparsers(dest="command") + + annotate = subparsers.add_parser( + "annotate", help="Draw / save user ROIs (standalone step)." + ) + annotate.add_argument("trial_table", help="Path to trial_table.h5.") + _add_silo_options(annotate) + + extract = subparsers.add_parser("extract", help="Run source extraction.") + extract.add_argument("trial_table", help="Path to trial_table.h5.") + _add_silo_options(extract) + + return parser + + +def _options_from_args( + args: argparse.Namespace, +) -> tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]: + """Build independent science, execution and annotation configuration.""" + return ( + BandSiloParams( + z_tol=args.z_tol, + decay_tau_s=args.decay_tau_s, + trace_beta_fraction=args.trace_beta_fraction, + trace_max_iter=args.trace_max_iter, + trace_tolerance=args.trace_tolerance, + trace_debias=args.trace_debias, + trace_event_threshold=args.trace_event_threshold, + random_seed=args.random_seed, + ), + ExecutionOptions( + max_workers=args.max_workers, + verbose=args.verbose, + max_trials=args.max_trials, + ), + AnnotationOptions( + enabled=args.draw_user_rois or args.command == "annotate", + interactive=args.interactive, + ), + ) + + +def _cmd_annotate( + args: argparse.Namespace, +) -> int: + """Handle ``giant annotate`` (standalone ROI annotation).""" + from .pipeline.annotate import annotate_rois + + params, execution, annotations = _options_from_args(args) + annotate_rois( + args.trial_table, + params, + microscope=args.microscope, + scan_mode=args.scan_mode, + execution=execution, + annotations=annotations, + ) + return 0 + + +def _cmd_extract(args: argparse.Namespace) -> int: + """Handle ``giant extract`` (source extraction).""" + from .pipeline.extract import extract_sources + + params, execution, annotations = _options_from_args(args) + extract_sources( + args.trial_table, + params, + microscope=args.microscope, + scan_mode=args.scan_mode, + execution=execution, + annotations=annotations, + ) + return 0 + def main(argv: Optional[Sequence[str]] = None) -> int: """Entry point for the ``giant`` console script. - Dispatches the ``organize``, ``register``, ``annotate``, and ``extract`` - subcommands to the corresponding pipeline stages. + Dispatches ``annotate`` and ``extract`` to their workflow services. Parameters ---------- @@ -21,10 +211,28 @@ def main(argv: Optional[Sequence[str]] = None) -> int: Returns ------- int - Process exit code. + Process exit code (``2`` when no subcommand is given). + + Raises + ------ + SystemExit + Invalid options and unsupported workflows print an argparse error + and exit with code two, without claiming the operation succeeded. """ - raise NotImplementedError + parser = build_parser() + args = parser.parse_args(argv) + handlers = { + "annotate": _cmd_annotate, + "extract": _cmd_extract, + } + if args.command is None: + parser.print_help() + return 2 + try: + return handlers[args.command](args) + except ValueError as exc: + parser.error(str(exc)) -if __name__ == "__main__": # pragma: no cover +if __name__ == "__main__": raise SystemExit(main()) diff --git a/src/giant_python/extraction/__init__.py b/src/giant_python/extraction/__init__.py new file mode 100644 index 0000000..2bcf7df --- /dev/null +++ b/src/giant_python/extraction/__init__.py @@ -0,0 +1 @@ +"""Source-extraction backends; import a backend explicitly when needed.""" diff --git a/src/giant_python/extraction/band/__init__.py b/src/giant_python/extraction/band/__init__.py new file mode 100644 index 0000000..3e3bb12 --- /dev/null +++ b/src/giant_python/extraction/band/__init__.py @@ -0,0 +1 @@ +"""Band-scan extraction kernels with no eager workflow or IO imports.""" diff --git a/src/giant_python/extraction/band/activity.py b/src/giant_python/extraction/band/activity.py new file mode 100644 index 0000000..3422843 --- /dev/null +++ b/src/giant_python/extraction/band/activity.py @@ -0,0 +1,334 @@ +"""Matched-filter activity, valid-column masking, and temporal smoothing.""" + +from __future__ import annotations + +from typing import List, Tuple + +import numpy as np +import scipy.ndimage as ndimage +import torch +from scipy import signal + +from ...progress import progress + + +def _plane_column_maps( + sel_pix_idxs: np.ndarray, num_fast_zs: int, plane_size: int +) -> Tuple[list, list, list]: + """Precompute per-z column masks, indices, and remaps into local space. + + Returns three lists (one entry per z): the boolean column mask, the column + indices of that plane's selected pixels, and a remap from global to local + (per-plane) column index (``-1`` outside the plane). + """ + sel_pix_z = sel_pix_idxs // plane_size + z_masks = [torch.from_numpy(sel_pix_z == z) for z in range(num_fast_zs)] + z_col_idxs = [torch.nonzero(m, as_tuple=False).squeeze(1) for m in z_masks] + z_remaps = [] + for z in range(num_fast_zs): + remap = torch.full((sel_pix_idxs.shape[0],), -1, dtype=torch.long) + if z_col_idxs[z].numel() > 0: + remap[z_col_idxs[z]] = torch.arange( + z_col_idxs[z].numel(), dtype=torch.long + ) + z_remaps.append(remap) + return z_masks, z_col_idxs, z_remaps + + +def _valid_sel_cols_for_motion( + motion_idx: int, + unique_motion_to_keep_yx: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + sel_pix_idxs: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + psf2d: np.ndarray, +) -> np.ndarray: + """Return a per-selected-pixel mask of columns valid for one motion bin. + + The valid region is the PSF-dilated reference support minus its + horizontally-eroded border (erosion width ``2 * psf_w - 1``), evaluated at + this bin's motion shift. + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + valid_pix_mask = np.zeros( + (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), dtype=bool + ) + valid_pix_mask[ + ref_d, + ref_r + int(unique_motion_to_keep_yx[motion_idx, 0]), + ref_c + int(unique_motion_to_keep_yx[motion_idx, 1]), + ] = True + valid_pix_mask = ndimage.binary_dilation( + valid_pix_mask, + structure=np.ones((1, psf2d.shape[0], psf2d.shape[1]), dtype=bool), + ) + valid_pix_mask = ndimage.binary_erosion( + valid_pix_mask, + # Erode each plane with PSF-height rows and + # (2 * PSF-width - 1) columns. + structure=np.ones( + (1, psf2d.shape[0], psf2d.shape[1] * 2 - 1), dtype=bool + ), + ) + valid_pix_idxs = np.flatnonzero(valid_pix_mask) + valid_lookup = np.zeros(num_fast_zs * plane_size, dtype=bool) + valid_lookup[valid_pix_idxs] = True + return valid_lookup[sel_pix_idxs] + + +def compute_rho( + residual: np.ndarray, + mot_inds_yx: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + h_mots: List[torch.Tensor], + d_mats: List[torch.Tensor], + d_mats_expanded: List[torch.Tensor], + sel_pix_idxs: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + psf2d: np.ndarray, + verbose: bool = False, +) -> np.ndarray: + """Compute the ``rho`` matched-filter response on the selected pixels. + + For each motion bin and z-plane, restricts ``H`` to that plane, forms the + normalized ``HD`` and ``HD_expanded`` projections, and projects the + residual onto ``HD - HD_expanded`` over the plane's valid columns. + + Parameters + ---------- + residual : ndarray of shape (n_superpixels, n_frames) + Noise-normalized residual from :func:`compute_residual`. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` from :func:`build_motion_h_matrices`. + d_mats, d_mats_expanded : list of torch.Tensor + Per-plane base/expanded convolution matrices from + :func:`build_convolution_matrix`. + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + psf2d : ndarray + The (cropped) PSF for this DMD. + verbose : bool + Show a per-motion-bin progress bar when set. + + Returns + ------- + ndarray of shape (n_sel, n_frames), float32 + The ``rho`` response (NaN where not computed). + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + z_masks, z_col_idxs, z_remaps = _plane_column_maps( + sel_pix_idxs, num_fast_zs, plane_size + ) + z_col_idxs_np = [zi.numpy() for zi in z_col_idxs] + + rho = np.full( + (len(sel_pix_idxs), residual.shape[1]), np.nan, dtype=np.float32 + ) + + for i in progress( + range(len(unique_motion_to_keep_yx)), + desc="Computing rho", + verbose=verbose, + ): + motion_frames = np.flatnonzero(mot_inds_yx == i) + if motion_frames.size == 0: + continue + + h_mot = h_mots[i].coalesce() + h_idxs = h_mot.indices() + h_vals = h_mot.values() + nrows, _ = h_mot.size() + residual_motion = residual[:, motion_frames].T + + valid_sel_cols = _valid_sel_cols_for_motion( + i, + unique_motion_to_keep_yx, + ref_d, + ref_r, + ref_c, + sel_pix_idxs, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + ) + + for z in range(num_fast_zs): + new_ncols = int(z_col_idxs[z].numel()) + if new_ncols == 0: + continue + + z_mask = z_masks[z] + keep_mask = z_mask[h_idxs[1]] + if keep_mask.sum().item() == 0: + # Unreachable defensive guard (mirrors the reference): a plane + # with selected pixels (new_ncols > 0) always has H columns in + # it, since both derive from the same in-plane PSF footprint. + continue + + remap = z_remaps[z] + new_rows = h_idxs[0, keep_mask] + new_cols = remap[h_idxs[1, keep_mask]] + new_idxs = torch.stack([new_rows, new_cols], dim=0) + new_vals = h_vals[keep_mask] + h_sub = torch.sparse_coo_tensor( + new_idxs, new_vals, (nrows, new_ncols), dtype=torch.float32 + ).coalesce() + + hd = torch.sparse.mm(h_sub, d_mats[z]) + hd = hd / torch.sum(hd, dim=0, keepdim=True) + # Center-surround matched filter. Both projections are + # column-normalized before subtracting, so ``hd_diff`` has + # zero-sum columns and is blind to spatially flat residual. + hd_expanded = torch.sparse.mm(h_sub, d_mats_expanded[z]) + hd_expanded = hd_expanded / torch.sum( + hd_expanded, dim=0, keepdim=True + ) + hd_diff = hd - hd_expanded + # TEMPORARY: whiten each filter column to unit L2 norm so every + # selected pixel's rho has the same noise variance. The L1 + # normalization above pins each column's *sum*, not its norm, so + # noise gain varies with where a column sits relative to the + # sparse superpixel reference-pixel lattice. Since the activity + # image accumulates rho^2 only at local maxima, that bias is + # amplified into a visible grid of bright reference pixels. + # Whitening also makes ``peak_th`` a true sigma threshold. + # To revert, delete the two lines below. + hd_norm = torch.linalg.norm(hd_diff, dim=0, keepdim=True) + hd_diff = hd_diff / hd_norm.clamp_min(1e-12) + + z_cols = z_col_idxs_np[z] + z_valid_mask = valid_sel_cols[z_cols] + if not np.any(z_valid_mask): + continue + z_valid_cols = z_cols[z_valid_mask] + local_valid_cols = remap[torch.from_numpy(z_valid_cols)].long() + + rho[np.ix_(z_valid_cols, motion_frames)] = ( + residual_motion @ hd_diff[:, local_valid_cols].numpy() + ).T + + return rho + + +def mask_high_nan_rho(rho: np.ndarray, thresh: float = 0.5) -> np.ndarray: + """NaN-out rows of ``rho`` that are mostly NaN and return the NaN fraction. + + ``rho`` is modified in place. Rows whose NaN fraction exceeds ``thresh`` + are set entirely to NaN. + + Parameters + ---------- + rho : ndarray of shape (n_sel, n_frames) + The rho response. + thresh : float + NaN-fraction threshold above which a row is dropped. + + Returns + ------- + ndarray of shape (n_sel,) + Per-row NaN fraction (before dropping). + """ + nan_ct = np.mean(np.isnan(rho), axis=1) + rho[nan_ct > thresh] = np.nan + return nan_ct + + +def decay_kernel_1d(decay_tau_s: float, align_hz: float) -> np.ndarray: + """Build the normalized causal exponential-decay smoothing kernel. + + Parameters + ---------- + decay_tau_s : float + Calcium-decay time constant, in seconds. + align_hz : float + Alignment/analysis rate, in Hz. + + Returns + ------- + ndarray + A 1-D kernel spanning ``3 * tau`` frames, summing to 1. + """ + decay_tau_frames = decay_tau_s * align_hz + k1d = np.exp( + np.linspace( + -np.ceil(decay_tau_frames * 3), + 0, + int(np.ceil(decay_tau_frames * 3) + 1), + ) + / decay_tau_frames + ) + return k1d / np.sum(k1d) + + +def smooth_rho( + rho: np.ndarray, k1d: np.ndarray, verbose: bool = False +) -> np.ndarray: + """NaN-aware temporal smoothing of ``rho`` with a 1-D decay kernel. + + Convolves along time only, dividing the smoothed values by the smoothed + valid-sample count and NaN-ing samples whose effective count drops below + ``0.75``. Rows are processed in chunks to bound temporary memory. + ``rho`` is modified in place and also returned. + + Parameters + ---------- + rho : ndarray of shape (n_sel, n_frames) + The rho response (NaN allowed). + k1d : ndarray + 1-D smoothing kernel from :func:`decay_kernel_1d`. + verbose : bool + Show a per-row-chunk progress bar when set. + + Returns + ------- + ndarray + The smoothed ``rho``. + """ + k2d = np.expand_dims(k1d, 0) + n_time = rho.shape[1] + bytes_per_row = max(1, n_time) * np.dtype(np.float32).itemsize * 3 + row_chunk = max(64, min(4096, int(200_000_000 // bytes_per_row))) + + for r0 in progress( + range(0, rho.shape[0], row_chunk), + desc="Smoothing rho", + verbose=verbose, + ): + r1 = min(r0 + row_chunk, rho.shape[0]) + rc = rho[r0:r1].copy() + row_has_data = np.any(np.isfinite(rc), axis=1) + if not np.any(row_has_data): + rho[r0:r1] = rc + continue + + rc_valid = rc[row_has_data] + rho_num = np.nan_to_num(rc_valid, nan=0.0) + rho_den = signal.convolve( + np.isfinite(rc_valid).astype(np.float32), k2d, mode="same" + ) + rho_num = signal.convolve(rho_num, k2d, mode="same") + valid_den = rho_den > 0.75 + np.divide(rho_num, rho_den, out=rho_num, where=valid_den) + rho_num[~valid_den] = np.nan + rc[row_has_data] = rho_num + rho[r0:r1] = rc + + return rho diff --git a/src/giant_python/extraction/band/annotation.py b/src/giant_python/extraction/band/annotation.py new file mode 100644 index 0000000..39a7218 --- /dev/null +++ b/src/giant_python/extraction/band/annotation.py @@ -0,0 +1,216 @@ +"""Standalone band annotation and shared load/draw/headless orchestration. + +Raw persistence lives in io.annotations, pure geometry in band.rois, and +toolkit code in gui.roi_editor. No toolkit is loaded on the headless path. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Optional, Union + +import numpy as np + +from ...io.annotations import read_annotations_h5, save_annotations_h5 +from ...models.params import ( + AnnotationInput, + BandParamsInput, + ExecutionInput, + resolve_band_options, +) +from ...models.trial_table import TrialTable +from ...progress import log +from . import geometry as geo +from . import inputs +from .inputs import ( + compute_keep_trials, + load_alignment_data_h5, + load_trial_table, +) +from .rois import compute_user_roi_geometry, resolve_annotation_records + + +def first_valid_trial(keep_trials: np.ndarray, dmd_ix: int) -> int: + """Return this 0-based DMD's first kept trial, or raise ValueError.""" + indexes = np.flatnonzero(np.asarray(keep_trials)[dmd_ix]) + if indexes.size == 0: + raise ValueError(f"No valid trials for DMD {dmd_ix + 1}") + return int(indexes[0]) + + +def motion_median_from_adata(a_data: dict) -> tuple: + """Round internal band motion then take per-axis (row,column,z) medians.""" + return ( + int(np.nanmedian(np.round(a_data["motionDSr"]))), + int(np.nanmedian(np.round(a_data["motionDSc"]))), + int(np.nanmedian(np.round(a_data["motionDSz"]))), + ) + + +def _stdin_is_tty() -> bool: + """Check for interactive stdin without importing a GUI toolkit.""" + return bool(getattr(sys.stdin, "isatty", lambda: False)()) + + +def resolve_interactivity(override: Optional[bool] = None) -> bool: + """Resolve override, then GIANT_HEADLESS, then terminal detection.""" + if override is not None: + return bool(override) + if os.environ.get("GIANT_HEADLESS"): + return False + return _stdin_is_tty() + + +def _empty_user_rois(n_dmds: int) -> dict: + """Create neutral per-path ROI selections without implying annotations.""" + keys = [f"DMD{d + 1}" for d in range(n_dmds)] + return { + "user_roi_masks": {k: [] for k in keys}, + "user_roi_superpixels": {k: [] for k in keys}, + "user_roi_labels": {k: [] for k in keys}, + "roi_records": {k: [] for k in keys}, + "annotated": False, + } + + +def load_annotations_h5( + annotations_path: str, n_dmds: int, user_roi_geo: dict +) -> tuple: + """Read and resolve band annotations; return the historical five-tuple. + + Missing/unreadable files and selections with no usable ROIs return + (False, {}, {}, {}, {}). Geometry validation is delegated to pure helpers. + """ + if not os.path.exists(annotations_path): + return False, {}, {}, {}, {} + try: + raw = read_annotations_h5(annotations_path) + except OSError as error: + print(f"Could not read {annotations_path}: {error}") + return False, {}, {}, {}, {} + return resolve_annotation_records(raw, n_dmds, user_roi_geo) + + +def resolve_user_rois( + annotation_dir: Union[str, Path], + n_dmds: int, + user_roi_geo: dict, + *, + interactive: bool, + ref_files: Optional[dict] = None, +) -> dict: + """Load usable annotations, or draw interactively or fail promptly.""" + path = os.path.join(str(annotation_dir), "annotations.h5") + skip, masks, superpixels, labels, records = load_annotations_h5( + path, n_dmds, user_roi_geo + ) + if skip: + return { + "user_roi_masks": masks, + "user_roi_superpixels": superpixels, + "user_roi_labels": labels, + "roi_records": records, + "annotated": True, + } + if not interactive: + raise RuntimeError( + "Annotations are enabled but no usable ROIs were found at " + f"{path!r}, and this process is not interactive. Run " + "`giant annotate ` first, or run in an " + "interactive session (or pass interactive=True) to draw now." + ) + return _draw_and_save_user_rois( + annotation_dir, n_dmds, user_roi_geo, ref_files + ) + + +def _draw_and_save_user_rois( + annotation_dir: Union[str, Path], + n_dmds: int, + user_roi_geo: dict, + ref_files: Optional[dict], +) -> dict: + """Launch the lazy ROI editor and save its records and masks.""" + from ...gui.roi_editor import run_user_roi_selection + + os.makedirs(str(annotation_dir), exist_ok=True) + selection = run_user_roi_selection(user_roi_geo, n_dmds) + save_annotations_h5( + str(annotation_dir), + selection["roi_records"], + selection["user_roi_masks"], + n_dmds, + ref_files=ref_files, + ) + selection["annotated"] = True + return selection + + +def build_user_roi_geometry(trial_table: dict, lookup: dict) -> tuple: + """Prepare per-DMD reference geometry and reference-file provenance.""" + user_roi_geo, ref_files = {}, {} + datadr = str(trial_table["datadr"]) + keep_trials = trial_table["keep_trials"] + for dmd_ix in range(int(trial_table["n_dmds"])): + key = f"DMD{dmd_ix + 1}" + ref_stack, _, ref_file = inputs.load_reference_stack( + datadr, trial_table["ref_stack"], dmd_ix + ) + smi = geo.build_subsample_matrix_inds( + lookup["allSuperPixelIDs"][key], lookup["sparseMaskInds"][key] + ) + fvt = first_valid_trial(keep_trials, dmd_ix) + a_data = load_alignment_data_h5(trial_table["fn_adata"][dmd_ix, fvt]) + user_roi_geo[key] = compute_user_roi_geometry( + ref_stack, + lookup["fastZ2RefZ"][key], + smi, + motion_median_from_adata(a_data), + ) + ref_files[key] = ref_file + return user_roi_geo, ref_files + + +def annotate_band_rois( + input: Union[str, Path, TrialTable], + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> str: + """Annotate a path or in-memory TrialTable; return annotations.h5's path. + + Existing usable annotations are loaded headlessly. No extraction or + source fitting occurs and in-memory trial tables are never reloaded. + Science, execution and annotation inputs are independently validated. + This explicit step requests annotation regardless of ``enabled``; + ``interactive`` still controls whether drawing is permitted. + """ + _, run, policy = resolve_band_options(params, execution, annotations) + log(f"Loading trial table {input}", run.verbose) + trial_table = load_trial_table(input) + trial_table["keep_trials"] = compute_keep_trials( + trial_table["fn_adata"], trial_table["filename"], trial_table["datadr"] + ) + annotation_dir = trial_table["annotation_save_dr"] + lookup = inputs.load_lookup_table( + Path(trial_table["moco_save_dr"]) / "bandRegLookupTable.h5", + trial_table["n_dmds"], + ) + log( + f"Loading reference geometry for {trial_table['n_dmds']} DMD path(s)", + run.verbose, + ) + user_roi_geo, ref_files = build_user_roi_geometry(trial_table, lookup) + resolve_user_rois( + annotation_dir, + trial_table["n_dmds"], + user_roi_geo, + interactive=resolve_interactivity(policy.interactive), + ref_files=ref_files, + ) + out_path = os.path.join(str(annotation_dir), "annotations.h5") + log(f"Annotations at {out_path}", run.verbose) + return out_path diff --git a/src/giant_python/extraction/band/background.py b/src/giant_python/extraction/band/background.py new file mode 100644 index 0000000..cb1cdaf --- /dev/null +++ b/src/giant_python/extraction/band/background.py @@ -0,0 +1,505 @@ +"""Band background interpolation, rolling baseline, and noise estimation.""" + +from __future__ import annotations + +from typing import Tuple + +import numpy as np +import pandas as pd +from scipy.interpolate import CubicSpline + +try: + import bottleneck as bn +except ImportError: + bn = None + + +def _collect_interp_columns( + cols, s_c, s_r, sel_idxs_by_col_all, sel_rows, r0, r1 +): + """Group a motion bin's reference pixels by column for interpolation. + + Columns with a single reference pixel become ``single_points`` (copied + verbatim); columns with several become sorted ``(rows, source-index)`` + entries for linear interpolation. + """ + rows_by_col = {} + idxs_by_col = {} + single_points = [] + for c in cols: + c_int = int(c) + if c_int not in sel_idxs_by_col_all: + continue + data_ix = np.flatnonzero(s_c == c) + if data_ix.size == 1: + rr = int(s_r[data_ix[0]]) + col_sel = sel_idxs_by_col_all[c_int] + rr_match = col_sel[sel_rows[col_sel] == rr] + if rr_match.size > 0 and (r0 <= rr <= r1): + single_points.append((rr_match, int(data_ix[0]))) + continue + rows = s_r[data_ix] + order = rows.argsort(kind="mergesort") + rows_by_col[c_int] = rows[order] + idxs_by_col[c_int] = data_ix[order] + return rows_by_col, idxs_by_col, single_points + + +# Preserve scientific parity with the reference interpolation branches. +def _interp_columns( # noqa: C901 + out, + rows_by_col, + idxs_by_col, + sel_idxs_by_col_all, + sel_rows, + data, + frames, + dtype, + method="linear", + spline_weight_cache=None, +): + """Interpolate each multi-point column onto its selected pixels. + + Selected rows that coincide with a reference row are copied exactly; those + strictly between reference rows use linear or natural-cubic interpolation; + those outside the sampled row range are left untouched (NaN). Cubic spline + weights depend only on row geometry, so they are computed once and applied + to all complete frames in one matrix multiplication. Linear interpolation + is retained for columns with fewer than three samples and frames containing + missing source values. + """ + if method not in {"linear", "cubic"}: + raise ValueError( + "interpolation method must be 'linear' or 'cubic', " + f"got {method!r}" + ) + for c_int, rows in rows_by_col.items(): + col_sel_pix_idxs = sel_idxs_by_col_all[c_int] + target_rows = sel_rows[col_sel_pix_idxs] + src_idx = idxs_by_col[c_int] + + pos = np.searchsorted(rows, target_rows, side="left") + in_bounds = pos < rows.size + exact = np.zeros_like(pos, dtype=bool) + exact[in_bounds] = rows[pos[in_bounds]] == target_rows[in_bounds] + + if np.any(exact): + exact_rows = np.flatnonzero(exact) + exact_src = src_idx[pos[exact]] + out[np.ix_(col_sel_pix_idxs[exact_rows], frames)] = data[ + exact_src + ][:, frames] + + lo = pos - 1 + hi = pos + interp_mask = (~exact) & (lo >= 0) & (hi < rows.size) + if not np.any(interp_mask): + continue + + interp_rows = np.flatnonzero(interp_mask) + lo_v = lo[interp_mask] + hi_v = hi[interp_mask] + row_lo = rows[lo_v].astype(dtype, copy=False) + row_hi = rows[hi_v].astype(dtype, copy=False) + denom = row_hi - row_lo + nonzero = denom != 0 + if not np.any(nonzero): + # Unreachable defensive guard (mirrors the reference): sorted + # distinct rows make searchsorted never straddle equal rows, so + # denom is always non-zero here. + continue + + interp_rows = interp_rows[nonzero] + lo_v = lo_v[nonzero] + hi_v = hi_v[nonzero] + row_lo = row_lo[nonzero] + row_hi = row_hi[nonzero] + alpha = ( + (target_rows[interp_rows].astype(dtype) - row_lo) + / (row_hi - row_lo) + )[:, None] + + target_indices = col_sel_pix_idxs[interp_rows] + linear_frame_mask = np.ones(frames.size, dtype=bool) + has_cubic_geometry = rows.size >= 3 and np.all(np.diff(rows) > 0) + if method == "cubic" and has_cubic_geometry: + source_values = data[src_idx][:, frames] + complete_frames = np.all(np.isfinite(source_values), axis=0) + if np.any(complete_frames): + source_rows = rows.astype(dtype, copy=False) + target_interp_rows = target_rows[interp_rows].astype( + dtype, copy=False + ) + origin = source_rows[0] + cache_key = ( + tuple(source_rows - origin), + tuple(target_interp_rows - origin), + np.dtype(dtype).str, + ) + spline_weights = ( + spline_weight_cache.get(cache_key) + if spline_weight_cache is not None + else None + ) + if spline_weights is None: + spline_weights = CubicSpline( + source_rows, + np.eye(rows.size, dtype=dtype), + axis=0, + bc_type="natural", + extrapolate=False, + )(target_interp_rows).astype(dtype, copy=False) + if spline_weight_cache is not None: + spline_weight_cache[cache_key] = spline_weights + cubic_values = ( + spline_weights @ source_values[:, complete_frames] + ) + out[np.ix_(target_indices, frames[complete_frames])] = ( + cubic_values + ) + linear_frame_mask = ~complete_frames + + if not np.any(linear_frame_mask): + continue + linear_frames = frames[linear_frame_mask] + vals_lo = data[src_idx[lo_v]][:, linear_frames] + vals_hi = data[src_idx[hi_v]][:, linear_frames] + linear_values = (1.0 - alpha) * vals_lo + alpha * vals_hi + out[np.ix_(target_indices, linear_frames)] = linear_values + + +def build_interp_data( + data: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + sel_pixels_2d: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + mot_inds_yx: np.ndarray, + height: int = 800, + width: int = 1280, + dtype=np.float32, + method: str = "cubic", +) -> Tuple[np.ndarray, Tuple[int, int], Tuple[int, int]]: + """Interpolate reference-pixel traces onto the selected-pixel grid. + + For each kept motion bin, the reference pixels (shifted by the bin's + motion) are linearly interpolated along rows, per column, onto the selected + pixels of the plane, filling one background estimate per selected pixel and + frame. Selected pixels outside the sampled row range remain NaN. + + Parameters + ---------- + data : ndarray of shape (n_ref, n_frames) + Reference-pixel traces for one plane. + ref_r, ref_c : ndarray + Reference-pixel row/column indices for this plane. + sel_pixels_2d : ndarray of shape (n_sel, 2) + Selected-pixel ``[row, col]`` coordinates for this plane. + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + height, width : int + Sensor row/column extents used to window valid shifts. + dtype : numpy dtype + Working/output dtype. + method : {"linear", "cubic"} + Interpolation along each column (default ``"cubic"``). Cubic mode uses + a natural spline with precomputed geometry weights and falls back to + linear for columns with fewer than three samples or frames with missing + samples. + + Returns + ------- + out : ndarray of shape (n_sel, n_frames) + Interpolated background estimate (NaN where unsampled). + row_bounds : tuple of int + ``(r0, r1)`` inclusive row window. + col_bounds : tuple of int + ``(c0, c1)`` inclusive column window. + """ + ref_r = np.asarray(ref_r, dtype=np.int32) + ref_c = np.asarray(ref_c, dtype=np.int32) + data = np.asarray(data, dtype=dtype) + if method not in {"linear", "cubic"}: + raise ValueError( + "interpolation method must be 'linear' or 'cubic', " + f"got {method!r}" + ) + + r0 = int(max(0, ref_r.min()) + unique_motion_to_keep_yx[:, 0].min()) + r1 = int(min(height, ref_r.max()) + unique_motion_to_keep_yx[:, 0].max()) + c0 = int(max(0, ref_c.min()) + unique_motion_to_keep_yx[:, 1].min()) + c1 = int(min(width, ref_c.max()) + unique_motion_to_keep_yx[:, 1].max()) + + out = np.full((sel_pixels_2d.shape[0], data.shape[1]), np.nan, dtype=dtype) + + sel_cols = sel_pixels_2d[:, 1] + sel_rows = sel_pixels_2d[:, 0] + unique_sel_cols = np.unique(sel_cols) + sel_idxs_by_col_all = { + int(c): np.flatnonzero(sel_cols == c) for c in unique_sel_cols + } + + frames_by_motion = [ + np.flatnonzero(mot_inds_yx == idx) + for idx in range(len(unique_motion_to_keep_yx)) + ] + spline_weight_cache = {} if method == "cubic" else None + + for m_idx, frames in enumerate(frames_by_motion): + if frames.size == 0: + continue + + s_r = ref_r + int(unique_motion_to_keep_yx[m_idx, 0]) + s_c = ref_c + int(unique_motion_to_keep_yx[m_idx, 1]) + + in_win = (s_r >= r0) & (s_r <= r1) & (s_c >= c0) & (s_c <= c1) + if not np.any(in_win): + continue + cols = np.unique(s_c[in_win]) + + rows_by_col, idxs_by_col, single_points = _collect_interp_columns( + cols, s_c, s_r, sel_idxs_by_col_all, sel_rows, r0, r1 + ) + _interp_columns( + out, + rows_by_col, + idxs_by_col, + sel_idxs_by_col_all, + sel_rows, + data, + frames, + dtype, + method, + spline_weight_cache, + ) + for rr_match, src_pos in single_points: + out[np.ix_(rr_match, frames)] = data[src_pos, frames] + + return out, (r0, r1), (c0, c1) + + +def baseline_window_frames(align_hz: float, baseline_window_s: float) -> int: + """Return the rolling-baseline window length in frames. + + Parameters + ---------- + align_hz : float + Alignment/analysis rate, in Hz. + baseline_window_s : float + Baseline window, in seconds. + + Returns + ------- + int + ``int(align_hz * baseline_window_s)``. + """ + return int(align_hz * baseline_window_s) + + +def compute_rolling_baseline( + interp_data: np.ndarray, baseline_window: int +) -> np.ndarray: + """Compute a NaN-aware rolling-median baseline of the interpolated data. + + Each frame's baseline is the median of its centered window, ignoring NaNs, + so windows with no valid samples stay NaN and edge windows are truncated to + the available samples. This is the fast moving-median analogue of the + former moving-mean baseline; the median is more robust to transient + activity spikes when estimating the background. + + The heavy lifting uses :func:`bottleneck.move_median` (a C sliding-window + median whose cost is close to the moving mean) when available, falling back + to pandas' skip-list rolling median otherwise. ``bottleneck`` computes a + *trailing* window, so the data is right-padded with NaNs by ``window // 2`` + and the result is sliced back to center each window on its frame. + + Parameters + ---------- + interp_data : ndarray of shape (n_sel, n_frames) + Interpolated background estimate (from :func:`build_interp_data`). + baseline_window : int + Rolling window length, in frames. + + Returns + ------- + ndarray + The rolling-median background estimate (NaN where no valid samples). + """ + n_frames = interp_data.shape[1] + window = int(min(max(baseline_window, 1), n_frames)) + + if bn is not None: + pad = window // 2 + if pad: + padded = np.concatenate( + [ + interp_data, + np.full( + (interp_data.shape[0], pad), + np.nan, + dtype=interp_data.dtype, + ), + ], + axis=1, + ) + else: + padded = interp_data + trailing = bn.move_median(padded, window=window, min_count=1, axis=1) + background = trailing[:, pad : pad + n_frames] + return background.astype(interp_data.dtype, copy=False) + + # Fallback (no bottleneck): pandas rolls along the row axis, so transpose + # to (n_frames, n_sel), roll each column, and transpose back. + background = ( + pd.DataFrame(interp_data.T) + .rolling(window=window, center=True, min_periods=1) + .median() + .to_numpy() + .T + ) + return background.astype(interp_data.dtype, copy=False) + + +def assemble_background( + interp_data_background: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + mot_inds_yx: np.ndarray, + sel_pix_idxs: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + n_super_pixels: int, + n_frames: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> np.ndarray: + """Map the selected-pixel background back onto superpixels per motion bin. + + For each kept motion bin and its frames, each superpixel's motion-shifted + reference pixel is looked up in ``sel_pix_idxs`` and its background trace + is copied onto that superpixel. + + Parameters + ---------- + interp_data_background : ndarray of shape (n_sel, n_frames) + Rolling-baseline background on the selected-pixel grid. + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + n_super_pixels, n_frames : int + Output shape. + dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + + Returns + ------- + ndarray of shape (n_super_pixels, n_frames), float32 + Per-superpixel background (NaN where unmapped). + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + background = np.full((n_super_pixels, n_frames), np.nan, dtype=np.float32) + + for motion_idx in range(len(unique_motion_to_keep_yx)): + motion_frames = np.flatnonzero(mot_inds_yx == motion_idx) + d_r = int(unique_motion_to_keep_yx[motion_idx, 0]) + d_c = int(unique_motion_to_keep_yx[motion_idx, 1]) + s_d = ref_d + s_r = ref_r + d_r + s_c = ref_c + d_c + + shifted_indices = s_d * plane_size + s_r * dmd_pixels_per_row + s_c + + bg_idxs = np.searchsorted(sel_pix_idxs, shifted_indices) + idxs_mask = bg_idxs < sel_pix_idxs.size + idxs_mask[idxs_mask] &= ( + sel_pix_idxs[bg_idxs[idxs_mask]] == shifted_indices[idxs_mask] + ) + + if np.any(idxs_mask): + valid_rows = np.flatnonzero(idxs_mask) + background[np.ix_(valid_rows, motion_frames)] = ( + interp_data_background[ + np.ix_(bg_idxs[valid_rows], motion_frames) + ] + ) + + return background + + +def fit_noise_variance_model( + low_res_data_norm: np.ndarray, + background: np.ndarray, + v_im: np.ndarray, + vif: float = 1.38, +) -> Tuple[np.ndarray, float, float]: + """Calibrate an affine noise-variance model and return per-sample std. + + Fits ``Var ~= Vk * (background * vIM) + Vb`` from the brightest pixels over + the first valid frames, then returns the predicted standard deviation for + every sample. + + Parameters + ---------- + low_res_data_norm : ndarray of shape (n_superpixels, n_frames) + Count-normalized channel-1 traces. + background : ndarray of shape (n_superpixels, n_frames) + Per-superpixel background estimate. + v_im : ndarray of shape (n_superpixels, n_frames) + Inverse count image (``1 / lowResDataCt``). + vif : float + Variance inflation factor. + + Returns + ------- + data_std : ndarray + Per-sample predicted standard deviation. + v_k : float + Fitted multiplicative variance coefficient. + v_b : float + Fitted additive variance floor. + """ + first_valid_frames = np.flatnonzero( + np.any(~np.isnan(background[:, :1000]), axis=0) + ) + var_im = np.nanvar(low_res_data_norm[:, first_valid_frames], axis=1) + v_b = np.nanpercentile(var_im, 5) * vif + var_pred = np.nanmean( + background[:, first_valid_frames], axis=1 + ) * np.nanmean(v_im[:, first_valid_frames], axis=1) + sel_bright = var_pred > np.nanpercentile(var_pred, 90) + v_k = np.nanpercentile( + (var_im[sel_bright] - (v_b / vif)) / var_pred[sel_bright], 10 + ) + data_std = np.sqrt(np.clip(v_k * background * v_im, 0, None) + v_b) + return data_std, v_k, v_b + + +def compute_residual( + low_res_data_norm: np.ndarray, + background: np.ndarray, + data_std: np.ndarray, +) -> np.ndarray: + """Return the background-subtracted, noise-normalized residual. + + Parameters + ---------- + low_res_data_norm : ndarray + Count-normalized traces. + background : ndarray + Per-superpixel background estimate. + data_std : ndarray + Per-sample predicted standard deviation. + + Returns + ------- + ndarray + ``(low_res_data_norm - background) / data_std``. + """ + return (low_res_data_norm - background) / data_std diff --git a/src/giant_python/extraction/band/geometry.py b/src/giant_python/extraction/band/geometry.py new file mode 100644 index 0000000..dd14cc4 --- /dev/null +++ b/src/giant_python/extraction/band/geometry.py @@ -0,0 +1,234 @@ +"""Pure DMD pixel geometry, selected-pixel support and PSF padding/cropping. + +Loading belongs to band.inputs; sparse projection to band.operators. +""" + +from __future__ import annotations + +from typing import Tuple + +import numpy as np +import scipy.ndimage as ndimage + + +def ref_pixs_to_drc( + ref_pixs: np.ndarray, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Map flat reference-pixel indices to DMD (depth, column, row) indices. + + Parameters + ---------- + ref_pixs : ndarray of int + Flat reference-pixel indices. + dmd_pixels_per_column, dmd_pixels_per_row : int + DMD geometry. + + Returns + ------- + ref_d, ref_c, ref_r : ndarray of int32 + Depth, column, and row indices. + """ + ref_pixs = np.asarray(ref_pixs, dtype=np.int64) + plane = int(dmd_pixels_per_column) * int(dmd_pixels_per_row) + npc = int(dmd_pixels_per_column) + ref_d = np.floor_divide(ref_pixs, plane).astype(np.int32) + ref_c = np.floor_divide( + ref_pixs - ref_d.astype(np.int64) * plane, npc + ).astype(np.int32) + ref_r = np.mod(ref_pixs, npc).astype(np.int32) + return ref_d, ref_c, ref_r + + +def build_subsample_matrix_inds( + all_super_pixel_ids: np.ndarray, + sparse_mask_inds: np.ndarray, +) -> np.ndarray: + """Build the superpixel -> reference-pixel index map for one DMD. + + For each superpixel, picks the element at index ``len(open_pixs) // 2`` + in sparse-mask order, for both odd and even counts; values are not sorted. + + Parameters + ---------- + all_super_pixel_ids : ndarray of shape (n_superpixels, 1) + Superpixel ids for this DMD. + sparse_mask_inds : ndarray of shape (N, 2) + ``[open_pixel (1-based), superpixel_id (1-based)]`` rows. + + Returns + ------- + ndarray of shape (n_superpixels, 2), int32 + ``[ref_pixel (0-based), superpixel_id (1-based)]`` per superpixel. + """ + num_super_pixels = all_super_pixel_ids.shape[0] + out = np.zeros((num_super_pixels, 2), dtype=np.int32) + for sp in range(num_super_pixels): + inds = np.where(sparse_mask_inds[:, 1] == sp + 1)[0] + open_pixs = sparse_mask_inds[inds, 0] - 1 + ref_pix = open_pixs[int(np.floor(len(open_pixs) / 2))] + out[sp, 0] = ref_pix + out[sp, 1] = sp + 1 + return out + + +def _pad_to(psf2d: np.ndarray, height: int, width: int) -> np.ndarray: + """Center-pad a 2-D PSF to ``(height, width)`` with its minimum value.""" + ph, pw = psf2d.shape + pad_r = (height - ph) // 2 + pad_c = (width - pw) // 2 + return np.pad( + psf2d, + ((pad_r, pad_r), (pad_c, pad_c)), + constant_values=np.min(psf2d), + ) + + +def build_combined_psf(psfs: list) -> np.ndarray: + """Stack per-DMD PSFs into a ``(n_dmds, H, W)`` array, center-padded.""" + height = max(p.shape[0] for p in psfs) + width = max(p.shape[1] for p in psfs) + combined = np.zeros((len(psfs), height, width), dtype=np.float32) + for i, p in enumerate(psfs): + combined[i] = _pad_to(p, height, width) + return combined + + +def threshold_and_crop_psf(psf2d: np.ndarray) -> np.ndarray: + """Zero values below ``max * exp(-3)`` and crop boundary zeros. + + Parameters + ---------- + psf2d : ndarray + A single-DMD PSF image. + + Returns + ------- + ndarray of float32 + The thresholded, tightly-cropped PSF. + """ + psf2d = psf2d.astype(np.float32, copy=True) + psf2d[psf2d < np.max(psf2d) * np.exp(-3)] = 0 + non_zero_rows = np.any(psf2d != 0, axis=1) + non_zero_cols = np.any(psf2d != 0, axis=0) + row_start, row_end = np.where(non_zero_rows)[0][[0, -1]] + col_start, col_end = np.where(non_zero_cols)[0][[0, -1]] + row_stop = row_end + 1 + col_stop = col_end + 1 + return psf2d[row_start:row_stop, col_start:col_stop] + + +def build_selected_pixel_mask( + unique_motion_to_keep_yx: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + psf2d: np.ndarray, +) -> Tuple[np.ndarray, np.ndarray]: + """Build the selected-pixel support mask over all kept motion shifts. + + Marks every reference pixel shifted by each kept 2-D motion bin, then + dilates in-plane by the PSF footprint. + + Parameters + ---------- + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + psf2d : ndarray + The (cropped) PSF for this DMD; its shape sets the dilation footprint. + + Returns + ------- + sel_pix_mask : ndarray of bool + ``(num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row)`` support. + sel_pix_idxs : ndarray of int + Flattened indices of the selected pixels. + """ + sel_pix_mask = np.zeros( + (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), dtype=bool + ) + for i in range(len(unique_motion_to_keep_yx)): + sel_pix_mask[ + ref_d, + ref_r + int(unique_motion_to_keep_yx[i, 0]), + ref_c + int(unique_motion_to_keep_yx[i, 1]), + ] = True + sel_pix_mask = ndimage.binary_dilation( + sel_pix_mask, + structure=np.ones((1, psf2d.shape[0], psf2d.shape[1]), dtype=bool), + ) + sel_pix_idxs = np.flatnonzero(sel_pix_mask) + return sel_pix_mask, sel_pix_idxs + + +def pixel_coords_from_idxs( + sel_pix_idxs: np.ndarray, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> np.ndarray: + """Convert flat selected-pixel indices to ``[z, row, col]`` coordinates. + + Parameters + ---------- + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices. + dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + + Returns + ------- + ndarray of shape (n_sel, 3), int32 + ``[z, row, col]`` per selected pixel. + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + pixel_coords = np.empty((len(sel_pix_idxs), 3), dtype=np.int32) + pixel_coords[:, 0] = sel_pix_idxs // plane_size + remainder = sel_pix_idxs % plane_size + pixel_coords[:, 1] = remainder // dmd_pixels_per_row + pixel_coords[:, 2] = remainder % dmd_pixels_per_row + return pixel_coords + + +def selected_pixels_2d_for_plane( + sel_pix_idxs: np.ndarray, + plane_z: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> Tuple[np.ndarray, np.ndarray]: + """Return the in-plane ``[row, col]`` coords of one z-plane's pixels. + + Parameters + ---------- + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices (all planes). + plane_z : int + The z-plane to select. + dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + + Returns + ------- + z_idxs : ndarray + Positions (into ``sel_pix_idxs``) of pixels in ``plane_z``. + sel_pixels_2d : ndarray of shape (n_plane, 2) + ``[row, col]`` coordinates of those pixels. + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + sel_pix_z = sel_pix_idxs // plane_size + sel_pix_remainder = sel_pix_idxs % plane_size + z_idxs = np.flatnonzero(sel_pix_z == plane_z) + sel_pixels_2d = np.column_stack( + [ + sel_pix_remainder[z_idxs] // dmd_pixels_per_row, + sel_pix_remainder[z_idxs] % dmd_pixels_per_row, + ] + ) + return z_idxs, sel_pixels_2d diff --git a/src/giant_python/extraction/band/inputs.py b/src/giant_python/extraction/band/inputs.py new file mode 100644 index 0000000..90489a5 --- /dev/null +++ b/src/giant_python/extraction/band/inputs.py @@ -0,0 +1,337 @@ +"""Band runtime input preparation and displacement-to-offset normalization. + +Shared IO is faithful to saved displacement. Only this backend boundary +negates offline motion into the reference offsets used by band numerics. +Lookup/reference/PSF loading and asset discovery also belong at this boundary; +PSF padding and cropping delegate to pure geometry functions. +""" + +from __future__ import annotations + +import os +from importlib import resources +from pathlib import Path +from typing import Optional, Tuple, Union + +import numpy as np + +from ...io.hdf5 import load_struct_h5, to_serializable +from ...io.slap2 import load_alignment_data_h5 as read_alignment_data_h5 +from ...io.tiff import read_tiff +from ...models.trial_table import TrialTable +from .geometry import build_combined_psf, threshold_and_crop_psf + + +def load_lookup_table(path: Union[str, Path], n_dmds: int) -> dict: + """Load the band-registration lookup table (``bandRegLookupTable.h5``). + + Parameters + ---------- + path : str or Path + Path to the lookup-table HDF5 file. + n_dmds : int + Number of DMD paths. + + Returns + ------- + dict + ``{"allSuperPixelIDs": {...}, "sparseMaskInds": {...}, + "fastZ2RefZ": {...}}``, each an inner dict keyed ``DMD{N}``. + + Raises + ------ + ValueError + If a requested DMD has no Path/DMD lookup group. + """ + lt = load_struct_h5(path) + all_super_pixel_ids = {} + sparse_mask_inds = {} + fastz_to_refz = {} + for d in range(n_dmds): + p = lt.get(f"Path{d + 1}", lt.get(f"DMD{d + 1}")) + if p is None: + raise ValueError( + f"Missing lookup-table group Path{d + 1} or DMD{d + 1}" + ) + key = f"DMD{d + 1}" + all_super_pixel_ids[key] = ( + np.asarray(p["allSuperPixelIDs"]).reshape(-1, 1).astype(np.int32) + ) + sparse_mask_inds[key] = np.asarray(p["sparseMaskInds"]).astype( + np.int32 + ) + fastz_to_refz[key] = ( + np.asarray(p["fastZ2RefZ"]).reshape(-1, 1).astype(np.int32) + ) + return { + "allSuperPixelIDs": all_super_pixel_ids, + "sparseMaskInds": sparse_mask_inds, + "fastZ2RefZ": fastz_to_refz, + } + + +def _ref_stack_group(ref_stack: dict, dmd_ix: int) -> dict: + """Return a Path/DMD reference subgroup; raise ValueError if missing.""" + group = ref_stack.get( + f"Path{dmd_ix + 1}", ref_stack.get(f"DMD{dmd_ix + 1}") + ) + if group is None: + raise ValueError( + f"Missing reference-stack group Path{dmd_ix + 1} " + f"or DMD{dmd_ix + 1}" + ) + return group + + +def find_reference_file( + datadr: Union[str, Path], dmd_ix: int +) -> Optional[str]: + """Find the REFERENCE tif for a DMD under ``datadr`` (recursive glob). + + Parameters + ---------- + datadr : str or Path + Raw-data directory. + dmd_ix : int + 0-based DMD index. + + Returns + ------- + str or None + Path to the first matching REFERENCE tif, or ``None`` if none found. + """ + patterns = ( + f"**/*DMD{dmd_ix + 1}_CONFIG2-REFERENCE*", + f"**/*DMD{dmd_ix + 1}-REFERENCE*", + ) + for pattern in patterns: + matches = list(Path(datadr).glob(pattern)) + if matches: + return str(matches[0]) + return None + + +def load_reference_stack( + datadr: Union[str, Path], + ref_stack_meta: dict, + dmd_ix: int, +) -> Tuple[Optional[np.ndarray], np.ndarray, Optional[str]]: + """Load the band reference stack for one DMD. + + Reads the REFERENCE tif from ``datadr`` but takes the channel list from the + trial table's embedded ref_stack metadata. The raw counts are scaled by + ``1/100`` and reshaped to ``[channels, z, y, x]``. + + Parameters + ---------- + datadr : str or Path + Raw-data directory. + ref_stack_meta : dict + The trial table's ``ref_stack`` sub-struct. + dmd_ix : int + 0-based DMD index. + + Returns + ------- + ref_stack : ndarray or None + ``[channels, z, y, x]`` stack, or ``None`` if no REFERENCE file found. + channels : ndarray + Channel indices for this DMD. + ref_file : str or None + The REFERENCE file used, or ``None``. + + Raises + ------ + ValueError + If the requested DMD has no Path/DMD reference metadata group. + """ + grp = _ref_stack_group(ref_stack_meta, dmd_ix) + channels = np.asarray(grp["channels"]).reshape(-1) + num_channels = len(channels) + + ref_file = find_reference_file(datadr, dmd_ix) + if ref_file is None: + return None, channels, None + + ref = read_tiff(ref_file) / 100 + ref = ref.reshape(-1, num_channels, ref.shape[1], ref.shape[2]) + ref = ref.transpose(1, 0, 2, 3) + return ref, channels, ref_file + + +def default_psf(dilation: int) -> np.ndarray: + """Load a bundled default PSF template (``assets/psfs/dil-NN.tif``). + + Parameters + ---------- + dilation : int + Dilation size selecting ``dil-{dilation:02d}.tif`` (e.g. ``17``). + + Returns + ------- + ndarray of float32 + The PSF template image. + """ + name = f"dil-{int(dilation):02d}.tif" + resource = ( + resources.files("giant_python") + .joinpath("assets") + .joinpath("psfs") + .joinpath(name) + ) + with resources.as_file(resource) as p: + return read_tiff(str(p)).astype(np.float32) + + +def load_psf(dilation: int, n_dmds: int) -> dict: + """Load per-DMD PSFs from bundled assets, thresholded and cropped. + + Each DMD uses the same ``dil-NN.tif`` template from + ``assets/psfs/``, center-padded to a common size when shapes differ. + + Parameters + ---------- + dilation : int + Dilation size selecting the bundled PSF template. + n_dmds : int + Number of DMD paths. + + Returns + ------- + dict + ``{"DMD{N}": psf2d}`` thresholded/cropped PSFs. + """ + psfs = [default_psf(dilation) for _ in range(n_dmds)] + psf_combined = build_combined_psf(psfs) + return { + f"DMD{i + 1}": threshold_and_crop_psf(psf_combined[i]) + for i in range(n_dmds) + } + + +def load_alignment_data_h5(path: Union[str, Path]) -> dict: + """Read band alignment, negating saved offline shifts exactly once. + + Online shifts and frame indexes are unchanged. Negation allocates arrays + and does not modify values owned by the raw reader. + """ + data = read_alignment_data_h5(path) + return { + **data, + **{ + key: None if data[key] is None else -data[key] + for key in ("motionDSr", "motionDSc", "motionDSz") + }, + } + + +def _resolve_fn_adata(fn_adata: np.ndarray, moco_save_dr: str) -> np.ndarray: + """Resolve alignment basenames, preserving empty entries and array rank.""" + n_dmds, n_trials = fn_adata.shape + out = np.empty((n_dmds, n_trials), dtype=object) + for d in range(n_dmds): + for t in range(n_trials): + entry = fn_adata[d, t] + out[d, t] = ( + "" + if entry in (None, "") + else os.path.join(moco_save_dr, str(entry)) + ) + return out + + +def load_trial_table(path: Union[str, Path, TrialTable]) -> dict: + """Prepare a structural band view from a path OR an already loaded table. + + Never reload a TrialTable. This does not probe files or read alignments; + callers retain control of ``compute_keep_trials`` and ``read_align_info``. + The view carries the original model as ``trial_table`` and its provenance + as ``source_path``; all historical flattened runtime keys are retained. + """ + tt = path if isinstance(path, TrialTable) else TrialTable.from_h5(path) + slap2 = tt.slap2_info + motion_correction = tt.motion_correction or {} + if slap2 is None or "fn_adata" not in motion_correction: + raise ValueError( + "load_trial_table expects a registered SLAP2 band trial table " + "(slap2_info + motion_correction/fn_adata); " + f"got {str(path)!r}." + ) + datadr = str(tt.datadr) if tt.datadr is not None else "" + result_dr = str(tt.savedr) if tt.savedr is not None else "" + moco_save_dr = os.path.join(result_dr, "motion_correction") + align_params = { + key: to_serializable(val) + for key, val in ( + motion_correction.get("align_params", {}) or {} + ).items() + } + filenames = np.atleast_2d(np.asarray(tt.filename, dtype=object)) + first_line = np.atleast_2d(np.asarray(slap2.first_line)) + last_line = np.atleast_2d(np.asarray(slap2.last_line)) + fn_adata = np.atleast_2d( + np.asarray(motion_correction["fn_adata"], dtype=object) + ) + n_dmds, n_trials = filenames.shape + return { + "trial_table": tt, + "source_path": tt.source_path, + "datadr": datadr, + "savedr": result_dr, + "moco_save_dr": moco_save_dr, + "annotation_save_dr": os.path.join(result_dr, "annotations"), + "src_extr_save_dr": os.path.join(result_dr, "source_extraction"), + "align_params": align_params, + "n_dmds": int(n_dmds), + "n_trials": int(n_trials), + "filename": filenames, + "first_line": first_line, + "last_line": last_line, + "fn_adata": _resolve_fn_adata(fn_adata, moco_save_dr), + "ref_stack": slap2.ref_stack or {}, + } + + +def compute_keep_trials( + fn_adata_abs: np.ndarray, + filename: np.ndarray, + datadr: Union[str, Path], +) -> np.ndarray: + """Keep trials only when both alignment and raw source files exist.""" + fn_adata_abs = np.atleast_2d(np.asarray(fn_adata_abs, dtype=object)) + filename = np.atleast_2d(np.asarray(filename, dtype=object)) + n_dmds, n_trials = fn_adata_abs.shape + keep = np.ones((n_dmds, n_trials), dtype=bool) + for d in range(n_dmds): + for t in range(n_trials): + adata = fn_adata_abs[d, t] + src = filename[d, t] + src_path = os.path.join(str(datadr), str(src)) if src else "" + if ( + not adata + or not os.path.exists(str(adata)) + or not src + or not os.path.exists(src_path) + ): + keep[d, t] = False + return keep + + +def read_align_info( + fn_adata_abs: np.ndarray, + keep_trials: np.ndarray, + n_dmds: int, +) -> tuple: + """Read first-kept-trial alignHz per DMD and the last channel count.""" + fn_adata_abs = np.atleast_2d(np.asarray(fn_adata_abs, dtype=object)) + keep_trials = np.atleast_2d(np.asarray(keep_trials)) + align_hz = {} + num_channels = None + for d in range(n_dmds): + valid = np.flatnonzero(keep_trials[d]) + if valid.size == 0: + continue + a_data = load_alignment_data_h5(fn_adata_abs[d, valid[0]]) + align_hz[f"DMD{d + 1}"] = a_data["alignHz"] + num_channels = a_data["numChannels"] + return align_hz, num_channels diff --git a/src/giant_python/extraction/band/localization.py b/src/giant_python/extraction/band/localization.py new file mode 100644 index 0000000..6608717 --- /dev/null +++ b/src/giant_python/extraction/band/localization.py @@ -0,0 +1,942 @@ +"""Superpixel-space NMF source localization. + +Ported from the source-localization block of +``extractSLAP2IntegrationSources.py`` (ref 2519-2908). Given the peak seeds +from :mod:`giant_python.numerics.peaks` and the noise-normalized residual / +per-motion ``H`` matrices from :mod:`giant_python.extraction.band.background` +and :mod:`giant_python.extraction.band.operators`, this +module fits Gaussian spatial profiles (``A``) and per-motion temporal weights +(``phi``) that reconstruct the residual in superpixel space. + +Each outer iteration: + +1. runs motion-binned multiplicative NMF (in random motion order) to get a + per-motion superpixel target ``X_mots`` (:func:`multiplicative_nmf`), +2. fits the parametric Gaussian profiles to those targets with Adam + (:func:`fit_gaussian_profiles`), +3. rebuilds ``A`` and least-squares-solves ``phi`` + (:func:`fit_phi_all_motions`), +4. sorts sources by explained variance, and every fourth iteration prunes + sources whose SNR falls below ``1/3``. + +The final per-source low-resolution SNR is returned alongside ``A``, ``phi``, +the fitted ``source_params`` ``[z, y, x, sigma_y, sigma_x, tilt]``, and the +(sorted/pruned) ``source_seeds``. + +This block is intentionally stochastic (random motion order via +``torch.randperm`` in NMF and Adam). When :func:`fit_sources` receives a +``seed``, each outer iteration draws from its own ``torch.Generator`` derived +from ``(seed, iteration)``, so a run is reproducible and independent of the +global torch RNG state; with ``seed=None`` the global RNG is used. The +``randperm`` calls are preserved in the same order as the reference, but the +unregularized QR temporal solve changes numerical results. The reference's +loss-only ``phi`` recomputation and ``print``/warning diagnostics are dropped +(they do not affect the outputs and consume no RNG). +""" + +from __future__ import annotations + +from typing import List, cast + +import numpy as np +import torch +from scipy.linalg import qr, solve_triangular + +from ...progress import progress +from .activity import smooth_rho + + +def sel_pix_gaussian_profile( + gaussian_params: torch.Tensor, pixel_coords_tensor: torch.Tensor +) -> torch.Tensor: + """Render normalized bivariate-Gaussian profiles over selected pixels. + + Parameters + ---------- + gaussian_params : torch.Tensor of shape (N, 6) + Per-source ``[z, mu_y, mu_x, sigma_y, sigma_x, tilt]`` (the tilt is + mapped to a correlation coefficient via ``tanh``). + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel ``[z, row, col]`` coordinates. + + Returns + ------- + torch.Tensor of shape (n_sel, N) + Column-normalized profiles; each source is confined to its own z-plane + and to within 3 sigma of its center. + """ + z_planes = gaussian_params[:, 0].unsqueeze(0) + y_means = gaussian_params[:, 1].unsqueeze(0) + x_means = gaussian_params[:, 2].unsqueeze(0) + y_sigmas = gaussian_params[:, 3].unsqueeze(0) + x_sigmas = gaussian_params[:, 4].unsqueeze(0) + corr_coef = torch.tanh(gaussian_params[:, 5].unsqueeze(0)) + + y_centered = pixel_coords_tensor[:, 1].unsqueeze(1) - y_means + x_centered = pixel_coords_tensor[:, 2].unsqueeze(1) - x_means + + z_score_y = y_centered / y_sigmas + z_score_x = x_centered / x_sigmas + + exponent = (-1 / (2 * (1 - corr_coef**2))) * ( + z_score_y**2 - 2 * corr_coef * z_score_x * z_score_y + z_score_x**2 + ) + profile = torch.exp(exponent) + profile = ( + profile + * ( + (torch.sqrt(z_score_y**2 + z_score_x**2) <= 3) + & (pixel_coords_tensor[:, 0].unsqueeze(1) == z_planes) + ).float() + ) + + return profile / torch.sum(profile, dim=0, keepdim=True) + + +def sel_pix_patch_profile( + patch_params: torch.Tensor, pixel_coords_tensor: torch.Tensor +) -> torch.Tensor: + """Render rectangular in-plane support patches over selected pixels. + + Parameters + ---------- + patch_params : torch.Tensor of shape (N, 5) + Per-source ``[z, mu_y, mu_x, y_radius, x_radius]``. + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel ``[z, row, col]`` coordinates. + + Returns + ------- + torch.Tensor of bool, shape (n_sel, N) + True where a pixel is inside a source's patch (same z-plane, within the + radii). + """ + z_planes = patch_params[:, 0].unsqueeze(0) + y_means = patch_params[:, 1].unsqueeze(0) + x_means = patch_params[:, 2].unsqueeze(0) + y_radii = patch_params[:, 3].unsqueeze(0) + x_radii = patch_params[:, 4].unsqueeze(0) + + y_centered = pixel_coords_tensor[:, 1].unsqueeze(1) - y_means + x_centered = pixel_coords_tensor[:, 2].unsqueeze(1) - x_means + + return ( + (torch.abs(y_centered) < y_radii) + & (torch.abs(x_centered) < x_radii) + & (pixel_coords_tensor[:, 0].unsqueeze(1) == z_planes) + ) + + +def init_source_params(source_seeds: np.ndarray) -> torch.Tensor: + """Initialize ``[z, y, x, sigma_y=1, sigma_x=1, tilt=0]`` per source. + + Parameters + ---------- + source_seeds : ndarray of shape (N, 3) + Peak seeds ``[z, mu_y, mu_x]``. + + Returns + ------- + torch.Tensor of shape (N, 6) + The initial source parameters. + """ + n = source_seeds.shape[0] + return torch.cat( + [ + torch.tensor(source_seeds, dtype=torch.float32), + torch.ones(n, 2, dtype=torch.float32), + torch.zeros(n, 1, dtype=torch.float32), + ], + dim=1, + ) + + +def build_a_patches( + source_seeds: np.ndarray, + pixel_coords_tensor: torch.Tensor, + sel_pix_idxs: np.ndarray, + n_pixels: int, + d_xy: float, +) -> torch.Tensor: + """Build the per-source rectangular support mask over the full pixel grid. + + Parameters + ---------- + source_seeds : ndarray of shape (N, 3) + Peak seeds ``[z, mu_y, mu_x]``. + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel coordinates. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices into the ``n_pixels`` grid. + n_pixels : int + Total number of image pixels (``z * rows * cols``). + d_xy : float + Half-width of the square support patch, in pixels. + + Returns + ------- + torch.Tensor of bool, shape (n_pixels, N) + The support mask. + """ + n_sources = source_seeds.shape[0] + a_patches = torch.zeros((n_pixels, n_sources), dtype=torch.bool) + patch_params = torch.cat( + [ + torch.tensor(source_seeds, dtype=torch.float32), + d_xy * torch.ones(n_sources, 2, dtype=torch.float32), + ], + dim=1, + ) + a_patches[sel_pix_idxs, :] = sel_pix_patch_profile( + patch_params, pixel_coords_tensor + ) + return a_patches + + +def build_x_support_mots( + h_mots: List[torch.Tensor], + a_patches: torch.Tensor, + sel_pix_idxs: np.ndarray, + n_motions: int, +) -> List[torch.Tensor]: + """Project each source's support patch into per-motion superpixel space. + + Parameters + ---------- + h_mots : list of torch.Tensor + Per-motion sparse superpixel<-pixel ``H`` matrices. + a_patches : torch.Tensor of bool, shape (n_pixels, N) + Support mask from :func:`build_a_patches`. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + n_motions : int + Number of motion bins. + + Returns + ------- + list of torch.Tensor of bool + One ``(n_superpixels, N)`` support mask per motion bin. + """ + return [ + torch.sparse.mm(h_mots[i], a_patches[sel_pix_idxs, :].float()) > 0 + for i in range(n_motions) + ] + + +def project_spatial_profiles( + source_params: torch.Tensor, + pixel_coords_tensor: torch.Tensor, + sel_pix_idxs: np.ndarray, + n_pixels: int, + a_patches: torch.Tensor, +) -> torch.Tensor: + """Build the mass-normalized Gaussian spatial profiles ``A``. + + The Gaussian profile is masked by the support patch and normalized so each + source's column sums to 1. + + Parameters + ---------- + source_params : torch.Tensor of shape (N, 6) + Source parameters. + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel coordinates. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + n_pixels : int + Total number of image pixels. + a_patches : torch.Tensor of bool, shape (n_pixels, N) + Support mask. + + Returns + ------- + torch.Tensor of shape (n_pixels, N) + The normalized spatial profiles. + """ + n_sources = source_params.shape[0] + a = torch.zeros((n_pixels, n_sources), dtype=torch.float32) + a[sel_pix_idxs, :] = sel_pix_gaussian_profile( + source_params, pixel_coords_tensor + ) + a[~a_patches] = 0 + mass = torch.sum(a, dim=0, keepdim=True) + mass[mass <= 0] = 1 + a /= mass + return a + + +def _motion_frames(mot_inds_yx: np.ndarray, i: int) -> np.ndarray: + """Return the frame indices belonging to motion bin ``i``.""" + return np.flatnonzero(mot_inds_yx == i) + + +def solve_phi_motion( + x: torch.Tensor, data_motion: torch.Tensor +) -> torch.Tensor: + """Least-squares temporal weights for one motion's superpixel profiles. + + Uses column-pivoted QR, ``X[:, piv] = Q R``, to return a basic solution + as in MATLAB's rectangular ``mldivide``: solve the leading independent + triangular block and set the remaining pivoted coefficients to zero. + Rank-deficient and underdetermined systems therefore have at most + ``rank(X)`` nonzero coefficients per frame, not a minimum-norm solution. + No normal equations or ridge regularization are used. + + Numerical rank uses the cutoff ``max(X.shape) * eps * abs(R[0, 0])``. + Pivot ties and borderline rank decisions can differ from MATLAB. + Inputs must be real float32/float64 CPU tensors without gradients; + ``x`` must be finite. Data frames containing any NaN or infinity return + NaN for all sources; only fully finite frames enter the solve. Missing + observations are not replaced with zeros or used for partial-frame fits. + Returns ``phiแต€`` (frames x sources), preserving the input dtype. + Shared by the NMF fits here and the high-res trace solve in + :mod:`giant_python.extraction.band.traces`. + """ + x_array = x.numpy() + data_array = data_motion.numpy() + phi = np.zeros((x.shape[1], data_motion.shape[1]), dtype=x_array.dtype) + if min(x.shape) == 0 or data_motion.shape[1] == 0: + return torch.from_numpy(phi.T) + + # SciPy rejects the entire RHS batch if even one frame is nonfinite. + # Preserve missing frames without contaminating the independent fits, + # including coefficients that would otherwise be zero for deficient X. + valid_frames = np.all(np.isfinite(data_array), axis=0) + phi[:, ~valid_frames] = np.nan + + q, r, piv = qr(x_array, mode="economic", pivoting=True) + diagonal = np.abs(np.diag(r)) + tolerance = max(x.shape) * np.finfo(x_array.dtype).eps * diagonal[0] + # Use a leading block: stop at the first numerically dependent column. + rank = int(np.count_nonzero(np.cumprod(diagonal > tolerance))) + if rank and np.any(valid_frames): + # Avoid copying the observation matrix in the usual all-finite case. + finite_data = ( + data_array if np.all(valid_frames) else data_array[:, valid_frames] + ) + phi[np.ix_(piv[:rank], valid_frames)] = solve_triangular( + r[:rank, :rank], q[:, :rank].T @ finite_data + ) + return torch.from_numpy(phi.T) + + +def fit_phi_all_motions( + a: torch.Tensor, + h_mots: List[torch.Tensor], + sel_pix_idxs: np.ndarray, + data_for_nmf: torch.Tensor, + mot_inds_yx: np.ndarray, + n_motions: int, + phi_low_res: torch.Tensor, +) -> torch.Tensor: + """Least-squares-solve the temporal weights ``phi`` for every motion bin. + + Parameters + ---------- + a : torch.Tensor of shape (n_pixels, N) + Spatial profiles. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + data_for_nmf : torch.Tensor of shape (n_superpixels, n_frames) + The residual data being reconstructed. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + n_motions : int + Number of motion bins. + phi_low_res : torch.Tensor of shape (n_frames, N) + Temporal weights, updated in place per motion bin and returned. + + Returns + ------- + torch.Tensor + ``phi_low_res`` with each motion's frames filled. + """ + for i in range(n_motions): + frames = _motion_frames(mot_inds_yx, i) + x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, :]) + phi_low_res[frames, :] = solve_phi_motion(x, data_for_nmf[:, frames]) + return phi_low_res + + +def multiplicative_nmf( + a: torch.Tensor, + h_mots: List[torch.Tensor], + x_support_mots: List[torch.Tensor], + sel_pix_idxs: np.ndarray, + data_for_nmf: torch.Tensor, + mot_inds_yx: np.ndarray, + n_motions: int, + sparse_fac: float, + max_iters: int, + generator: torch.Generator | None = None, +) -> List[torch.Tensor]: + """Motion-binned multiplicative NMF, producing per-motion spatial targets. + + Visits motion bins in a random order (``torch.randperm``). For each bin it + projects ``A`` into superpixel space, initializes ``phi`` by projected + least squares, then alternates multiplicative updates of ``phi`` and the + spatial + target ``X`` under the support constraint, periodically sparsifying ``X``. + + Parameters + ---------- + a : torch.Tensor of shape (n_pixels, N) + Current spatial profiles. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + x_support_mots : list of torch.Tensor of bool + Per-motion superpixel support masks. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + data_for_nmf : torch.Tensor of shape (n_superpixels, n_frames) + Residual data. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index. + n_motions : int + Number of motion bins. + sparse_fac : float + Sparsification floor subtracted from the max-normalized ``X``. + max_iters : int + Multiplicative-update iterations per motion bin. + generator : torch.Generator or None + RNG for the motion order; ``None`` uses the global torch RNG. + + Returns + ------- + list of torch.Tensor + Per-motion fitted superpixel spatial targets ``X_mots``. + """ + x_mots: List[torch.Tensor | None] = [None] * n_motions + shuffled_indices = torch.randperm(n_motions, generator=generator) + for idx in shuffled_indices: + # randperm uses an integer dtype, so item() returns a Python int. + i = cast(int, idx.item()) + frames = _motion_frames(mot_inds_yx, i) + + x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, :]) + x[~x_support_mots[i]] = 0 + x = x / torch.norm(x, dim=0, keepdim=True) + + phi = torch.clamp(solve_phi_motion(x, data_for_nmf[:, frames]), min=0) + + for iter_idx in range(max_iters): + numerator = x.T @ data_for_nmf[:, frames] + denominator = (x.T @ x) @ phi.T + 1e-10 + phi = torch.clamp(phi * (numerator / denominator).T, min=0) + + numerator = data_for_nmf[:, frames] @ phi + denominator = x @ (phi.T @ phi) + 1e-10 + x = torch.clamp(x * (numerator / denominator), min=0) + + x = x * x_support_mots[i].float() + + norms = torch.norm(x, dim=0, keepdim=True) + x = torch.where(norms > 0, x / norms, x) + phi = torch.where(norms > 0, phi * norms, phi) + + if iter_idx % 3 == 0: + x_max = torch.max(x, dim=0, keepdim=True)[0] + x = torch.where(x_max > 0, x / x_max, x) + x = torch.clamp(x, min=sparse_fac) - sparse_fac + norms = torch.norm(x, dim=0, keepdim=True) + x = torch.where(norms > 0, x / norms, x) + + x_mots[i] = x + # randperm visits every slot exactly once, including the empty case. + return cast(List[torch.Tensor], x_mots) + + +def fit_gaussian_profiles( + source_params: torch.Tensor, + source_seeds: np.ndarray, + x_mots: List[torch.Tensor], + h_mots: List[torch.Tensor], + pixel_coords_tensor: torch.Tensor, + a_patches: torch.Tensor, + sel_pix_idxs: np.ndarray, + d_xy: float, + num_epochs: int, + n_motions: int, + learning_rate: float, + gd_tol: float, + generator: torch.Generator | None = None, +) -> torch.Tensor: + """Adam-fit the Gaussian source parameters to the per-motion NMF targets. + + Optimizes location, scale, and tilt to make the projected Gaussian profiles + match ``X_mots`` (one motion per epoch, in random rounds). Locations are + clamped to +/- ``d_xy`` of the seed and sigmas to ``[0.3, 5]``. Stops early + when consecutive full-round losses differ by less than ``gd_tol``. + + Parameters + ---------- + source_params : torch.Tensor of shape (N, 6) + Current source parameters (z fixed; loc/scale/tilt optimized). + source_seeds : ndarray of shape (N, 3) + Peak seeds (for the location clamp bounds). + x_mots : list of torch.Tensor + Per-motion NMF spatial targets. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel coordinates. + a_patches : torch.Tensor of bool + Support mask. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + d_xy : float + Location clamp half-width. + num_epochs : int + Maximum optimization epochs (one motion each). + n_motions : int + Number of motion bins (epochs per round). + learning_rate : float + Base Adam learning rate (scaled per parameter group). + gd_tol : float + Convergence tolerance on consecutive round losses. + generator : torch.Generator or None + RNG for the motion order; ``None`` uses the global torch RNG. + + Returns + ------- + torch.Tensor of shape (N, 6) + The updated source parameters. + """ + optim_loc = source_params[:, 1:3].clone().requires_grad_(True) + optim_scale = source_params[:, 3:5].clone().requires_grad_(True) + optim_tilt = source_params[:, 5].unsqueeze(1).clone().requires_grad_(True) + optimizer = torch.optim.Adam( + [ + {"params": optim_loc, "lr": 10 * learning_rate}, + {"params": optim_scale, "lr": 0.1 * learning_rate}, + {"params": optim_tilt, "lr": 0.1 * learning_rate}, + ] + ) + + z_col = source_params[:, 0].unsqueeze(1) + loc_min = torch.from_numpy(source_seeds[:, 1:3] - 1.5).float() + loc_max = torch.from_numpy(source_seeds[:, 1:3] + 1.5).float() + + losses = [] + shuffled_indices = None + for epoch in range(num_epochs): + optimizer.zero_grad() + + a_step = sel_pix_gaussian_profile( + torch.cat([z_col, optim_loc, optim_scale, optim_tilt], dim=1), + pixel_coords_tensor, + ) + a_step = a_step * a_patches[sel_pix_idxs, :].float() + mass = torch.sum(a_step, dim=0, keepdim=True) + mass[mass <= 0] = 1 + a_step = a_step / mass + + if epoch % n_motions == 0: + shuffled_indices = torch.randperm(n_motions, generator=generator) + # Epoch zero initializes the permutation before any index is read. + i = cast( + int, + cast(torch.Tensor, shuffled_indices)[epoch % n_motions].item(), + ) + + x_step = torch.sparse.mm(h_mots[i], a_step) + norms = torch.norm(x_step, dim=0, keepdim=True) + x_step = torch.where(norms > 0, x_step / norms, x_step) + loss = torch.sum((x_step - x_mots[i]) ** 2) + + loss.backward() + optimizer.step() + + with torch.no_grad(): + optim_loc.clamp_(min=loc_min, max=loc_max) + optim_scale[:, 0].clamp_(min=0.3, max=2) + optim_scale[:, 1].clamp_(min=0.3, max=2) + + if epoch % n_motions == n_motions - 1: + total_loss = 0 + for i in range(n_motions): + x_step = torch.sparse.mm(h_mots[i], a_step) + norms = torch.norm(x_step, dim=0, keepdim=True) + x_step = torch.where(norms > 0, x_step / norms, x_step) + total_loss += torch.sum((x_step - x_mots[i]) ** 2) + # This branch completes a nonempty motion round: loss is Tensor. + losses.append(cast(torch.Tensor, total_loss).item()) + if ( + epoch // n_motions > 0 + and abs(losses[-1] - losses[-2]) < gd_tol + ): + break + + return torch.cat( + [z_col, optim_loc.detach(), optim_scale.detach(), optim_tilt.detach()], + dim=1, + ) + + +def variance_sortorder( + phi_low_res: torch.Tensor, n_sources: int +) -> np.ndarray: + """Return the descending-explained-variance source ordering.""" + phi = phi_low_res[:, :n_sources].numpy() + variance = np.nansum((phi - np.nanmean(phi, axis=0)) ** 2, axis=0) + return np.argsort(-variance) + + +def reorder_sources( + order, + source_params: torch.Tensor, + source_seeds: np.ndarray, + a: torch.Tensor, + a_patches: torch.Tensor, + x_support_mots: List[torch.Tensor], + phi_low_res: torch.Tensor, +): + """Apply a source ordering / selection to every per-source array. + + Parameters + ---------- + order : array-like of int + Source indices in the new order (a permutation for sorting, or a subset + for pruning). + source_params, source_seeds, a, a_patches, phi_low_res : array/tensor + Per-source state to reindex along the source axis. + x_support_mots : list of torch.Tensor + Per-motion support masks (reindexed along the source axis). + + Returns + ------- + tuple + The reindexed ``(source_params, source_seeds, a, a_patches, + x_support_mots, phi_low_res)``. + """ + source_params = source_params[order, :] + source_seeds = source_seeds[order, :] + a = a[:, order] + a_patches = a_patches[:, order] + x_support_mots = [support[:, order] for support in x_support_mots] + phi_low_res = phi_low_res[:, order] + return ( + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) + + +def compute_source_snr( + a: torch.Tensor, + h_mots: List[torch.Tensor], + sel_pix_idxs: np.ndarray, + data_for_nmf: torch.Tensor, + phi_low_res: torch.Tensor, + mot_inds_yx: np.ndarray, + n_motions: int, + n_sources: int, +) -> torch.Tensor: + """Per-source low-resolution SNR (explained variance / residual variance). + + Reconstructs the residual from ``A`` and ``phi``, then for each source sums + the squared explained signal and squared residual over the superpixels the + source contributes to. + + Parameters + ---------- + a : torch.Tensor of shape (n_pixels, N) + Spatial profiles. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + data_for_nmf : torch.Tensor of shape (n_superpixels, n_frames) + Residual data. + phi_low_res : torch.Tensor of shape (n_frames, N) + Temporal weights. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index. + n_motions, n_sources : int + Number of motion bins / sources. + + Returns + ------- + torch.Tensor of shape (N,) + Per-source SNR (``varExp / varResidual``). + """ + residual_recon = torch.full_like(data_for_nmf, float("nan")) + for i in range(n_motions): + frames = _motion_frames(mot_inds_yx, i) + x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, :]) + residual_recon[:, frames] = ( + data_for_nmf[:, frames] - x @ phi_low_res[frames, :].T + ) + + var_exp = torch.zeros(n_sources, dtype=torch.float32) + var_residual = torch.zeros(n_sources, dtype=torch.float32) + for j in range(n_sources): + for i in range(n_motions): + frames = _motion_frames(mot_inds_yx, i) + x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, j].unsqueeze(1)) + contributing = (x > 0).nonzero()[:, 0] + var_exp[j] += torch.sum( + torch.sum( + x[contributing] * phi_low_res[frames, j].unsqueeze(-1).T, + dim=0, + ) + ** 2 + ) + var_residual[j] += torch.sum( + torch.sum(residual_recon[contributing][:, frames], dim=0) ** 2 + ) + # Ratio of accumulated squared signals, not a peak-amplitude SNR. + return var_exp / var_residual + + +def fit_sources( + source_seeds: np.ndarray, + residual: np.ndarray, + h_mots: List[torch.Tensor], + unique_motion_to_keep_yx: np.ndarray, + mot_inds_yx: np.ndarray, + sel_pix_idxs: np.ndarray, + pixel_coords: np.ndarray, + n_pixels: int, + d_xy: float, + sparse_fac: float, + outer_loop_iters: int = 10, + mult_nmf_max_iters: int = 10, + learning_rate: float = 0.01, + gd_tol: float = 1e-4, + verbose: bool = False, + *, + temporal_kernel: np.ndarray | None = None, + seed: int | None = None, +) -> dict: + """Localize sources by superpixel-space NMF + Gaussian-profile fitting. + + Parameters + ---------- + source_seeds : ndarray of shape (N, 3) + Peak seeds ``[z, mu_y, mu_x]`` from :mod:`giant_python.numerics.peaks`. + residual : ndarray of shape (n_superpixels, n_frames) + Noise-normalized residual to reconstruct. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + unique_motion_to_keep_yx : ndarray of shape (n_motions, 2) + Kept 2-D motion vectors (only its length is used). + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + sel_pix_idxs : ndarray of int + Flat selected-pixel indices into the ``n_pixels`` grid. + pixel_coords : ndarray of shape (n_sel, 3) + Selected-pixel ``[z, row, col]`` coordinates. + n_pixels : int + Total number of image pixels. + d_xy : float + Support-patch half-width / location clamp radius. + sparse_fac : float + NMF sparsification floor. + outer_loop_iters : int + Number of outer NMF/fit iterations. + mult_nmf_max_iters : int + Multiplicative-update iterations per motion bin. + learning_rate : float + Base Adam learning rate. + gd_tol : float + Adam convergence tolerance. + verbose : bool + Show a progress bar over the outer NMF/fit iterations when set. + temporal_kernel : ndarray, optional + Temporal matched-filter kernel, normally from ``decay_kernel_1d``. + Applied with the activity image's NaN-aware ``smooth_rho`` before + motion binning, without compensating for motion differences. Leaves + ``residual`` unchanged. Frames with insufficient filter coverage in + any row are excluded from NMF and SNR; their temporal weights remain + NaN. Motion bins with no remaining frames are also excluded. NMF, + low-resolution temporal weights and SNR all use the filtered data. + Raises ValueError if no usable frames remain. If omitted, use the + unfiltered residual. + seed : int or None + Nonnegative seed for the random motion order. Each outer iteration + uses a generator seeded from ``(seed, iteration)``, so a change in + the number of Adam epochs in one iteration does not shift the draws + of later iterations. ``None`` uses the global torch RNG. + + Returns + ------- + dict + ``A`` (n_pixels, N), ``phi_low_res`` (n_frames, N), ``source_params`` + (N, 6), ``source_seeds`` (N, 3), ``source_snr`` (N,), and + ``n_sources``. + """ + n_sources = source_seeds.shape[0] + n_motions = unique_motion_to_keep_yx.shape[0] + n_frames = residual.shape[1] + pixel_coords_tensor = torch.tensor(pixel_coords, dtype=torch.float32) + nmf_residual = residual.astype(np.float32, copy=False) + if temporal_kernel is not None: + # smooth_rho operates in place. Dropped frames must not contribute + # even if the caller supplied finite values for them. + nmf_residual = nmf_residual.copy() + nmf_residual[:, mot_inds_yx < 0] = np.nan + nmf_residual = smooth_rho( + nmf_residual, temporal_kernel, verbose=verbose + ) + # The shared filter retains NaNs at low-coverage samples. The dense + # NMF solves cannot accept these, so omit affected frames rather + # than interpreting missing observations as zero residuals. + valid_frames = np.all(np.isfinite(nmf_residual), axis=0) + retained = (mot_inds_yx >= 0) & valid_frames + if not np.any(retained): + raise ValueError( + "No usable frames remain after NaN-aware temporal filtering" + ) + kept_motions = np.unique(mot_inds_yx[retained]) + filtered_motion_inds = np.full_like(mot_inds_yx, -1) + filtered_motion_inds[retained] = np.searchsorted( + kept_motions, mot_inds_yx[retained] + ) + mot_inds_yx = filtered_motion_inds + h_mots = [h_mots[i] for i in kept_motions] + n_motions = len(h_mots) + data_for_nmf = torch.from_numpy(nmf_residual) + num_epochs = n_motions * 5 + + source_params = init_source_params(source_seeds) + a_patches = build_a_patches( + source_seeds, pixel_coords_tensor, sel_pix_idxs, n_pixels, d_xy + ) + x_support_mots = build_x_support_mots( + h_mots, a_patches, sel_pix_idxs, n_motions + ) + a = project_spatial_profiles( + source_params, pixel_coords_tensor, sel_pix_idxs, n_pixels, a_patches + ) + phi_low_res = torch.full( + (n_frames, n_sources), float("nan"), dtype=torch.float32 + ) + + for outer_loop_iter in progress( + range(outer_loop_iters), + desc="Localizing sources (NMF)", + verbose=verbose, + ): + generator = None + if seed is not None: + iter_seed = np.random.SeedSequence( + [seed, outer_loop_iter] + ).generate_state(1)[0] + generator = torch.Generator().manual_seed(int(iter_seed)) + x_mots = multiplicative_nmf( + a, + h_mots, + x_support_mots, + sel_pix_idxs, + data_for_nmf, + mot_inds_yx, + n_motions, + sparse_fac, + mult_nmf_max_iters, + generator, + ) + source_params = fit_gaussian_profiles( + source_params, + source_seeds, + x_mots, + h_mots, + pixel_coords_tensor, + a_patches, + sel_pix_idxs, + d_xy, + num_epochs, + n_motions, + learning_rate, + gd_tol, + generator, + ) + a = project_spatial_profiles( + source_params, + pixel_coords_tensor, + sel_pix_idxs, + n_pixels, + a_patches, + ) + phi_low_res = fit_phi_all_motions( + a, + h_mots, + sel_pix_idxs, + data_for_nmf, + mot_inds_yx, + n_motions, + phi_low_res, + ) + + order = variance_sortorder(phi_low_res, n_sources) + ( + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) = reorder_sources( + order, + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) + + if (outer_loop_iter + 1) % 4 == 3: + snr = compute_source_snr( + a, + h_mots, + sel_pix_idxs, + data_for_nmf, + phi_low_res, + mot_inds_yx, + n_motions, + n_sources, + ) + keep = (snr > 1 / 3).nonzero()[:, 0] + n_sources = keep.shape[0] + ( + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) = reorder_sources( + keep, + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) + + source_snr = compute_source_snr( + a, + h_mots, + sel_pix_idxs, + data_for_nmf, + phi_low_res, + mot_inds_yx, + n_motions, + n_sources, + ).numpy() + + return { + "A": a, + "phi_low_res": phi_low_res, + "source_params": source_params, + "source_seeds": source_seeds, + "source_snr": source_snr, + "n_sources": n_sources, + } diff --git a/src/giant_python/extraction/band/motion_binning.py b/src/giant_python/extraction/band/motion_binning.py new file mode 100644 index 0000000..91ff94d --- /dev/null +++ b/src/giant_python/extraction/band/motion_binning.py @@ -0,0 +1,110 @@ +"""Rounded motion bins and low-resolution frame-selection policy.""" + +from __future__ import annotations + +from typing import Tuple + +import numpy as np + + +def bin_motion( + motion_r: np.ndarray, motion_c: np.ndarray, motion_z: np.ndarray +) -> Tuple[np.ndarray, np.ndarray]: + """Group frames into unique rounded (row, col, z) motion bins. + + Parameters + ---------- + motion_r, motion_c, motion_z : ndarray + Per-frame downsampled motion in rows, columns, and z. + + Returns + ------- + unique_motion : ndarray of shape (n_bins, 3) + The unique rounded ``[row, col, z]`` motion vectors. + mot_inds : ndarray of shape (n_frames,) + Index of each frame's motion bin into ``unique_motion``. + """ + unique_motion, mot_inds = np.unique( + np.round(np.stack((motion_r, motion_c, motion_z), axis=1)), + axis=0, + return_inverse=True, + ) + # np.unique's inverse is 1-D on numpy<2 and 2.1+, but 2-D on 2.0; flatten + # so downstream bincount/isin behave identically across versions. + return unique_motion, np.reshape(mot_inds, -1) + + +def select_motion_bins( + unique_motion: np.ndarray, + mot_inds: np.ndarray, + motion_z: np.ndarray, + z_tol: float = 1.5, + min_frames: int = 100, +) -> Tuple[np.ndarray, np.ndarray]: + """Keep motion bins near the median z with enough frames. + + A bin is kept when its z is within ``z_tol`` of the median z (over all + frames) and it contains more than ``min_frames`` frames. After filtering, + the remaining frames are treated as having no z motion. + + Parameters + ---------- + unique_motion : ndarray of shape (n_bins, 3) + Unique motion vectors from :func:`bin_motion`. + mot_inds : ndarray of shape (n_frames,) + Per-frame motion-bin indices. + motion_z : ndarray + Per-frame z motion (used for the median). + z_tol : float + Maximum absolute z deviation from the median, in microns. + min_frames : int + Minimum frame count (strictly greater) for a bin to be kept. + + Returns + ------- + mot_inds_to_keep : ndarray + Indices of the kept bins into ``unique_motion``. + frames_to_keep : ndarray of bool + Per-frame mask of frames belonging to a kept bin. + """ + median_z = np.median(motion_z) + bin_counts = np.bincount(mot_inds, minlength=unique_motion.shape[0]) + keep_mask = (np.abs(unique_motion[:, 2] - median_z) <= z_tol) & ( + bin_counts > min_frames + ) + mot_inds_to_keep = np.nonzero(keep_mask)[0] + frames_to_keep = np.isin(mot_inds, mot_inds_to_keep) + return mot_inds_to_keep, frames_to_keep + + +def bin_motion_yx( + motion_r: np.ndarray, + motion_c: np.ndarray, + frames_to_keep: np.ndarray, +) -> Tuple[np.ndarray, np.ndarray]: + """Bin kept frames into unique rounded (row, col) motion bins. + + Parameters + ---------- + motion_r, motion_c : ndarray + Per-frame downsampled row/column motion. + frames_to_keep : ndarray of bool + Frames to bin; dropped frames get index ``-1``. + + Returns + ------- + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Unique rounded ``[row, col]`` motion vectors over the kept frames. + mot_inds_yx : ndarray of shape (n_frames,), int32 + Per-frame bin index into ``unique_motion_to_keep_yx``; ``-1`` for + frames not kept. + """ + mot_inds_yx = -1 * np.ones((frames_to_keep.shape[0],), dtype=np.int32) + stacked = np.round( + np.stack((motion_r, motion_c), axis=1)[frames_to_keep, :] + ) + unique_motion_to_keep_yx, inv = np.unique( + stacked, axis=0, return_inverse=True + ) + mot_inds_yx[frames_to_keep] = np.reshape(inv, -1) + return unique_motion_to_keep_yx, mot_inds_yx diff --git a/src/giant_python/extraction/band/operators.py b/src/giant_python/extraction/band/operators.py new file mode 100644 index 0000000..492affd --- /dev/null +++ b/src/giant_python/extraction/band/operators.py @@ -0,0 +1,321 @@ +"""PSF kernels and sparse/dense band projection operators.""" + +from __future__ import annotations + +from typing import Any, List, Tuple, cast + +import numpy as np +import torch +from scipy.interpolate import RectBivariateSpline + +from .geometry import ref_pixs_to_drc + + +def expand_psf( + psf2d: np.ndarray, ex_fac: int = 2 +) -> Tuple[torch.Tensor, Tuple[int, int], torch.Tensor, Tuple[int, int]]: + """Normalize a PSF and build a center-aligned higher-resolution copy. + + The expanded PSF is an ``ex_fac``x up-sampled cubic-spline interpolation + that keeps the center aligned. Localization constructs its center/surround + filter separately with :func:`gaussian_kernel_2d`. + + Parameters + ---------- + psf2d : ndarray + The (cropped) PSF for this DMD. + ex_fac : int + Expansion factor along each axis. + + Returns + ------- + psf_tensor : torch.Tensor + Sum-normalized PSF. + psf_center : tuple of int + ``(row, col)`` center of ``psf_tensor``. + psf_tensor_expanded : torch.Tensor + Sum-normalized expanded PSF. + psf_center_expanded : tuple of int + ``(row, col)`` center of ``psf_tensor_expanded``. + """ + psf_arr = np.asarray(psf2d, dtype=np.float32) + psf_tensor = torch.from_numpy(psf_arr).float() + psf_tensor = psf_tensor / torch.sum(psf_tensor) + psf_shape = psf_arr.shape + psf_center = (psf_shape[0] // 2, psf_shape[1] // 2) + + expanded_center_y = (psf_shape[0] * ex_fac - 1) / 2 + expanded_center_x = (psf_shape[1] * ex_fac - 1) / 2 + orig_y = torch.linspace( + -expanded_center_y, expanded_center_y, psf_shape[0] + ) + orig_x = torch.linspace( + -expanded_center_x, expanded_center_x, psf_shape[1] + ) + expanded_y = torch.linspace( + -expanded_center_y, expanded_center_y, psf_shape[0] * ex_fac + ) + expanded_x = torch.linspace( + -expanded_center_x, expanded_center_x, psf_shape[1] * ex_fac + ) + interp_spline = RectBivariateSpline( + orig_y.numpy(), orig_x.numpy(), psf_tensor.numpy() + ) + psf_tensor_expanded = torch.from_numpy( + interp_spline(expanded_y.numpy(), expanded_x.numpy()) + ).float() + psf_tensor_expanded = psf_tensor_expanded / torch.sum(psf_tensor_expanded) + psf_shape_expanded = psf_tensor_expanded.shape + psf_center_expanded = ( + psf_shape_expanded[0] // 2, + psf_shape_expanded[1] // 2, + ) + return psf_tensor, psf_center, psf_tensor_expanded, psf_center_expanded + + +def shrink_psf( + psf2d: np.ndarray, scale_y: float = 1.0, scale_x: float = 0.75 +) -> np.ndarray: + """Spatially shrink a PSF on the same grid (TEMPORARY). + + Resamples the PSF on a grid whose coordinates are divided by the per-axis + scale so a ``scale < 1`` narrows the PSF's spatial footprint along that + axis while keeping the same array shape and center; the result is + re-normalized to sum to 1. Used only for the temporary activity-image + experiment. + + Parameters + ---------- + psf2d : ndarray + The (cropped) PSF for this DMD. + scale_y : float + Spatial shrink factor along the row (y) axis (``< 1`` shrinks, + ``1`` is a no-op). + scale_x : float + Spatial shrink factor along the column (x) axis (``< 1`` shrinks, + ``1`` is a no-op). + + Returns + ------- + ndarray of shape ``psf2d.shape``, float32 + The shrunk, sum-normalized PSF. + """ + psf_arr = np.asarray(psf2d, dtype=np.float32) + psf_shape = psf_arr.shape + center_y = (psf_shape[0] - 1) / 2 + center_x = (psf_shape[1] - 1) / 2 + orig_y = np.arange(psf_shape[0], dtype=np.float64) - center_y + orig_x = np.arange(psf_shape[1], dtype=np.float64) - center_x + interp_spline = RectBivariateSpline(orig_y, orig_x, psf_arr) + shrunk = interp_spline(orig_y / scale_y, orig_x / scale_x) + shrunk = np.clip(shrunk, 0.0, None).astype(np.float32) + total = shrunk.sum() + if total > 0: + shrunk = shrunk / total + return shrunk + + +def gaussian_kernel_2d( + sigma: float, truncate: float = 4.0 +) -> Tuple[torch.Tensor, Tuple[int, int]]: + """Build a sum-normalized isotropic 2-D Gaussian kernel (TEMPORARY). + + Used by the temporary difference-of-Gaussians activity-image experiment, + which replaces the PSF-derived ``D``/``D_expanded`` convolution matrices + with an isotropic center/surround pair. The subtraction itself still + happens in :func:`compute_rho`, after per-column normalization. + + Parameters + ---------- + sigma : float + Gaussian standard deviation, in pixels. + truncate : float + Kernel half-width, in standard deviations. Truncation lowers the + realized standard deviation slightly, so the default is generous. + + Returns + ------- + kernel : torch.Tensor + Sum-normalized ``(2 * radius + 1, 2 * radius + 1)`` kernel. + center : tuple of int + ``(row, col)`` center index of ``kernel``. + """ + radius = int(np.ceil(truncate * sigma)) + coords = np.arange(-radius, radius + 1, dtype=np.float64) + g_1d = np.exp(-(coords**2) / (2.0 * sigma**2)) + kernel = np.outer(g_1d, g_1d).astype(np.float32) + kernel /= kernel.sum() + return torch.from_numpy(kernel), (radius, radius) + + +def build_convolution_matrix( + sel_pixels_2d: np.ndarray, + psf_tensor: torch.Tensor, + psf_center: Tuple[int, int], +) -> torch.Tensor: + """Build a dense PSF-convolution matrix for one plane's selected pixels. + + Entry ``(t, s)`` is the PSF weight for the offset from source pixel ``s`` + to target pixel ``t`` (zero where the offset falls outside the PSF). Used + with both the base and expanded PSFs to form ``D``/``D_expanded``. + + Parameters + ---------- + sel_pixels_2d : ndarray of shape (n_plane, 2) + In-plane ``[row, col]`` coordinates. + psf_tensor : torch.Tensor + Sum-normalized PSF (base or expanded). + psf_center : tuple of int + ``(row, col)`` center of ``psf_tensor``. + + Returns + ------- + torch.Tensor + Dense ``(n_plane, n_plane)`` convolution matrix. + """ + n = sel_pixels_2d.shape[0] + if n == 0: + return torch.zeros((0, 0), dtype=torch.float32) + src_rows = sel_pixels_2d[:, 0][np.newaxis, :] + src_cols = sel_pixels_2d[:, 1][np.newaxis, :] + tgt_rows = sel_pixels_2d[:, 0][:, np.newaxis] + tgt_cols = sel_pixels_2d[:, 1][:, np.newaxis] + rel_rows = tgt_rows - src_rows + psf_center[0] + rel_cols = tgt_cols - src_cols + psf_center[1] + psf_np = psf_tensor.numpy() + psf_shape = psf_np.shape + valid_mask = ( + (rel_rows >= 0) + & (rel_rows < psf_shape[0]) + & (rel_cols >= 0) + & (rel_cols < psf_shape[1]) + ) + out = np.zeros((n, n), dtype=np.float32) + out[valid_mask] = psf_np[rel_rows[valid_mask], rel_cols[valid_mask]] + return torch.from_numpy(out) + + +def build_motion_h_matrices( + sparse_h_inds: np.ndarray, + sparse_h_vals: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + sel_pix_idxs: np.ndarray, + num_super_pixels: int, + dmd_pixels_per_row: int, +) -> List[torch.Tensor]: + """Build one sparse superpixel<-selected-pixel ``H`` per motion bin. + + Each motion bin shifts the base ``H`` columns (image pixels) by its 2-D + motion, then remaps them into selected-pixel column space. + + Parameters + ---------- + sparse_h_inds : ndarray of shape (2, nnz) + Base ``H`` COO row/column indices (from ``geometry.build_sparse_h``). + sparse_h_vals : ndarray of shape (nnz,) + Base ``H`` values. + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + num_super_pixels : int + Number of superpixels (``H`` row count). + dmd_pixels_per_row : int + Grid geometry (column-shift stride). + + Returns + ------- + list of torch.Tensor + One sparse COO ``(num_super_pixels, n_sel)`` tensor per motion bin. + """ + n_mot = len(unique_motion_to_keep_yx) + h_mots: List[torch.Tensor | None] = [None] * n_mot + base_sparse_rows = sparse_h_inds[0] + base_sparse_cols = sparse_h_inds[1] + motion_shifts = unique_motion_to_keep_yx[:, 0].astype( + np.int64, copy=False + ) * dmd_pixels_per_row + unique_motion_to_keep_yx[:, 1].astype( + np.int64, copy=False + ) + shifted_inds = np.empty((2, base_sparse_cols.shape[0]), dtype=np.int64) + shifted_inds[0] = base_sparse_rows + for i, pix_shift in enumerate(motion_shifts): + shifted_cols = base_sparse_cols + pix_shift + shifted_inds[1] = np.searchsorted(sel_pix_idxs, shifted_cols) + # PyTorch accepts NumPy COO arrays despite its narrower stubs. + h_mots[i] = torch.sparse_coo_tensor( + cast(Any, shifted_inds), + cast(Any, sparse_h_vals), + (num_super_pixels, sel_pix_idxs.shape[0]), + dtype=torch.float32, + ) + # Every slot is populated by the corresponding motion shift above. + return cast(List[torch.Tensor], h_mots) + + +def build_sparse_h( + subsample_matrix_inds: np.ndarray, + psf2d: np.ndarray, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> Tuple[np.ndarray, np.ndarray]: + """Build the sparse ``H`` PSF-convolution matrix (COO index/value arrays). + + ``H`` projects image space (flattened ``z*(rows*cols) + row*cols + col``) + into superpixel space by convolving each superpixel's reference pixel with + the PSF. + + Parameters + ---------- + subsample_matrix_inds : ndarray of shape (n_superpixels, 2) + ``[ref_pixel (0-based), superpixel_id (1-based)]`` per superpixel. + psf2d : ndarray + The (thresholded/cropped) PSF for this DMD. + dmd_pixels_per_column, dmd_pixels_per_row : int + DMD plane geometry. + + Returns + ------- + sparse_h_inds : ndarray of shape (2, nnz), int32 + Row (superpixel) / column (image-pixel) indices of non-zero entries. + sparse_h_vals : ndarray of shape (nnz,), float32 + The corresponding PSF weights. + """ + ref_d, ref_c, ref_r = ref_pixs_to_drc( + subsample_matrix_inds[:, 0], dmd_pixels_per_column, dmd_pixels_per_row + ) + psf2d = np.asarray(psf2d, dtype=np.float32) + psf_h, psf_w = psf2d.shape + filter_size = psf_h * psf_w + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + num_super_pixels = subsample_matrix_inds.shape[0] + + row_offsets = np.arange(psf_h, dtype=np.int32) - (psf_h // 2) + col_offsets = np.arange(psf_w, dtype=np.int32) - (psf_w // 2) + row_offsets_flat = np.broadcast_to( + row_offsets.reshape(-1, 1), (psf_h, psf_w) + ).ravel() + col_offsets_flat = np.broadcast_to( + col_offsets.reshape(1, -1), (psf_h, psf_w) + ).ravel() + psf_vals_flat = psf2d.ravel() + + sparse_h_inds = np.zeros( + (2, num_super_pixels * filter_size), dtype=np.int32 + ) + sparse_h_vals = np.zeros( + (num_super_pixels * filter_size,), dtype=np.float32 + ) + sparse_h_inds[0] = np.repeat(subsample_matrix_inds[:, 1] - 1, filter_size) + for sp in range(num_super_pixels): + start = sp * filter_size + end = (sp + 1) * filter_size + rows = ref_r[sp] + row_offsets_flat + cols = ref_c[sp] + col_offsets_flat + sparse_h_inds[1, start:end] = ( + ref_d[sp] * plane_size + rows * dmd_pixels_per_row + cols + ).astype(np.int32, copy=False) + sparse_h_vals[start:end] = psf_vals_flat + + non_zero_mask = sparse_h_vals != 0 + return sparse_h_inds[:, non_zero_mask], sparse_h_vals[non_zero_mask] diff --git a/src/giant_python/extraction/band/result_assembly.py b/src/giant_python/extraction/band/result_assembly.py new file mode 100644 index 0000000..41fb7ac --- /dev/null +++ b/src/giant_python/extraction/band/result_assembly.py @@ -0,0 +1,215 @@ +"""Canonical band numerical output assembly; persistence belongs to shared IO. + +Internal reference offsets become MATLAB displacement once here. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import numpy as np +from torch import Tensor + +from ...models.experiment_summary import ( + FrameInfo, + PathSummary, + Source, + UserRoi, + Visualizations, +) +from ...numerics.baseline import assemble_dff +from .types import TrialTraceResult + + +@dataclass +class PathResult: + """One path's numerical outputs, geometry, and assembly window lengths. + + ``a`` is (pixels, sources), ``source_params`` is (sources, 6), and + trace_results contains named trial records. + Reference offsets use the INTERNAL band convention; + online shifts retain their saved displacement convention. + No tensor copies or numerical normalization occur in this container. + """ + + a: Tensor + source_params: Tensor + source_snr: np.ndarray + act_im: np.ndarray + mean_im: np.ndarray + act_im_peaks: np.ndarray + trace_results: list[TrialTraceResult] + z_depths: np.ndarray + num_fast_zs: int + dmd_pixels_per_column: int + dmd_pixels_per_row: int + num_channels: int + denoise_window: int + baseline_window: int + draw_user_rois: bool = False + soma_masks: Optional[list] = None + soma_labels: Optional[list] = None + yx_shape: tuple = (0, 0) + + +def _with_nan_channel(arr: np.ndarray, num_channels: int) -> np.ndarray: + """Map frames/sources to sources/channels/frames, filling channel 0.""" + out = np.full( + (arr.shape[1], num_channels, arr.shape[0]), np.nan, dtype=arr.dtype + ) + out[:, 0, :] = arr.T + return out + + +def _dmd_user_rois(user_rois: Optional[dict], key: str) -> tuple: + """Select one DMD's resolved ROI masks, labels and superpixels.""" + if user_rois is None: + return None, None, [] + return ( + user_rois["user_roi_masks"].get(key), + user_rois["user_roi_labels"].get(key), + user_rois["user_roi_superpixels"].get(key, []), + ) + + +def assemble_path_summary(pr: PathResult, path_index: int = 0) -> PathSummary: + """Concatenate trials and assemble dF/F with unchanged numerical ordering. + + Offline reference offsets are negated once into MATLAB displacement. + Online shifts, frame lines, input arrays, and interpolation are unchanged. + Populated model entries share array views; empty templates retain layout. + """ + results = pr.trace_results + d_f = np.concatenate([r.d_f for r in results], axis=0) + # F = dF_ls + intermediate F0 + backprojected fit residual. + f0_ls = np.concatenate( + [ + r.f0_ls if r.f_residual is None else r.f0_ls + r.f_residual + for r in results + ], + axis=0, + ) + f_full, f0, d_f, _ = assemble_dff( + d_f, f0_ls, pr.denoise_window, pr.baseline_window + ) + profiles = pr.a.numpy().T.reshape( + -1, pr.num_fast_zs, pr.dmd_pixels_per_column, pr.dmd_pixels_per_row + ) + coords = pr.source_params[:, :3].numpy() + df_channels = _with_nan_channel(d_f, pr.num_channels) + # Keep the fitted convolution trace itself, not a second + # baseline-subtracted version: otherwise it would no longer correspond + # to the fitted events. + # Legacy manually constructed records have no transient outputs. + denoised = np.concatenate( + [ + ( + r.d_f_denoised + if r.d_f_denoised is not None + else np.full_like(r.d_f, np.nan) + ) + for r in results + ], + axis=0, + ) + events = np.concatenate( + [ + r.events if r.events is not None else np.full_like(r.d_f, np.nan) + for r in results + ], + axis=0, + ) + denoised_channels = _with_nan_channel(denoised, pr.num_channels) + event_channels = _with_nan_channel(events, pr.num_channels) + f0_channels = _with_nan_channel(f0, pr.num_channels) + snr = pr.source_snr.reshape(-1, 1) + sources = [ + Source( + profile=profiles[j], + coords=coords[j], + df_ls=df_channels[j], + df_denoised=denoised_channels[j], + events=event_channels[j], + f0=f0_channels[j], + snr=snr[j, 0], + ) + for j in range(profiles.shape[0]) + ] + frame_info = FrameInfo( + trial_num_frames=np.concatenate( + [[len(r.frame_line_idxs)] for r in results] + ).reshape(-1, 1), + discard_frames=np.any(np.isnan(f_full), axis=1).reshape(-1, 1), + frame_line_idxs=np.concatenate( + [r.frame_line_idxs for r in results], axis=0 + ).reshape(-1, 1), + offlineXshifts=-np.concatenate( + [r.reference_offsets[1] for r in results], axis=0 + ).reshape(-1, 1), + offlineYshifts=-np.concatenate( + [r.reference_offsets[0] for r in results], axis=0 + ).reshape(-1, 1), + offlineZshifts=-np.concatenate( + [r.reference_offsets[2] for r in results], axis=0 + ).reshape(-1, 1), + onlineXshifts=np.concatenate( + [r.online_shifts[1] for r in results], axis=0 + ).reshape(-1, 1), + onlineYshifts=np.concatenate( + [r.online_shifts[0] for r in results], axis=0 + ).reshape(-1, 1), + onlineZshifts=np.concatenate( + [r.online_shifts[2] for r in results], axis=0 + ).reshape(-1, 1), + ) + annotation_enabled = False + user_rois = [] + empty_roi_arrays = {} + if pr.draw_user_rois and pr.soma_masks is not None: + annotation_enabled = True + masks = ( + np.stack([np.asarray(m, dtype=bool) for m in pr.soma_masks]) + if len(pr.soma_masks) > 0 + else np.zeros((0, pr.num_fast_zs, *pr.yx_shape), dtype=bool) + ) + fluorescence = np.concatenate( + [r.user_roi_f for r in results], axis=0 + ).transpose(1, 2, 0) + labels = np.asarray(pr.soma_labels, dtype=object).reshape(-1) + user_rois = [ + UserRoi( + label=str(label), + mask=masks[j] if j < masks.shape[0] else None, + f=fluorescence[j] if j < fluorescence.shape[0] else None, + ) + for j, label in enumerate(labels) + ] + if not user_rois: + empty_roi_arrays = {"mask": masks, "F": fluorescence} + return PathSummary( + path_index=path_index, + sources=sources, + user_rois=user_rois, + visualizations=Visualizations( + act_im=pr.act_im, mean_im=pr.mean_im, act_im_peaks=pr.act_im_peaks + ), + frame_info=frame_info, + global_f=np.concatenate([r.global_f for r in results], axis=0).T, + z_depths=pr.z_depths, + annotation_enabled=annotation_enabled, + _empty_source_arrays=( + {} + if sources + else { + "profiles": profiles, + "coords": coords, + "dF_ls": df_channels, + "dF_denoised": denoised_channels, + "events": event_channels, + "F0": f0_channels, + "SNR": snr, + } + ), + _empty_roi_arrays=empty_roi_arrays, + ) diff --git a/src/giant_python/extraction/band/rois.py b/src/giant_python/extraction/band/rois.py new file mode 100644 index 0000000..849e3fe --- /dev/null +++ b/src/giant_python/extraction/band/rois.py @@ -0,0 +1,158 @@ +"""Pure band ROI geometry, annotation validation, and superpixel mapping.""" + +import numpy as np + + +def user_roi_superpixel_lists_from_masks( + roi_masks: list, + sp_fastz: np.ndarray, + sp_rows: np.ndarray, + sp_cols: np.ndarray, +) -> list: + """Return indices of superpixels in each ROI, preserving overlap/order.""" + return [ + np.flatnonzero(np.asarray(mask)[sp_fastz, sp_rows, sp_cols]) + for mask in roi_masks + ] + + +def compute_user_roi_geometry( + ref: np.ndarray, + fastz_to_refz: np.ndarray, + subsample_matrix_inds: np.ndarray, + motion_median: tuple, +) -> dict: + """Prepare reference display and mask geometry with internal band offsets. + + Reference is (channels,z,Y,X); fast-Z lookup is 1-based. Reference pixel + decomposition retains the original column-major convention and indexes. + """ + avg_motion_r, avg_motion_c, avg_motion_z = motion_median + num_ref_z = ref.shape[1] + yx_shape = (ref.shape[2], ref.shape[3]) + plane = yx_shape[0] * yx_shape[1] + ch_means = [np.nanmean(ref[c]) for c in range(ref.shape[0])] + best_ch = int(np.argmax(ch_means)) if len(ch_means) > 0 else 0 + z_map = np.array(fastz_to_refz + avg_motion_z).reshape(-1) - 1 + num_fast_z = z_map.shape[0] + ref_pix = subsample_matrix_inds[:, 0] + sp_fastz = ref_pix // plane + sp_cols = avg_motion_c + (ref_pix - sp_fastz * plane) // yx_shape[0] + sp_rows = avg_motion_r + ref_pix % yx_shape[0] + sp_mask = np.zeros((num_fast_z, *yx_shape), dtype=bool) + valid = ( + (sp_rows >= 0) + & (sp_rows < yx_shape[0]) + & (sp_cols >= 0) + & (sp_cols < yx_shape[1]) + & (sp_fastz >= 0) + & (sp_fastz < num_fast_z) + ) + if np.any(valid): + sp_mask[sp_fastz[valid], sp_rows[valid], sp_cols[valid]] = True + return { + "ref": ref, + "num_ref_z": num_ref_z, + "yx_shape": yx_shape, + "best_ch": best_ch, + "z_map": z_map, + "num_fast_z": num_fast_z, + "sp_fastz": sp_fastz, + "sp_rows": sp_rows, + "sp_cols": sp_cols, + "sp_mask": sp_mask, + } + + +def _empty_path_selection(dmd_key: str, out: dict) -> None: + """Record a neutral selection for one invalid/missing path.""" + for values in out.values(): + values[dmd_key] = [] + + +def _read_roi_records(grp: dict, expected: tuple) -> tuple: + """Validate decoded ROI records against the current band mask geometry. + + Takes a plain decoded mapping, NOT an HDF5 handle. Missing masks become + empty masks, missing records are skipped, and bad shapes reject the path. + """ + n_rois = int(np.asarray(grp.get("n_rois", 0)).reshape(-1)[0]) + roi_masks, labels, recs = [], [], [] + for i in range(n_rois): + record = grp.get(f"roi_{i:03d}") + if record is None: + continue + label = str(record.get("label", f"ROI{i+1}")) + roi_type = str(record.get("type", "polygon")) + if "mask" in record: + mask = np.asarray(record["mask"]) + if mask.shape != expected: + return roi_masks, labels, recs, True + roi_masks.append(mask > 0) + else: + roi_masks.append(np.zeros(expected, dtype=bool)) + labels.append(label) + rec: dict[str, str | np.ndarray] = {"type": roi_type, "label": label} + if "position" in record: + rec["position"] = np.asarray(record["position"]) + recs.append(rec) + return roi_masks, labels, recs, False + + +def _load_one_path( + raw: dict, dmd_ix: int, user_roi_geo: dict, out: dict +) -> bool: + """Resolve one decoded path against current geometry, without disk IO.""" + key = f"DMD{dmd_ix + 1}" + path_key = f"Path{dmd_ix + 1}" + geo = user_roi_geo[key] + expected = (geo["num_fast_z"], geo["yx_shape"][0], geo["yx_shape"][1]) + group = raw.get(path_key) + if group is None: + _empty_path_selection(key, out) + return False + masks, labels, records, bad_shape = _read_roi_records(group, expected) + if bad_shape: + print( + f"annotations.h5: {path_key} ROI mask shape mismatch " + f"(expected {expected}); treating this path as no selection." + ) + _empty_path_selection(key, out) + return False + out["user_roi_masks"][key] = masks + out["user_roi_superpixels"][key] = user_roi_superpixel_lists_from_masks( + masks, geo["sp_fastz"], geo["sp_rows"], geo["sp_cols"] + ) + out["user_roi_labels"][key] = labels + out["roi_records"][key] = records + if len(records) > 0: + print( + f"Loaded {len(records)} ROI(s) for {path_key} from annotations.h5" + ) + return True + return False + + +def resolve_annotation_records( + raw: dict, n_dmds: int, user_roi_geo: dict +) -> tuple: + """Return selection status, masks, superpixels, labels, and ROI records.""" + out = { + "user_roi_masks": {}, + "user_roi_superpixels": {}, + "user_roi_labels": {}, + "roi_records": {}, + } + any_valid = False + for dmd_ix in range(n_dmds): + if _load_one_path(raw, dmd_ix, user_roi_geo, out): + any_valid = True + if not any_valid: + return False, {}, {}, {}, {} + return ( + True, + out["user_roi_masks"], + out["user_roi_superpixels"], + out["user_roi_labels"], + out["roi_records"], + ) diff --git a/src/giant_python/extraction/band/summary_images.py b/src/giant_python/extraction/band/summary_images.py new file mode 100644 index 0000000..610b45f --- /dev/null +++ b/src/giant_python/extraction/band/summary_images.py @@ -0,0 +1,684 @@ +"""Summary/visualization images: the mean image and the activity image. + +Ported from ``extractSLAP2IntegrationSources.py``. Groups the two per-DMD +summary images written to ``experiment_summary.h5`` / consumed downstream: + +* :func:`compute_mean_image` (ref 1913-1918) scatters the kept-frame mean + trace onto the reference-pixel grid, producing the ``mean_im`` visualization. +* :func:`accumulate_activity_image` / :func:`finalize_activity_image` (ref + 2358-2504) turn the smoothed ``rho`` from + :mod:`giant_python.extraction.band.activity` into the ``act_im`` activity + image that Phase-5 peak detection localizes sources in. + +The activity image is built by sweeping ``rho`` in temporal batches, finding +spatio-temporal local maxima (a voxel greater than its 6 spatial neighbors in +the same frame and its temporal neighbors), and accumulating their squared +value; it is then masked where data was never valid, has a local NaN-median +subtracted, and is zeroed at vertical support edges. + +The reference's optional ``profile_activity_map`` timing instrumentation is +intentionally dropped (it only prints timings and does not affect output). +""" + +from __future__ import annotations + +from typing import cast + +import numpy as np +import scipy.ndimage as ndimage +from numpy.typing import NDArray + +from ...numerics.morphology import fast_dilation +from ...progress import progress + + +def compute_mean_image( + low_res_data_norm: np.ndarray, + unique_motion: np.ndarray, + mot_inds: np.ndarray, + frames_to_keep: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + num_channels: int, + low_res_data2_norm: np.ndarray | None = None, +) -> np.ndarray: + """Scatter the kept-frame mean trace onto the reference pixel grid. + + The mean over kept frames is placed at each superpixel's reference pixel, + shifted by the most common motion bin, producing the ``mean_im`` + visualization ``(channels, z, rows, cols)``. + + Parameters + ---------- + low_res_data_norm : ndarray of shape (n_superpixels, n_frames) + Count-normalized channel-1 low-res traces. + unique_motion : ndarray of shape (n_bins, 3) + Unique motion vectors from + :func:`giant_python.extraction.band.motion_binning.bin_motion`. + mot_inds : ndarray + Per-frame motion-bin indices (used to pick the most common bin). + frames_to_keep : ndarray of bool + Frames contributing to the mean. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Output grid geometry. + num_channels : int + Number of acquisition channels (>= 2 fills the second channel). + low_res_data2_norm : ndarray, optional + Count-normalized channel-2 traces (used when ``num_channels >= 2``). + + Returns + ------- + ndarray of float32 + ``mean_im`` with shape + ``(num_channels, num_fast_zs, dmd_pixels_per_column, + dmd_pixels_per_row)``. + """ + mean_im = np.full( + (num_channels, num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), + np.nan, + dtype=np.float32, + ) + most_common_mot = np.argmax(np.bincount(mot_inds)) + sel_r = ref_r + int(unique_motion[most_common_mot, 0]) + sel_c = ref_c + int(unique_motion[most_common_mot, 1]) + mean_im[0, ref_d, sel_r, sel_c] = np.nanmean( + low_res_data_norm[:, frames_to_keep], axis=1 + ) + if num_channels >= 2: + # The two-channel input contract requires the second-channel array. + mean_im[1, ref_d, sel_r, sel_c] = np.nanmean( + cast(np.ndarray, low_res_data2_norm)[:, frames_to_keep], axis=1 + ) + return mean_im + + +def accumulate_activity_image( + rho: np.ndarray, + sel_pix_idxs: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + batch_size: int = 1000, + verbose: bool = False, +) -> np.ndarray: + """Accumulate squared spatio-temporal local maxima of ``rho``. + + ``rho`` (selected pixels x frames) is scattered into a padded spatial grid + one temporal batch at a time. Within each batch, a voxel is a local maximum + if it strictly exceeds its two temporal neighbors and its four in-plane + neighbors and is not adjacent (after a 3x3 dilation) to any NaN voxel. The + squared values of local maxima are summed into ``act_im``. + + Parameters + ---------- + rho : ndarray of shape (n_sel, n_frames) + Smoothed rho response (NaN allowed). + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices into the spatial grid. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Spatial grid geometry. + batch_size : int + Number of frames processed per batch (capped at ``n_frames``). + verbose : bool + Show a per-batch progress bar when set. + + Returns + ------- + ndarray of float32 + ``act_im`` with shape + ``(num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row)``. + """ + n_frames = rho.shape[1] + batch_size = min(batch_size, n_frames) + num_batches = int(np.ceil(n_frames / batch_size)) + + spatial_coords = np.unravel_index( + sel_pix_idxs, (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row) + ) + depth_indices = spatial_coords[0][None, :].astype(np.intp, copy=False) + row_indices = spatial_coords[1][None, :].astype(np.intp, copy=False) + col_indices = spatial_coords[2][None, :].astype(np.intp, copy=False) + + dilation_struct = np.ones((3, 3), dtype=np.uint8) + act_im = np.zeros( + (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), + dtype=np.float32, + ) + temporal_pad = 1 + + # Crop to a padded ROI containing all selected pixels; all-NaN regions + # outside it can never produce a local maximum. + row_min = int(np.min(row_indices)) + row_max = int(np.max(row_indices)) + col_min = int(np.min(col_indices)) + col_max = int(np.max(col_indices)) + row_start = max(0, row_min - 1) + row_stop = min(dmd_pixels_per_column, row_max + 2) + col_start = max(0, col_min - 1) + col_stop = min(dmd_pixels_per_row, col_max + 2) + roi_h = row_stop - row_start + roi_w = col_stop - col_start + + row_indices_roi = (row_indices - row_start).astype(np.intp, copy=False) + col_indices_roi = (col_indices - col_start).astype(np.intp, copy=False) + + prealloc_size = int(min(batch_size + 2 * temporal_pad, n_frames)) + batch_rho = np.empty( + (prealloc_size, num_fast_zs, roi_h, roi_w), dtype=np.float32 + ) + nan_mask = np.empty_like(batch_rho, dtype=bool) + interior_shape = ( + max(prealloc_size - 2, 0), + num_fast_zs, + max(roi_h - 2, 0), + max(roi_w - 2, 0), + ) + local_maxima_core = np.empty(interior_shape, dtype=bool) + compare_tmp_core = np.empty(interior_shape, dtype=bool) + batch_rho_pow2_core = np.empty(interior_shape, dtype=np.float32) + time_indices_pre = np.arange(prealloc_size)[:, None] + + for batch_idx in progress( + range(num_batches), desc="Creating activity map", verbose=verbose + ): + batch_start = batch_idx * batch_size + batch_end = min(batch_start + batch_size, n_frames) + padded_start = max(0, batch_start - temporal_pad) + padded_end = min(n_frames, batch_end + temporal_pad) + curr_size = padded_end - padded_start + + br = batch_rho[:curr_size] + nm = nan_mask[:curr_size] + + br.fill(0) + nm.fill(True) + time_indices = time_indices_pre[:curr_size] + batch_vals = rho[:, padded_start:padded_end].T + br[time_indices, depth_indices, row_indices_roi, col_indices_roi] = ( + batch_vals + ) + nm[time_indices, depth_indices, row_indices_roi, col_indices_roi] = ( + np.isnan(batch_vals) + ) + + br[nm] = 0 + core_t = curr_size - 2 + if core_t <= 0: + continue + + dilated_nan_mask = fast_dilation(nm, dilation_struct) + + center = br[1:-1, :, 1:-1, 1:-1] + lmc = local_maxima_core[:core_t] + ctmp = compare_tmp_core[:core_t] + b3c = batch_rho_pow2_core[:core_t] + + np.greater(center, br[:-2, :, 1:-1, 1:-1], out=lmc) + np.greater(center, br[2:, :, 1:-1, 1:-1], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.greater(center, br[1:-1, :, :-2, 1:-1], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.greater(center, br[1:-1, :, 2:, 1:-1], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.greater(center, br[1:-1, :, 1:-1, :-2], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.greater(center, br[1:-1, :, 1:-1, 2:], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.logical_not(dilated_nan_mask[1:-1, :, 1:-1, 1:-1], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + + np.multiply(center, center, out=b3c) + np.multiply(b3c, lmc, out=b3c, casting="unsafe") + r_lo, r_hi = row_start + 1, row_stop - 1 + c_lo, c_hi = col_start + 1, col_stop - 1 + act_im[:, r_lo:r_hi, c_lo:c_hi] += np.sum( + b3c, axis=0, dtype=np.float32 + ) + + return act_im + + +def finalize_activity_image( + act_im: np.ndarray, + sel_pix_idxs: np.ndarray, + nan_ct: np.ndarray, + ref_r: np.ndarray | None = None, + ref_c: np.ndarray | None = None, + ref_d: np.ndarray | None = None, + phase_window_height: int = 7, + phase_window_width: int = 11, + phase_tolerance: float = 0.0, + phase_min_samples: int = 5, +) -> np.ndarray: + """Subtract the local NaN-median and suppress vertical support edges. + + Pixels whose rho was mostly NaN (``nan_ct > 0.5``) or that were never + selected are set to NaN. When superpixel reference centers are supplied, + an axis-aligned window includes only valid pixels with similar signed + vertical offsets from their nearest same-column reference center. + No snake identity or shared trajectory is assumed. Without reference + centers, a fixed 5x11 footprint is used. The output is ``act_im - median`` + in the original activity units. With geometry, neighborhoods + below ``phase_min_samples`` use the mean of finite first-pass medians + in the full window, regardless of phase. Empty fallback + windows and pixels without reference geometry remain NaN. + Finally, finite input pixels within valid support are set to zero if the + centered vertical window in their own column crosses invalid/nonfinite + input or an image boundary. This overrides undefined medians at these + edges without changing the filter samples. + Horizontal window clipping alone does not trigger this suppression. + + Parameters + ---------- + act_im : ndarray + Accumulated activity image (masked in place before subtraction). + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices into ``act_im``'s grid. + nan_ct : ndarray of shape (n_sel,) + Per-selected-pixel rho NaN fraction from + :func:`giant_python.extraction.band.activity.mask_high_nan_rho`. + ref_r, ref_c : ndarray, optional + Reference-center row and column for each superpixel, expressed in + the activity-image coordinate frame. The pipeline shifts these by + the dominant retained motion bin as an approximation for the + multi-motion image. Multiple bands can occupy the same column. + ref_d : ndarray, optional + Depth of each reference center. Required with geometry for multi-plane + images; omitted centers belong to plane zero in single-plane images. + phase_window_height, phase_window_width : int + Positive odd spatial window dimensions for phase matching (7x11). + phase_tolerance : float + Maximum signed-phase difference, in row pixels (default 0, exact + equality). Midpoint ties choose the smaller reference row. + phase_min_samples : int + Minimum finite phase-matched samples for first-pass medians + (default 5); smaller neighborhoods use the full-window fallback. + + Returns + ------- + ndarray + Median-subtracted activity in original units, NaN outside valid + support or where the median is undefined. Valid vertical support-edge + pixels are zero. + """ + nan_mask = np.full_like(act_im, True, dtype=bool) + valid_sel_pix = np.flatnonzero(nan_ct <= 0.5) + nan_mask[np.unravel_index(sel_pix_idxs[valid_sel_pix], nan_mask.shape)] = ( + False + ) + act_im[nan_mask] = np.nan + + if ref_r is not None and ref_c is not None: + med_act_im = phase_matched_nanmedian( + act_im, + np.flatnonzero(~nan_mask), + ref_r, + ref_c, + ref_d=ref_d, + height=phase_window_height, + width=phase_window_width, + phase_tolerance=phase_tolerance, + min_samples=phase_min_samples, + ) + else: + med_act_im = ndimage.generic_filter( + act_im, np.nanmedian, size=(1, 5, 11) + ) + # Check original input support, not median support. The vertical-only + # footprint preserves horizontal edges and never combines depth planes. + valid_input = (~nan_mask) & np.isfinite(act_im) + window_height = ( + phase_window_height if ref_r is not None and ref_c is not None else 5 + ) + # With no output override, ndimage preserves the boolean input dtype. + vertical_interior = cast( + NDArray[np.bool_], + ndimage.minimum_filter1d( + valid_input, + size=window_height, + axis=1, + mode="constant", + cval=0, + ), + ) + vertical_edge = valid_input & ~vertical_interior + act_im = act_im - med_act_im + act_im[nan_mask] = np.nan + act_im[vertical_edge] = 0 + return act_im + + +# Preserve scientific parity of phase assignment and frozen fallback passes. +def phase_matched_nanmedian( # noqa: C901 + image: np.ndarray, + valid_pix_idxs: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + ref_d: np.ndarray | None = None, + height: int = 7, + width: int = 11, + phase_tolerance: float = 0.0, + min_samples: int = 5, + chunk_size: int = 4096, +) -> np.ndarray: + """Local median conditioned on signed reference-band phase. + + For each selected pixel, phase is ``row - nearest_reference_row`` using + only reference centers in the same column and depth. Distances are + vertical, not Euclidean, so adjacent-column center spacing cannot change + the sign/phase. Different bands may overlap in column space and have + different curvature. Reference coordinates must already include motion. + + All nearest-center assignments are retained; exact distance ties choose + the smaller reference row. Duplicate centers are deduplicated; exact + coincident branches cannot be distinguished from these coordinates alone. + Columns without centers have no phase: no interpolation across gaps or + extrapolation beyond scanned columns is attempted. Multi-plane images + require ``ref_d`` to avoid borrowing geometry from a different plane. + + Within a clipped, axis-aligned ``height x width`` window, take only finite + pixels in ``valid_pix_idxs`` whose signed phase differs by at most + ``phase_tolerance`` (default 0, exact equality). The center itself is + included if finite. There is no + branch-membership restriction: nearby different snakes can contribute + when their phases match. At least ``min_samples`` (default 5) are required + for first-pass statistics. Otherwise, use the nanmean of finite first-pass + medians in the full spatial window, regardless of phase. Fallback values + are never reused by other windows; an empty fallback remains NaN. + This is a phase-conditioned spatial background, not a + reconstruction or separation of contributions at physical crossings. + + Non-selected/nonfinite pixels and + pixels without geometry remain NaN. Sparse phase/support arrays and chunked + gathers bound working memory; only output images are dense. + """ + image = np.asarray(image) + if image.ndim != 3 or any(size == 0 for size in image.shape): + raise ValueError( + "image must have non-empty shape (depth, row, column)" + ) + for name, value in (("height", height), ("width", width)): + if ( + not isinstance(value, (int, np.integer)) + or value < 1 + or value % 2 == 0 + ): + raise ValueError(f"{name} must be a positive odd integer") + for name, value in ( + ("min_samples", min_samples), + ("chunk_size", chunk_size), + ): + if not isinstance(value, (int, np.integer)) or value < 1: + raise ValueError(f"{name} must be a positive integer") + if not np.isfinite(phase_tolerance) or phase_tolerance < 0: + raise ValueError("phase_tolerance must be finite and nonnegative") + + ref_r = np.asarray(ref_r, dtype=float).reshape(-1) + ref_c = np.asarray(ref_c).reshape(-1) + if not ref_r.size or ref_r.size != ref_c.size: + raise ValueError("ref_r and ref_c must be non-empty and equally sized") + if not np.all(np.isfinite(ref_r)): + raise ValueError("reference rows must be finite") + if ref_d is None: + if image.shape[0] != 1: + raise ValueError("ref_d is required for multi-plane images") + ref_d = np.zeros(ref_r.size, dtype=np.intp) + else: + ref_d = np.asarray(ref_d).reshape(-1) + if ref_d.size != ref_r.size: + raise ValueError("ref_d must have the same size as ref_r") + for coordinates in (ref_c, ref_d): + if not np.all(np.isfinite(coordinates)) or np.any( + coordinates != np.rint(coordinates) + ): + raise ValueError( + "reference columns and depths must be finite integers" + ) + ref_c, ref_d = ref_c.astype(np.intp), ref_d.astype(np.intp) + + selected = np.asarray(valid_pix_idxs) + if selected.ndim != 1 or ( + selected.size and not np.issubdtype(selected.dtype, np.integer) + ): + raise ValueError( + "valid_pix_idxs must be a one-dimensional integer array" + ) + if np.any(selected < 0) or np.any(selected >= image.size): + raise ValueError("valid_pix_idxs contains an out-of-bounds index") + selected = np.unique(selected.astype(np.intp)) + result = np.full( + image.shape, np.nan, dtype=np.result_type(image.dtype, np.float32) + ) + if not selected.size: + return result + + nz, nr, nc = image.shape + in_bounds = (ref_d >= 0) & (ref_d < nz) & (ref_c >= 0) & (ref_c < nc) + # Group references by (depth, column); unlike the legacy filter, do not + # collapse all rows in a column to one shared trajectory. + references = {} + for depth, column, row in zip( + ref_d[in_bounds], ref_c[in_bounds], ref_r[in_bounds] + ): + references.setdefault(int(depth * nc + column), []).append(row) + depths, rows, cols = np.unravel_index(selected, image.shape) + keys = depths * nc + cols + order = np.argsort(keys, kind="stable") + boundaries = np.r_[ + 0, np.flatnonzero(np.diff(keys[order])) + 1, selected.size + ] + phase = np.full(selected.size, np.nan) + for start, stop in zip(boundaries[:-1], boundaries[1:]): + positions = order[start:stop] + centers = references.get(int(keys[positions[0]])) + if centers is None: + continue + centers = np.unique(centers) + pixel_rows = rows[positions] + insertion = np.searchsorted(centers, pixel_rows) + left = np.clip(insertion - 1, 0, centers.size - 1) + right = np.clip(insertion, 0, centers.size - 1) + nearest = np.where( + np.abs(pixel_rows - centers[left]) + <= np.abs(pixel_rows - centers[right]), + left, + right, + ) + phase[positions] = pixel_rows - centers[nearest] + + # Lookup both support membership and phase using sorted selected indices; + # finite unselected image pixels must never enter the sample population. + row_offsets, col_offsets = np.meshgrid( + np.arange(-(height // 2), height // 2 + 1), + np.arange(-(width // 2), width // 2 + 1), + indexing="ij", + ) + row_offsets, col_offsets = row_offsets.ravel(), col_offsets.ravel() + values = image.ravel()[selected] + outputs = np.flatnonzero(np.isfinite(phase) & np.isfinite(values)) + + def windows(output_positions): + """Yield chunked spatial lookups and exact selected-pixel support.""" + for start in range(0, output_positions.size, chunk_size): + positions = output_positions[start : start + chunk_size] + sample_r = rows[positions, None] + row_offsets + sample_c = cols[positions, None] + col_offsets + inside = ( + (sample_r >= 0) + & (sample_r < nr) + & (sample_c >= 0) + & (sample_c < nc) + ) + sample_flat = ( + depths[positions, None] * nr + sample_r + ) * nc + sample_c + lookup = np.minimum( + np.searchsorted(selected, sample_flat), selected.size - 1 + ) + yield positions, lookup, inside & (selected[lookup] == sample_flat) + + for positions, lookup, support in windows(outputs): + accept = ( + support + & np.isfinite(values[lookup]) + & ( + np.abs(phase[lookup] - phase[positions, None]) + <= phase_tolerance + ) + ) + enough = np.count_nonzero(accept, axis=1) >= min_samples + if not np.any(enough): + continue + samples = np.where( + accept[enough], values[lookup[enough]], np.nan + ).astype(result.dtype) + medians = np.nanmedian(samples, axis=1) + flat = selected[positions[enough]] + result.ravel()[flat] = medians + + # Advanced indexing freezes first-pass values: fallback estimates must + # not propagate or depend on chunk size / iteration order. + first_medians = result.ravel()[selected] + missing = outputs[~np.isfinite(first_medians[outputs])] + for positions, lookup, support in windows(missing): + accept = support & np.isfinite(first_medians[lookup]) + counts = np.count_nonzero(accept, axis=1) + enough = counts > 0 + # Equivalent to nanmean, without empty-window warnings. + totals = np.sum( + np.where(accept, first_medians[lookup], 0), + axis=1, + dtype=np.float64, + ) + result.ravel()[selected[positions[enough]]] = ( + totals[enough] / counts[enough] + ) + return result + + +def snake_aligned_nanmedian( + image: np.ndarray, + valid_pix_idxs: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + height: int = 5, + width: int = 11, + chunk_size: int = 100_000, +) -> np.ndarray: + """Return local medians along the band trajectory. + + The parallel snakes have the same trajectory and differ by a nearly fixed + vertical spacing. The trajectory is unwrapped modulo that spacing so its + row does not jump when an outer snake is absent from a column. Disconnected + column runs are kept separate. For each valid output pixel, this function + gathers ``height * width`` samples translated by the local trajectory. + Work is restricted to valid pixels and chunked to bound temporary memory. + ``ref_r`` and ``ref_c`` must already be expressed in ``image`` coordinates; + this helper does not apply motion correction itself. + + The output is NaN at unrequested pixels and all-NaN footprints. + """ + if height < 1 or width < 1 or height % 2 == 0 or width % 2 == 0: + raise ValueError("height and width must be positive odd integers") + + ref_r = np.asarray(ref_r).reshape(-1) + ref_c = np.asarray(ref_c).reshape(-1).astype(np.intp, copy=False) + if ref_r.size == 0 or ref_r.size != ref_c.size: + raise ValueError("ref_r and ref_c must be non-empty and equally sized") + + n_rows, n_cols = image.shape[-2:] + in_bounds = (ref_c >= 0) & (ref_c < n_cols) + columns = np.unique(ref_c[in_bounds]) + if columns.size == 0: + raise ValueError("no reference centers fall inside the image") + + rows_by_column = [np.sort(ref_r[ref_c == column]) for column in columns] + spacing_samples = [ + np.diff(rows) for rows in rows_by_column if rows.size > 1 + ] + row_spacings = ( + np.concatenate(spacing_samples) + if spacing_samples + else np.empty(0, dtype=float) + ) + snake_spacing = ( + float(np.median(row_spacings[row_spacings > 0])) + if np.any(row_spacings > 0) + else None + ) + + run_starts = np.r_[0, np.flatnonzero(np.diff(columns) > 1) + 1] + run_stops = np.r_[run_starts[1:], columns.size] + center_rows = np.array([rows[0] for rows in rows_by_column], dtype=float) + segment_at_reference = np.empty(columns.size, dtype=np.intp) + for segment, (start, stop) in enumerate(zip(run_starts, run_stops)): + segment_at_reference[start:stop] = segment + if snake_spacing is None: + center_rows[start:stop] = [ + np.median(rows) for rows in rows_by_column[start:stop] + ] + continue + for index in range(start + 1, stop): + delta = center_rows[index] - center_rows[index - 1] + center_rows[index] -= ( + np.rint(delta / snake_spacing) * snake_spacing + ) + + all_columns = np.arange(n_cols) + insertions = np.searchsorted(columns, all_columns) + right = np.clip(insertions, 0, columns.size - 1) + left = np.clip(insertions - 1, 0, columns.size - 1) + nearest = np.where( + np.abs(all_columns - columns[left]) + <= np.abs(columns[right] - all_columns), + left, + right, + ) + trajectory = center_rows[nearest] + trajectory_segment = segment_at_reference[nearest] + + result = np.full_like( + image, np.nan, dtype=np.result_type(image, np.float32) + ) + col_offsets = np.arange(-(width // 2), width // 2 + 1) + row_offsets = np.arange(-(height // 2), height // 2 + 1) + + for start in range(0, len(valid_pix_idxs), chunk_size): + flat = valid_pix_idxs[start : start + chunk_size] + depths, rows, cols = np.unravel_index(flat, image.shape) + sample_cols = cols[:, None] + col_offsets[None, :] + valid_cols = (sample_cols >= 0) & (sample_cols < n_cols) + clipped_cols = np.clip(sample_cols, 0, n_cols - 1) + valid_cols &= ( + trajectory_segment[clipped_cols] == trajectory_segment[cols, None] + ) + path_delta = np.rint( + trajectory[clipped_cols] - trajectory[cols, None] + ).astype(np.intp) + path_rows = rows[:, None] + path_delta + + sample_rows = path_rows[:, :, None] + row_offsets[None, None, :] + sample_cols = np.broadcast_to( + clipped_cols[:, :, None], sample_rows.shape + ) + sample_depths = np.broadcast_to( + depths[:, None, None], sample_rows.shape + ) + valid = ( + valid_cols[:, :, None] + & (sample_rows >= 0) + & (sample_rows < n_rows) + ) + samples = np.full(sample_rows.shape, np.nan, dtype=result.dtype) + samples[valid] = image[ + sample_depths[valid], sample_rows[valid], sample_cols[valid] + ] + medians = np.nanmedian(samples, axis=(1, 2)) + result.ravel()[flat] = medians + return result diff --git a/src/giant_python/extraction/band/traces.py b/src/giant_python/extraction/band/traces.py new file mode 100644 index 0000000..a5830a2 --- /dev/null +++ b/src/giant_python/extraction/band/traces.py @@ -0,0 +1,813 @@ +"""High-resolution per-trial trace extraction. + +Ported from ``get_high_res_traces`` in +``extractSLAP2IntegrationSources.py`` (ref 844-1011). For one trial this reads +the high-res superpixel activity, interpolates the alignment motion/background +onto the trial's downsample grid, bins the frames by motion (matching the +low-res motion bins used during NMF), and โ€” per motion bin โ€” projects the fixed +spatial profiles ``A_final`` into superpixel space and extracts per-source +temporal traces (``phi``, stored as ``dF_ls``) via least squares or +normalized profile-weighted averages. Background (``F0``) is the +backprojection of each projected profile onto the background, and a +backprojection of the fit residual (``data - background - x @ phi``) is kept +for the final F0 estimate. Transient fitting +adds separate denoised traces and events, jointly across the trial with fixed +spatial profiles; F0 is never fitted with events. The production workflow +always calculates both least-squares and transient traces. It also computes the +global and per-user-ROI fluorescence. + +The pure compute (:func:`compute_high_res_traces`) is factored out of the IO +wrapper (:func:`get_high_res_traces`, which reads the SLAP2 file lazily via +:mod:`giant_python.extraction.band.trial_data`) so the numerics can be used +without ``slap2_utils``. The workflow owns per-trial scheduling. Results use +``TrialTraceResult``, with denoised traces, events and the residual +backprojection after the original fields. +""" + +from __future__ import annotations + +from typing import List, Optional, Tuple, cast + +import numpy as np +import torch + +from ...numerics.interpolation import nearest_interp +from .geometry import build_selected_pixel_mask, ref_pixs_to_drc +from .localization import solve_phi_motion +from .operators import build_motion_h_matrices +from .transient import TransientOptions, fit_transient_traces +from .trial_data import read_band_trial_data +from .types import TrialTraceResult + + +def _empty_trace_result( + n_sources: int, num_channels: int, n_soma: int +) -> TrialTraceResult: + """Return the all-empty result tuple for a skipped trial. + + Parameters + ---------- + n_sources : int + Number of sources (column count of the empty ``phi``/``F0``). + num_channels : int + Number of acquisition channels. + n_soma : int + Number of user/soma ROIs. + + Returns + ------- + tuple + Same record as :func:`compute_high_res_traces`, with zero + frames. + """ + return TrialTraceResult( + np.full((0, n_sources), np.nan, dtype=np.float32), + np.full((0, n_sources), np.nan, dtype=np.float32), + np.full((0,), np.nan), + np.full((0,), np.nan), + np.full((0, num_channels), np.nan, dtype=np.float32), + (np.full((0,), np.nan), np.full((0,), np.nan), np.full((0,), np.nan)), + ( + np.full((0,), 0, dtype=np.int16), + np.full((0,), 0, dtype=np.int16), + np.full((0,), 0, dtype=np.int16), + ), + np.full((0, n_soma, num_channels), np.nan, dtype=np.float32), + np.full((0, n_sources), np.nan, dtype=np.float32), + np.full((0, n_sources), np.nan, dtype=np.float32), + np.full((0, n_sources), np.nan, dtype=np.float32), + ) + + +def _interp_trial_alignment( + frames: np.ndarray, a_data: dict, background_ds: np.ndarray +): + """Interpolate alignment motion/background onto the trial's frame grid. + + Motion and per-superpixel background are linearly interpolated from the + alignment downsample grid to ``frames``; the online shifts use + nearest-neighbor interpolation. + + Parameters + ---------- + frames : ndarray + The trial's high-res downsample frame line positions. + a_data : dict + Alignment data (``DSframes``, ``motionDSr/c/z``, + ``onlineYshift/Xshift/Zshift``). + background_ds : ndarray of shape (n_superpixels, n_ds) + Per-superpixel low-res background for this trial. + + Returns + ------- + tuple + ``(motion_r, motion_c, motion_z, background, online_y, online_x, + online_z)``. + """ + ds = a_data["DSframes"] + motion_r = np.interp(frames, ds, a_data["motionDSr"]) + motion_c = np.interp(frames, ds, a_data["motionDSc"]) + motion_z = np.interp(frames, ds, a_data["motionDSz"]) + background = np.array( + [ + np.interp(frames, ds, background_ds[i]) + for i in range(background_ds.shape[0]) + ] + ) + online_y = nearest_interp(frames, ds, a_data["onlineYshift"]) + online_x = nearest_interp(frames, ds, a_data["onlineXshift"]) + online_z = nearest_interp(frames, ds, a_data["onlineZshift"]) + return ( + motion_r, + motion_c, + motion_z, + background, + online_y, + online_x, + online_z, + ) + + +def _bin_trial_motion( + motion_r: np.ndarray, + motion_c: np.ndarray, + motion_z: np.ndarray, + median_z: float, + unique_motion_ds: np.ndarray, + mot_inds_to_keep_ds: np.ndarray, + z_tol: float = 1.5, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Bin trial frames by motion and match them to the low-res motion bins. + + Frames within ``z_tol`` um of the median z are rebinned into 2-D (row, col) + motion bins; each retained low-res bin is matched to its trial bin so the + same PSF geometry is reused. + + Parameters + ---------- + motion_r, motion_c, motion_z : ndarray + Per-frame interpolated motion. + median_z : float + Median z across the recording (from the low-res pass). + unique_motion_ds : ndarray of shape (n_ds_bins, 2+) + The low-res kept 2-D motion vectors. + mot_inds_to_keep_ds : ndarray + Indices of the low-res bins to keep. + z_tol : float, optional + Maximum absolute rounded z offset from the median, in um (default 1.5). + + Returns + ------- + unique_motion : ndarray of shape (n_bins, 2) + Trial 2-D motion bins (over the z-kept frames). + mot_inds : ndarray of shape (n_frames,), int32 + Per-frame trial-bin index (``-1`` for dropped frames). + mot_inds_to_keep : ndarray + Trial-bin indices matching a kept low-res bin. + frames_to_keep : ndarray of bool + Frames belonging to a matched bin. + """ + unique_motion3, mot_inds = np.unique( + np.round(np.stack((motion_r, motion_c, motion_z), axis=1)), + axis=0, + return_inverse=True, + ) + mot_inds = np.reshape(mot_inds, -1) + frames_to_keep = np.isin( + mot_inds, + np.flatnonzero(np.abs(unique_motion3[:, 2] - median_z) <= z_tol), + ) + + mot_inds = -1 * np.ones((len(mot_inds),), dtype=np.int32) + stacked = np.round( + np.stack((motion_r, motion_c), axis=1)[frames_to_keep, :] + ) + unique_motion, inv = np.unique(stacked, axis=0, return_inverse=True) + mot_inds[frames_to_keep] = np.reshape(inv, -1) + + mot_inds_to_keep = -1 * np.ones_like(mot_inds_to_keep_ds, dtype=np.int64) + for i, motion_idx_ds in enumerate(mot_inds_to_keep_ds): + matches = np.flatnonzero( + np.all( + unique_motion[:, :2] == unique_motion_ds[motion_idx_ds, :2], + axis=1, + ) + ) + if len(matches) > 0: + mot_inds_to_keep[i] = matches[0] + mot_inds_to_keep = mot_inds_to_keep[mot_inds_to_keep != -1] + + frames_to_keep = np.isin(mot_inds, mot_inds_to_keep) + return unique_motion, mot_inds, mot_inds_to_keep, frames_to_keep + + +def _weighted_trace_average( + x: torch.Tensor, data: torch.Tensor +) -> torch.Tensor: + """Average superpixels using each projected source profile's intensities. + + ``x`` is (superpixels, sources), ``data`` is (superpixels, frames), and + the result is (frames, sources). Normalize in superpixel space for each + motion bin: projection through H need not preserve the profile's mass. + Sources with no observed profile support return NaN, not a zero signal. + Unlike least squares, overlapping sources are not de-mixed. + """ + mass = x.sum(dim=0) + weights = x / mass.masked_fill(mass <= 0, float("nan")) + return (weights.T @ data).T + + +def _backproject(x: torch.Tensor, data: torch.Tensor) -> torch.Tensor: + """Backproject each projected source profile onto a superpixel movie. + + Returns ``x_j^T data / x_j^T x_j`` as (frames, sources): the per-source + coefficient in the same units as the least-squares/denoised ``phi``, but + without de-mixing overlapping sources. Zero profiles return zero, like + the basic least-squares solution; nonfinite samples propagate as NaN. + """ + norm = (x * x).sum(dim=0) + return (data.T @ x) / norm.masked_fill(norm <= 0, 1.0) + + +def _validate_transient_settings( + transient_options, simple_trace_extraction, analyze_hz +): + """Validate mutually exclusive trace modes and the transient grid rate.""" + if transient_options is not None: + if simple_trace_extraction: + raise ValueError( + "Simple and convolution trace extraction are mutually " + "exclusive" + ) + if not isinstance(transient_options, TransientOptions): + raise TypeError( + "transient_options must be TransientOptions or None" + ) + from ...models.params import _positive + + _positive("analyze_hz", analyze_hz) + + +def _solve_trial_phi_f0( + data: np.ndarray, + background: np.ndarray, + unique_motion: np.ndarray, + mot_inds: np.ndarray, + mot_inds_to_keep: np.ndarray, + sparse_h_inds: np.ndarray, + sparse_h_vals: np.ndarray, + sel_pix_idxs: np.ndarray, + a_final: torch.Tensor, + num_super_pixels: int, + dmd_pixels_per_row: int, + simple_trace_extraction: bool = False, + *, + transient_options: Optional[TransientOptions] = None, + analyze_hz: Optional[float] = None, + return_transient: bool = False, +) -> tuple: + """Extract per-source traces, background and residual backprojections. + + For each kept motion bin, shifts ``H`` by the bin's motion, projects + ``A_final`` into superpixel space, solves for the source temporal + weights (from the background-subtracted residual), and backprojects the + profiles onto the background (``F0``) and onto the fit residual + ``data - background - x @ phi``. + + Parameters + ---------- + data : ndarray of shape (n_superpixels, n_frames) + Count-normalized activity. + background : ndarray of shape (n_superpixels, n_frames) + Interpolated background. + unique_motion : ndarray of shape (n_bins, 2) + Trial 2-D motion bins. + mot_inds : ndarray of shape (n_frames,) + Per-frame trial-bin index. + mot_inds_to_keep : ndarray + Trial-bin indices to solve. + sparse_h_inds, sparse_h_vals : ndarray + Base sparse ``H`` (from ``geometry.build_sparse_h``). + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + a_final : torch.Tensor of shape (n_pixels, n_sources) + Fixed source spatial profiles. + num_super_pixels : int + Superpixel count (``H`` row count). + dmd_pixels_per_row : int + Grid geometry (column-shift stride). + simple_trace_extraction : bool, optional + Use normalized profile-intensity weighted averages for both dF and F0 + instead of least-squares dF and backprojected F0, with a zero + residual term (default False). + transient_options : TransientOptions or None + Add benchmark-style constrained dF, keeping separate QR dF and + backprojected F0. Mutually + exclusive with simple extraction. A failed fit raises RuntimeError. + analyze_hz : float or None + Nominal grid rate, required for transient fitting; line indices are + not seconds. Missing frames must retain their positions in the grid. + return_transient : bool + Also return denoised dF, events and the residual backprojection after + phi and f0. Missing-frame events are NaN; the initial state is not + counted as a new event. + + Returns + ------- + phi, f0 : torch.Tensor of shape (n_frames, n_sources) + Per-source dF and background backprojection (NaN where no motion bin + applies, where input frames contain nonfinite samples, or where + weighted averaging has no profile support). Outside simple + extraction, F0 is :func:`_backproject` of the background, not a + least-squares fit. The optional residual term is the backprojection + of ``data - background - x @ phi`` (zero for simple extraction). + """ + _validate_transient_settings( + transient_options, simple_trace_extraction, analyze_hz + ) + n_sources = a_final.shape[1] + n_frames = data.shape[1] + phi = torch.full((n_frames, n_sources), float("nan"), dtype=torch.float32) + f0 = torch.full((n_frames, n_sources), float("nan"), dtype=torch.float32) + denoised = torch.full_like(phi, float("nan")) + events = torch.full_like(phi, float("nan")) + f_residual = torch.full_like( + phi, 0.0 if simple_trace_extraction else float("nan") + ) + residual = data - background + + # One shifted superpixel<-pixel H per trial motion bin (same builder the + # NMF pass uses); only the matched bins are solved below. + h_mots = build_motion_h_matrices( + sparse_h_inds, + sparse_h_vals, + unique_motion, + sel_pix_idxs, + num_super_pixels, + dmd_pixels_per_row, + ) + + extract = ( + _weighted_trace_average + if simple_trace_extraction + else solve_phi_motion + ) + designs = {} + for motion_idx in mot_inds_to_keep: + motion_frames = np.flatnonzero(mot_inds == motion_idx) + x = torch.sparse.mm(h_mots[motion_idx], a_final[sel_pix_idxs, :]) + # F0 is never fitted with transient events. + residual_frames = torch.from_numpy( + residual[:, motion_frames].astype(np.float32) + ) + phi[motion_frames, :] = extract(x, residual_frames) + if transient_options is not None: + designs[int(motion_idx)] = x.detach().cpu().numpy() + background_frames = torch.from_numpy( + background[:, motion_frames].astype(np.float32) + ) + if simple_trace_extraction: + f0[motion_frames, :] = extract(x, background_frames) + else: + f0[motion_frames, :] = _backproject(x, background_frames) + f_residual[motion_frames, :] = _backproject( + x, residual_frames - x @ phi[motion_frames, :].T + ) + + if transient_options is not None and n_sources: + residual32 = residual.astype(np.float32) + valid = np.isin(mot_inds, mot_inds_to_keep) + valid &= np.all(np.isfinite(residual32), axis=0) + # Preserve the empty/all-invalid behavior of the existing extractor. + if np.any(valid): + result = fit_transient_traces( + residual32, + designs, + mot_inds, + valid, + analyze_hz, + options=transient_options, + ) + if not result.metadata["converged"]: + raise RuntimeError( + "Convolution trace extraction did not converge: " + f"{result.metadata['message']}. Increase trace_max_iter " + "or inspect the fit with fit_transient_traces()." + ) + denoised = torch.from_numpy(result.d_f.astype(np.float32)) + events = torch.from_numpy(result.events.astype(np.float32)) + events[~valid] = float("nan") + + if return_transient: + return phi, f0, denoised, events, f_residual + return phi, f0 + + +def _compute_global_f( + data: np.ndarray, + data2: Optional[np.ndarray], + frames_to_keep: np.ndarray, + num_channels: int, +) -> np.ndarray: + """Sum fluorescence over all superpixels per channel and frame. + + Parameters + ---------- + data : ndarray of shape (n_superpixels, n_frames) + Channel-1 activity. + data2 : ndarray or None + Channel-2 activity (used when ``num_channels >= 2``). + frames_to_keep : ndarray of bool + Dropped frames are set to NaN. + num_channels : int + Number of acquisition channels. + + Returns + ------- + ndarray of shape (n_frames, num_channels) + Global fluorescence. + """ + channel_sums = [np.sum(data, axis=0)] + if num_channels >= 2: + # Two-channel acquisitions supply data2; do not coerce/copy it. + channel_sums.append(np.sum(cast(np.ndarray, data2), axis=0)) + global_f = np.stack(channel_sums, axis=-1) + global_f[~frames_to_keep, :] = np.nan + return global_f + + +def _compute_f_soma( + data: np.ndarray, + data2: Optional[np.ndarray], + soma_sps: List[np.ndarray], + frames_to_keep: np.ndarray, + num_channels: int, +) -> np.ndarray: + """Sum fluorescence over each user-ROI's superpixels per channel. + + Parameters + ---------- + data : ndarray of shape (n_superpixels, n_frames) + Channel-1 activity. + data2 : ndarray or None + Channel-2 activity (used when ``num_channels >= 2``). + soma_sps : list of ndarray + Per-ROI superpixel index arrays. + frames_to_keep : ndarray of bool + Dropped frames are set to NaN. + num_channels : int + Number of acquisition channels. + + Returns + ------- + ndarray of shape (n_frames, n_rois, num_channels) + Per-ROI fluorescence. + """ + n_frames = data.shape[1] + f_soma = np.full( + (n_frames, len(soma_sps), num_channels), np.nan, dtype=np.float32 + ) + for i, roi_sps in enumerate(soma_sps): + f_soma[:, i, 0] = np.nansum(data[roi_sps, :], axis=0) + if num_channels >= 2: + f_soma[:, i, 1] = np.nansum( + cast(np.ndarray, data2)[roi_sps, :], axis=0 + ) + f_soma[~frames_to_keep, :, :] = np.nan + return f_soma + + +def compute_high_res_traces( + data: np.ndarray, + data2: Optional[np.ndarray], + background_ds: np.ndarray, + frames: np.ndarray, + a_data: dict, + subsample_matrix_inds: np.ndarray, + sparse_h_inds: np.ndarray, + sparse_h_vals: np.ndarray, + a_final: torch.Tensor, + unique_motion_ds: np.ndarray, + mot_inds_to_keep_ds: np.ndarray, + median_z: float, + psf2d: np.ndarray, + num_super_pixels: int, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + num_channels: int, + soma_sps: List[np.ndarray], + z_tol: float = 1.5, + simple_trace_extraction: bool = False, + *, + transient_options: Optional[TransientOptions] = None, + analyze_hz: Optional[float] = None, +) -> TrialTraceResult: + """Compute one trial's high-res source traces from loaded arrays. + + Parameters + ---------- + data : ndarray of shape (n_superpixels, n_frames) + Count-normalized channel-1 activity. + data2 : ndarray or None + Count-normalized channel-2 activity. + background_ds : ndarray of shape (n_superpixels, n_ds) + Per-superpixel low-res background for this trial. + frames : ndarray + The trial's downsample frame line positions. + a_data : dict + Alignment data for this trial. + subsample_matrix_inds : ndarray + Superpixel reference-pixel map (from ``geometry``). + sparse_h_inds, sparse_h_vals : ndarray + Base sparse ``H``. + a_final : torch.Tensor of shape (n_pixels, n_sources) + Fixed source spatial profiles. + unique_motion_ds : ndarray + Low-res kept 2-D motion vectors. + mot_inds_to_keep_ds : ndarray + Indices of the low-res bins to keep. + median_z : float + Median z (from the low-res pass). + psf2d : ndarray + The (cropped) PSF for this DMD. + num_super_pixels, num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row \ +: int + Geometry. + num_channels : int + Number of acquisition channels. + soma_sps : list of ndarray + Per-ROI superpixel index arrays. + z_tol : float, optional + Maximum absolute rounded z offset from the median, in um (default 1.5). + simple_trace_extraction : bool, optional + Average superpixels weighted by normalized source profiles instead + of least-squares fitting dF and backprojecting F0; the residual term + is zero (default False). + transient_options : TransientOptions or None + Add constrained dF and events, retaining QR dF, backprojected F0 and + auxiliary data. + Fits nonnegative sparse events plus an unpenalized initial state. + Public dF retains missing-frame NaNs; nonconvergence raises + RuntimeError. + analyze_hz : float or None + Required nominal rate for transient fitting on the uniform full grid. + Acquisition averaging is approximated, not explicitly modeled. + + Returns + ------- + TrialTraceResult + Named source traces and auxiliary data. Source trace arrays are + ``(n_frames, n_sources)``; denoised traces/events are separate from + d_f, and ``f_residual`` is the backprojected fit residual. + """ + if transient_options is not None: + frame_grid = np.asarray(frames) + if frame_grid.ndim != 1 or not np.all(np.isfinite(frame_grid)): + raise ValueError("frames must be a finite chronological vector") + # Reader ceil-rounds line positions, so nominally uniform samples + # can have alternating line spacings. Use the explicit nominal rate. + steps = np.diff(frame_grid) + if np.any(steps <= 0): + raise ValueError( + "Convolution traces require chronological frames; retain gaps" + ) + ( + motion_r, + motion_c, + motion_z, + background, + online_y, + online_x, + online_z, + ) = _interp_trial_alignment(frames, a_data, background_ds) + + ( + unique_motion, + mot_inds, + mot_inds_to_keep, + frames_to_keep, + ) = _bin_trial_motion( + motion_r, + motion_c, + motion_z, + median_z, + unique_motion_ds, + mot_inds_to_keep_ds, + z_tol=z_tol, + ) + + ref_d, ref_c, ref_r = ref_pixs_to_drc( + subsample_matrix_inds[:, 0], dmd_pixels_per_column, dmd_pixels_per_row + ) + _, sel_pix_idxs = build_selected_pixel_mask( + unique_motion, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + ) + + phi, f0, denoised, events, f_residual = _solve_trial_phi_f0( + data, + background, + unique_motion, + mot_inds, + mot_inds_to_keep, + sparse_h_inds, + sparse_h_vals, + sel_pix_idxs, + a_final, + num_super_pixels, + dmd_pixels_per_row, + simple_trace_extraction=simple_trace_extraction, + transient_options=transient_options, + analyze_hz=analyze_hz, + return_transient=True, + ) + + global_f = _compute_global_f(data, data2, frames_to_keep, num_channels) + f_soma = _compute_f_soma( + data, data2, soma_sps, frames_to_keep, num_channels + ) + + return TrialTraceResult( + phi.numpy(), + f0.numpy(), + frames, + sel_pix_idxs, + global_f, + (motion_r, motion_c, motion_z), + (online_y, online_x, online_z), + f_soma, + denoised.numpy(), + events.numpy(), + f_residual.numpy(), + ) + + +def _load_high_res_trial_data( + trial_ix: int, + dmd_ix: int, + samp_freq: float, + super_pixel_ids: np.ndarray, + datadr: str, + trial_table: dict, + num_channels: int, +): + """Read one trial's high-res activity via the SLAP2 reader. + + Returns + ------- + tuple + ``(data, data2, a_data, frames)`` with count-normalized activity. + """ + # The reader returns None only for skipped trials; keep_trial is True. + result = cast( + dict, + read_band_trial_data( + trial_ix, + True, + dmd_ix, + samp_freq, + super_pixel_ids, + datadr, + trial_table, + num_channels, + all_channels=True, + ), + ) + data = result["data"] / result["data_count"] + data2 = None + if num_channels >= 2: + data2 = result["data2"] / result["data_count2"] + return data, data2, result["alignment"], result["ds_frames"] + + +def get_high_res_traces( + trial_info: tuple, + dmd_ix: int, + samp_freq: float, + super_pixel_ids: np.ndarray, + datadr: str, + trial_table: dict, + subsample_matrix_inds: np.ndarray, + sparse_h_inds: np.ndarray, + sparse_h_vals: np.ndarray, + a_final, + unique_motion_ds: np.ndarray, + mot_inds_to_keep_ds: np.ndarray, + median_z: float, + psf2d: np.ndarray, + num_super_pixels: int, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + num_channels: int, + soma_sps: List[np.ndarray], + z_tol: float = 1.5, + simple_trace_extraction: bool = False, + *, + transient_options: Optional[TransientOptions] = None, +) -> TrialTraceResult: + """Read and extract one trial's high-res source traces. + + Skipped trials (``keep_trial`` False) return the empty result immediately; + otherwise the trial is read (via ``slap2_utils``) and passed to + :func:`compute_high_res_traces`. + + Parameters + ---------- + trial_info : tuple + ``(trial_ix, keep_trial, background_ds)`` for this trial. + dmd_ix : int + 0-based DMD index. + samp_freq : float + Analysis sample rate, in Hz. + super_pixel_ids : ndarray + Superpixel id lookup for this DMD. + datadr : str + Raw-data directory. + trial_table : dict + Normalized trial table. + subsample_matrix_inds, sparse_h_inds, sparse_h_vals : ndarray + Geometry / sparse ``H``. + a_final : ndarray or torch.Tensor + Fixed source spatial profiles ``(n_pixels, n_sources)``. + unique_motion_ds, mot_inds_to_keep_ds : ndarray + Low-res motion bins / kept indices. + median_z : float + Median z (from the low-res pass). + psf2d : ndarray + The (cropped) PSF for this DMD. + num_super_pixels, num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row \ +: int + Geometry. + num_channels : int + Number of acquisition channels. + soma_sps : list of ndarray + Per-ROI superpixel index arrays. + z_tol : float, optional + Maximum absolute rounded z offset from the median, in um (default 1.5). + simple_trace_extraction : bool, optional + Average superpixels weighted by normalized source profiles instead + of least-squares fitting dF and backprojecting F0; the residual term + is zero (default False). + transient_options : TransientOptions or None + Add constrained dF/events alongside QR dF, backprojected F0 and + unchanged profiles. + The reader's samp_freq supplies the nominal convolution rate. + + Returns + ------- + tuple + The named result of :func:`compute_high_res_traces` (empty for + skipped trials). + """ + trial_ix, keep_trial, background_ds = trial_info + + if not isinstance(a_final, torch.Tensor): + a_final = torch.tensor(a_final, dtype=torch.float32) + n_sources = a_final.shape[1] + + if not keep_trial: + return _empty_trace_result(n_sources, num_channels, len(soma_sps)) + + data, data2, a_data, frames = _load_high_res_trial_data( + trial_ix, + dmd_ix, + samp_freq, + super_pixel_ids, + datadr, + trial_table, + num_channels, + ) + + return compute_high_res_traces( + data, + data2, + background_ds, + frames, + a_data, + subsample_matrix_inds, + sparse_h_inds, + sparse_h_vals, + a_final, + unique_motion_ds, + mot_inds_to_keep_ds, + median_z, + psf2d, + num_super_pixels, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + num_channels, + soma_sps, + z_tol=z_tol, + simple_trace_extraction=simple_trace_extraction, + transient_options=transient_options, + analyze_hz=samp_freq, + ) diff --git a/src/giant_python/extraction/band/transient.py b/src/giant_python/extraction/band/transient.py new file mode 100644 index 0000000..4b9c921 --- /dev/null +++ b/src/giant_python/extraction/band/transient.py @@ -0,0 +1,325 @@ +"""Fixed-profile, output-grid sparse transient fitting. + +Port of the standalone trace benchmark, not a localization/NMF solver. +Fits 0.5 * ||r - X K s||^2 + beta * sum(s[1:]), s >= 0, using +float64 sufficient statistics and bounded L-BFGS-B. The first coefficient +is an unpenalized initial state. Trials must be solved independently. +Reader averaging is not modeled exactly; events are not spike times. +""" + +import warnings +from dataclasses import dataclass + +import numpy as np +from scipy.optimize import minimize +from scipy.signal import fftconvolve, lfilter + +from ...models.params import _nonnegative, _positive + + +@dataclass(frozen=True) +class TransientOptions: + """Settings for trace-only fitting with fixed spatial profiles. + + The workflow supplies tau_s from the shared BandSiloParams.decay_tau_s. + ``beta`` overrides the trial-relative ``beta_fraction`` when supplied. + ``debias`` optionally refits selected events without the L1 penalty. + ``event_threshold`` is in absolute coefficient units, not scaled units. + """ + + tau_s: float = 0.15 + beta_fraction: float = 0.01 + beta: float | None = None + max_iter: int = 500 + tolerance: float = 1e-7 + debias: bool = False + event_threshold: float = 1e-6 + + def __post_init__(self): + """Reject invalid solver settings before fitting or reading data.""" + _positive("tau_s", self.tau_s) + _positive("max_iter", self.max_iter, integer=True) + _positive("tolerance", self.tolerance) + _nonnegative("beta_fraction", self.beta_fraction) + _nonnegative("event_threshold", self.event_threshold) + if self.beta is not None: + _nonnegative("beta", self.beta) + if not isinstance(self.debias, bool): + raise ValueError("debias must be a bool") + + +@dataclass +class TransientResult: + """Trace, latent predictions/events, and convergence diagnostics.""" + + d_f: np.ndarray + metadata: dict + events: np.ndarray + initial_state: np.ndarray + model_trace: np.ndarray + + +class TransientOperator: + """Causal convolution and exact adjoint without a dense temporal matrix.""" + + def __init__(self, analyze_hz, tau_s, kernel=None): + """Use a unit-peak exponential or a nonnegative causal kernel.""" + _positive("analyze_hz", analyze_hz) + _positive("tau_s", tau_s) + self.gamma = float(np.exp(-1 / (analyze_hz * tau_s))) + self.kernel = ( + None if kernel is None else np.asarray(kernel, dtype=float) + ) + if self.kernel is not None and ( + self.kernel.ndim != 1 + or not self.kernel.size + or not np.all(np.isfinite(self.kernel)) + or np.any(self.kernel < 0) + or self.kernel[0] <= 0 + ): + raise ValueError( + "kernel must be finite, nonnegative, with kernel[0]>0" + ) + + def forward(self, events): + """Convolve all sources in chronological order, including gaps.""" + if self.kernel is None: + return lfilter([1.0], [1.0, -self.gamma], events, axis=0) + return fftconvolve(events, self.kernel[:, None], mode="full", axes=0)[ + : len(events) + ] + + def adjoint(self, traces): + """Apply the exact transpose, including finite-trial boundaries.""" + return self.forward(traces[::-1])[::-1] + + +def _temporal_quadratic(normal, groups, trace): + """Half squared error minus its data constant, and the trace gradient.""" + gradient = np.zeros_like(trace) + value = 0.0 + for motion, (gram, rhs) in normal.items(): + ix = groups[motion] + fit = trace[ix] + gt = fit @ gram + value += np.sum(fit * (0.5 * gt - rhs.T)) + gradient[ix] = gt - rhs.T + return float(value), gradient + + +def _prepare_transient_system(residual, designs, motion_indices, valid): + """Validate arrays and build motion-specific sufficient statistics.""" + residual = np.asarray(residual) + motion_indices = np.asarray(motion_indices) + valid = np.asarray(valid, dtype=bool).copy() + if residual.ndim != 2: + raise ValueError("residual must have shape (superpixels, frames)") + n_sp, n_frames = residual.shape + if valid.shape != (n_frames,) or motion_indices.shape != (n_frames,): + raise ValueError( + "motion_indices and valid must have one entry per frame" + ) + if not designs: + raise ValueError("Transient fitting requires projected profiles") + arrays = {m: np.asarray(x) for m, x in designs.items()} + first = next(iter(arrays.values())) + if first.ndim != 2: + raise ValueError("designs must have shape (superpixels, sources)") + n_sources = first.shape[1] + for x in arrays.values(): + if x.shape != (n_sp, n_sources) or not np.all(np.isfinite(x)): + raise ValueError( + "Projected profiles must be finite with matching shapes" + ) + valid &= np.all(np.isfinite(residual), axis=0) + valid &= np.isin(motion_indices, list(arrays)) + if not np.any(valid) or n_sources == 0: + raise ValueError( + "Transient fitting requires observed frames and sources" + ) + groups, normal = {}, {} + for motion, x in arrays.items(): + ix = np.flatnonzero(valid & (motion_indices == motion)) + if ix.size: + xd = x.astype(np.float64) + groups[motion] = ix + normal[motion] = ( + xd.T @ xd, + xd.T @ residual[:, ix].astype(np.float64), + ) + return (n_frames, n_sources), valid, groups, normal + + +def fit_transient_traces( + residual, + designs, + motion_indices, + valid, + analyze_hz, + *, + options=None, + kernel=None, +): + """Fit one trial using fixed motion-specific (superpixel, source) designs. + + Residual is (superpixels, frames). The complete nominal output grid must + be retained, in chronological order: never compress out gaps or group + disjoint motion frames before convolution. Nonfinite observations reject + the entire frame, as in QR. ``valid`` also excludes unmatched motion/z. + Public dF is NaN there, while model_trace contains latent gap predictions. + + Low-level nonconvergence returns diagnostics and warns. Production callers + must check ``metadata['converged']`` before saving a result. + """ + options = TransientOptions() if options is None else options + if not isinstance(options, TransientOptions): + raise TypeError("options must be TransientOptions or None") + op = TransientOperator(analyze_hz, options.tau_s, kernel) + shape, valid, groups, normal = _prepare_transient_system( + residual, designs, motion_indices, valid + ) + rhs_trace = np.zeros(shape) + for motion, (_, rhs) in normal.items(): + rhs_trace[groups[motion]] = rhs.T + event_rhs = op.adjoint(rhs_trace) + beta_scale = float(max(0.0, np.max(event_rhs[1:], initial=0.0))) + penalty = float( + options.beta + if options.beta is not None + else options.beta_fraction * beta_scale + ) + diagonals = np.concatenate([np.diag(g) for g, _ in normal.values()]) + positive = diagonals[diagonals > 0] + if not positive.size: + raise ValueError("No observable profile support") + gram_scale = float(np.max(positive)) + amplitude_scale = max(float(np.max(np.abs(rhs_trace))) / gram_scale, 1e-12) + objective_scale = gram_scale * amplitude_scale**2 + + def objective(flat, event_penalty=penalty): + """Evaluate the scaled objective and its analytic event gradient.""" + coefficients = flat.reshape(shape) * amplitude_scale + trace = op.forward(coefficients) + value, trace_gradient = _temporal_quadratic(normal, groups, trace) + gradient = op.adjoint(trace_gradient).copy() + value += event_penalty * coefficients[1:].sum() + gradient[1:] += event_penalty + return ( + value / objective_scale, + (gradient * amplitude_scale / objective_scale).ravel(), + ) + + optimizer_options = { + "maxiter": options.max_iter, + "ftol": options.tolerance, + "gtol": options.tolerance, + "maxls": 40, + "maxcor": 10, + } + result = minimize( + objective, + np.zeros(np.prod(shape)), + method="L-BFGS-B", + jac=True, + bounds=[(0.0, None)] * int(np.prod(shape)), + options=optimizer_options, + ) + selection = { + "converged": bool(result.success), + "iterations": int(result.nit), + "message": str(result.message), + "objective_without_data_constant": float(result.fun * objective_scale), + } + debias_info = { + "requested": options.debias, + "performed": False, + "event_threshold": float(options.event_threshold), + } + if options.debias and result.success: + active = ( + result.x.reshape(shape) * amplitude_scale > options.event_threshold + ) + active[0] = True + free = np.flatnonzero(active.ravel()) + debias_info["retained_events"] = int(active[1:].sum()) + + def restricted_objective(reduced): + """Refit selected events only; the initial state remains free.""" + full = np.zeros(np.prod(shape)) + full[free] = reduced + value, gradient = objective(full, event_penalty=0.0) + return value, gradient[free] + + refit = minimize( + restricted_objective, + result.x[free], + method="L-BFGS-B", + jac=True, + bounds=[(0.0, None)] * len(free), + options=optimizer_options, + ) + full = np.zeros(np.prod(shape)) + full[free] = refit.x + refit.x = full + result = refit + debias_info.update( + performed=True, + converged=bool(refit.success), + iterations=int(refit.nit), + message=str(refit.message), + ) + elif options.debias: + debias_info["skip_reason"] = "Support-selection fit did not converge" + coefficients = result.x.reshape(shape) * amplitude_scale + model_trace = op.forward(coefficients) + phi = model_trace.copy() + phi[~valid] = np.nan + events = coefficients.copy() + initial = events[0].copy() + events[0] = 0 + converged = bool( + result.success + and np.isfinite(result.fun) + and np.all(np.isfinite(model_trace)) + ) + if not converged: + warnings.warn( + f"Transient solver did not converge: {result.message}", + RuntimeWarning, + stacklevel=2, + ) + return TransientResult( + phi, + { + "method": "transient", + "tau_s": options.tau_s, + "beta": penalty, + "beta_fraction": ( + options.beta_fraction if options.beta is None else None + ), + "beta_scale": beta_scale, + "converged": converged, + "iterations": int(result.nit), + "message": str(result.message), + "kernel": ( + "exponential" + if kernel is None + else np.asarray(kernel).tolist() + ), + "sampling_model": ( + "nominal output grid; acquisition averaging not modeled " + "exactly" + ), + "selection": selection, + "debias": debias_info, + "final_event_penalty": ( + 0.0 if debias_info["performed"] else penalty + ), + "objective_without_data_constant": float( + result.fun * objective_scale + ), + }, + events, + initial, + model_trace, + ) diff --git a/src/giant_python/extraction/band/trial_data.py b/src/giant_python/extraction/band/trial_data.py new file mode 100644 index 0000000..3fa0c63 --- /dev/null +++ b/src/giant_python/extraction/band/trial_data.py @@ -0,0 +1,355 @@ +"""Read low-resolution superpixel activity for band-scan trials. + +Ported from ``extractSLAP2IntegrationSources.py`` (``get_trial_data`` ref +715-842 and the per-trial low-res caching block ref 1851-1892). Vendor file +construction belongs to shared SLAP2 IO; alignment inputs normalize motion +at the band boundary. The original reader lifetime and accumulation order are +unchanged. Shared interpolation and morphology utilities live in +:mod:`giant_python.numerics`. +""" + +from __future__ import annotations + +import os +from typing import Optional + +import numpy as np + +from ...io.slap2 import open_slap2_file +from .inputs import load_alignment_data_h5 + +__all__ = [ + "accumulate_superpixel_data", + "assemble_lowres_data", + "compute_ds_frames", + "load_lowres_data", + "read_band_trial_data", + "save_lowres_data", +] + + +def compute_ds_frames( + first_line: float, last_line: float, dt: float +) -> np.ndarray: + """Return the downsampled-frame line grid for one trial. + + Parameters + ---------- + first_line, last_line : float + First/last acquisition line (1-based, inclusive). + dt : float + Downsample step, in lines (``1 / samp_freq / line_period_s``). + + Returns + ------- + ndarray + ``ceil`` of the line positions of each downsampled frame. + """ + return np.ceil(np.arange(first_line, last_line + 1, dt)) + + +def _build_time_windows( + ds_frames: np.ndarray, dt: float, lines_per_cycle: int, num_cycles: int +): + """Return per-frame time windows and their line/cycle index arrays.""" + dt_read = max(3 * dt, lines_per_cycle) + max_line = num_cycles * lines_per_cycle + time_windows = [ + np.arange( + max(1, np.floor(f - dt_read)), + min(np.ceil(f + dt_read), max_line) + 1, + ) + for f in ds_frames + ] + line_indices = [(tw - 1) % lines_per_cycle + 1 for tw in time_windows] + cycle_indices = [ + np.floor((tw - 1) / lines_per_cycle) + 1 for tw in time_windows + ] + return time_windows, line_indices, cycle_indices + + +def _build_line_cycle_cache(line_indices: list, cycle_indices: list): + """Return unique (line, cycle) arrays and their ``(line, cycle)`` map.""" + pairs = set() + for lines, cycles in zip(line_indices, cycle_indices): + for li, ci in zip(lines, cycles): + pairs.add((int(li), int(ci))) + all_lines = np.array([p[0] for p in pairs]) + all_cycles = np.array([p[1] for p in pairs]) + mapping = { + (int(li), int(ci)): i + for i, (li, ci) in enumerate(zip(all_lines, all_cycles)) + } + return all_lines, all_cycles, mapping + + +def _accumulate_line( + acc: dict, + line_data: np.ndarray, + positions: np.ndarray, + z_idx: np.ndarray, + super_pixel_ids: np.ndarray, + weight: float, + ds_ix: int, +) -> None: + """Accumulate one line's weighted superpixel data into ``acc`` in place.""" + lookup_values = positions * 100 + z_idx + matching_mask = np.isin(super_pixel_ids, lookup_values) + matching_indices = np.where(matching_mask)[0] + if len(matching_indices) == 0: + return + value_to_pos = dict( + zip(lookup_values.astype(np.uint32), range(len(lookup_values))) + ) + matched = [ + value_to_pos[int(super_pixel_ids[idx])] for idx in matching_indices + ] + acc["data"][matching_indices, ds_ix] += line_data[matched, 0] * weight + acc["data_count"][matching_indices, ds_ix] += weight + if acc["data2"] is not None: + acc["data2"][matching_indices, ds_ix] += line_data[matched, 1] * weight + acc["data_count2"][matching_indices, ds_ix] += weight + + +def accumulate_superpixel_data( + data_file, + ds_frames: np.ndarray, + dt: float, + super_pixel_ids: np.ndarray, + num_channels: int, + all_channels: bool, +) -> dict: + """Accumulate weighted per-superpixel activity over the DS-frame grid. + + Port of the core loop of ``get_trial_data``: for each downsampled frame, + weight nearby acquisition lines by ``exp(-|dframe|/dt)`` and scatter their + superpixel line data into the ``(n_superpixels, n_ds_frames)`` accumulator. + + Parameters + ---------- + data_file : object + SLAP2 data-file-like object exposing ``header['linesPerCycle']``, + ``numCycles``, ``getLineData``, ``lineDataNumElements``, + ``lineSuperPixelIDs``, and ``lineFastZIdxs``. + ds_frames : ndarray + Downsampled-frame line grid from :func:`compute_ds_frames`. + dt : float + Downsample step, in lines. + super_pixel_ids : ndarray + Superpixel id lookup for this DMD, shape ``(n_superpixels, 1)``. + num_channels : int + Number of acquisition channels. + all_channels : bool + Read all channels (``getLineData(..., None)``) vs channel 1 only. + + Returns + ------- + dict + ``{"data", "data_count", "data2", "data_count2"}``. The second-channel + entries are ``None`` unless ``all_channels`` and ``num_channels >= 2``. + """ + lines_per_cycle = data_file.header["linesPerCycle"] + n_ds = len(ds_frames) + num_sp = super_pixel_ids.shape[0] + sp_flat = np.asarray(super_pixel_ids).reshape(-1) + read_second = all_channels and num_channels >= 2 + + time_windows, line_indices, cycle_indices = _build_time_windows( + ds_frames, dt, lines_per_cycle, data_file.numCycles + ) + all_lines, all_cycles, mapping = _build_line_cycle_cache( + line_indices, cycle_indices + ) + all_line_data = data_file.getLineData( + all_lines, all_cycles, 1 if not all_channels else None + ) + + acc = { + "data": np.zeros((num_sp, n_ds), dtype=np.float32), + "data_count": np.zeros((num_sp, n_ds), dtype=np.float32), + "data2": ( + np.zeros((num_sp, n_ds), dtype=np.float32) if read_second else None + ), + "data_count2": ( + np.zeros((num_sp, n_ds), dtype=np.float32) if read_second else None + ), + } + + for ds_ix in range(n_ds): + weights = np.exp(-np.abs(ds_frames[ds_ix] - time_windows[ds_ix]) / dt) + frame_lines = line_indices[ds_ix] + frame_cycles = cycle_indices[ds_ix] + for i, li in enumerate(frame_lines): + line_idx = int(li) + if data_file.lineDataNumElements[line_idx - 1] == 0: + continue + cache_idx = mapping[(line_idx, int(frame_cycles[i]))] + _accumulate_line( + acc, + all_line_data[cache_idx], + data_file.lineSuperPixelIDs[line_idx - 1], + data_file.lineFastZIdxs[line_idx - 1], + sp_flat, + weights[i], + ds_ix, + ) + return acc + + +def read_band_trial_data( + trial_ix: int, + keep_trial: bool, + dmd_ix: int, + samp_freq: float, + super_pixel_ids: np.ndarray, + datadr: str, + trial_table: dict, + num_channels: int, + all_channels: bool = True, +) -> Optional[dict]: + """Read one band trial's low-res superpixel activity. + + Opens the trial's SLAP2 file, builds the downsampled-frame grid, + accumulates weighted superpixel activity, and loads the matching alignment + data. Counts are returned unnormalized (the caller forms + ``data / data_count``); activity is divided by 100 to undo the acquisition + scaling. + + Parameters + ---------- + trial_ix : int + Trial index into ``trial_table`` columns. + keep_trial : bool + When False, the trial is skipped and ``None`` is returned. + dmd_ix : int + 0-based DMD index into ``trial_table`` rows. + samp_freq : float + Analysis/alignment sample rate, in Hz. + super_pixel_ids : ndarray + Superpixel id lookup for this DMD. + datadr : str + Raw-data directory holding the SLAP2 files. + trial_table : dict + Normalized trial table (``filename``, ``first_line``, ``last_line``, + ``fn_adata``). + num_channels : int + Number of acquisition channels. + all_channels : bool + Read all channels (default) vs channel 1 only. + + Returns + ------- + dict or None + ``{"data", "data_count", "alignment", "ds_frames", "data2", + "data_count2"}`` for kept trials, else ``None``. + """ + if not keep_trial: + return None + + source_fn = trial_table["filename"][dmd_ix, trial_ix] + first_line = trial_table["first_line"][dmd_ix, trial_ix] + last_line = trial_table["last_line"][dmd_ix, trial_ix] + + data_file = open_slap2_file(os.path.join(datadr, source_fn)) + dt = 1.0 / samp_freq / data_file.metaData.linePeriod_s + ds_frames = compute_ds_frames(first_line, last_line, dt) + + acc = accumulate_superpixel_data( + data_file, ds_frames, dt, super_pixel_ids, num_channels, all_channels + ) + alignment = load_alignment_data_h5( + trial_table["fn_adata"][dmd_ix, trial_ix] + ) + + data2 = acc["data2"] + return { + "data": acc["data"] / 100.0, + "data_count": acc["data_count"], + "alignment": alignment, + "ds_frames": ds_frames, + "data2": None if data2 is None else data2 / 100.0, + "data_count2": acc["data_count2"], + } + + +def assemble_lowres_data(results: list, num_channels: int) -> dict: + """Concatenate per-trial reader results into low-res arrays. + + Port of the concatenation block that stitches per-trial outputs into the + per-DMD low-res dataset. ``None`` results (skipped trials) are dropped. + + Parameters + ---------- + results : list + Per-trial dicts from :func:`read_band_trial_data` (or ``None``). + num_channels : int + Number of acquisition channels (>= 2 adds second-channel arrays). + + Returns + ------- + dict + ``lowResData``, ``lowResDataCt``, ``lowResMotionR/C/Z``, + ``lowResTrialID`` (and ``lowResData2``/``lowResDataCt2`` when + ``num_channels >= 2``). + """ + kept = [(i, r) for i, r in enumerate(results) if r is not None] + arrays = { + "lowResData": np.concatenate([r["data"] for _, r in kept], axis=1), + "lowResDataCt": np.concatenate( + [r["data_count"] for _, r in kept], axis=1 + ), + "lowResMotionR": np.concatenate( + [r["alignment"]["motionDSr"] for _, r in kept], axis=0 + ), + "lowResMotionC": np.concatenate( + [r["alignment"]["motionDSc"] for _, r in kept], axis=0 + ), + "lowResMotionZ": np.concatenate( + [r["alignment"]["motionDSz"] for _, r in kept], axis=0 + ), + "lowResTrialID": np.concatenate( + [np.ones_like(r["ds_frames"]) * i for i, r in kept], axis=0 + ), + } + if num_channels >= 2: + arrays["lowResData2"] = np.concatenate( + [r["data2"] for _, r in kept], axis=1 + ) + arrays["lowResDataCt2"] = np.concatenate( + [r["data_count2"] for _, r in kept], axis=1 + ) + return arrays + + +def save_lowres_data(path: str, arrays: dict) -> None: + """Save assembled low-res arrays to an ``.npz`` cache.""" + np.savez(path, **arrays) + + +def load_lowres_data(path: str, num_channels: int) -> dict: + """Load an ``.npz`` low-res cache into a dict of arrays. + + Parameters + ---------- + path : str + Path to the ``lowres_data_DMD{N}.npz`` cache. + num_channels : int + Number of acquisition channels (>= 2 also loads the second channel). + + Returns + ------- + dict + The cached low-res arrays. + """ + keys = [ + "lowResData", + "lowResDataCt", + "lowResMotionR", + "lowResMotionC", + "lowResMotionZ", + "lowResTrialID", + ] + if num_channels >= 2: + keys += ["lowResData2", "lowResDataCt2"] + with np.load(path) as data_arrays: + return {k: data_arrays[k] for k in keys} diff --git a/src/giant_python/extraction/band/types.py b/src/giant_python/extraction/band/types.py new file mode 100644 index 0000000..0fd5f07 --- /dev/null +++ b/src/giant_python/extraction/band/types.py @@ -0,0 +1,42 @@ +"""Small runtime records used by band extraction, without copying arrays.""" + +from dataclasses import dataclass +from typing import NamedTuple + +import numpy as np + + +class TrialTraceResult(NamedTuple): + """One trial's named traces in a compact, immutable record. + + Source arrays are (frames, sources); global fluorescence is + (frames, channels) and ROI fluorescence is (frames, ROIs, channels). + Reference offsets and online shifts are (row, column, z) arrays, each + with one value per frame. Only output assembly negates reference offsets + into saved MATLAB displacement. Skipped trials have zero frames. + ``d_f`` is the LS result in production; ``d_f_denoised`` and ``events`` + contain separate convolution outputs. ``f_residual`` is the source + backprojection of ``data - background - x @ d_f``, added with ``f0_ls`` to + ``d_f`` before the final F0 estimate. Optional defaults permit legacy + eight-argument construction, but the record now has eleven fields. + """ + + d_f: np.ndarray + f0_ls: np.ndarray + frame_line_idxs: np.ndarray + selected_pixels: np.ndarray + global_f: np.ndarray + reference_offsets: tuple[np.ndarray, np.ndarray, np.ndarray] + online_shifts: tuple[np.ndarray, np.ndarray, np.ndarray] + user_roi_f: np.ndarray + d_f_denoised: np.ndarray | None = None + events: np.ndarray | None = None + f_residual: np.ndarray | None = None + + +@dataclass(frozen=True) +class ResolvedAcquisition: + """Acquisition metadata resolved once without mutating science options.""" + + num_channels: int + align_hz: dict[str, float] diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py new file mode 100644 index 0000000..1c0f8f0 --- /dev/null +++ b/src/giant_python/extraction/band/workflow.py @@ -0,0 +1,610 @@ +"""Canonical band extraction driver, from prepared inputs to eager results. + +Numerical ordering follows the original BandSILo driver. Acquisition metadata, +execution policy and annotations are separate from scientific configuration; +result assembly and persistence belong to their canonical owners. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Optional, Union + +import numpy as np + +from ...io.experiment_summary import write_summary +from ...io.hdf5 import to_serializable +from ...models.experiment_summary import ExperimentSummary, PathSummary +from ...models.params import ( + AnnotationInput, + AnnotationOptions, + BandParamsInput, + BandSiloParams, + ExecutionInput, + ExecutionOptions, + resolve_band_options, +) +from ...models.trial_table import TrialTable +from ...numerics.peaks import get_act_im_peaks +from ...parallel import map_trials +from ...progress import log, progress +from . import activity +from . import background as bg +from . import geometry as geo +from . import inputs, motion_binning, operators +from . import summary_images as si +from . import trial_data as td +from .annotation import ( + build_user_roi_geometry, + resolve_interactivity, + resolve_user_rois, +) +from .inputs import compute_keep_trials, load_trial_table, read_align_info +from .localization import fit_sources +from .result_assembly import ( + PathResult, + _dmd_user_rois, + assemble_path_summary, +) +from .traces import get_high_res_traces +from .types import ResolvedAcquisition + + +def _resolve_session_user_rois( + result_dr: str, + trial_table: dict, + lookup: dict, + annotations: AnnotationOptions, +) -> Optional[dict]: + """Load session annotations or invoke the configured drawing fallback.""" + if not annotations.enabled: + return None + user_roi_geo, ref_files = build_user_roi_geometry(trial_table, lookup) + return resolve_user_rois( + Path(result_dr) / "annotations", + trial_table["n_dmds"], + user_roi_geo, + interactive=resolve_interactivity(annotations.interactive), + ref_files=ref_files, + ) + + +def extract_band_sources( + input: Union[str, Path, TrialTable], + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> ExperimentSummary: + """Extract all paths from an exact filename or an already loaded table. + + Loaded models are prepared directly, without inferring or reopening a + conventional filename. Channel inference is local; caller scientific and + policy objects remain unchanged. Results are persisted under the table's + savedr/source_extraction directory and returned with every path intact. + """ + params, run, roi = resolve_band_options(params, execution, annotations) + log(f"Loading trial table {input}", run.verbose) + trial_table = load_trial_table(input) + result_dr = str(trial_table["savedr"]) + src_extr_dr = Path(result_dr) / "source_extraction" + src_extr_dr.mkdir(parents=True, exist_ok=True) + output_path = src_extr_dr / "experiment_summary.h5" + + lookup = inputs.load_lookup_table( + Path(result_dr) / "motion_correction" / "bandRegLookupTable.h5", + trial_table["n_dmds"], + ) + psf = inputs.load_psf(params.psf_dilation, trial_table["n_dmds"]) + trial_table["keep_trials"] = compute_keep_trials( + trial_table["fn_adata"], trial_table["filename"], trial_table["datadr"] + ) + align_hz, num_channels = read_align_info( + trial_table["fn_adata"], + trial_table["keep_trials"], + trial_table["n_dmds"], + ) + if num_channels is None: + raise ValueError( + "num_channels could not be determined from the alignment data " + "(no kept trials?); provide acquisition metadata for a kept trial." + ) + acquisition = ResolvedAcquisition(num_channels, align_hz) + user_rois = _resolve_session_user_rois(result_dr, trial_table, lookup, roi) + + n_dmds = trial_table["n_dmds"] + log(f"Extracting sources for {n_dmds} DMD path(s)", run.verbose) + paths: list[PathSummary] = [] + for dmd_ix in range(n_dmds): + paths.append( + assemble_path_summary( + _process_dmd( + dmd_ix, + trial_table, + params, + lookup, + psf, + user_rois, + acquisition=acquisition, + execution=run, + annotations=roi, + ), + path_index=dmd_ix, + ) + ) + + params_serializable = to_serializable( + _params_dict(params, acquisition, roi) + ) + log(f"Writing {output_path}", run.verbose) + summary = ExperimentSummary(paths=paths, params=params_serializable) + write_summary(summary, output_path) + return summary + + +def _params_dict( + params: BandSiloParams, + acquisition: ResolvedAcquisition, + annotations: AnnotationOptions, +) -> dict: + """Historical persisted parameter names, with locally resolved metadata.""" + return { + "numChannels": acquisition.num_channels, + "analyzeHz": params.analyze_hz, + "denoiseWindow_s": params.denoise_window_s, + "baselineWindow_s": params.baseline_window_s, + "backgroundInterpolation": "cubic", + "decayTau_s": params.decay_tau_s, + "dXY": params.d_xy, + "sparse_fac": params.sparse_fac, + "vif": params.vif, + "peakth": params.peakth, + "peak_buffer": params.peak_buffer, + "trace_beta_fraction": params.trace_beta_fraction, + "trace_max_iter": params.trace_max_iter, + "trace_tolerance": params.trace_tolerance, + "trace_debias": params.trace_debias, + "trace_event_threshold": params.trace_event_threshold, + "random_seed": params.random_seed, + "draw_user_rois": annotations.enabled, + } + + +def _process_dmd( + dmd_ix: int, + trial_table: dict, + params: BandSiloParams, + lookup: dict, + psf: dict, + user_rois: Optional[dict] = None, + *, + acquisition: ResolvedAcquisition, + execution: ExecutionOptions, + annotations: AnnotationOptions, +) -> PathResult: + """Read, localize and extract one path without changing numerical order.""" + key = f"DMD{dmd_ix + 1}" + log(f"Processing {key}", execution.verbose) + soma_masks, soma_labels, soma_sps = _dmd_user_rois(user_rois, key) + datadr = str(trial_table["datadr"]) + n_trials = trial_table["filename"].shape[1] + if execution.max_trials is not None: + n_trials = min(n_trials, execution.max_trials) + log(f"Debug: limiting to first {n_trials} trial(s)", execution.verbose) + num_channels = acquisition.num_channels + align_hz = acquisition.align_hz[key] + keep_trials = trial_table["keep_trials"] + + ref_stack, _, _ = inputs.load_reference_stack( + datadr, trial_table["ref_stack"], dmd_ix + ) + dmd_pixels_per_column = ref_stack.shape[2] + dmd_pixels_per_row = ref_stack.shape[3] + yx_shape = (dmd_pixels_per_column, dmd_pixels_per_row) + fastz_to_refz = lookup["fastZ2RefZ"][key] + num_fast_zs = fastz_to_refz.shape[0] + all_sp_ids = lookup["allSuperPixelIDs"][key] + num_super_pixels = all_sp_ids.shape[0] + n_pixels = num_fast_zs * dmd_pixels_per_column * dmd_pixels_per_row + + smi = geo.build_subsample_matrix_inds( + all_sp_ids, lookup["sparseMaskInds"][key] + ) + psf2d = psf[key] + sparse_h_inds, sparse_h_vals = operators.build_sparse_h( + smi, psf2d, dmd_pixels_per_column, dmd_pixels_per_row + ) + ref_d, ref_c, ref_r = geo.ref_pixs_to_drc( + smi[:, 0], dmd_pixels_per_column, dmd_pixels_per_row + ) + + def _read(trial_ix): + """Read one low-resolution trial in acquisition order.""" + return td.read_band_trial_data( + trial_ix, + bool(keep_trials[dmd_ix, trial_ix]), + dmd_ix, + align_hz, + all_sp_ids, + datadr, + trial_table, + num_channels, + ) + + low = td.assemble_lowres_data( + map_trials( + _read, + range(n_trials), + execution.max_workers, + desc=f"{key} reading trials" if execution.verbose else None, + ), + num_channels, + ) + low_res_data_norm = low["lowResData"] / low["lowResDataCt"] + v_im = 1.0 / low["lowResDataCt"] + low2 = ( + low["lowResData2"] / low["lowResDataCt2"] + if num_channels >= 2 + else None + ) + unique_motion, mot_inds = motion_binning.bin_motion( + low["lowResMotionR"], low["lowResMotionC"], low["lowResMotionZ"] + ) + median_z = np.median(low["lowResMotionZ"]) + _, frames_to_keep = motion_binning.select_motion_bins( + unique_motion, mot_inds, low["lowResMotionZ"], z_tol=params.z_tol + ) + mean_im = si.compute_mean_image( + low_res_data_norm, + unique_motion, + mot_inds, + frames_to_keep, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + num_channels, + low2, + ) + umyx, mot_inds_yx = motion_binning.bin_motion_yx( + low["lowResMotionR"], low["lowResMotionC"], frames_to_keep + ) + _, sel_pix_idxs = geo.build_selected_pixel_mask( + umyx, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + ) + pixel_coords = geo.pixel_coords_from_idxs( + sel_pix_idxs, dmd_pixels_per_column, dmd_pixels_per_row + ) + background = _estimate_background( + low_res_data_norm, + umyx, + mot_inds_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + align_hz, + params, + execution=execution, + ) + data_std, _, _ = bg.fit_noise_variance_model( + low_res_data_norm, background, v_im, params.vif + ) + residual = bg.compute_residual(low_res_data_norm, background, data_std) + act_im, act_im_peaks, a, source_params, source_snr = _localize( + residual, + umyx, + mot_inds_yx, + sel_pix_idxs, + pixel_coords, + n_pixels, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + sparse_h_inds, + sparse_h_vals, + align_hz, + params, + execution=execution, + exclusion_mask=( + np.logical_or.reduce(soma_masks) + if soma_masks is not None and len(soma_masks) > 0 + else None + ), + ) + + # Trace-only options: never passed to localization or its QR solver. + from .transient import TransientOptions + + transient_options = TransientOptions( + tau_s=params.decay_tau_s, + beta_fraction=params.trace_beta_fraction, + max_iter=params.trace_max_iter, + tolerance=params.trace_tolerance, + debias=params.trace_debias, + event_threshold=params.trace_event_threshold, + ) + + def _traces(trial_ix): + """Extract high-res traces using this trial's background slice.""" + return get_high_res_traces( + ( + trial_ix, + bool(keep_trials[dmd_ix, trial_ix]), + background[:, low["lowResTrialID"] == trial_ix], + ), + dmd_ix, + params.analyze_hz, + all_sp_ids, + datadr, + trial_table, + smi, + sparse_h_inds, + sparse_h_vals, + a, + umyx, + np.arange(umyx.shape[0]), + median_z, + psf2d, + num_super_pixels, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + num_channels, + soma_sps, + z_tol=params.z_tol, + transient_options=transient_options, + ) + + trace_results = map_trials( + _traces, + range(n_trials), + execution.max_workers, + desc=f"{key} extracting traces" if execution.verbose else None, + ) + trace_results = [r for r in trace_results if r.d_f.shape[0] > 0] + return PathResult( + a=a, + source_params=source_params, + source_snr=source_snr, + act_im=act_im, + mean_im=mean_im, + act_im_peaks=act_im_peaks, + trace_results=trace_results, + z_depths=fastz_to_refz, + num_fast_zs=num_fast_zs, + dmd_pixels_per_column=dmd_pixels_per_column, + dmd_pixels_per_row=dmd_pixels_per_row, + num_channels=num_channels, + denoise_window=int( + np.ceil(params.denoise_window_s * params.analyze_hz) + ), + baseline_window=int( + np.ceil(params.baseline_window_s * params.analyze_hz) + ), + draw_user_rois=annotations.enabled, + soma_masks=soma_masks, + soma_labels=soma_labels, + yx_shape=yx_shape, + ) + + +def _estimate_background( + low_res_data_norm, + umyx, + mot_inds_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + align_hz, + params, + *, + execution=None, +): + """Compose interpolation and rolling baseline on the original grid.""" + execution = ExecutionOptions() if execution is None else execution + log("Estimating background", execution.verbose) + interp_data = np.full( + (sel_pix_idxs.shape[0], low_res_data_norm.shape[1]), + np.nan, + dtype=np.float32, + ) + for z in progress( + range(num_fast_zs), + desc="Interpolating planes", + verbose=execution.verbose, + ): + z_idxs, sel_2d = geo.selected_pixels_2d_for_plane( + sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row + ) + if z_idxs.size == 0: + continue + ref_z = np.flatnonzero(ref_d == z) + interp_data[z_idxs], _, _ = bg.build_interp_data( + low_res_data_norm[ref_z], + ref_r[ref_z], + ref_c[ref_z], + sel_2d, + umyx, + mot_inds_yx, + method="cubic", + ) + baseline_window = bg.baseline_window_frames( + align_hz, params.baseline_window_s + ) + interp_background = bg.compute_rolling_baseline( + interp_data, baseline_window + ) + return bg.assemble_background( + interp_background, + umyx, + mot_inds_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + low_res_data_norm.shape[0], + low_res_data_norm.shape[1], + dmd_pixels_per_column, + dmd_pixels_per_row, + ) + + +def _localize( + residual, + umyx, + mot_inds_yx, + sel_pix_idxs, + pixel_coords, + n_pixels, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + sparse_h_inds, + sparse_h_vals, + align_hz, + params, + *, + execution=None, + exclusion_mask=None, +): + """Compose rho, activity and localization; preserve DoG and RNG order.""" + execution = ExecutionOptions() if execution is None else execution + log("Computing activity image and localizing sources", execution.verbose) + h_mots = operators.build_motion_h_matrices( + sparse_h_inds, + sparse_h_vals, + umyx, + sel_pix_idxs, + num_super_pixels, + dmd_pixels_per_row, + ) + # compute_rho normalizes center/surround columns separately. The PSF still + # determines its valid-column dilation/erosion mask, not the DoG widths. + dog_center_sd = 0.9 + dog_surround_sd = 4.5 + center_kernel, center_kernel_ctr = operators.gaussian_kernel_2d( + dog_center_sd + ) + surround_kernel, surround_kernel_ctr = operators.gaussian_kernel_2d( + dog_surround_sd + ) + d_mats, d_mats_exp = [], [] + for z in progress( + range(num_fast_zs), + desc="Computing D matrices", + verbose=execution.verbose, + ): + _, sel_2d = geo.selected_pixels_2d_for_plane( + sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row + ) + d_mats.append( + operators.build_convolution_matrix( + sel_2d, center_kernel, center_kernel_ctr + ) + ) + d_mats_exp.append( + operators.build_convolution_matrix( + sel_2d, surround_kernel, surround_kernel_ctr + ) + ) + rho = activity.compute_rho( + residual, + mot_inds_yx, + umyx, + h_mots, + d_mats, + d_mats_exp, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + verbose=execution.verbose, + ) + nan_ct = activity.mask_high_nan_rho(rho) + temporal_kernel = activity.decay_kernel_1d(params.decay_tau_s, align_hz) + rho = activity.smooth_rho( + rho, + temporal_kernel, + verbose=execution.verbose, + ) + act_im = si.accumulate_activity_image( + rho, + sel_pix_idxs, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + verbose=execution.verbose, + ) + # Only final median geometry follows the dominant retained motion bin. + # Dropped frames (-1) do not vote; rho/NMF retain original scan centers. + motion_counts = np.bincount(mot_inds_yx[mot_inds_yx >= 0]) + dominant_motion = umyx[np.argmax(motion_counts)] + act_im = si.finalize_activity_image( + act_im, + sel_pix_idxs, + nan_ct, + ref_r=ref_r + int(dominant_motion[0]), + ref_c=ref_c + int(dominant_motion[1]), + ref_d=ref_d, + ) + source_seeds = get_act_im_peaks( + act_im, + peak_th=params.peakth, + buffer_size=params.peak_buffer, + exclusion_mask=exclusion_mask, + ) + act_im_peaks = source_seeds.copy() + print(f"Number of detected sources: {act_im_peaks.shape[0]}") + result = fit_sources( + source_seeds, + residual, + h_mots, + umyx, + mot_inds_yx, + sel_pix_idxs, + pixel_coords, + n_pixels, + params.d_xy, + params.sparse_fac, + verbose=execution.verbose, + temporal_kernel=temporal_kernel, + seed=params.random_seed, + ) + return ( + act_im, + act_im_peaks, + result["A"], + result["source_params"], + result["source_snr"], + ) diff --git a/src/giant_python/gui/__init__.py b/src/giant_python/gui/__init__.py index a1538d6..32c7c48 100644 --- a/src/giant_python/gui/__init__.py +++ b/src/giant_python/gui/__init__.py @@ -1,10 +1,4 @@ -"""Interactive GUI for ROI annotation. +"""Parameter and ROI editors with toolkit imports deferred until use. -Import-isolated: the headless pipeline never imports this package, so -``pip install giant-python`` works on a server without GUI dependencies. The -GUI extras (e.g. napari) are pulled in via ``pip install giant-python[gui]``. +Use the public pipeline annotation service for the complete ROI workflow. """ - -from .draw_rois import DrawROIs, annotate_rois - -__all__ = ["annotate_rois", "DrawROIs"] diff --git a/src/giant_python/gui/draw_rois.py b/src/giant_python/gui/draw_rois.py deleted file mode 100644 index 2c55e91..0000000 --- a/src/giant_python/gui/draw_rois.py +++ /dev/null @@ -1,51 +0,0 @@ -"""ROI annotation GUI (port of drawROIs.m / annotateROIs.m). - -Intended to be backed by napari. Kept in an isolated subpackage so the -headless pipeline never imports GUI dependencies. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Union - - -class DrawROIs: - """Interactive ROI annotation tool (port of the drawROIs.m class). - - Lets the user draw polygon / circle / ellipse ROIs over the mean and - activity images and saves them to annotations.h5. - - Parameters - ---------- - trial_table : TrialTable - The session trial table (provides image paths and save directory). - """ - - def __init__(self, trial_table: "object") -> None: - self.trial_table = trial_table - - def show(self) -> None: - """Launch the annotation GUI.""" - raise NotImplementedError - - def save_rois(self, path: Union[str, Path]) -> None: - """Save drawn ROIs to annotations.h5. - - Parameters - ---------- - path : str or Path - Destination path for annotations.h5. - """ - raise NotImplementedError - - -def annotate_rois(trial_table: "object") -> None: - """Launch the ROI annotation GUI for a session (port of annotateROIs.m). - - Parameters - ---------- - trial_table : TrialTable - The session trial table. - """ - raise NotImplementedError diff --git a/src/giant_python/gui/parameters_gui.py b/src/giant_python/gui/parameters_gui.py new file mode 100644 index 0000000..1fc463b --- /dev/null +++ b/src/giant_python/gui/parameters_gui.py @@ -0,0 +1,143 @@ +"""Existing tkinter BandSILo parameter form, with lazy toolkit imports.""" + +from dataclasses import asdict, replace +from typing import Optional + +import numpy as np + +from ..models.params import ( + AnnotationInput, + AnnotationOptions, + BandParamsInput, + BandSiloParams, + ExecutionInput, + ExecutionOptions, + resolve_band_options, +) + +_PARAM_GUI_FIELDS = ( + ("analyze_hz", "Analyze Hz:"), + ("decay_tau_s", "Decay Tau (s):"), + ("baseline_window_s", "Baseline Window (s):"), + ("denoise_window_s", "Denoise Window (s):"), + ("vif", "Variance Inflation Factor (VIF):"), + ("d_xy", "dXY:"), + ("peakth", "Peak Threshold:"), + ("peak_buffer", "Peak Buffer:"), + ("max_workers", "Max Workers:"), + ("z_tol", "Z Tolerance (ยตm):"), +) + + +def run_parameter_gui( + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> Optional[tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]]: + """Return split option objects, or None on cancellation, without mutation. + + Retains the original form and natural-log sparse-factor control. Toolkit + code is imported only when this function is actually invoked. + """ + import tkinter as tk + from tkinter import messagebox, ttk + + base, run, roi = resolve_band_options(params, execution, annotations) + values = {**asdict(base), **asdict(run), **asdict(roi)} + root = tk.Tk() + root.title("SLAP2 Analysis Parameters") + frame = ttk.Frame(root, padding="10") + frame.grid(row=0, column=0) + entries = {} + row = 0 + for attr, label in _PARAM_GUI_FIELDS: + ttk.Label(frame, text=label).grid(row=row, column=0, sticky=tk.W) + var = tk.StringVar(value=str(values[attr])) + ttk.Entry(frame, textvariable=var, width=15).grid(row=row, column=1) + entries[attr] = var + row += 1 + ttk.Label(frame, text="Sparse Factor (log):").grid( + row=row, column=0, sticky=tk.W + ) + sparse_var = tk.StringVar(value=str(float(np.log(base.sparse_fac)))) + ttk.Entry(frame, textvariable=sparse_var, width=15).grid(row=row, column=1) + row += 1 + ttk.Label(frame, text="LS and convolution traces are always saved.").grid( + row=row, column=0, columnspan=2, sticky=tk.W + ) + row += 1 + for attr, label in ( + ("trace_beta_fraction", "Trace Event Penalty Fraction:"), + ("trace_max_iter", "Trace Maximum Iterations:"), + ("trace_tolerance", "Trace Optimizer Tolerance:"), + ("trace_event_threshold", "Trace Debias Event Threshold:"), + ): + ttk.Label(frame, text=label).grid(row=row, column=0, sticky=tk.W) + var = tk.StringVar(value=str(values[attr])) + ttk.Entry(frame, textvariable=var, width=15).grid(row=row, column=1) + entries[attr] = var + row += 1 + ttk.Label(frame, text="Debias Convolution Traces?").grid( + row=row, column=0, sticky=tk.W + ) + debias_var = tk.BooleanVar(value=base.trace_debias) + ttk.Checkbutton(frame, variable=debias_var).grid(row=row, column=1) + row += 1 + ttk.Label(frame, text="Draw User ROIs?").grid( + row=row, column=0, sticky=tk.W + ) + draw_rois_var = tk.BooleanVar(value=roi.enabled) + ttk.Checkbutton(frame, variable=draw_rois_var).grid(row=row, column=1) + row += 1 + result: dict[ + str, + Optional[tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]], + ] = {"options": None} + + def on_ok(): + """Validate controls and return a replacement parameter instance.""" + try: + updates = { + "analyze_hz": float(entries["analyze_hz"].get()), + "decay_tau_s": float(entries["decay_tau_s"].get()), + "baseline_window_s": float(entries["baseline_window_s"].get()), + "denoise_window_s": float(entries["denoise_window_s"].get()), + "vif": float(entries["vif"].get()), + "d_xy": int(entries["d_xy"].get()), + "peakth": float(entries["peakth"].get()), + "peak_buffer": int(entries["peak_buffer"].get()), + "z_tol": float(entries["z_tol"].get()), + "sparse_fac": float(np.exp(float(sparse_var.get()))), + "trace_beta_fraction": float( + entries["trace_beta_fraction"].get() + ), + "trace_max_iter": int(entries["trace_max_iter"].get()), + "trace_tolerance": float(entries["trace_tolerance"].get()), + "trace_event_threshold": float( + entries["trace_event_threshold"].get() + ), + "trace_debias": bool(debias_var.get()), + } + result["options"] = ( + replace(base, **updates), + replace(run, max_workers=int(entries["max_workers"].get())), + replace(roi, enabled=bool(draw_rois_var.get())), + ) + root.destroy() + except ValueError as error: + messagebox.showerror("Error", f"Invalid parameter values: {error}") + + def on_cancel(): + """Discard edits and close the dialog.""" + root.destroy() + + button_frame = ttk.Frame(frame) + button_frame.grid(row=row, column=0, columnspan=2, pady=10) + ttk.Button(button_frame, text="OK", command=on_ok).pack(side=tk.LEFT) + ttk.Button(button_frame, text="Cancel", command=on_cancel).pack( + side=tk.LEFT + ) + root.grab_set() + root.mainloop() + return result["options"] diff --git a/src/giant_python/gui/roi_editor.py b/src/giant_python/gui/roi_editor.py new file mode 100644 index 0000000..abd5995 --- /dev/null +++ b/src/giant_python/gui/roi_editor.py @@ -0,0 +1,194 @@ +"""The existing OpenCV rectangle selector and tkinter label prompt. + +Accepts prepared band reference geometry and returns in-memory records/masks. +There is no file IO here, and tkinter/cv2 are imported only on drawing paths. +""" + +import numpy as np + +from ..extraction.band.rois import user_roi_superpixel_lists_from_masks + + +def _ask_roi_label( + roi_num: int, + path_num: int, + plane_num: int, +) -> str: + """Prompt for one ROI label, defaulting to ROI{n} on cancel or failure.""" + import tkinter as tk + from tkinter import simpledialog + + text = None + try: + root = tk.Tk() + root.withdraw() + root.attributes("-topmost", True) + text = simpledialog.askstring( + "ROI label", + f"Label for ROI {roi_num} " + f"(Path {path_num}, fast-Z plane {plane_num}):", + parent=root, + ) + root.destroy() + except Exception as error: # noqa: BLE001 - GUI best-effort + print(f"Could not prompt for ROI label ({error}); using default.") + if not text or not text.strip(): + return f"ROI{roi_num}" + return text.strip() + + +def run_user_roi_selection(user_roi_geo: dict, n_dmds: int) -> dict: + """Select ROIs on reference images and return per-DMD selections.""" + import cv2 + + out = { + "user_roi_masks": {}, + "user_roi_superpixels": {}, + "user_roi_labels": {}, + "roi_records": {}, + } + for dmd_ix in range(n_dmds): + key = f"DMD{dmd_ix + 1}" + geo = user_roi_geo[key] + roi_masks, roi_records = _select_rois_for_dmd(cv2, dmd_ix, geo) + out["user_roi_masks"][key] = roi_masks + out["user_roi_superpixels"][key] = ( + user_roi_superpixel_lists_from_masks( + roi_masks, geo["sp_fastz"], geo["sp_rows"], geo["sp_cols"] + ) + ) + out["user_roi_labels"][key] = [ + record["label"] for record in roi_records + ] + out["roi_records"][key] = roi_records + return out + + +def _normalize_plane( + plane: np.ndarray, +) -> np.ndarray: + """Percentile-normalize a reference plane to an 8-bit image.""" + im = np.nan_to_num(plane, nan=0.0) + vmin = np.percentile(im, 1) + vmax = np.percentile(im, 99.5) + if not np.isfinite(vmin): + vmin = float(np.nanmin(im)) if np.any(np.isfinite(im)) else 0.0 + if not np.isfinite(vmax): + vmax = float(np.nanmax(im)) if np.any(np.isfinite(im)) else 1.0 + if vmax <= vmin: + vmax = vmin + 1.0 + im8 = np.clip((im - vmin) / (vmax - vmin), 0, 1) + return (im8 * 255).astype(np.uint8) + + +def _select_rois_for_dmd(cv2, dmd_ix: int, g: dict): + """Run the per-DMD interactive loop, returning masks and records.""" + ref = g["ref"] + num_ref_z = g["num_ref_z"] + yx_shape = g["yx_shape"] + best_ch = g["best_ch"] + z_map = g["z_map"] + num_fast_z = g["num_fast_z"] + sp_mask = g["sp_mask"] + roi_masks = [] + roi_records = [] + roi_union = np.zeros((num_fast_z, *yx_shape), dtype=bool) + window_name = f"Select user ROI(s) DMD{dmd_ix + 1}" + cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) + cv2.resizeWindow(window_name, 800, 500) + has_z_trackbar = num_fast_z > 1 + if has_z_trackbar: + cv2.createTrackbar("z", window_name, 0, num_fast_z - 1, lambda v: None) + curr_fz = 0 + while True: + if has_z_trackbar: + curr_fz = int( + np.clip( + cv2.getTrackbarPos("z", window_name), 0, num_fast_z - 1 + ) + ) + refz = int(np.clip(z_map[curr_fz], 0, max(0, num_ref_z - 1))) + im8 = _normalize_plane(ref[best_ch, refz]) + disp = _compose_display(cv2, im8, sp_mask[curr_fz], roi_union[curr_fz]) + cv2.imshow(window_name, disp) + keycode = cv2.waitKey(50) & 0xFF + if keycode == ord("e"): + _edit_rois( + cv2, + im8, + sp_mask[curr_fz], + curr_fz, + dmd_ix, + num_fast_z, + yx_shape, + roi_masks, + roi_union, + roi_records, + ) + elif keycode == ord("n"): + curr_fz = min(curr_fz + 1, num_fast_z - 1) + if has_z_trackbar: + cv2.setTrackbarPos("z", window_name, curr_fz) + elif keycode == ord("p"): + curr_fz = max(curr_fz - 1, 0) + if has_z_trackbar: + cv2.setTrackbarPos("z", window_name, curr_fz) + elif keycode in (27, ord("q")): + break + cv2.destroyWindow(window_name) + return roi_masks, roi_records + + +def _compose_display(cv2, im8, sp_plane, roi_plane): + """Overlay the superpixel mask in green and ROI union in red.""" + disp = cv2.cvtColor(im8, cv2.COLOR_GRAY2BGR) + if np.any(sp_plane): + sp_color = np.zeros_like(disp) + sp_color[sp_plane] = (0, 255, 0) + disp = cv2.addWeighted(disp, 0.75, sp_color, 0.25, 0) + if roi_plane.any(): + roi_color = np.zeros_like(disp) + roi_color[roi_plane] = (0, 0, 255) + disp = cv2.addWeighted(disp, 0.7, roi_color, 0.3, 0) + return disp + + +def _edit_rois( + cv2, + im8, + sp_plane, + curr_fz, + dmd_ix, + num_fast_z, + yx_shape, + roi_masks, + roi_union, + roi_records, +): + """Append selected rectangles as polygon records and overlapping masks.""" + edit_name = f"Edit ROIs z={curr_fz + 1}" + edit_disp = _compose_display( + cv2, im8, sp_plane, np.zeros(yx_shape, dtype=bool) + ) + rois = cv2.selectROIs( + edit_name, edit_disp, showCrosshair=True, fromCenter=False + ) + cv2.resizeWindow(edit_name, 800, 500) + if rois is not None and len(rois) > 0: + for x, y, w, h in rois: + roi_num = len(roi_masks) + 1 + y_stop, x_stop = y + h, x + w + mask = np.zeros((num_fast_z, *yx_shape), dtype=bool) + mask[curr_fz, y:y_stop, x:x_stop] = True + roi_masks.append(mask) + roi_union[curr_fz, y:y_stop, x:x_stop] = True + label = _ask_roi_label(roi_num, dmd_ix + 1, curr_fz + 1) + position = np.array( + [[y, x], [y, x + w], [y + h, x + w], [y + h, x]], + dtype=np.float64, + ) + roi_records.append( + {"type": "polygon", "label": label, "position": position} + ) + print(f"Added {len(rois)} ROI(s) at fast-Z {curr_fz + 1}") + cv2.destroyWindow(edit_name) diff --git a/src/giant_python/io/__init__.py b/src/giant_python/io/__init__.py index 539b30d..ac64315 100644 --- a/src/giant_python/io/__init__.py +++ b/src/giant_python/io/__init__.py @@ -6,20 +6,52 @@ transposition when porting from MATLAB. """ -from .hdf5 import load_struct_h5, save_struct_h5 +from importlib import import_module +from typing import TYPE_CHECKING + +from .hdf5 import load_struct_h5 from .slap2 import ( - get_online_motion, - read_integration_trial_data, - ref_pixs_to_drc, + Slap2Reader, + load_alignment_data_h5, + open_slap2_file, ) -from .tiff import scanimagetiff_data_wrapper, scanimagetiff_wrapper +from .tiff import read_tiff + +if TYPE_CHECKING: + from .annotations import read_annotations_h5, save_annotations_h5 + from .experiment_summary import ( + read_summary, + write_summary, + ) + from .trial_table import read_trial_table + +_SCHEMA_EXPORTS = { + "read_trial_table": ".trial_table", + "read_annotations_h5": ".annotations", + "save_annotations_h5": ".annotations", + "read_summary": ".experiment_summary", + "write_summary": ".experiment_summary", +} + + +def __getattr__(name): + """Load model codecs lazily, avoiding schema/model import cycles.""" + if name not in _SCHEMA_EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(import_module(_SCHEMA_EXPORTS[name], __name__), name) + globals()[name] = value + return value + __all__ = [ - "get_online_motion", + "Slap2Reader", + "load_alignment_data_h5", "load_struct_h5", - "read_integration_trial_data", - "ref_pixs_to_drc", - "save_struct_h5", - "scanimagetiff_data_wrapper", - "scanimagetiff_wrapper", + "open_slap2_file", + "read_annotations_h5", + "read_summary", + "read_tiff", + "read_trial_table", + "save_annotations_h5", + "write_summary", ] diff --git a/src/giant_python/io/annotations.py b/src/giant_python/io/annotations.py new file mode 100644 index 0000000..73e61f9 --- /dev/null +++ b/src/giant_python/io/annotations.py @@ -0,0 +1,91 @@ +"""Raw ROI records, masks and provenance IO without band geometry or UI.""" + +from pathlib import Path +from typing import Optional, Union + +import h5py +import numpy as np + +from .hdf5 import load_struct_h5 + + +def _h5_scalar_str(ds: h5py.Dataset) -> str: + """Read a scalar string, including legacy one-element string arrays.""" + value = ds[()] + if isinstance(value, np.ndarray): + value = value.reshape(-1)[0] + if isinstance(value, bytes): + return value.decode("utf-8", "replace") + return str(value) + + +def read_annotations_h5(path: Union[str, Path]) -> dict: + """Read raw Path groups including n_rois, masks, records and provenance. + + No geometry validation or superpixel mapping occurs here. Missing and + unreadable files raise OSError; annotation orchestration decides policy. + The shared decoder honors row_major and leaves coordinate indexes alone. + """ + return load_struct_h5(path) + + +def save_annotations_h5( + dr: str, + roi_records_by_dmd: dict, + user_roi_masks: dict, + n_dmds: int, + ref_files: Optional[dict] = None, +) -> str: + """Write dr/annotations.h5 and return its path, preserving the band schema. + + Rectangles are polygon records with zero-based [y,x] vertices. Masks are + (fast-Z,Y,X), may overlap, and are stored as compressed uint8. The caller + owns directory creation and the decision to replace an existing file. + """ + path = str(Path(dr) / "annotations.h5") + str_dt = h5py.string_dtype(encoding="utf-8") + ref_files = ref_files or {} + with h5py.File(path, "w") as f: + f["row_major"] = 1 + f["coords_zero_indexed"] = 1 + for dmd_ix in range(n_dmds): + key = f"DMD{dmd_ix + 1}" + group = f.create_group(f"Path{dmd_ix + 1}") + ref_file = ref_files.get(key) + group.create_dataset( + "dr", + data=str(Path(ref_file).parent) if ref_file else "", + dtype=str_dt, + ) + group.create_dataset( + "fn", + data=Path(ref_file).name if ref_file else "", + dtype=str_dt, + ) + records = roi_records_by_dmd.get(key, []) + group.create_dataset("n_rois", data=len(records)) + mask_list = user_roi_masks.get(key, []) + for i, record in enumerate(records): + _write_roi_group(group, i, record, mask_list, str_dt) + return path + + +def _write_roi_group(grp, i: int, rec: dict, mask_list: list, str_dt) -> None: + """Write one roi_### group with the established optional-field policy.""" + group = grp.create_group(f"roi_{i:03d}") + roi_type = rec.get("type", "polygon") + group.create_dataset("type", data=roi_type, dtype=str_dt) + group.create_dataset( + "label", data=rec.get("label", f"ROI{i + 1}"), dtype=str_dt + ) + if i < len(mask_list): + group.create_dataset( + "mask", + data=np.asarray(mask_list[i]).astype(np.uint8), + compression="gzip", + shuffle=True, + ) + if roi_type == "polygon" and "position" in rec: + group.create_dataset( + "position", data=np.asarray(rec["position"], dtype=np.float64) + ) diff --git a/src/giant_python/io/experiment_summary.py b/src/giant_python/io/experiment_summary.py new file mode 100644 index 0000000..359197d --- /dev/null +++ b/src/giant_python/io/experiment_summary.py @@ -0,0 +1,361 @@ +"""Codec for the experiment-summary schema emitted by band extraction. + +The canonical interface is ``write_summary(summary, path) -> None`` and +``read_summary(path) -> ExperimentSummary``. +There is no motion-sign conversion here. Existing params/row_major are left +untouched; supplied paths are replaced, and unrelated paths are retained. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path +from typing import Union + +import h5py +import numpy as np + +from ..models.experiment_summary import ( + FRAME_INFO_KEYS, + ExperimentSummary, + FrameInfo, + PathSummary, + Source, + UserRoi, + Visualizations, +) +from .hdf5 import load_struct_h5, write_dict_to_h5group + +_TEMPORAL_KEYS = ("dF_ls", "dF_denoised", "events", "F0", "SNR") +_SOURCE_KEYS = ("profiles", "coords", *_TEMPORAL_KEYS) + + +def _decode_path(assembled: dict, path_index: int) -> PathSummary: + """Decode schema arrays into an eager path, sharing populated arrays. + + This schema-only adapter does not concatenate traces or compute baselines. + Empty-array templates preserve dimensions/dtypes absent from empty lists. + """ + sources = [ + Source( + profile=assembled["profiles"][j], + coords=assembled["coords"][j], + df_ls=assembled["dF_ls"][j], + df_denoised=( + assembled["dF_denoised"][j] + if "dF_denoised" in assembled + else None + ), + events=assembled["events"][j] if "events" in assembled else None, + f0=assembled["F0"][j], + snr=assembled["SNR"][j, 0], + ) + for j in range(assembled["profiles"].shape[0]) + ] + rois = assembled.get("user_rois") + user_rois = [] + if rois is not None: + labels = np.asarray(rois["labels"], dtype=object).reshape(-1) + user_rois = [ + UserRoi( + label=str(label), + mask=rois["mask"][j] if j < rois["mask"].shape[0] else None, + f=rois["F"][j] if j < rois["F"].shape[0] else None, + fsvd=( + rois["Fsvd"][j] + if "Fsvd" in rois and j < rois["Fsvd"].shape[0] + else None + ), + ) + for j, label in enumerate(labels) + ] + return PathSummary( + path_index=path_index, + sources=sources, + user_rois=user_rois, + visualizations=Visualizations( + mean_im=assembled["mean_im"], + act_im=assembled["act_im"], + act_im_peaks=assembled["act_im_peaks"], + ), + frame_info=FrameInfo( + **{key: assembled[key] for key in FRAME_INFO_KEYS} + ), + global_f=assembled["global_f"], + z_depths=assembled["Z_depths"], + annotation_enabled=assembled.get( + "annotation_enabled", rois is not None + ), + _empty_source_arrays=( + {} + if sources + else { + key: assembled[key] for key in _SOURCE_KEYS if key in assembled + } + ), + _empty_roi_arrays=( + {key: rois[key] for key in ("mask", "F", "Fsvd") if key in rois} + if rois is not None and not user_rois + else {} + ), + ) + + +def _stack_required_arrays( + arrays: Iterable[np.ndarray | None], field: str +) -> np.ndarray: + """Stack arrays; identify missing fields by their 0-based index.""" + required: list[np.ndarray] = [] + for index, array in enumerate(arrays): + if array is None: + raise ValueError(f"{field}[{index}] is required; got None") + required.append(array) + return np.stack(required) + + +def _optional_array(array, reference: np.ndarray, field: str) -> np.ndarray: + """Validate a supplied trace, or mark unavailable samples with NaNs.""" + if array is None: + dtype = ( + reference.dtype + if np.issubdtype(reference.dtype, np.inexact) + else np.dtype(float) + ) + return np.full(reference.shape, np.nan, dtype=dtype) + array = np.asarray(array) + if array.shape != reference.shape: + raise ValueError( + f"{field} must have shape {reference.shape}; got {array.shape}" + ) + return array + + +def _stack_optional_arrays( + arrays: Iterable[np.ndarray | None], reference: np.ndarray, field: str +) -> np.ndarray: + """Check each optional entry against its corresponding required trace.""" + return np.stack( + [ + _optional_array(array, reference[index], f"{field}[{index}]") + for index, array in enumerate(arrays) + ] + ) + + +def _empty_geometry( + path: PathSummary, field: str +) -> tuple[tuple[int, ...], np.ndarray]: + """Infer empty tensor dimensions from the saved images and global trace.""" + mean = np.asarray(path.visualizations.mean_im) + global_f = np.asarray(path.global_f) + if mean.ndim != 4 or global_f.ndim != 2: + raise ValueError( + f"Empty {field} require shape templates or " + "mean_im and global_f geometry" + ) + return mean.shape[1:], global_f + + +def _encode_path(path: PathSummary) -> dict: + """Expose band-schema arrays for persistence, without scientific changes. + + Populated lists are stacked so edits to public Source/UserRoi entries are + reflected. Empty results retain their original tensor dimensions/dtypes. + Missing optional traces are NaN, never inferred from other signals. + Required fields and optional shapes are checked before path replacement. + """ + out = { + "Z_depths": path.z_depths, + "global_f": path.global_f, + "mean_im": path.visualizations.mean_im, + "act_im": path.visualizations.act_im, + "act_im_peaks": path.visualizations.act_im_peaks, + **path.frame_info.to_dict(), + } + if path.sources: + out.update( + { + "profiles": _stack_required_arrays( + (s.profile for s in path.sources), "Source.profile" + ), + "coords": _stack_required_arrays( + (s.coords for s in path.sources), "Source.coords" + ), + "dF_ls": _stack_required_arrays( + (s.df_ls for s in path.sources), "Source.df_ls" + ), + "F0": _stack_required_arrays( + (s.f0 for s in path.sources), "Source.f0" + ), + "SNR": np.asarray( + [np.nan if s.snr is None else s.snr for s in path.sources] + ).reshape(-1, 1), + } + ) + for key, field in ( + ("dF_denoised", "df_denoised"), + ("events", "events"), + ): + out[key] = _stack_optional_arrays( + (getattr(s, field) for s in path.sources), + out["dF_ls"], + f"Source.{field}", + ) + elif path._empty_source_arrays: + out.update(path._empty_source_arrays) + else: + spatial_shape, global_f = _empty_geometry(path, "sources") + out.update( + { + "profiles": np.empty((0, *spatial_shape), dtype=np.float32), + "coords": np.empty((0, 3), dtype=np.int32), + "dF_ls": np.empty((0, *global_f.shape), dtype=global_f.dtype), + "F0": np.empty((0, *global_f.shape), dtype=global_f.dtype), + "SNR": np.empty((0, 1), dtype=np.float32), + } + ) + if not path.sources: + for key in ("dF_denoised", "events"): + out[key] = _optional_array(out.get(key), out["dF_ls"], key) + if not path.annotation_enabled and path.user_rois: + raise ValueError("Set annotation_enabled=True to serialize user ROIs") + if path.user_rois: + out["user_rois"] = { + "mask": _stack_required_arrays( + (roi.mask for roi in path.user_rois), "UserRoi.mask" + ), + "labels": [roi.label for roi in path.user_rois], + "F": _stack_required_arrays( + (roi.f for roi in path.user_rois), "UserRoi.f" + ), + } + out["user_rois"]["Fsvd"] = _stack_optional_arrays( + (roi.fsvd for roi in path.user_rois), + out["user_rois"]["F"], + "UserRoi.fsvd", + ) + else: + if path._empty_roi_arrays: + empty = path._empty_roi_arrays + else: + # Both disabled and enabled-but-empty annotations have full schema. + spatial_shape, global_f = _empty_geometry(path, "ROIs") + empty = { + "mask": np.zeros((0, *spatial_shape), dtype=bool), + "F": np.empty((0, *global_f.shape), dtype=global_f.dtype), + } + out["user_rois"] = { + **empty, + "labels": [], + "Fsvd": _optional_array( + empty.get("Fsvd"), empty["F"], "UserRoi.Fsvd" + ), + } + if path.z_depths is None: + raise ValueError("Z_depths is required; got None") + out["Z_depths"] = np.asarray(path.z_depths).reshape(-1, 1) + return out + + +def _write_path_group(f: h5py.File, path: PathSummary) -> None: + """Replace one Path{index+1} group without array sign conversion.""" + assembled = _encode_path(path) + name = f"Path{path.path_index + 1}" + if name in f: + del f[name] + group = f.create_group(name) + group.create_dataset("Z_depths", data=assembled["Z_depths"]) + sources = group.create_group("sources") + spatial = sources.create_group("spatial") + for key in ("profiles", "coords"): + spatial.create_dataset(key, data=assembled[key]) + temporal = sources.create_group("temporal") + for key in _TEMPORAL_KEYS: + temporal.create_dataset(key, data=assembled[key]) + frame_group = group.create_group("frame_info") + for key in FRAME_INFO_KEYS: + frame_group.create_dataset(key, data=assembled[key]) + group.create_group("global").create_dataset( + "F", data=assembled["global_f"] + ) + vis = group.create_group("visualizations") + for key in ("act_im", "mean_im", "act_im_peaks"): + vis.create_dataset(key, data=assembled[key]) + rois = assembled["user_rois"] + roi_group = group.create_group("user_rois") + roi_group.attrs["annotation_enabled"] = bool(path.annotation_enabled) + for key in ("mask", "F", "Fsvd"): + roi_group.create_dataset(key, data=rois[key]) + roi_group.create_dataset( + "labels", + data=np.asarray(rois["labels"], dtype=object).reshape(-1, 1), + dtype=h5py.string_dtype(encoding="utf-8"), + ) + + +def write_summary(summary: ExperimentSummary, path: Union[str, Path]) -> None: + """Persist models, retaining existing params, flags and unrelated paths. + + Path identity comes exclusively from PathSummary.path_index. Required + source/ROI arrays and supplied optional trace shapes are validated before + replacing each existing path group. Disabled annotations still emit empty + ROI datasets; their state is recorded in a group attribute. + """ + indexes = [item.path_index for item in summary.paths] + if len(set(indexes)) != len(indexes): + raise ValueError("Summary contains duplicate path_index values") + with h5py.File(path, "a") as f: + if "row_major" not in f: + f["row_major"] = 1 + if "params" not in f: + write_dict_to_h5group( + f.create_group("params"), + summary.params or {}, + ) + for item in summary.paths: + _write_path_group(f, item) + + +def read_summary(path: Union[str, Path]) -> ExperimentSummary: + """Read emitted band paths eagerly, respecting row_major and saved signs. + + Unrecognized root groups are ignored. Paths are sorted by their numeric + identity, not lexical order. This is not a general standard-SILo codec. + Legacy missing optional traces remain None; without annotation metadata, + ROI group presence retains its historical enabled/disabled meaning. + """ + raw = load_struct_h5(path) + names = sorted( + (key for key in raw if key.startswith("Path") and key[4:].isdigit()), + key=lambda key: int(key[4:]), + ) + # The shared struct loader intentionally ignores attributes. Legacy files + # used group presence alone to distinguish disabled/empty annotations. + with h5py.File(path, "r") as f: + annotation_enabled = { + name: ( + bool( + f[name]["user_rois"].attrs.get("annotation_enabled", True) + ) + if "user_rois" in f[name] + else False + ) + for name in names + } + paths = [] + for name in names: + group = raw[name] + sources = group["sources"] + rois = group.get("user_rois") + assembled = { + "Z_depths": group["Z_depths"], + **sources["spatial"], + **sources["temporal"], + **group["frame_info"], + **group["visualizations"], + "global_f": group["global"]["F"], + "user_rois": rois, + "annotation_enabled": annotation_enabled[name], + } + paths.append(_decode_path(assembled, int(name[4:]) - 1)) + return ExperimentSummary(paths=paths, params=raw.get("params")) diff --git a/src/giant_python/io/hdf5.py b/src/giant_python/io/hdf5.py index 44a2ee8..9e3fedd 100644 --- a/src/giant_python/io/hdf5.py +++ b/src/giant_python/io/hdf5.py @@ -1,11 +1,12 @@ -"""Generic struct <-> HDF5 (de)serialization (port of load/saveStructFromH5). +"""MATLAB-style HDF5 decoding and experiment-parameter group writing. This module is the foundation of MATLAB interoperability: the model -``from_h5`` / ``to_h5`` methods build on these helpers. Round-trip tests -against real MATLAB-produced ``.h5`` files are the acceptance criterion. +``from_h5`` / ``to_h5`` methods build on these helpers. + +Serialization conventions +------------------------- +Shared with GIAnT-MATLAB; see the package README. -Serialization conventions (shared with GIAnT-MATLAB; see the package README) ------------------------------------------------------------------------- * Writers set ``/row_major`` to ``1`` (Python / h5py) or ``0`` (MATLAB). * Dimension tuples in the README match h5py ``shape`` when ``row_major=1``. * Vectors documented as ``1 x N`` or ``N x 1`` are written as rank-2 @@ -15,7 +16,93 @@ """ from pathlib import Path -from typing import Union +from typing import Union, cast + +import h5py +import numpy as np + + +def _is_string_dataset(ds: h5py.Dataset) -> bool: + """Return whether an h5py dataset holds (fixed/variable-length) strings.""" + if h5py.check_string_dtype(ds.dtype) is not None: + return True + return ds.dtype.kind in ("S", "U", "O") + + +def _decode_one(x) -> str: + """Decode a single bytes/str scalar to ``str``.""" + if isinstance(x, (bytes, np.bytes_)): + return x.decode("utf-8", "replace") + return str(x) + + +def _decode_strings(val): + """Decode a scalar or ndarray of (variable-length) strings to ``str``. + + Parameters + ---------- + val : bytes, str, or ndarray + Raw value read from a string dataset. + + Returns + ------- + str or ndarray of object + Decoded scalar string, or an object array of decoded strings. + """ + if isinstance(val, (bytes, str, np.bytes_, np.str_)): + return _decode_one(val) + arr = np.asarray(val, dtype=object) + if arr.size == 0: + return arr + return np.vectorize(_decode_one, otypes=[object])(arr) + + +def _read_dataset(ds: h5py.Dataset, row_major: bool): + """Read one dataset in MATLAB ``size()`` orientation. + + Scalar strings/paths (e.g. ``datadr``) are returned as plain ``str``; + string grids (e.g. ``filename``, ``fn_adata``) stay as object arrays with + MATLAB shape. Column-major files have their 2-D+ axes transposed to recover + the README orientation. + + Parameters + ---------- + ds : h5py.Dataset + The dataset to read. + row_major : bool + Whether the file was written row-major (``True``) or column-major. + + Returns + ------- + object + The decoded value (``str``, scalar, or ndarray). + """ + val = ds[()] + if _is_string_dataset(ds): + val = _decode_strings(val) + if isinstance(val, np.ndarray): + if (not row_major) and val.ndim >= 2: + val = val.T + if val.size == 1: + return val.reshape(-1)[0] + return val + val = np.asarray(val) + if (not row_major) and val.ndim >= 2: + val = val.T + return val + + +def _read_group(grp: h5py.Group, row_major: bool) -> dict: + """Read an h5py group into a nested dict (skipping ``row_major``).""" + out = {} + for key, item in grp.items(): + if key == "row_major": + continue + if isinstance(item, h5py.Group): + out[key] = _read_group(item, row_major) + else: + out[key] = _read_dataset(item, row_major) + return out def load_struct_h5(path: Union[str, Path]) -> dict: @@ -25,6 +112,11 @@ def load_struct_h5(path: Union[str, Path]) -> dict: ``row_major`` / axis-permutation conventions required for MATLAB compatibility. Corresponds to loadStructFromH5.m in GIAnT-MATLAB. + Groups become nested dicts, string datasets become ``str`` (scalars) or + object arrays (grids), and numeric datasets are returned in MATLAB + ``size()`` orientation. The ``/row_major`` flag (absent => column-major) + selects whether 2-D+ axes are transposed. + Parameters ---------- path : str or Path @@ -35,24 +127,76 @@ def load_struct_h5(path: Union[str, Path]) -> dict: dict Nested dictionary mirroring the HDF5 group hierarchy. """ - raise NotImplementedError + with h5py.File(path, "r") as f: + row_major = False + if "row_major" in f: + flag = cast(h5py.Dataset, f["row_major"]) + row_major = bool(np.asarray(flag[()]).reshape(-1)[0]) + return _read_group(f, row_major) -def save_struct_h5(struct: dict, path: Union[str, Path]) -> None: - """Write a (possibly nested) dict to an HDF5 file as a MATLAB-style struct. +def to_serializable(val): + """Convert nested NumPy values to ordinary Python parameter values.""" + if isinstance(val, dict): + return {k: to_serializable(v) for k, v in val.items()} + if isinstance(val, (list, tuple)): + return type(val)(to_serializable(v) for v in val) + if isinstance(val, np.ndarray): + return val.tolist() + if isinstance(val, np.generic): + return val.item() + return val - Recursively writes nested dictionaries to HDF5 groups/datasets so that the - result is readable by GIAnT-MATLAB. Corresponds to saveStructToH5.m in - GIAnT-MATLAB. - Sets ``/row_major = 1``. Rank-2 vector shapes (``1 x N`` / ``N x 1``) are - preserved on disk; singleton dimensions are not squeezed. +def _write_preserved_array(grp: h5py.Group, key: str, value) -> None: + """Write a NumPy value with its original shape and numeric dtype.""" + arr = np.asarray(value) + if arr.dtype.kind in ("U", "S", "O"): + grp.create_dataset( + key, + data=np.asarray(_decode_strings(arr), dtype=object), + dtype=h5py.string_dtype(encoding="utf-8"), + ) + else: + grp.create_dataset(key, data=arr) - Parameters - ---------- - struct : dict - Nested dictionary to serialize. - path : str or Path - Destination path for the HDF5 file. + +def _write_sequence(grp: h5py.Group, key: str, value) -> None: + """Skip empty sequences and preserve persisted element stringification.""" + arr = np.asarray(value) + if arr.size == 0: + return + if arr.dtype.kind in ("U", "S", "O"): + grp.create_dataset( + key, + data=np.array([str(x) for x in value], dtype=object), + dtype=h5py.string_dtype(encoding="utf-8"), + ) + else: + grp.create_dataset(key, data=arr) + + +def write_dict_to_h5group(grp: h5py.Group, d: dict) -> None: + """Write params, skipping empty sequences/None and stringifying unknowns. + + This is deliberately NOT a general lossless struct serializer. Its policy + preserves the existing experiment-summary params contract. Decoded arrays + retain their dtype/rank (including uint8 boolean parameter scalars). """ - raise NotImplementedError + str_dt = h5py.string_dtype(encoding="utf-8") + for key, v in d.items(): + key = str(key) + if isinstance(v, dict): + write_dict_to_h5group(grp.create_group(key), v) + elif isinstance(v, (np.ndarray, np.generic)): + _write_preserved_array(grp, key, v) + elif isinstance(v, bool): + grp.create_dataset(key, data=np.uint8(v)) + elif isinstance(v, str): + grp.create_dataset(key, data=v, dtype=str_dt) + elif isinstance(v, (int, float)): + grp.create_dataset(key, data=v) + elif isinstance(v, (list, tuple)): + _write_sequence(grp, key, v) + elif v is not None: + grp.create_dataset(key, data=str(v), dtype=str_dt) diff --git a/src/giant_python/io/slap2.py b/src/giant_python/io/slap2.py index b762d2f..61d8223 100644 --- a/src/giant_python/io/slap2.py +++ b/src/giant_python/io/slap2.py @@ -1,99 +1,77 @@ -"""SLAP2 binary data I/O (``.dat`` / ``.meta``) and online-motion retrieval. +"""SLAP2 binary reader access and saved alignment-data decoding. -Intended home for the SLAP2 data-file reader (equivalent of MATLAB's -``slap2.Slap2DataFile``) plus the online motion-offset retrieval ported from -getOnlineMotion.m. Also hosts the integration (band-scan) superpixel readers -used by the integration source-extraction backend. +Band geometry and superpixel reduction belong to the extraction.band backend, +not this raw acquisition adapter. """ -from typing import Optional, Tuple +from pathlib import Path +from typing import Any, Protocol, Union import numpy as np +from .hdf5 import load_struct_h5 -def ref_pixs_to_drc( - ref_pixs: np.ndarray, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, -) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - """Map flat reference pixel indices to DMD (depth, column, row) indices. - Port of ``ref_pixs_to_drc`` from ``extractSLAP2IntegrationSources.py``. +def _reshape_1d(src: dict, key: str): + """Flatten an optional raw alignment vector without altering its sign.""" + value = src.get(key) + return None if value is None else np.asarray(value).reshape(-1) - Parameters - ---------- - ref_pixs : ndarray of int - Flat reference-pixel indices. - dmd_pixels_per_column, dmd_pixels_per_row : int - DMD geometry. - Returns - ------- - ref_d, ref_c, ref_r : ndarray of int - Depth, column, and row indices. +def load_alignment_data_h5(path: Union[str, Path]) -> dict: + """Read alignment vectors with the original saved displacement sign. + + Flatten DS/offline/online vectors and expose numChannels/alignHz scalars. + No coordinate normalization or motion negation belongs in this reader. + Band numerical consumers must use extraction.band.inputs instead. """ - raise NotImplementedError - - -def read_integration_trial_data( - data_file: object, - ds_frames: np.ndarray, - super_pixel_ids: np.ndarray, - activity_channel: Optional[int] = None, -) -> Tuple[np.ndarray, np.ndarray]: - """Read superpixel-binned activity for one integration (band-scan) trial. - - Accumulates weighted line data into per-superpixel time series over the - downsampled frame grid. Port of ``get_trial_data`` from - ``extractSLAP2IntegrationSources.py``. - - Parameters - ---------- - data_file : object - SLAP2 data-file object exposing line/superpixel access. - ds_frames : ndarray - Downsampled frame (line-index) grid. - super_pixel_ids : ndarray - Superpixel id lookup for the DMD being read. - activity_channel : int, optional - Channel to read; ``None`` reads all channels. - - Returns - ------- - data : ndarray of shape (n_superpixels, n_ds_frames) - Weighted-mean superpixel activity. - data_count : ndarray of shape (n_superpixels, n_ds_frames) - Accumulated weights (for normalization). + data = load_struct_h5(path) + online = data.get("slap2", {}) or {} + return { + "DSframes": _reshape_1d(data, "DSframes"), + "motionDSr": _reshape_1d(data, "motionDSr"), + "motionDSc": _reshape_1d(data, "motionDSc"), + "motionDSz": _reshape_1d(data, "motionDSz"), + "onlineYshift": _reshape_1d(online, "onlineMotionYshift"), + "onlineXshift": _reshape_1d(online, "onlineMotionXshift"), + "onlineZshift": _reshape_1d(online, "onlineMotionZshift"), + "numChannels": int(np.asarray(data["numChannels"]).reshape(-1)[0]), + "alignHz": float(np.asarray(data["alignHz"]).reshape(-1)[0]), + } + + +class Slap2Reader(Protocol): + """Minimal raw-reader surface consumed by the existing band reducer. + + Line/cycle indexes passed to getLineData remain 1-based. The owning + caller retains the reader for the entire batched reduction; this protocol + deliberately promises neither a close method nor context-manager support. """ - raise NotImplementedError - - -def get_online_motion( - data_file: object, - ds_frames: np.ndarray, -) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - """Retrieve online motion correction offsets from a SLAP2 data file. - - Reads the per-frame X, Y, and Z motion correction shifts that were - computed online during SLAP2 acquisition by inspecting per-line headers - in the data file. Corresponds to getOnlineMotion.m in GIAnT-MATLAB. - - Parameters - ---------- - data_file : object - SLAP2 data file object exposing line-header access (e.g. a - ``Slap2DataFile`` instance). - ds_frames : ndarray of int - Line indices of the downsampled frames for which to retrieve - offsets. - - Returns - ------- - online_x_shift : ndarray of shape (n_frames,) - Online motion correction X (column) shifts, in pixels. - online_y_shift : ndarray of shape (n_frames,) - Online motion correction Y (row) shifts, in pixels. - online_z_shift : ndarray of shape (n_frames,) - Online motion correction Z (axial) shifts, in micrometres. + + header: dict + metaData: Any + numCycles: int + lineDataNumElements: Any + lineSuperPixelIDs: Any + lineFastZIdxs: Any + + def getLineData(self, line_indices, cycle_indices, channels=None): + """Read a batch using the vendor reader's unchanged contract.""" + + +def open_slap2_file(path: Union[str, Path]) -> Slap2Reader: + """Lazily open DataFile/MultiDataFiles; return ownership to the caller. + + Retains the original CYCLE detection and module reload. No reads, close, + copying, or early resource disposal are performed here. """ - raise NotImplementedError + import importlib + import re + + import slap2_utils + + importlib.reload(slap2_utils) + path = str(path) + if re.search(r"CYCLE\d+", path): + return slap2_utils.MultiDataFiles(path) + return slap2_utils.DataFile(path) diff --git a/src/giant_python/io/tiff.py b/src/giant_python/io/tiff.py index 30c3177..43625f6 100644 --- a/src/giant_python/io/tiff.py +++ b/src/giant_python/io/tiff.py @@ -1,61 +1,17 @@ -"""ScanImage TIFF reading (port of ScanImageTiffWrapper / ...DataWrapper). - -Also the intended home for the fast TIFF writers (ports of Fast_Tiff_Write / -Fast_BigTiff_Write) used to emit registered movies. -""" +"""Native TIFF array reading without acquisition-specific transformations.""" from pathlib import Path -from typing import List, Tuple, Union +from typing import Union import numpy as np -def scanimagetiff_wrapper( - fn: Union[str, Path], -) -> Tuple[np.ndarray, List[str]]: - """Read image data and per-frame metadata from a ScanImage TIFF file. - - Opens the file with a ScanImage TIFF reader, extracts the image stack - via :func:`scanimagetiff_data_wrapper`, and returns the per-frame - metadata strings. Corresponds to ScanImageTiffWrapper.m in GIAnT-MATLAB. +def read_tiff(path: Union[str, Path]) -> np.ndarray: + """Read a TIFF eagerly, preserving native dtype, rank and page ordering. - Parameters - ---------- - fn : str or Path - Path to the ScanImage TIFF file. - - Returns - ------- - data : ndarray - Image stack read from the file. - meta : str - File-level metadata string embedded in the TIFF. + Reference intensity scaling, channel reshaping and PSF cropping are + caller policies, not generic TIFF decoding. Import tifffile only on use. """ - raise NotImplementedError - + import tifffile -def scanimagetiff_data_wrapper( - reader: object, - tif_file: Union[str, Path], -) -> np.ndarray: - """Read a ScanImage TIFF stack, falling back to standard TIFF reading. - - Attempts to read image data via *reader*. On failure, reads all pages of - the TIFF using standard image I/O. Corresponds to - ScanImageTiffDataWrapper.m in GIAnT-MATLAB. - - Parameters - ---------- - reader : object - ScanImage TIFF reader object (e.g. from the ``scanimage-tiff-reader`` - Python package). - tif_file : str or Path - Path to the ScanImage TIFF file used to construct *reader*. - - Returns - ------- - ndarray - Image stack in the native on-disk datatype, shaped - ``(W, H, n_frames)`` to match ScanImage convention. - """ - raise NotImplementedError + return tifffile.imread(path) diff --git a/src/giant_python/io/trial_table.py b/src/giant_python/io/trial_table.py new file mode 100644 index 0000000..bc2b3f3 --- /dev/null +++ b/src/giant_python/io/trial_table.py @@ -0,0 +1,48 @@ +"""Faithful trial-table schema decoding, without runtime file discovery.""" + +from pathlib import Path +from typing import Optional, Union + +import numpy as np + +from .hdf5 import load_struct_h5 + + +def slap2_info_from_raw(slap2: Optional[dict]): + """Decode recognized SLAP2 metadata; keep absent metadata absent.""" + from ..models.trial_table import Slap2Info + + if not slap2: + return None + names = ( + "ref_stack", + "first_line", + "last_line", + "trial_start_time_inferred", + "trial_end_time_from_pc", + ) + return Slap2Info(**{key: slap2[key] for key in names if key in slap2}) + + +def read_trial_table(path: Union[str, Path]): + """Return an eager TrialTable with source provenance, no path resolution. + + ``source_path`` records the input filename and is not part of disk data. + Numeric ranks are retained; the filename string grid is always rank two. + """ + from ..models.trial_table import TrialTable + + raw = load_struct_h5(path) + return TrialTable( + datadr=Path(raw["datadr"]) if raw.get("datadr") else None, + savedr=Path(raw["savedr"]) if raw.get("savedr") else None, + filename=np.atleast_2d( + np.asarray(raw.get("filename", []), dtype=object) + ), + true_trial_ix=raw.get("true_trial_ix", []), + epoch=raw.get("epoch", []), + slap2_info=slap2_info_from_raw(raw.get("slap2_info")), + motion_correction=raw.get("motion_correction"), + source_extraction=raw.get("source_extraction"), + source_path=Path(path).absolute(), + ) diff --git a/src/giant_python/math/__init__.py b/src/giant_python/math/__init__.py deleted file mode 100644 index e0d7c6a..0000000 --- a/src/giant_python/math/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Numerical kernels for GIAnT-Python. - -Pure, side-effect-free functions ported closely from GIAnT-MATLAB so they can -be validated against MATLAB outputs function-by-function. Watch MATLAB<->NumPy -axis-order and 0- vs 1-based indexing differences when porting. -""" - -from .baseline import compute_f0 -from .interpolation import interp_frame -from .peaks import ( - detect_peaks_2d, - gaussian_peaks_integrated, - get_act_im_peaks, -) -from .registration import ( - dft_register_clipped, - xcorr2_nans, - xcorr2_nans3d, - xcorr2_nans_weighted, -) - -__all__ = [ - "compute_f0", - "detect_peaks_2d", - "dft_register_clipped", - "gaussian_peaks_integrated", - "get_act_im_peaks", - "interp_frame", - "xcorr2_nans", - "xcorr2_nans3d", - "xcorr2_nans_weighted", -] diff --git a/src/giant_python/math/baseline.py b/src/giant_python/math/baseline.py deleted file mode 100644 index 7550644..0000000 --- a/src/giant_python/math/baseline.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Fluorescence baseline (F0) estimation. - -Shared by both source-extraction backends. Ports ``computeF0.m`` (GIAnT-MATLAB) -/ ``compute_f0`` from ``extractSLAP2IntegrationSources.py``. -""" - -import numpy as np - - -def compute_f0( - f_in: np.ndarray, - denoise_window: int, - hull_window: int, -) -> np.ndarray: - """Estimate the slowly varying baseline F0 of fluorescence traces. - - Median-filters to reduce noise, applies a rolling convex-hull-like min - envelope on decimated grids, discards doubtful samples near NaNs, then - smooths/fills and PCHIP-interpolates back to the full time grid. - - Parameters - ---------- - f_in : ndarray of shape (T, ...) - Fluorescence with time along axis 0; NaNs allowed. - denoise_window : int - Median-filter window, in time samples. - hull_window : int - Window controlling the convex-hull-like min envelope. - - Returns - ------- - ndarray - Baseline F0, same shape as *f_in*. - """ - raise NotImplementedError diff --git a/src/giant_python/math/deconv.py b/src/giant_python/math/deconv.py deleted file mode 100644 index 0f2205b..0000000 --- a/src/giant_python/math/deconv.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Temporal deconvolution of fluorescence traces into events (SILo).""" - -import numpy as np - - -def deconvolve_trace( - trace: np.ndarray, - tau_s: float, - frame_rate_hz: float, -) -> np.ndarray: - """Deconvolve a fluorescence trace into an inferred event train. - - Removes the indicator decay kinetics (single-exponential with time - constant *tau_s*) to recover an event/spike train. Corresponds to the - deconvolution step in GIAnT-MATLAB source extraction. - - Parameters - ---------- - trace : ndarray of shape (n_frames,) - Denoised dF/F trace. - tau_s : float - Indicator decay time constant, in seconds. - frame_rate_hz : float - Acquisition/analysis frame rate, in Hz. - - Returns - ------- - ndarray of shape (n_frames,) - Inferred non-negative event train. - """ - raise NotImplementedError diff --git a/src/giant_python/math/interpolation.py b/src/giant_python/math/interpolation.py deleted file mode 100644 index de6ef52..0000000 --- a/src/giant_python/math/interpolation.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Frame interpolation with photon-count weighting (port of interpFrame.m).""" - -from typing import Tuple - -import numpy as np - - -def interp_frame( - m1: np.ndarray, - view_c: np.ndarray, - view_r: np.ndarray, - freshness: np.ndarray, -) -> Tuple[np.ndarray, np.ndarray]: - """Interpolate an image and compute the per-pixel variance multiplier. - - Performs bilinear interpolation of a 2-D image at arbitrary coordinates, - weighting each contributing pixel by its *freshness* (effective photon - count). Also returns a variance multiplier proportional to the local - measurement noise. Corresponds to interpFrame.m in GIAnT-MATLAB. - - Parameters - ---------- - m1 : ndarray of shape (H, W) - Source image to interpolate. - view_c : ndarray - Column coordinates at which to evaluate the interpolation. - view_r : ndarray - Row coordinates at which to evaluate the interpolation. - freshness : ndarray of shape (H, W) - Effective sample count for each pixel of *m1*. - - Returns - ------- - im : ndarray - Interpolated image evaluated at (*view_r*, *view_c*). - v_im : ndarray - Variance multiplier; multiply a pixel intensity by this value to - obtain a quantity proportional to the pixel's variance. - """ - raise NotImplementedError diff --git a/src/giant_python/math/nmf.py b/src/giant_python/math/nmf.py deleted file mode 100644 index 1feded3..0000000 --- a/src/giant_python/math/nmf.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Non-negative matrix factorization for source decomposition (SILo).""" - -from typing import Tuple - -import numpy as np - - -def nmf_decompose( - activity: np.ndarray, - n_sources: int, - lambda_: float = 0.1, - phi: float = 0.0, - n_iter: int = 100, -) -> Tuple[np.ndarray, np.ndarray]: - """Decompose pixel activity into spatial profiles and temporal traces. - - Sparse, spatially regularized non-negative matrix factorization used by - the SILo source-extraction stage. Corresponds to the NMF refinement step - in GIAnT-MATLAB source extraction. - - Parameters - ---------- - activity : ndarray of shape (n_pixels, n_frames) - Per-pixel activity matrix. - n_sources : int - Number of sources to extract. - lambda_ : float - Sparsity weight. - phi : float - Spatial-coherence weight. - n_iter : int - Number of update iterations. - - Returns - ------- - spatial : ndarray of shape (n_pixels, n_sources) - Spatial profiles. - temporal : ndarray of shape (n_sources, n_frames) - Temporal traces. - """ - raise NotImplementedError diff --git a/src/giant_python/math/peaks.py b/src/giant_python/math/peaks.py deleted file mode 100644 index dd357d4..0000000 --- a/src/giant_python/math/peaks.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Activity-image peak detection via integrated-Gaussian fitting. - -Shared by both source-extraction backends (standard and integration). Ports -``getActImPeaks.m`` (GIAnT-MATLAB) / ``get_act_im_peaks`` and its -integrated-Gaussian helpers from ``extractSLAP2IntegrationSources.py``. -""" - -from typing import Optional - -import numpy as np - - -def gaussian_peaks_integrated( - theta: np.ndarray, - yx: np.ndarray, -) -> np.ndarray: - """Evaluate integrated isotropic 2-D Gaussians over unit pixels. - - Port of ``gaussianPeaksIntegrated`` from getActImPeaks.m. - - Parameters - ---------- - theta : ndarray of shape (N, 4) - Per-peak parameters ``[amp, mu_y, mu_x, sigma]``. - yx : ndarray of shape (M, 2) - Pixel-centre coordinates ``[y, x]``. - - Returns - ------- - ndarray of shape (M,) - Predicted values at the given pixel centres. - """ - raise NotImplementedError - - -def detect_peaks_2d( - act_im_2d: np.ndarray, - exclusion_mask: np.ndarray, - mu_bg: float, - sigma_bg: float, - peak_thresh: float, - peak_th: float, -) -> np.ndarray: - """Detect Gaussian peaks in a single activity-image plane. - - Performs initial local-maximum detection followed by iterative - residual peak finding with bounded Levenberg-Marquardt curve fitting. - - Parameters - ---------- - act_im_2d : ndarray of shape (H, W) - One plane of the activity image; may contain NaNs. - exclusion_mask : ndarray of shape (H, W) - Pixels to exclude from detection. - mu_bg, sigma_bg : float - Background mean and (MAD-normalised) standard deviation. - peak_thresh : float - Absolute amplitude threshold (``mu_bg + peak_th * sigma_bg``). - peak_th : float - Threshold in MAD-normalised standard deviations. - - Returns - ------- - ndarray of shape (N, 4) - Fitted peaks ``[amp, mu_y, mu_x, sigma]`` (``(0, 4)`` if none). - """ - raise NotImplementedError - - -def get_act_im_peaks( - act_im: np.ndarray, - peak_th: float = 3.0, - exclusion_mask: Optional[np.ndarray] = None, -) -> np.ndarray: - """Find Gaussian peaks across a 3-D ``(Z, H, W)`` activity image. - - Background statistics and the detection threshold are computed once - across all planes for uniform sensitivity, then each plane is processed - independently via :func:`detect_peaks_2d`. - - Parameters - ---------- - act_im : ndarray of shape (Z, H, W) - Activity image; may contain NaNs. - peak_th : float - Threshold in MAD-normalised standard deviations. - exclusion_mask : ndarray, optional - Mask broadcastable to ``(Z, H, W)`` of pixels to exclude. - - Returns - ------- - ndarray of shape (N, 3) - Source seeds ``[z, mu_y, mu_x]`` (``(0, 3)`` if none detected). - """ - raise NotImplementedError diff --git a/src/giant_python/math/registration.py b/src/giant_python/math/registration.py deleted file mode 100644 index 3cf2eab..0000000 --- a/src/giant_python/math/registration.py +++ /dev/null @@ -1,124 +0,0 @@ -"""FFT-based and NaN-aware cross-correlation registration kernels. - -Ports of dftregistration_clipped.m, xcorr2_nans.m, xcorr2_nans3d.m, and -xcorr2_nans_weighted.m from GIAnT-MATLAB. -""" - -from typing import Tuple - -import numpy as np - - -def xcorr2_nans_weighted( - frame: np.ndarray, - freshness: np.ndarray, - template: np.ndarray, - shifts_center: np.ndarray, - d_shift: float, -) -> Tuple[np.ndarray, float]: - """Perform weighted, NaN-aware local normalised cross-correlation. - - Estimates the sub-pixel translation between *frame* and *template* by - evaluating the weighted normalised cross-correlation over a local search - window of +/-*d_shift* pixels centred on *shifts_center*. Corresponds to - xcorr2_nans_weighted.m in GIAnT-MATLAB. - - Parameters - ---------- - frame : ndarray of shape (H, W) - Floating image to align; may contain NaNs. - freshness : ndarray of shape (H, W) - Per-pixel effective sample count (photon-count weighting) for - *frame*. - template : ndarray of shape (H, W) - Fixed reference image. - shifts_center : ndarray of shape (2,) - ``[row, col]`` centre of the local search window. - d_shift : float - Half-width of the search window in pixels (rounded to integer). - - Returns - ------- - motion : ndarray of shape (2,) - Estimated ``[row, col]`` shift of *frame* relative to *template*. - r : float - Peak normalised cross-correlation coefficient. - """ - raise NotImplementedError - - -def xcorr2_nans( - frame: np.ndarray, - template: np.ndarray, -) -> np.ndarray: - """Compute NaN-aware 2-D normalised cross-correlation. - - Port of xcorr2_nans.m in GIAnT-MATLAB. - - Parameters - ---------- - frame : ndarray of shape (H, W) - Image to align; may contain NaNs. - template : ndarray of shape (H, W) - Fixed reference image. - - Returns - ------- - ndarray - Cross-correlation surface. - """ - raise NotImplementedError - - -def xcorr2_nans3d( - frame: np.ndarray, - template: np.ndarray, -) -> np.ndarray: - """Compute NaN-aware 3-D normalised cross-correlation. - - Port of xcorr2_nans3d.m in GIAnT-MATLAB. - - Parameters - ---------- - frame : ndarray of shape (Z, H, W) - Volume to align; may contain NaNs. - template : ndarray of shape (Z, H, W) - Fixed reference volume. - - Returns - ------- - ndarray - Cross-correlation volume. - """ - raise NotImplementedError - - -def dft_register_clipped( - buf_fft: np.ndarray, - target_fft: np.ndarray, - usfac: int = 1, - max_shift: float = np.inf, -) -> np.ndarray: - """Estimate sub-pixel translation by upsampled DFT cross-correlation. - - Port of dftregistration_clipped.m in GIAnT-MATLAB: efficient sub-pixel - image registration via upsampled DFT, with the shift clipped to - ``+/-max_shift``. - - Parameters - ---------- - buf_fft : ndarray - FFT of the image to register. - target_fft : ndarray - FFT of the reference image. - usfac : int - Upsampling factor (sub-pixel resolution = ``1 / usfac``). - max_shift : float - Maximum allowed shift magnitude, in pixels. - - Returns - ------- - ndarray - Estimated ``[row, col]`` shift. - """ - raise NotImplementedError diff --git a/src/giant_python/math/variance.py b/src/giant_python/math/variance.py deleted file mode 100644 index fa15e11..0000000 --- a/src/giant_python/math/variance.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Poisson variance modeling and activity-image construction (SILo).""" - -import numpy as np - - -def activity_image( - movie: np.ndarray, - photon_scale: float, -) -> np.ndarray: - """Build a heteroscedastic-noise-normalized activity image from a movie. - - Uses a Poisson-based variance model (variance proportional to mean photon - count, via *photon_scale*) to weight temporal fluctuations, yielding an - image that highlights active sources. Corresponds to the activity-image - construction in GIAnT-MATLAB source extraction. - - Parameters - ---------- - movie : ndarray of shape (n_frames, H, W) - Motion-corrected movie. - photon_scale : float - Photons per digital unit, used to scale the variance model. - - Returns - ------- - ndarray of shape (H, W) - Activity image. - """ - raise NotImplementedError diff --git a/src/giant_python/models/__init__.py b/src/giant_python/models/__init__.py index 3257a91..90ce112 100644 --- a/src/giant_python/models/__init__.py +++ b/src/giant_python/models/__init__.py @@ -1,27 +1,33 @@ """Typed data models for GIAnT-Python. -These dataclasses are the single source of truth for the on-disk HDF5 schema -shared with GIAnT-MATLAB. Each model knows how to (de)serialize itself via -``from_h5`` / ``to_h5`` so that files written by either toolbox remain -interchangeable. +Scientific configuration is distinct from run and annotation policy. +Implemented model codecs delegate to shared IO. """ -from .alignment import AlignmentData -from .experiment import ( +from .experiment_summary import ( ExperimentSummary, + FrameInfo, + PathSummary, Source, UserRoi, Visualizations, ) -from .params import AlignParams, SiloParams, set_params +from .params import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + resolve_band_options, +) from .trial_table import Slap2Info, TrialTable __all__ = [ - "AlignmentData", - "AlignParams", + "AnnotationOptions", + "BandSiloParams", + "ExecutionOptions", "ExperimentSummary", - "set_params", - "SiloParams", + "FrameInfo", + "PathSummary", + "resolve_band_options", "Slap2Info", "Source", "TrialTable", diff --git a/src/giant_python/models/alignment.py b/src/giant_python/models/alignment.py deleted file mode 100644 index 4499eb9..0000000 --- a/src/giant_python/models/alignment.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Alignment data model (mirrors the ``aData`` struct / *_ALIGNMENTDATA.h5).""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from pathlib import Path -from typing import Optional, Union - -import numpy as np - - -@dataclass -class AlignmentData: - """Per-trial motion-correction output written to ``*_ALIGNMENTDATA.h5``. - - Mirrors the ``aData`` struct in GIAnT-MATLAB. Field presence depends on - which registration backend wrote the file (StripRegistration, - MultiRoiRegistration, or BandRegistration); see the GIAnT-python README - Pipeline Outputs section. - - Attributes - ---------- - num_channels : int or None - Number of channels in the recording (``numChannels``). - frametime : float or None - Seconds per downsampled frame. - align_hz : float or None - Frame rate (Hz) at which alignment was performed (``alignHz``). - motion_ds_c, motion_ds_r : ndarray or None - Per-downsampled-frame column / row shifts (``motionDSc`` / - ``motionDSr``). - motion_ds_z : ndarray or None - Per-downsampled-frame Z shift (``motionDSz``); always written by - BandRegistration; by MultiRoiRegistration only when - ``refStackTemplate`` is enabled; never by StripRegistration. - mean_im : ndarray or None - Per-channel mean registered image, shaped ``(channels, rows, cols)`` - (``meanIM``); not written by BandRegistration. - rec_neg_err : ndarray or None - Reconstruction error per frame (``recNegErr``); not written by - BandRegistration. - motion_c, motion_r : ndarray or None - Column / row shifts upsampled to raw frame rate (``motionC`` / - ``motionR``); StripRegistration / Bergamo only. - motion_z : ndarray or None - Reserved (``motionZ``); not written by any current backend. - brightness_ds : ndarray or None - Per-channel brightness/scaling at the selected motion shift, shaped - ``(nDSframes, channels)`` (``brightnessDS``); BandRegistration only. - log_likelihood_ds : ndarray or None - Peak log-likelihood of the motion match per downsampled frame - (``logLikelihoodDS``); BandRegistration only. - ds_frames : ndarray or None - 1-indexed line indices of the downsampled frames (``DSframes``); - SLAP2 only (MultiRoiRegistration and BandRegistration). - registration_failed : bool - Whether registration failed for this trial (``registrationFailed``); - SLAP2 only. - slap2 : dict or None - SLAP2-only sub-struct. Always may include - ``onlineMotion{X,Y,Z}shift``; MultiRoiRegistration also writes - ``varFacDS``, ``Z_depths``, ``cropRow``/``cropCol``, - ``viewC``/``viewR``, ``trimRows``/``trimCols``. - """ - - num_channels: Optional[int] = None - frametime: Optional[float] = None - align_hz: Optional[float] = None - motion_ds_c: Optional[np.ndarray] = None - motion_ds_r: Optional[np.ndarray] = None - motion_ds_z: Optional[np.ndarray] = None - mean_im: Optional[np.ndarray] = None - rec_neg_err: Optional[np.ndarray] = None - motion_c: Optional[np.ndarray] = None - motion_r: Optional[np.ndarray] = None - motion_z: Optional[np.ndarray] = None - brightness_ds: Optional[np.ndarray] = None - log_likelihood_ds: Optional[np.ndarray] = None - ds_frames: Optional[np.ndarray] = None - registration_failed: bool = False - slap2: Optional[dict] = field(default=None) - - @classmethod - def from_h5(cls, path: Union[str, Path]) -> "AlignmentData": - """Load alignment data from a ``*_ALIGNMENTDATA.h5`` file. - - Parameters - ---------- - path : str or Path - Path to the alignment data file. - - Returns - ------- - AlignmentData - The deserialized alignment data. - """ - raise NotImplementedError - - def to_h5(self, path: Union[str, Path]) -> None: - """Write alignment data to a ``*_ALIGNMENTDATA.h5`` file. - - Parameters - ---------- - path : str or Path - Destination path for the alignment data file. - """ - raise NotImplementedError diff --git a/src/giant_python/models/experiment.py b/src/giant_python/models/experiment_summary.py similarity index 51% rename from src/giant_python/models/experiment.py rename to src/giant_python/models/experiment_summary.py index 067474d..b751c54 100644 --- a/src/giant_python/models/experiment.py +++ b/src/giant_python/models/experiment_summary.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from pathlib import Path from typing import Optional, Union @@ -22,8 +22,11 @@ class Source: coords : ndarray or None 0-indexed ``[z, y, x]`` location. df_ls, df_denoised, events : ndarray or None - Least-squares dF, denoised dF, and deconvolved events, - shaped ``(channels, frames)``. + Least-squares dF after baseline re-estimation, the fitted convolution + trace (without another baseline subtraction), and nonnegative event + amplitudes, shaped ``(channels, frames)``. Missing observations and + unprocessed channels are NaN. Events exclude each trial's initial + state. f0 : ndarray or None Per-channel baseline. snr : float or None @@ -36,7 +39,7 @@ class Source: df_denoised: Optional[np.ndarray] = None events: Optional[np.ndarray] = None f0: Optional[np.ndarray] = None - snr: Optional[float] = None + snr: Optional[Union[float, np.floating]] = None @dataclass @@ -69,7 +72,8 @@ class UserRoi: Rotation angle (ellipse ROIs). fsvd : ndarray or None ROI signal after SVD / projection, shaped ``(channels, total frames)`` - (``experiment_summary.h5`` ``user_rois/Fsvd``). + (``experiment_summary.h5`` ``user_rois/Fsvd``). Band extraction does + not compute this signal; the writer uses NaN when unavailable. f : ndarray or None Raw or baseline-corrected ROI fluorescence, shaped ``(channels, total frames)`` @@ -109,70 +113,86 @@ class Visualizations: @dataclass -class ExperimentSummary: - """Source-extraction output (``exptSummary`` / experiment_summary.h5). +class FrameInfo: + """Eager, typed frame bookkeeping. - The headline result object returned by the extraction stage. Note that - heavy arrays are intended to be disk-backed / lazily loaded; the - :class:`~giant_python.pipeline.pipeline.Pipeline` facade holds a reference - to this object but does not pin its pixel data in memory. + Offline shifts carry saved MATLAB displacement, NOT private band + reference offsets. No conversion occurs in this model or its codec. + Frame line indexes remain 1-based; emitted vectors retain rank two. + ``to_dict`` returns a shallow mapping, sharing the original arrays. + """ - Attributes - ---------- - sources : list of list of Source - Extracted sources, indexed ``[path][source]`` (one inner list per - imaging path / ``Path{n}`` group). - user_rois : list of list of UserRoi - Manually drawn ROIs and their traces, indexed ``[path][roi]``. - visualizations : Visualizations - Summary images (single-path convenience view; see also - per-path structure in ``experiment_summary.h5``). - z_depths : ndarray or None - Z depths per imaging plane (``Path{n}/Z_depths``; SLAP2 only), - shaped ``(fastz,)``. - global_f : ndarray or None - Whole-field fluorescence per channel (``Path{n}/global/F``), shaped - ``(channels, total frames)``. - frame_info : dict or None - Trial/frame bookkeeping for the stitched time series - (``Path{n}/frame_info``): ``offline{X,Y,Z}shifts``, - ``online{X,Y,Z}shifts``, ``trial_num_frames``, ``frame_line_idxs`` - (1-indexed), and ``discard_frames``. - params : dict or None - The SILo parameters used to produce this summary. ``activityChannel`` - is 1-indexed into the recording's channels. + trial_num_frames: Optional[np.ndarray] = None + discard_frames: Optional[np.ndarray] = None + frame_line_idxs: Optional[np.ndarray] = None + offlineXshifts: Optional[np.ndarray] = None + offlineYshifts: Optional[np.ndarray] = None + offlineZshifts: Optional[np.ndarray] = None + onlineXshifts: Optional[np.ndarray] = None + onlineYshifts: Optional[np.ndarray] = None + onlineZshifts: Optional[np.ndarray] = None + + def to_dict(self) -> dict: + """Return all frame fields without copying or changing their arrays.""" + return {item.name: getattr(self, item.name) for item in fields(self)} + + +FRAME_INFO_KEYS = tuple(item.name for item in fields(FrameInfo)) + + +@dataclass +class PathSummary: + """All eager output for one imaging path, identified by a 0-based index. + + Arrays remain NumPy-backed in memory; Source/UserRoi entries may be views + of assembled arrays. ``annotation_enabled`` distinguishes disabled from + enabled-but-empty annotations (saved as an ROI group attribute). Private + empty-array templates retain dimensions and dtypes that cannot be inferred + from empty lists; they never duplicate populated source or ROI tensors. """ - sources: list = field(default_factory=list) - user_rois: list = field(default_factory=list) + path_index: int = 0 + sources: list[Source] = field(default_factory=list) + user_rois: list[UserRoi] = field(default_factory=list) visualizations: Visualizations = field(default_factory=Visualizations) - z_depths: Optional[np.ndarray] = None + frame_info: FrameInfo = field(default_factory=FrameInfo) global_f: Optional[np.ndarray] = None - frame_info: Optional[dict] = None + z_depths: Optional[np.ndarray] = None + annotation_enabled: bool = False + _empty_source_arrays: dict = field(default_factory=dict, repr=False) + _empty_roi_arrays: dict = field(default_factory=dict, repr=False) + + def __post_init__(self): + """Validate path identity without copying any numerical arrays.""" + if ( + not isinstance(self.path_index, (int, np.integer)) + or self.path_index < 0 + ): + raise ValueError( + "path_index must be a nonnegative, 0-based integer" + ) + + +@dataclass +class ExperimentSummary: + """Eager, in-memory extraction output with explicit per-path ownership. + + This is not a lazy or disk-backed model. All imaging data belongs to + entries in ``paths``; only extraction parameters are shared. + """ + + paths: list[PathSummary] = field(default_factory=list) params: Optional[dict] = None @classmethod def from_h5(cls, path: Union[str, Path]) -> "ExperimentSummary": - """Load an experiment summary from experiment_summary.h5. + """Read the currently emitted band-summary schema eagerly.""" + from ..io.experiment_summary import read_summary - Parameters - ---------- - path : str or Path - Path to the experiment_summary.h5 file. - - Returns - ------- - ExperimentSummary - The deserialized experiment summary. - """ - raise NotImplementedError + return read_summary(path) def to_h5(self, path: Union[str, Path]) -> None: - """Write this experiment summary to experiment_summary.h5. - - Parameters - ---------- - path : str or Path - Destination path for the experiment_summary.h5 file. - """ - raise NotImplementedError + """Write band-schema output using the shared IO codec; return None.""" + from ..io.experiment_summary import write_summary + + write_summary(self, path) diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 1c395d0..b145489 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -1,123 +1,244 @@ -"""Typed parameter models and the legacy ``set_params`` helper. +"""Scientific configuration and independent execution and annotation policy. -The dataclasses here replace the untyped parameter dicts used in -GIAnT-MATLAB with validated, autocompleting parameter objects. The -:func:`set_params` function is retained as a dict-based compatibility shim -(a direct port of setParams.m) for code paths that still pass raw dicts. +``BandSiloParams`` contains only band science. +``ExecutionOptions`` controls scheduling/debug limits; ``AnnotationOptions`` +controls ROI use and GUI policy. Resolution copies inputs without retuning +the existing algorithm. """ from __future__ import annotations -from dataclasses import dataclass -from typing import Optional +import math +from dataclasses import asdict, dataclass, fields +from numbers import Integral, Real +from typing import Optional, Union + + +def _positive(name: str, value: object, *, integer: bool = False) -> None: + """Reject nonfinite, nonpositive, boolean, or nonnumeric values.""" + kind = Integral if integer else Real + if ( + isinstance(value, bool) + or not isinstance(value, kind) + or not math.isfinite(value) + or value <= 0 + ): + label = "positive integer" if integer else "positive finite number" + raise ValueError(f"{name} must be a {label}") + + +def _nonnegative(name: str, value: object) -> None: + """Reject nonfinite, negative, boolean, or nonnumeric values.""" + if ( + isinstance(value, bool) + or not isinstance(value, Real) + or not math.isfinite(value) + or value < 0 + ): + raise ValueError(f"{name} must be a finite nonnegative number") @dataclass -class AlignParams: - """Motion-correction parameters (mirrors the registration params struct). +class BandSiloParams: + """Scientific settings for the implemented SLAP2 band algorithm. Attributes ---------- - align_hz : float - Downsampled alignment rate, in Hz. - maxshift : float - Maximum allowed shift, in pixels. - clip_shift : float or None - Clip threshold for per-frame shift changes. - alpha : float or None - Regularization weight. - n_workers : int - Number of parallel workers for per-trial processing. - overwrite_existing : bool - Recompute trials whose outputs already exist. - is_revolt : bool - Bergamo ReVolt acquisition mode. + analyze_hz : float + Temporal analysis rate in Hz. + decay_tau_s : float + Decay time constant in seconds, shared by localization filtering and + convolution-constrained trace extraction. Saved as decayTau_s. + baseline_window_s, denoise_window_s : float + Baseline and temporal denoising windows in seconds. + vif : float + Noise-model variance inflation factor. + d_xy : int + Spatial downsampling / superpixel spacing in pixels. + sparse_fac : float + NMF sparsity factor, not its logarithm. + peakth : float + Peak threshold. Seven preserves the current implementation; this + configuration split does not retune the algorithm. + peak_buffer : int + Peak exclusion buffer diameter in pixels. + psf_dilation : int + Positive dilation size selecting the bundled PSF template. Template + availability is checked by the backend when loading the PSF. + z_tol : float + Positive z tolerance in microns: maximum absolute motion-bin z + deviation from the median frame z for low-resolution frame selection. + trace_beta_fraction : float + Nonnegative event penalty relative to each trial's adjoint scale. + trace_max_iter : int + Maximum L-BFGS-B iterations per temporal optimization stage. + trace_tolerance : float + Positive optimizer objective/gradient tolerance. + trace_debias : bool + Refit selected events without L1 shrinkage (default False). + trace_event_threshold : float + Absolute event amplitude threshold for optional debias support. + random_seed : int + Nonnegative seed for the random motion-bin order used by source + localization (NMF and Adam). Each DMD path and outer iteration + draws from its own generator derived from this seed. + + Notes + ----- + Scheduling, trial limits, logging and annotation policy are deliberately + separate from science. Numerical defaults preserve the band algorithm. + Sparse-superpixel background interpolation always uses cubic splines. + Channel count is inferred from acquisition metadata; source activity + extraction always uses channel zero. Production extraction always saves + least-squares dF_ls alongside convolution dF_denoised and events. """ - align_hz: float = 80.0 - maxshift: float = 20.0 - clip_shift: Optional[float] = None - alpha: Optional[float] = None - n_workers: int = 1 - overwrite_existing: bool = False - is_revolt: bool = False + analyze_hz: float = 100.0 + decay_tau_s: float = 0.15 + baseline_window_s: float = 4.0 + denoise_window_s: float = 1.0 + vif: float = 1.38 + d_xy: int = 5 + sparse_fac: float = math.exp(-3.0) + peakth: float = 7.0 + peak_buffer: int = 3 + psf_dilation: int = 17 + z_tol: float = 1.5 + trace_beta_fraction: float = 0.01 + trace_max_iter: int = 500 + trace_tolerance: float = 1e-7 + trace_debias: bool = False + trace_event_threshold: float = 1e-6 + random_seed: int = 0 + + def __post_init__(self) -> None: + """Validate values supported by the existing numerical path.""" + for name in ( + "analyze_hz", + "decay_tau_s", + "baseline_window_s", + "denoise_window_s", + "vif", + "sparse_fac", + "peakth", + "z_tol", + "trace_tolerance", + ): + _positive(name, getattr(self, name)) + for name in ("d_xy", "peak_buffer", "psf_dilation", "trace_max_iter"): + _positive(name, getattr(self, name), integer=True) + if not isinstance(self.trace_debias, bool): + raise ValueError("trace_debias must be a bool") + for name in ("trace_beta_fraction", "trace_event_threshold"): + _nonnegative(name, getattr(self, name)) + if ( + isinstance(self.random_seed, bool) + or not isinstance(self.random_seed, Integral) + or self.random_seed < 0 + ): + raise ValueError("random_seed must be a nonnegative integer") @dataclass -class SiloParams: - """Source-extraction (SILo) parameters (mirrors the SILo params struct). +class ExecutionOptions: + """Run policy, separate from scientific configuration. Attributes ---------- - microscope : str - ``"slap2"`` or ``"bergamo"``. - scan_mode : str - SLAP2 scan mode selecting the extraction backend: - ``"standard"`` (pixel-movie SILo) or ``"integration"`` (the - band-scan / superpixel variant, paired with BandRegistration). - sigma_px : float - Spatial smoothing sigma, in pixels. - lambda_ : float - NMF sparsity weight (``lambda`` in MATLAB; renamed to avoid the - Python keyword). - phi : float or None - NMF spatial-coherence weight. - tau_s : float or None - Deconvolution time constant, in seconds. - photon_scale : float or None - Photons per digital unit (variance modeling). - peakth : float or None - Activity-image peak detection threshold. - activity_channel : int - Channel index used for activity detection. - draw_user_rois : bool - Whether to prompt for user ROIs before extraction. - analyze_hz : float or None - Temporal analysis rate, in Hz. + max_workers : int + Positive number of worker processes; preserves the default of six. + verbose : bool + Enable stage messages and progress bars. + max_trials : int or None + Positive debug limit per DMD. ``None`` processes all trials. """ - microscope: str = "slap2" - scan_mode: str = "standard" - sigma_px: float = 1.5 - lambda_: float = 0.1 - phi: Optional[float] = None - tau_s: Optional[float] = None - photon_scale: Optional[float] = None - peakth: Optional[float] = None - activity_channel: int = 0 - draw_user_rois: bool = False - analyze_hz: Optional[float] = None - - -def set_params( - fn_name: str, - params_in: Optional[dict] = None, -) -> dict: - """Return a parameter dictionary for the named GIAnT function. + max_workers: int = 6 + verbose: bool = False + max_trials: Optional[int] = None + + def __post_init__(self) -> None: + """Validate scheduling and debug limits.""" + _positive("max_workers", self.max_workers, integer=True) + if self.max_trials is not None: + _positive("max_trials", self.max_trials, integer=True) + if not isinstance(self.verbose, bool): + raise ValueError("verbose must be a bool") + + +@dataclass +class AnnotationOptions: + """ROI inclusion and GUI policy, separate from scientific settings. + + Attributes + ---------- + enabled : bool + Include user ROIs during extraction. + interactive : bool or None + Allow or forbid drawing; ``None`` retains backend auto-detection. + """ + + enabled: bool = False + interactive: Optional[bool] = None + + def __post_init__(self) -> None: + """Validate the explicit and automatic annotation policies.""" + if not isinstance(self.enabled, bool): + raise ValueError("enabled must be a bool") + if self.interactive is not None and not isinstance( + self.interactive, bool + ): + raise ValueError("interactive must be a bool or None") - Populates default parameters for the specified function and merges any - caller-supplied overrides. Corresponds to setParams.m in GIAnT-MATLAB. - Prefer the typed :class:`AlignParams` / :class:`SiloParams` models for new - code; this shim exists for dict-based compatibility. + +BandParamsInput = Optional[Union[BandSiloParams, dict]] +ExecutionInput = Optional[Union[ExecutionOptions, dict]] +AnnotationInput = Optional[Union[AnnotationOptions, dict]] + + +def _option_values( + option: object, + model: type[Union[BandSiloParams, ExecutionOptions, AnnotationOptions]], +) -> dict: + """Copy a typed option or validate a strictly scoped dictionary.""" + if option is None: + return {} + if isinstance(option, model): + return asdict(option) + if isinstance(option, dict): + allowed = {item.name for item in fields(model)} + unknown = option.keys() - allowed + if unknown: + raise TypeError(f"Unknown {model.__name__} fields: {unknown}") + return dict(option) + raise TypeError(f"Expected {model.__name__}, dict, or None") + + +def resolve_band_options( + params: BandParamsInput = None, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]: + """Resolve independent band science, execution and annotation options. Parameters ---------- - fn_name : str - Name of the GIAnT function whose parameters should be configured. - Supported values: ``"SILo"``, ``"MultiRoiRegistration"``, - ``"StripRegistration"``, ``"BandRegistration"``. - params_in : dict, optional - Caller-supplied parameter overrides. Any key present in *params_in* - takes precedence over the corresponding default value. + params : BandSiloParams, dict or None + Scientific settings only. Routing, execution, annotation and unknown + fields raise ``TypeError`` rather than being dispatched elsewhere. + execution : ExecutionOptions, dict or None + Run options or a dictionary containing only execution fields. + annotations : AnnotationOptions, dict or None + Annotation policy or a dictionary containing only annotation fields. Returns ------- - dict - Parameter dictionary with defaults merged with *params_in*. - - Raises - ------ - ValueError - If *fn_name* is not a recognised GIAnT function name. + tuple of BandSiloParams, ExecutionOptions, AnnotationOptions + Fresh validated objects. Caller inputs are never mutated, including + when channel count or GUI interactivity is later resolved. """ - raise NotImplementedError + return ( + BandSiloParams(**_option_values(params, BandSiloParams)), + ExecutionOptions(**_option_values(execution, ExecutionOptions)), + AnnotationOptions(**_option_values(annotations, AnnotationOptions)), + ) diff --git a/src/giant_python/models/trial_table.py b/src/giant_python/models/trial_table.py index 065b02e..ea96bae 100644 --- a/src/giant_python/models/trial_table.py +++ b/src/giant_python/models/trial_table.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import Optional, Union +import numpy as np + @dataclass class Slap2Info: @@ -19,17 +21,21 @@ class Slap2Info: ref_stack : dict Reference stack(s), keyed by DMD path, each with image, channels, Zs, and the DMD-pixel-to-sample transform. - first_line, last_line : list of int + first_line, last_line : ndarray or list First/last scanned line index for each trial. - trial_start_time_inferred, trial_end_time_from_pc : list of float + trial_start_time_inferred, trial_end_time_from_pc : ndarray or list Inferred and PC-reported trial timing. """ ref_stack: dict = field(default_factory=dict) - first_line: list = field(default_factory=list) - last_line: list = field(default_factory=list) - trial_start_time_inferred: list = field(default_factory=list) - trial_end_time_from_pc: list = field(default_factory=list) + first_line: Union[np.ndarray, list] = field(default_factory=list) + last_line: Union[np.ndarray, list] = field(default_factory=list) + trial_start_time_inferred: Union[np.ndarray, list] = field( + default_factory=list + ) + trial_end_time_from_pc: Union[np.ndarray, list] = field( + default_factory=list + ) @dataclass @@ -48,34 +54,46 @@ class TrialTable: Directory containing the raw data files. savedr : Path or None Directory in which results are written. - filename : list + filename : ndarray or list File names, shaped ``(n_dmds, n_trials)``. - true_trial_ix : list + true_trial_ix : ndarray or list Trial indices, shaped ``(n_dmds, n_trials)``. - epoch : list + epoch : ndarray or list Epoch numbers, shaped ``(n_dmds, n_trials)``. slap2_info : Slap2Info or None SLAP2-only metadata (``None`` for Bergamo). motion_correction : dict or None Populated by the registration stage (registered filenames, alignment - data filenames, per-trial failure flags, and the ``AlignParams`` used). + data filenames, per-trial failure flags, and registration parameters). source_extraction : dict or None Populated by the extraction stage (analysis params, raw input files). """ datadr: Optional[Path] = None savedr: Optional[Path] = None - filename: list = field(default_factory=list) - true_trial_ix: list = field(default_factory=list) - epoch: list = field(default_factory=list) + filename: Union[np.ndarray, list] = field(default_factory=list) + true_trial_ix: Union[np.ndarray, list] = field(default_factory=list) + epoch: Union[np.ndarray, list] = field(default_factory=list) slap2_info: Optional[Slap2Info] = None motion_correction: Optional[dict] = None source_extraction: Optional[dict] = None + # Provenance only; never a trial-table HDF5 field. + source_path: Optional[Path] = field( + default=None, repr=False, compare=False + ) @classmethod def from_h5(cls, path: Union[str, Path]) -> "TrialTable": """Load a trial table from a MATLAB-compatible trial_table.h5 file. + Faithful deserialization: the on-disk group hierarchy (README + ``trial_table.h5`` schema) is mirrored into the model with no path + resolution or file probing. ``slap2_info`` becomes a + :class:`Slap2Info`; ``motion_correction`` / ``source_extraction`` stay + as nested dicts (present only once those stages have run). Callers that + need a runtime view (absolute ``fn_adata`` paths, kept-trial mask, per- + DMD ``alignHz``) resolve it separately. + Parameters ---------- path : str or Path @@ -86,14 +104,6 @@ def from_h5(cls, path: Union[str, Path]) -> "TrialTable": TrialTable The deserialized trial table. """ - raise NotImplementedError + from ..io.trial_table import read_trial_table - def to_h5(self, path: Union[str, Path]) -> None: - """Write this trial table to a MATLAB-compatible trial_table.h5 file. - - Parameters - ---------- - path : str or Path - Destination path for the trial_table.h5 file. - """ - raise NotImplementedError + return read_trial_table(path) diff --git a/src/giant_python/numerics/__init__.py b/src/giant_python/numerics/__init__.py new file mode 100644 index 0000000..9755622 --- /dev/null +++ b/src/giant_python/numerics/__init__.py @@ -0,0 +1,4 @@ +"""Implemented baseline, peak, interpolation and morphology kernels. + +Import individual modules as needed. +""" diff --git a/src/giant_python/numerics/baseline.py b/src/giant_python/numerics/baseline.py new file mode 100644 index 0000000..67b56b0 --- /dev/null +++ b/src/giant_python/numerics/baseline.py @@ -0,0 +1,193 @@ +"""Shared baseline (F0) estimation and dF/F assembly. + +Ported from ``extractSLAP2IntegrationSources.py`` (``_movmean_nan`` / +``computeF0`` ref 217-305, and the dF/F assembly ref 2972-2978). +:func:`compute_f0` estimates a slowly-varying fluorescence baseline via a +rolling median plus a decimated-grid rolling-min ("convex-hull"-like) +envelope, interpolated back to the full time grid with PCHIP. +:func:`assemble_dff` combines the per-trial +least-squares traces into ``F``, re-estimates ``F0``, and forms ``dF``/``dFF``. +""" + +from __future__ import annotations + +from typing import Tuple + +import numpy as np +import pandas as pd +from scipy.interpolate import PchipInterpolator + + +def _movmean_nan(x: np.ndarray, window: int) -> np.ndarray: + """Centered, NaN-ignoring moving mean (like MATLAB ``smoothdata movmean``). + + Uses partial windows at the edges and returns NaN where a window contains + no valid samples. + + Parameters + ---------- + x : ndarray + 1-D input (NaNs allowed). + window : int + Window length in samples (clamped to >= 1). + + Returns + ------- + ndarray of float + The smoothed signal. + """ + window = max(int(window), 1) + if window == 1: + return x.astype(float, copy=True) + + kernel = np.ones(window, dtype=float) + valid = ~np.isnan(x) + x_filled = np.where(valid, x, 0.0) + + num = np.convolve(x_filled, kernel, mode="same") + den = np.convolve(valid.astype(float), kernel, mode="same") + + return num / np.where(den == 0.0, np.nan, den) + + +def _compute_f0_column( + f0_col: np.ndarray, + sample_times: np.ndarray, + n_samps_in_hull: int, + total_frames: int, +) -> np.ndarray: + """Estimate the baseline for one already-median-filtered trace column. + + Builds a set of decimated-grid interpolations of the trace, takes their + per-sample minimum (the rolling-min envelope), discards samples with too + few contributions, smooths/fills the gaps, and PCHIP-interpolates the + envelope back onto the full ``total_frames`` grid. + """ + f00 = np.full((sample_times.shape[0], n_samps_in_hull), np.nan) + for dix in range(n_samps_in_hull, 0, -1): + start = dix - 1 + xi = sample_times[start::n_samps_in_hull] + f00[:, dix - 1] = np.interp( + sample_times, xi, f0_col[xi], left=np.nan, right=np.nan + ) + ff = np.nanmin(f00, axis=1) + + doubt = np.sum(~np.isnan(f00), axis=1) < int(np.ceil(n_samps_in_hull / 2)) + if np.sum(~doubt) > 2: + ff[doubt] = np.nan + + win = 2 * int(np.ceil(n_samps_in_hull / 2.0)) + 1 + fill = _movmean_nan(ff, win) + nan_mask = np.isnan(ff) + ff[nan_mask] = fill[nan_mask] + ff = _movmean_nan(ff, win) + + nan_mask = np.isnan(ff) + if np.any(nan_mask): + ff = np.interp(sample_times, sample_times[~nan_mask], ff[~nan_mask]) + + pchip = PchipInterpolator(sample_times, ff, extrapolate=True) + return pchip(np.arange(total_frames)) + + +def compute_f0( + f_in: np.ndarray, denoise_window: int, hull_window: int +) -> np.ndarray: + """Estimate a slowly-varying fluorescence baseline ``F0``. + + Port of ``computeF0``: (1) a centered rolling-median denoise, (2) a + rolling convex-hull-like min envelope on decimated grids, (3) discard of + doubtful samples near NaNs, smoothing/filling, and PCHIP back to the full + time grid. + + Parameters + ---------- + f_in : ndarray of shape (T, ...) + Fluorescence with time along axis 0 (NaNs allowed). + denoise_window : int + Rolling-median window (time samples). + hull_window : int + Window controlling the convex-hull-like envelope. + + Returns + ------- + ndarray + The baseline ``F0``, same shape as ``f_in``. + """ + f = np.asarray(f_in) + orig_shape = f.shape + if f.ndim == 1: + f = f[:, None] + else: + f = f.reshape(f.shape[0], -1) + + total_frames = f.shape[0] + if total_frames < 4: + return np.ones_like(f_in, dtype=float) * np.nanmean( + f_in, axis=0, keepdims=True + ) + + hull_window = int(min(hull_window, total_frames // 4)) + delta_des = max(4.0, denoise_window / 6.0) + + sample_times = np.rint( + np.linspace( + 0, total_frames - 1, num=int(np.ceil(total_frames / delta_des) + 1) + ) + ).astype(int) + n_samps_in_hull = int(np.ceil(hull_window / delta_des)) + + f0 = ( + pd.DataFrame(f) + .rolling(window=denoise_window, center=True, min_periods=1) + .median() + .to_numpy() + ) + + # ``to_numpy()`` can hand back a read-only view (pandas copy-on-write), so + # take a writable, owned copy before assigning the per-column baselines. + f0 = np.array(f0, dtype=float).reshape(f0.shape[0], -1) + for cix in range(f0.shape[1]): + if np.all(np.isnan(f0[:, cix])): + continue + f0[:, cix] = _compute_f0_column( + f0[:, cix], sample_times, n_samps_in_hull, total_frames + ) + + return f0.reshape(orig_shape) + + +def assemble_dff( + d_f_ls: np.ndarray, + f0_ls: np.ndarray, + denoise_window: int, + hull_window: int, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Assemble ``F``, re-estimate ``F0``, and form ``dF``/``dFF``. + + Reconstructs total fluorescence ``F = dF_ls + F0_ls`` from the per-motion + source traces (least squares or weighted averages) and background terms + (for band extraction, the backprojected background plus the backprojected + fit residual), re-estimates the baseline with + :func:`compute_f0`, and computes ``dF = F - F0`` and ``dFF = dF / + clip(F0, 1e-4, inf)``. + + Parameters + ---------- + d_f_ls : ndarray of shape (T, n_sources) + Concatenated per-source dF traces (``phi``). + f0_ls : ndarray of shape (T, n_sources) + Concatenated per-source background terms in the same extraction scale. + denoise_window, hull_window : int + Windows passed to :func:`compute_f0`. + + Returns + ------- + f, f0, d_f, d_ff : ndarray + Total fluorescence, baseline, baseline-subtracted, and dF/F. + """ + f = d_f_ls + f0_ls + f0 = compute_f0(f, denoise_window, hull_window) + d_f = f - f0 + d_ff = d_f / np.clip(f0, 1e-4, None) + return f, f0, d_f, d_ff diff --git a/src/giant_python/numerics/interpolation.py b/src/giant_python/numerics/interpolation.py new file mode 100644 index 0000000..5933975 --- /dev/null +++ b/src/giant_python/numerics/interpolation.py @@ -0,0 +1,30 @@ +"""Shared nearest-neighbor interpolation.""" + +import numpy as np + + +def nearest_interp( + x: np.ndarray, xp: np.ndarray, yp: np.ndarray +) -> np.ndarray: + """Nearest-neighbor interpolation (MATLAB ``interp1(...,'nearest')``). + + Parameters + ---------- + x : ndarray + Query points. + xp : ndarray + Sample positions (assumed sorted ascending). + yp : ndarray + Sample values aligned with ``xp``. + + Returns + ------- + ndarray + ``yp`` sampled at the nearest ``xp`` for each ``x``. + """ + if len(xp) == 1: + return yp + x_bds = xp[:-1] / 2.0 + xp[1:] / 2.0 + idx = np.searchsorted(x_bds, x, side="left") + idx = np.clip(idx, 0, len(xp) - 1) + return yp[idx] diff --git a/src/giant_python/numerics/morphology.py b/src/giant_python/numerics/morphology.py new file mode 100644 index 0000000..4e4abbb --- /dev/null +++ b/src/giant_python/numerics/morphology.py @@ -0,0 +1,60 @@ +"""Shared binary morphology on trailing image axes.""" + +from __future__ import annotations + +from typing import Optional + +import numpy as np + + +def fast_dilation( + mask: np.ndarray, + kernel: Optional[np.ndarray] = None, + iterations: int = 1, +) -> np.ndarray: + """Binary-dilate the trailing two axes of ``mask``. + + The common 3x3 / single-iteration case is handled with a pure-numpy shift + accumulation (fast, no OpenCV); other cases fall back to per-slice + ``cv2.dilate`` over all leading axes. + + Parameters + ---------- + mask : ndarray + Boolean (or 0/1) array; the last two axes are the image plane. + kernel : ndarray, optional + Structuring element; defaults to a 7x7 block. + iterations : int + Number of dilation iterations (generic path only). + + Returns + ------- + ndarray of bool + The dilated mask. + """ + if kernel is None: + kernel = np.ones((7, 7), np.uint8) + + if iterations == 1 and kernel.shape == (3, 3) and np.all(kernel): + m = mask.astype(bool, copy=False) + pad = [(0, 0)] * (m.ndim - 2) + [(1, 1), (1, 1)] + p = np.pad(m, pad, mode="constant", constant_values=False) + out = np.zeros_like(m, dtype=bool) + h, w = m.shape[-2], m.shape[-1] + for dr in range(3): + r_end = dr + h + for dc in range(3): + c_end = dc + w + out |= p[..., dr:r_end, dc:c_end] + return out + + import cv2 + + out = np.empty_like(mask, dtype=bool) + for idx in np.ndindex(mask.shape[:-2]): + out[idx] = cv2.dilate( + mask[idx].astype(np.uint8, copy=False), + kernel, + iterations=iterations, + ).astype(bool, copy=False) + return out diff --git a/src/giant_python/numerics/peaks.py b/src/giant_python/numerics/peaks.py new file mode 100644 index 0000000..4c63715 --- /dev/null +++ b/src/giant_python/numerics/peaks.py @@ -0,0 +1,848 @@ +"""Shared Gaussian peak detection in the activity image. + +Ported from the peak-detection kernels of +``extractSLAP2IntegrationSources.py`` (ref 307-712, originally +``getActImPeaks.m``). :func:`get_act_im_peaks` finds source seeds in the 3-D +(Z, H, W) activity image produced by +:mod:`giant_python.extraction.band.summary_images`; those seeds initialize the +Phase-6 NMF localization. + +Detection fits integrated axis-aligned 2-D Gaussians (each Gaussian integrated +over unit pixels) to the activity image with a bounded Levenberg-Marquardt +solver (:func:`_lsq_curvefit`, using the analytical Jacobian from +:func:`_gaussian_peaks_integrated_val_jac`). Per plane, an initial set of local +maxima is fit jointly, then residual peaks are added one per connected +component per round until none exceed the threshold, and finally +under-amplitude peaks are removed. An optional exclusion mask (e.g. user soma +ROIs) suppresses detection in masked regions. + +The reference's nested helpers (``_make_bounds``/``_peak_mask``/ +``_buffer_mask`` and the per-component new-peak step) are lifted to +module-level functions for independent reuse. +""" + +from __future__ import annotations + +from typing import Tuple, cast + +import numpy as np +import scipy.ndimage as ndimage +from scipy.special import erf + +# Amplitude rescale applied to raw activity-image peak values when seeding the +# Gaussian fit (matches the reference ``AMP_SCALE = 1 / 0.75``). +_AMP_SCALE = 1.0 / 0.75 + +# MAD -> Gaussian-sigma scale factor (median absolute deviation / 0.6745). +_MAD_TO_SIGMA = 0.67449 + +# Minimum finite-pixel count for an independent background estimate. +_MIN_COMPONENT_PIXELS = 100 + + +def _validate_theta(theta: np.ndarray) -> None: + """Require explicit amplitude, center and both axis-width columns.""" + if theta.ndim != 2 or theta.shape[1] != 5: + raise ValueError( + "Gaussian parameters must have shape (N, 5): " + "[amp, mu_y, mu_x, sigma_y, sigma_x]; " + "repeat the width for equal-width Gaussians" + ) + + +def gaussian_peaks_integrated( + theta: np.ndarray, yxdata: np.ndarray +) -> np.ndarray: + """Evaluate integrated axis-aligned 2-D Gaussians at selected pixels. + + Each Gaussian is integrated over unit pixels on a regular grid (port of + ``gaussianPeaksIntegrated``). Only the ``M`` requested pixels are evaluated + (the full grid is never materialized). + + Parameters + ---------- + theta : ndarray of shape (N, 5) + Per-Gaussian ``[amp, mu_y, mu_x, sigma_y, sigma_x]``. Equal widths + must be supplied explicitly in both width columns. + yxdata : ndarray of shape (M, 2) + Pixel-center ``[y, x]`` coordinates. + + Returns + ------- + ndarray of shape (M,) + Predicted value at each pixel: ``sum_n A_n * Iy[y, n] * Ix[x, n]``. + """ + _validate_theta(theta) + x_int = yxdata[:, 1].astype(np.intp) + y_int = yxdata[:, 0].astype(np.intp) + x_min = int(x_int.min()) + x_max = int(x_int.max()) + y_min = int(y_int.min()) + y_max = int(y_int.max()) + x_idx = x_int - x_min + y_idx = y_int - y_min + + amp = theta[:, 0] + my = theta[:, 1] + mx = theta[:, 2] + sy = np.maximum(theta[:, 3], np.finfo(float).eps) + sx = np.maximum(theta[:, 4], np.finfo(float).eps) + + c = np.sqrt(np.pi / 2) + rt2 = np.sqrt(2.0) + + xc = np.arange(x_min, x_max + 1, dtype=float) + xl = (xc - 0.5)[:, np.newaxis] + xr = (xc + 0.5)[:, np.newaxis] + ix = ( + c + * sx[np.newaxis, :] + * ( + erf((xr - mx[np.newaxis, :]) / (rt2 * sx[np.newaxis, :])) + - erf((xl - mx[np.newaxis, :]) / (rt2 * sx[np.newaxis, :])) + ) + ) + + yc = np.arange(y_min, y_max + 1, dtype=float) + yb = (yc - 0.5)[:, np.newaxis] + yt = (yc + 0.5)[:, np.newaxis] + iy = ( + c + * sy[np.newaxis, :] + * ( + erf((yt - my[np.newaxis, :]) / (rt2 * sy[np.newaxis, :])) + - erf((yb - my[np.newaxis, :]) / (rt2 * sy[np.newaxis, :])) + ) + ) + + return (iy[y_idx, :] * ix[x_idx, :]) @ amp + + +def _gaussian_peaks_integrated_val_jac( + theta: np.ndarray, yxdata: np.ndarray +) -> Tuple[np.ndarray, np.ndarray]: + """Integrated Gaussians with their analytical Jacobian. + + Shares the integrated-profile intermediates between the forward value and + the Jacobian so the LM solver needs one evaluation per iteration instead of + ``P*N`` finite differences, where ``P`` is the number of parameters. + + Parameters + ---------- + theta : ndarray of shape (N, 5) + Per-Gaussian ``[amp, mu_y, mu_x, sigma_y, sigma_x]``. + yxdata : ndarray of shape (M, 2) + Pixel-center ``[y, x]`` coordinates. + + Returns + ------- + val : ndarray of shape (M,) + Forward value at each pixel. + jac : ndarray of shape (M, 5*N) + Derivatives in the same order as ``theta.ravel()``. + """ + _validate_theta(theta) + x_int = yxdata[:, 1].astype(np.intp) + y_int = yxdata[:, 0].astype(np.intp) + m = len(x_int) + n = theta.shape[0] + + x_min = int(x_int.min()) + x_max = int(x_int.max()) + y_min = int(y_int.min()) + y_max = int(y_int.max()) + x_idx = x_int - x_min + y_idx = y_int - y_min + + amp = theta[:, 0] + my = theta[:, 1] + mx = theta[:, 2] + sy = np.maximum(theta[:, 3], np.finfo(float).eps) + sx = np.maximum(theta[:, 4], np.finfo(float).eps) + + c = np.sqrt(np.pi / 2) + rt2 = np.sqrt(2.0) + inv_rt2sx = 1.0 / (rt2 * sx[np.newaxis, :]) + inv_rt2sy = 1.0 / (rt2 * sy[np.newaxis, :]) + + xc = np.arange(x_min, x_max + 1, dtype=float) + xl = (xc - 0.5)[:, np.newaxis] + xr = (xc + 0.5)[:, np.newaxis] + ux_l = (xl - mx[np.newaxis, :]) * inv_rt2sx + ux_r = (xr - mx[np.newaxis, :]) * inv_rt2sx + erf_ux_l = erf(ux_l) + erf_ux_r = erf(ux_r) + ix = c * sx[np.newaxis, :] * (erf_ux_r - erf_ux_l) + + yc = np.arange(y_min, y_max + 1, dtype=float) + yb = (yc - 0.5)[:, np.newaxis] + yt = (yc + 0.5)[:, np.newaxis] + uy_b = (yb - my[np.newaxis, :]) * inv_rt2sy + uy_t = (yt - my[np.newaxis, :]) * inv_rt2sy + erf_uy_b = erf(uy_b) + erf_uy_t = erf(uy_t) + iy = c * sy[np.newaxis, :] * (erf_uy_t - erf_uy_b) + + iy_m = iy[y_idx, :] + ix_m = ix[x_idx, :] + + dval_da = iy_m * ix_m + val = dval_da @ amp + + exp_uyb2 = np.exp(-(uy_b**2)) + exp_uyt2 = np.exp(-(uy_t**2)) + d_iy_dmy_m = (exp_uyb2 - exp_uyt2)[y_idx, :] + dval_dmy = amp[np.newaxis, :] * d_iy_dmy_m * ix_m + + exp_uxl2 = np.exp(-(ux_l**2)) + exp_uxr2 = np.exp(-(ux_r**2)) + d_ix_dmx_m = (exp_uxl2 - exp_uxr2)[x_idx, :] + dval_dmx = amp[np.newaxis, :] * iy_m * d_ix_dmx_m + + sqrt2 = rt2 + d_iy_ds_m = ( + iy / sy[np.newaxis, :] + sqrt2 * (uy_b * exp_uyb2 - uy_t * exp_uyt2) + )[y_idx, :] + d_ix_ds_m = ( + ix / sx[np.newaxis, :] + sqrt2 * (ux_l * exp_uxl2 - ux_r * exp_uxr2) + )[x_idx, :] + dval_dsy = amp[np.newaxis, :] * d_iy_ds_m * ix_m + dval_dsx = amp[np.newaxis, :] * iy_m * d_ix_ds_m + + jac = np.empty((m, 5 * n), dtype=float) + jac[:, 0::5] = dval_da + jac[:, 1::5] = dval_dmy + jac[:, 2::5] = dval_dmx + jac[:, 3::5] = dval_dsy + jac[:, 4::5] = dval_dsx + + return val, jac + + +def _lsq_curvefit( + theta0: np.ndarray, + xdata: np.ndarray, + ydata: np.ndarray, + lb_flat: np.ndarray, + ub_flat: np.ndarray, + max_nfev: int = 5000, + *, + shared_widths: bool = False, +) -> np.ndarray: + """Bounded Levenberg-Marquardt fit of integrated Gaussians. + + Reimplements the subset of MATLAB ``lsqcurvefit`` (trust-region-reflective + defaults) used by ``getActImPeaks.m``, but with the analytical Jacobian so + each iteration is a single value+Jacobian evaluation. + + Parameters + ---------- + theta0 : ndarray of shape (N, 5) + Initial ``[amp, mu_y, mu_x, sigma_y, sigma_x]`` per Gaussian. + xdata : ndarray of shape (M, 2) + Pixel-center ``[y, x]`` coordinates. + ydata : ndarray of shape (M,) + Observed values (``act_im(sel) - mu_bg``). + lb_flat, ub_flat : ndarray of shape (5*N,) + Row-major-flattened lower/upper bounds. + max_nfev : int + Cap on value+Jacobian evaluations. + shared_widths : bool + Constrain width updates to be equal for the isotropic warm start. + Both initial width columns and their bounds must already match. + + Returns + ------- + ndarray of shape (N, 5) + Optimized parameters. + """ + _validate_theta(theta0) + if lb_flat.shape != (theta0.size,) or ub_flat.shape != (theta0.size,): + raise ValueError("Gaussian bounds must have shape (5*N,)") + if shared_widths and ( + not np.array_equal(theta0[:, 3], theta0[:, 4]) + or not np.array_equal(lb_flat[3::5], lb_flat[4::5]) + or not np.array_equal(ub_flat[3::5], ub_flat[4::5]) + ): + raise ValueError( + "Shared-width fitting requires equal y/x widths and bounds" + ) + x = np.clip(theta0.ravel().copy(), lb_flat, ub_flat) + + val, jac = _gaussian_peaks_integrated_val_jac(x.reshape(-1, 5), xdata) + r = val - ydata + cost = np.dot(r, r) + nfev = 1 + + lam = 1e-2 # initial damping (MATLAB InitDamping default) + + for _ in range(400): # MATLAB MaxIter default + if nfev >= max_nfev: + break + + if shared_widths: + # Optimize one width degree of freedom, keeping five-column + # parameters throughout. The chain rule ties the two widths. + full_jac = jac.reshape(len(r), -1, 5) + fit_jac = full_jac[:, :, :4].copy() + fit_jac[:, :, 3] = full_jac[:, :, 3] + full_jac[:, :, 4] + fit_jac = fit_jac.reshape(len(r), -1) + else: + fit_jac = jac + jtj = fit_jac.T @ fit_jac + jtr = fit_jac.T @ r + diag_jtj = np.maximum(np.diag(jtj), 1e-8) + + delta = np.linalg.solve(jtj + lam * np.diag(diag_jtj), -jtr) + if shared_widths: + updates = delta.reshape(-1, 4) + delta = np.column_stack([updates, updates[:, 3]]).ravel() + x_new = np.clip(x + delta, lb_flat, ub_flat) + + val_new, jac_new = _gaussian_peaks_integrated_val_jac( + x_new.reshape(-1, 5), xdata + ) + r_new = val_new - ydata + cost_new = np.dot(r_new, r_new) + nfev += 1 + + if cost_new < cost: + step_norm = np.max(np.abs(x_new - x)) + rel_cost_drop = (cost - cost_new) / max(cost, 1.0) + + x = x_new + r = r_new + jac = jac_new + cost = cost_new + lam = max(lam * 0.1, 1e-10) + + if step_norm < 1e-6 or rel_cost_drop < 1e-6: + break + else: + lam = min(lam * 10.0, 1e10) + + return x.reshape(-1, 5) + + +def _fit_isotropic_then_anisotropic( + theta0: np.ndarray, + xdata: np.ndarray, + ydata: np.ndarray, + lb_flat: np.ndarray, + ub_flat: np.ndarray, + max_nfev: int = 5000, +) -> np.ndarray: + """Fit shared widths first, then relax to independent y/x widths. + + All peaks in the supplied fit region are optimized jointly in both stages. + Both stages use five-column parameters. The first stage ties the two + widths and constrains their shared sigma to the intersection of the y/x + bounds (normally [0.35, 1] pixels). + The second stage starts from that solution with sigma_y == sigma_x and + restores the original five-parameter bounds. Each stage has its own + ``max_nfev`` budget. + + This is an initialization preference, not an anisotropy penalty: widths + can separate whenever that lowers the residual sum of squares. The LM + solver accepts only improving steps, retaining the isotropic solution + when relaxation cannot improve it. + """ + _validate_theta(theta0) + if lb_flat.shape != (theta0.size,) or ub_flat.shape != (theta0.size,): + raise ValueError("Gaussian bounds must have shape (5*N,)") + if theta0.shape[0] == 0: + return theta0.copy() + + lb = lb_flat.reshape(-1, 5) + ub = ub_flat.reshape(-1, 5) + iso_lb = lb.copy() + iso_ub = ub.copy() + iso_lb[:, 3] = np.maximum(lb[:, 3], lb[:, 4]) + iso_ub[:, 3] = np.minimum(ub[:, 3], ub[:, 4]) + iso_lb[:, 4] = iso_lb[:, 3] + iso_ub[:, 4] = iso_ub[:, 3] + if np.any(iso_lb[:, 3] > iso_ub[:, 3]): + raise ValueError( + "Isotropic fitting requires overlapping y/x width bounds" + ) + + clipped = np.clip(theta0, lb, ub) + iso0 = clipped.copy() + iso0[:, 3] = np.sqrt(clipped[:, 3] * clipped[:, 4]) + iso0[:, 4] = iso0[:, 3] + isotropic = _lsq_curvefit( + iso0, + xdata, + ydata, + iso_lb.ravel(), + iso_ub.ravel(), + max_nfev=max_nfev, + shared_widths=True, + ) + return _lsq_curvefit( + isotropic, xdata, ydata, lb_flat, ub_flat, max_nfev=max_nfev + ) + + +def _make_bounds( + plocs: np.ndarray, height: int, width: int +) -> Tuple[np.ndarray, np.ndarray]: + """Return flattened LM bounds for peaks located at ``plocs``. + + Amplitude is unbounded above; means are constrained to +/- 1.5 px of the + seed (clipped to the image); sigma_y to ``[0.35, 8.0]`` and sigma_x to + ``[0.35, 1.0]``, in pixels. + + Parameters + ---------- + plocs : ndarray of shape (N, 2) + Seed ``[y, x]`` locations. + height, width : int + Image extents (for clipping the mean bounds). + + Returns + ------- + lb_flat, ub_flat : ndarray of shape (5*N,) + Row-major-flattened lower/upper bounds. + """ + n = plocs.shape[0] + lb = np.column_stack( + [ + np.zeros(n), + np.maximum(0, plocs[:, 0] - 1.5), + np.maximum(0, plocs[:, 1] - 1.5), + np.ones(n) * 0.35, + np.ones(n) * 0.35, + ] + ) + ub = np.column_stack( + [ + np.full(n, np.inf), + np.minimum(height - 1, plocs[:, 0] + 1.5), + np.minimum(width - 1, plocs[:, 1] + 1.5), + np.full(n, 8.0), + np.full(n, 1.0), + ] + ) + return lb.ravel(), ub.ravel() + + +def _peak_mask(tf: np.ndarray, height: int, width: int) -> np.ndarray: + """Return a boolean image marking each Gaussian's rounded center.""" + pim = np.zeros((height, width), dtype=bool) + if tf.shape[0] > 0: + iy = np.clip(np.round(tf[:, 1]).astype(int), 0, height - 1) + ix = np.clip(np.round(tf[:, 2]).astype(int), 0, width - 1) + pim[iy, ix] = True + return pim + + +def _buffer_mask(pim: np.ndarray, buffer_size: int) -> np.ndarray: + """Dilate the peak mask by a ``buffer_size`` square (identity if <= 0).""" + if buffer_size > 0: + return ndimage.binary_dilation( + pim, structure=np.ones((buffer_size, buffer_size)) + ) + return pim + + +def _process_cc_new_peak( + new_peak: tuple[tuple[int, int], int], + thetaf: np.ndarray, + p_locs: np.ndarray, + labeled_full: np.ndarray, + act_im_2d: np.ndarray, + mu_bg: float, + reject_mask: np.ndarray, + height: int, + width: int, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Add and fit one new residual peak within its connected component. + + Appends the candidate to the parameter set, refits every Gaussian whose + center lands in the component, and โ€” if the new peak barely moved (its + center stayed near-integer) โ€” rejects it (refitting the others without it + and marking its pixel so it is not re-proposed). + + Parameters + ---------- + new_peak : tuple + ``((pY, pX), cc_label)`` for the candidate. + thetaf : ndarray of shape (N, 5) + Current Gaussian parameters. + p_locs : ndarray of shape (N, 2) + Current seed locations. + labeled_full : ndarray + Connected-component labels of the current support. + act_im_2d : ndarray + The plane's activity image. + mu_bg : float + Background median subtracted from the fit target. + reject_mask : ndarray of bool + Mutated in place: the rejected peak's pixel is set True. + height, width : int + Image extents. + + Returns + ------- + thetaf, p_locs, cc_mask, cc_yx + Updated parameters/locations plus the component mask and its pixel + coordinates (for the caller's per-round fit-image update). + """ + (py_new, px_new), cc_label = new_peak + amp_new = act_im_2d[py_new, px_new] * _AMP_SCALE + + n_before = thetaf.shape[0] + thetaf = np.vstack( + [thetaf, [amp_new, float(py_new), float(px_new), 0.5, 0.5]] + ) + p_locs = np.vstack([p_locs, [float(py_new), float(px_new)]]) + new_idx = n_before + + cc_mask = labeled_full == cc_label + cc_yx = np.column_stack(np.where(cc_mask)).astype(float) + cc_vals = act_im_2d[cc_mask] - mu_bg + + iy = np.clip(np.round(thetaf[:, 1]).astype(int), 0, height - 1) + ix = np.clip(np.round(thetaf[:, 2]).astype(int), 0, width - 1) + in_cc = cc_mask[iy, ix] + + lb_cc, ub_cc = _make_bounds(p_locs[in_cc], height, width) + thetaf[in_cc] = _fit_isotropic_then_anisotropic( + thetaf[in_cc], cc_yx, cc_vals, lb_cc, ub_cc, max_nfev=5000 + ) + + mu_y_new = thetaf[new_idx, 1] + mu_x_new = thetaf[new_idx, 2] + if ( + abs(mu_y_new - round(mu_y_new)) < 1e-3 + and abs(mu_x_new - round(mu_x_new)) < 1e-3 + ): + refit_mask = in_cc.copy() + refit_mask[new_idx] = False + if np.any(refit_mask): + lb_rf, ub_rf = _make_bounds(p_locs[refit_mask], height, width) + thetaf[refit_mask] = _fit_isotropic_then_anisotropic( + thetaf[refit_mask], cc_yx, cc_vals, lb_rf, ub_rf, max_nfev=5000 + ) + thetaf = np.delete(thetaf, new_idx, axis=0) + p_locs = np.delete(p_locs, new_idx, axis=0) + reject_mask[py_new, px_new] = True + + return thetaf, p_locs, cc_mask, cc_yx + + +def _refine_residual_peaks( + thetaf: np.ndarray, + p_locs: np.ndarray, + act_im_2d: np.ndarray, + exclusion_mask: np.ndarray, + mu_bg: float, + sigma_bg: float, + peak_th: float, + buffer_size: int, + height: int, + width: int, + buffer_mask: np.ndarray, + res_im: np.ndarray, + fit_support: np.ndarray, + act_sel_pix: np.ndarray, + fit_im: np.ndarray, +) -> np.ndarray: + """Iteratively add residual peaks, one per connected component per round. + + Each round finds the strongest normalized-residual pixel in every component + and, for those above ``peak_th``, adds and fits a new Gaussian (see + :func:`_process_cc_new_peak`). The fit image, residual, support, and buffer + are rebuilt once per round. Terminates when no component yields a new peak. + + Parameters + ---------- + thetaf, p_locs : ndarray + Current Gaussian parameters and seed locations. + act_im_2d, exclusion_mask : ndarray + The plane's activity image and its exclusion mask. + mu_bg, sigma_bg : float + Background median and MAD-scaled sigma. + peak_th : float + Residual threshold (in sigma) for accepting a new peak. + buffer_size : int + Peak-buffer dilation size. + height, width : int + Image extents. + buffer_mask, res_im, fit_support, act_sel_pix, fit_im : ndarray + Initial per-round state from the first fit. + + Returns + ------- + ndarray of shape (N, 5) + The refined Gaussian parameters. + """ + reject_mask = np.zeros((height, width), dtype=bool) + # With output omitted, label returns (label array, component count). + labeled_full, _ = cast(tuple[np.ndarray, int], ndimage.label(act_sel_pix)) + + while True: + e = res_im.copy() + e[buffer_mask | exclusion_mask | reject_mask | ~fit_support] = -np.inf + e[np.isnan(e)] = -np.inf + + n_labels = int(labeled_full.max()) + if n_labels == 0: + break + + label_ids = list(range(1, n_labels + 1)) + max_vals = ndimage.maximum(e, labeled_full, label_ids) + max_pos = ndimage.maximum_position(e, labeled_full, label_ids) + + new_peaks = [ + (max_pos[i], label_ids[i]) + for i in range(n_labels) + if max_vals[i] > peak_th + ] + if not new_peaks: + break + + modified_ccs = [] + for new_peak in new_peaks: + thetaf, p_locs, cc_mask, cc_yx = _process_cc_new_peak( + new_peak, + thetaf, + p_locs, + labeled_full, + act_im_2d, + mu_bg, + reject_mask, + height, + width, + ) + modified_ccs.append((cc_mask, cc_yx)) + + p_im = _peak_mask(thetaf, height, width) + buffer_mask = _buffer_mask(p_im, buffer_size) + for cc_mask, cc_yx in modified_ccs: + fit_im[cc_mask] = gaussian_peaks_integrated(thetaf, cc_yx) + res_im = (act_im_2d - fit_im - mu_bg) / sigma_bg + fit_support = fit_im > 1e-3 + + act_sel_pix = ndimage.binary_dilation(p_im, structure=np.ones((9, 9))) + act_sel_pix &= ~np.isnan(act_im_2d) + labeled_full, _ = cast( + tuple[np.ndarray, int], ndimage.label(act_sel_pix) + ) + + return thetaf + + +def detect_peaks_2d( + act_im_2d: np.ndarray, + exclusion_mask: np.ndarray, + mu_bg: float, + sigma_bg: float, + peak_thresh: float, + peak_th: float, + buffer_size: int = 0, +) -> np.ndarray: + """Detect Gaussian peaks in one activity-image plane. + + Finds initial local maxima (above ``peak_thresh``), fits them jointly, then + iteratively adds residual peaks (:func:`_refine_residual_peaks`), and + finally drops peaks whose amplitude falls below a sigma-adjusted threshold. + Every fit/refit optimizes shared widths first, then allows anisotropy. + + Parameters + ---------- + act_im_2d : ndarray of shape (H, W) + One activity-image plane (may contain NaNs). + exclusion_mask : ndarray of bool, shape (H, W) + Pixels to exclude from detection. + mu_bg, sigma_bg : float + Background median and MAD-scaled sigma for the supplied region. + peak_thresh : float + Absolute detection threshold ``mu_bg + peak_th * sigma_bg``. + peak_th : float + Threshold in sigma for residual peaks. + buffer_size : int + If > 0, detected peaks are buffered by a ``buffer_size`` square when + suppressing nearby residual peaks. + + Returns + ------- + ndarray of shape (N, 5) + ``[amp, mu_y, mu_x, sigma_y, sigma_x]`` per peak, or ``(0, 5)`` if + none. Widths are bounded to ``[0.35, 8]`` in y and ``[0.35, 1]`` in x. + """ + height, width = act_im_2d.shape + empty = np.zeros((0, 5)) + + explored = act_im_2d.copy() + explored[exclusion_mask | np.isnan(explored)] = -np.inf + + rank8 = ndimage.rank_filter(explored, rank=7, size=3) + rank9 = ndimage.maximum_filter(explored, size=3) + p_tmp = (rank8 > peak_thresh) & (explored == rank9) + + if not np.any(p_tmp): + return empty + + py, px = np.where(p_tmp) + amp = act_im_2d[py, px] * _AMP_SCALE + n_peaks = len(py) + + act_sel_pix = ndimage.binary_dilation(p_tmp, structure=np.ones((9, 9))) + act_sel_pix &= ~np.isnan(act_im_2d) + + thetaf = np.column_stack( + [ + amp, + py.astype(float), + px.astype(float), + 0.5 * np.ones(n_peaks), + 0.5 * np.ones(n_peaks), + ] + ) + p_locs = np.column_stack([py.astype(float), px.astype(float)]) + + sel_yx = np.column_stack(np.where(act_sel_pix)).astype(float) + sel_vals = act_im_2d[act_sel_pix] - mu_bg + + lb_f, ub_f = _make_bounds(p_locs, height, width) + thetaf = _fit_isotropic_then_anisotropic( + thetaf, sel_yx, sel_vals, lb_f, ub_f, max_nfev=5000 + ) + + p_im = _peak_mask(thetaf, height, width) + buffer_mask = _buffer_mask(p_im, buffer_size) + + fit_im = np.zeros((height, width), dtype=float) + fit_im[act_sel_pix] = gaussian_peaks_integrated(thetaf, sel_yx) + res_im = (act_im_2d - fit_im - mu_bg) / sigma_bg + fit_support = fit_im > 1e-3 + + thetaf = _refine_residual_peaks( + thetaf, + p_locs, + act_im_2d, + exclusion_mask, + mu_bg, + sigma_bg, + peak_th, + buffer_size, + height, + width, + buffer_mask, + res_im, + fit_support, + act_sel_pix, + fit_im, + ) + + if thetaf.shape[0] > 0: + sy = thetaf[:, 3] + sx = thetaf[:, 4] + adj_thresh = peak_thresh / ( + np.pi + / 2 + * sy + * sx + * erf(1 / (np.sqrt(2) * sy)) + * erf(1 / (np.sqrt(2) * sx)) + ) + thetaf = thetaf[thetaf[:, 0] >= adj_thresh] + + return thetaf + + +def get_act_im_peaks( + act_im: np.ndarray, + peak_th: float = 3.0, + exclusion_mask: np.ndarray | None = None, + buffer_size: int = 0, +) -> np.ndarray: + """Find Gaussian source seeds in a 3-D (Z, H, W) activity image. + + Each plane is split into 4-connected components of finite pixels. Regions + with fewer than 100 pixels or zero MAD are skipped. Background statistics + (median and MAD-scaled sigma) and the detection threshold are computed + independently for each remaining component. Detection and fitting are + restricted to that component (:func:`detect_peaks_2d`). Exclusion masks + suppress peak detection but do not affect connectivity or background stats. + + Gaussians have independent, axis-aligned standard deviations, bounded to + ``0.35 <= sigma_y <= 8`` and ``0.35 <= sigma_x <= 1`` pixels. + Each fit starts with an optimized isotropic Gaussian (shared sigma in + [0.35, 1]) before relaxing to independent widths. This favors isotropic + local minima through initialization without an explicit shape penalty. + + Parameters + ---------- + act_im : ndarray of shape (Z, H, W) + Activity image; nonfinite pixels separate valid-pixel components. + peak_th : float + Threshold in component-local MAD-normalized standard deviations. + exclusion_mask : None or ndarray + ``None``, a 2-D ``(H, W)`` mask applied to every plane, or a 3-D + ``(Z, H, W)`` per-plane mask (e.g. user soma ROIs). + buffer_size : int + If > 0, detected peaks are buffered by a ``buffer_size`` square when + suppressing nearby residual peaks. + + Returns + ------- + ndarray of shape (N, 3) + Source seeds ``[z, mu_y, mu_x]``, or ``(0, 3)`` if none detected. + """ + n_z, height, width = act_im.shape + empty = np.zeros((0, 3)) + + if exclusion_mask is None: + excl_planes = [np.zeros((height, width), dtype=bool)] * n_z + elif exclusion_mask.ndim == 2: + excl_planes = [exclusion_mask.astype(bool)] * n_z + else: + excl_planes = [exclusion_mask[z].astype(bool) for z in range(n_z)] + + source_seeds_list = [] + for z in range(n_z): + labels, _ = cast( + tuple[np.ndarray, int], ndimage.label(np.isfinite(act_im[z])) + ) + counts = np.bincount(labels.ravel()) + for label, bounds in enumerate(ndimage.find_objects(labels), start=1): + if bounds is None or counts[label] < _MIN_COMPONENT_PIXELS: + continue + + component = labels[bounds] == label + plane_crop = act_im[z][bounds] + valid_vals = plane_crop[component] + mu_bg = float(np.median(valid_vals)) + sigma_bg = ( + float(np.median(np.abs(valid_vals - mu_bg))) / _MAD_TO_SIGMA + ) + if sigma_bg <= 0: + continue + + # NaN-mask other regions, including any inside this bounding box, + # so neither the initial fit nor residual refinement uses them. + thetaf = detect_peaks_2d( + np.where(component, plane_crop, np.nan), + excl_planes[z][bounds], + mu_bg, + sigma_bg, + mu_bg + peak_th * sigma_bg, + peak_th, + buffer_size=buffer_size, + ) + if thetaf.shape[0] > 0: + z_col = np.full(thetaf.shape[0], z, dtype=float) + source_seeds_list.append( + np.column_stack( + [ + z_col, + thetaf[:, 1] + bounds[0].start, + thetaf[:, 2] + bounds[1].start, + ] + ) + ) + + return np.vstack(source_seeds_list) if source_seeds_list else empty diff --git a/src/giant_python/parallel.py b/src/giant_python/parallel.py index a6a102f..12915f5 100644 --- a/src/giant_python/parallel.py +++ b/src/giant_python/parallel.py @@ -5,19 +5,33 @@ trials without each stage reimplementing the plumbing. """ -from typing import Callable, Iterable, List, TypeVar +from typing import Callable, Iterable, List, Optional, TypeVar, cast T = TypeVar("T") R = TypeVar("R") +def _maybe_bar(trials: list, desc: Optional[str]): + """Wrap ``trials`` in a tqdm bar when ``desc`` is set, else as-is.""" + if desc is None: + return trials + from tqdm import tqdm + + return tqdm(trials, desc=desc) + + def map_trials( func: Callable[[T], R], trials: Iterable[T], n_workers: int = 1, + desc: Optional[str] = None, ) -> List[R]: """Apply *func* to each trial, optionally in parallel. + Runs serially when ``n_workers <= 1`` (or a single trial), otherwise fans + the trials out over a joblib process pool. Results are returned in input + order regardless of backend. + Parameters ---------- func : callable @@ -26,10 +40,33 @@ def map_trials( The trials to process. n_workers : int Number of parallel workers. ``1`` runs serially. + desc : str, optional + When set, display a tqdm progress bar with this label as the trials are + dispatched (used by the verbose band pipeline). ``None`` shows nothing. Returns ------- list Results in input order. """ - raise NotImplementedError + trials = list(trials) + if n_workers <= 1 or len(trials) <= 1: + return [func(t) for t in _maybe_bar(trials, desc)] + + from joblib import Parallel, delayed + + # ``return_as="generator"`` yields results (in submission order) as each + # trial *completes*, so wrapping this generator in tqdm tracks real + # progress. Wrapping the dispatch generator instead would fill the bar as + # soon as tasks are submitted, long before the workers finish. + results = cast( + Iterable[R], + Parallel(n_jobs=n_workers, return_as="generator")( + delayed(func)(t) for t in trials + ), + ) + if desc is not None: + from tqdm import tqdm + + results = tqdm(results, desc=desc, total=len(trials)) + return list(results) diff --git a/src/giant_python/pipeline/__init__.py b/src/giant_python/pipeline/__init__.py index 110b418..28c68ce 100644 --- a/src/giant_python/pipeline/__init__.py +++ b/src/giant_python/pipeline/__init__.py @@ -1,53 +1,12 @@ -"""Pipeline orchestration for GIAnT-Python. +"""Implemented band annotation/extraction services and a chainable facade.""" -Thin stage classes (the "C" layer) that compose the functional/dataclass core -into the three pipeline stages, plus the :class:`Pipeline` facade that owns the -shared session state and exposes ``run_all``. The underlying stage *functions* -are also exported for power users who want to skip the classes. -""" - -from .base import Stage -from .extract import ( - IntegrationSourceExtractor, - SourceExtractor, - StandardSourceExtractor, - extract_integration_sources, - silo, -) -from .organize import ( - TrialTableBuilder, - build_trial_table, - build_trial_table_slap2, - verify_files, -) +from .annotate import annotate_rois +from .extract import extract_band_sources, extract_sources from .pipeline import Pipeline -from .register import ( - IntegrationRegistration, - MotionCorrector, - MultiRoiRegistration, - StripRegistration, - integration_registration, - multi_roi_registration, - strip_registration, -) __all__ = [ - "build_trial_table", - "build_trial_table_slap2", - "extract_integration_sources", - "integration_registration", - "IntegrationRegistration", - "IntegrationSourceExtractor", - "MotionCorrector", - "MultiRoiRegistration", - "multi_roi_registration", + "annotate_rois", + "extract_band_sources", + "extract_sources", "Pipeline", - "silo", - "SourceExtractor", - "StandardSourceExtractor", - "Stage", - "StripRegistration", - "strip_registration", - "TrialTableBuilder", - "verify_files", ] diff --git a/src/giant_python/pipeline/annotate.py b/src/giant_python/pipeline/annotate.py new file mode 100644 index 0000000..0e2f2a2 --- /dev/null +++ b/src/giant_python/pipeline/annotate.py @@ -0,0 +1,55 @@ +"""Public ROI annotation service with lazy band/GUI imports.""" + +from __future__ import annotations + +from typing import Optional + +from ..models.params import ( + AnnotationInput, + BandParamsInput, + ExecutionInput, +) +from .extract import TrialTableInput, _require_band_mode + + +def annotate_rois( + input: TrialTableInput, + params: BandParamsInput = None, + *, + microscope: str = "slap2", + scan_mode: Optional[str] = "band", + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> str: + """Load or draw and save band ROIs through the shared annotation service. + + Parameters + ---------- + input : str, Path or TrialTable + Exact filename or loaded trial table; never reconstructed from a + results directory. + params : BandSiloParams, dict or None + Scientific configuration only. + microscope : str + Currently only ``'slap2'`` is supported. + scan_mode : str or None + Same routing policy as ``extract_sources``. + execution : ExecutionOptions, dict or None + Run policy, including logging. + annotations : AnnotationOptions, dict or None + ROI inclusion and GUI policy. Calling this explicit annotation + step enables ROI handling for this call even if ``enabled=False``. + + Returns + ------- + str + Path to the written or existing annotation file. + """ + _require_band_mode(microscope, scan_mode) + if input is None: + raise ValueError("A trial-table path or TrialTable is required") + from ..extraction.band.annotation import annotate_band_rois + + return annotate_band_rois( + input, params, execution=execution, annotations=annotations + ) diff --git a/src/giant_python/pipeline/base.py b/src/giant_python/pipeline/base.py deleted file mode 100644 index a61589d..0000000 --- a/src/giant_python/pipeline/base.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Abstract base class for pipeline stages.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any - - -class Stage(ABC): - """Base class for a single pipeline stage. - - A stage bundles its parameters and exposes a single :meth:`run` method. - Concrete stages (``TrialTableBuilder``, ``MotionCorrector``, - ``SourceExtractor``) delegate the actual computation to the functional - core and the math kernels. Stages are independently usable for partial - re-runs; the :class:`~giant_python.pipeline.pipeline.Pipeline` facade - composes them for the common end-to-end case. - """ - - @abstractmethod - def run(self, *args: Any, **kwargs: Any) -> Any: - """Execute the stage and return its result.""" - raise NotImplementedError diff --git a/src/giant_python/pipeline/extract.py b/src/giant_python/pipeline/extract.py index 34a1d7a..ef2f5fe 100644 --- a/src/giant_python/pipeline/extract.py +++ b/src/giant_python/pipeline/extract.py @@ -1,16 +1,7 @@ -"""Stage 3 - source extraction / SILo (port of SILo.m). - -Exposes the porting-faithful extraction functions plus a -:class:`SourceExtractor` base with one backend per SLAP2 scan mode -(standard pixel-movie SILo / integration band-scan SILo), selected via -:meth:`SourceExtractor.for_scan_mode` -- mirroring the -:class:`~giant_python.pipeline.register.MotionCorrector` backend pattern. - -Both backends share the same algorithmic skeleton (activity image -> peak -detection -> source localization -> trace extraction -> baseline/dF -> -deconvolution); the shared steps live as pure kernels in -:mod:`giant_python.math` (``peaks``, ``baseline``, ``deconv``) so that only -the scan-mode-specific data loading and geometry differ between backends. +"""Canonical standalone extraction services. + +Only SLAP2 band extraction is supported. Backend imports are lazy; no input +table is reloaded here. """ from __future__ import annotations @@ -18,126 +9,111 @@ from pathlib import Path from typing import Optional, Union -from ..models.params import SiloParams -from .base import Stage +from ..models.experiment_summary import ExperimentSummary +from ..models.params import ( + AnnotationInput, + BandParamsInput, + ExecutionInput, +) +from ..models.trial_table import TrialTable +TrialTableInput = Union[str, Path, TrialTable] -def silo( - dr_or_path_to_trial_table: Optional[Union[str, Path]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform standard (pixel-movie) source extraction and localization. - Detects active synaptic boutons or spines from registered movies using - peak-finding on a per-trial activity image followed by NMF-based trace - refinement across trials. Results are saved as an experiment summary - file. Corresponds to SILo.m in GIAnT-MATLAB. +def _require_band_mode( + microscope: str, + scan_mode: Optional[str], +) -> None: + """Reject unsupported routing before loading data or a backend.""" + if microscope != "slap2": + raise ValueError( + "Only microscope='slap2', scan_mode='band' is supported" + ) + mode = "band" if scan_mode is None else scan_mode + if mode != "band": + raise ValueError( + f"Source extraction for scan_mode={mode!r} is unsupported; " + "only 'band' is supported" + ) + + +def extract_sources( + input: TrialTableInput, + params: BandParamsInput = None, + *, + microscope: str = "slap2", + scan_mode: Optional[str] = "band", + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> ExperimentSummary: + """Extract sources through the supported workflow service. Parameters ---------- - dr_or_path_to_trial_table : str or Path, optional - Either a directory containing a trial table file, or the full path - to a trial table file. - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params` under ``"SILo"``. + input : str, Path or TrialTable + Exact trial-table filename or a loaded table, passed through intact. + params : BandSiloParams, dict or None + Scientific settings only. + microscope : str + Currently only ``'slap2'`` is supported. + scan_mode : str or None + Explicit routing, defaulting to band. ``None`` also selects band. + execution : ExecutionOptions, dict or None + Run policy, independent of science. + annotations : AnnotationOptions, dict or None + ROI inclusion and interactivity policy. + + Returns + ------- + ExperimentSummary + Eager extraction results, also persisted by the backend. + + Raises + ------ + ValueError + If an unsupported microscope or scan mode is requested. """ - raise NotImplementedError + _require_band_mode(microscope, scan_mode) + return extract_band_sources( + input, params, execution=execution, annotations=annotations + ) -def extract_integration_sources( - path_to_trial_table: Optional[Union[str, Path]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform integration (band-scan) source extraction for SLAP2. +def extract_band_sources( + input: TrialTableInput, + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> ExperimentSummary: + """Perform band-scan source extraction for SLAP2. The band-scan variant of SILo: operates on superpixel / DMD-geometry - data and the integration lookup table produced by - :func:`giant_python.pipeline.register.integration_registration` - (``fnAdataInt``), rather than on a reconstructed pixel movie. Shares the - peak-detection, baseline, and deconvolution kernels with :func:`silo`. - Corresponds to ``extractSLAP2IntegrationSources.py`` in - ophys-slap2-analysis. - - Parameters - ---------- - path_to_trial_table : str or Path, optional - Full path to the trial table file (with integration alignment data). - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params`. - """ - raise NotImplementedError - + data and the externally produced band registration lookup table + (``fnAdataInt``), rather than on a reconstructed pixel movie. Corresponds + to ``extractSLAP2IntegrationSources.py`` in ophys-slap2-analysis. -class SourceExtractor(Stage): - """Stage 3 base class: extract sources from motion-corrected trials. - - Use :meth:`for_scan_mode` to obtain the appropriate backend. Concrete - subclasses wrap the corresponding extraction function. + Delegates lazily to ``giant_python.extraction.band.workflow``. All option + resolution belongs to that service. Parameters ---------- - params : SiloParams, optional - Source-extraction parameters; defaults to :class:`SiloParams` - defaults. - """ - - def __init__(self, params: Optional[SiloParams] = None) -> None: - self.params = params or SiloParams() - - @staticmethod - def for_scan_mode( - scan_mode: str, - params: Optional[SiloParams] = None, - ) -> "SourceExtractor": - """Return the extraction backend for the given SLAP2 scan mode. - - Parameters - ---------- - scan_mode : str - ``"standard"`` (-> :class:`StandardSourceExtractor`) or - ``"integration"`` (-> :class:`IntegrationSourceExtractor`). - params : SiloParams, optional - Source-extraction parameters. - - Returns - ------- - SourceExtractor - The appropriate backend instance. - """ - raise NotImplementedError - - def run(self, trial_table: "object") -> "object": - """Extract sources and write experiment_summary.h5. - - Parameters - ---------- - trial_table : TrialTable - A trial table whose trials have been motion-corrected. - - Returns - ------- - ExperimentSummary - The extracted sources and summary (also persisted to disk). - """ - raise NotImplementedError - - -class StandardSourceExtractor(SourceExtractor): - """Standard pixel-movie SILo backend (wraps :func:`silo`).""" - - def run(self, trial_table: "object") -> "object": - """Extract from pixel movies. See :meth:`SourceExtractor.run`.""" - raise NotImplementedError - - -class IntegrationSourceExtractor(SourceExtractor): - """Integration (band-scan) SILo backend. - - Wraps :func:`extract_integration_sources`. + input : str, Path or TrialTable + Trial-table path or loaded metadata with band alignment data. + params : BandSiloParams, dict or None + Scientific settings only. + execution : ExecutionOptions, dict or None + Run policy, independent of scientific settings. + annotations : AnnotationOptions, dict or None + ROI inclusion and GUI policy. + + Returns + ------- + ExperimentSummary + The extracted sources and summary (also written to disk). """ + if input is None: + raise ValueError("A trial-table path or TrialTable is required") + from ..extraction.band.workflow import extract_band_sources as extract - def run(self, trial_table: "object") -> "object": - """Extract from integration data. See :meth:`SourceExtractor.run`.""" - raise NotImplementedError + return extract(input, params, execution=execution, annotations=annotations) diff --git a/src/giant_python/pipeline/organize.py b/src/giant_python/pipeline/organize.py deleted file mode 100644 index 1f59ef0..0000000 --- a/src/giant_python/pipeline/organize.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Stage 1 - trial organization (ports of buildTrialTable[SLAP2].m). - -Builds the trial table that threads through the rest of the pipeline, plus the -``verify_files`` integrity check. Exposes both the porting-faithful functions -and the :class:`TrialTableBuilder` stage class. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Optional, Tuple, Union - -import numpy as np - -from .base import Stage - - -def build_trial_table( - dr: Optional[Union[str, Path]] = None, - fns: Optional[Union[str, list]] = None, - save_dr: Optional[Union[str, Path]] = None, -) -> dict: - """Organize multi-trial recordings and metadata for Bergamo recordings. - - Groups TIFF or HDF5 recording files into epochs and trials, populating - a trial table that is saved to disk. Corresponds to buildTrialTable.m - in GIAnT-MATLAB. - - Parameters - ---------- - dr : str or Path, optional - Directory containing TIFF or HDF5 recording files. - fns : str or list, optional - Filename or list of filenames to include. Pass ``True`` to - auto-select all non-registered TIFF files in *dr*. - save_dr : str or Path, optional - Directory in which to save the trial table. Defaults to *dr*. - - Returns - ------- - dict - Trial table with keys ``filename``, ``trueTrialIx``, ``epoch``, - ``trialEndTimeFromPC``, and ``trialStartTimeInferred``. - """ - raise NotImplementedError - - -def build_trial_table_slap2( - dr: Optional[Union[str, Path]] = None, - save_dr: Optional[Union[str, Path]] = None, -) -> dict: - """Organize multi-trial recordings for the SLAP2 data processing pipeline. - - Groups DMD1 and DMD2 DAT files into epochs and trials, loads reference - stacks, and populates a trial table saved to disk. Corresponds to - buildTrialTableSLAP2.m in GIAnT-MATLAB. - - Parameters - ---------- - dr : str or Path, optional - Directory containing DAT recording files and reference TIFF stacks. - save_dr : str or Path, optional - Directory in which to save the trial table. Defaults to *dr*. - - Returns - ------- - dict - Trial table with keys ``filename``, ``firstLine``, ``lastLine``, - ``trueTrialIx``, ``epoch``, ``refStack``, ``trialEndTimeFromPC``, - ``trialStartTimeInferred``, ``datadr``, and ``savedr``. - """ - raise NotImplementedError - - -def verify_files( - fn: Union[str, Path], - dr: Union[str, Path], - params: dict, -) -> Tuple[dict, np.ndarray]: - """Verify that all files referenced by a trial table exist on disk. - - Loads the trial table and checks for the registered TIFF or HDF5 movies, - alignment data files, and raw source data files for each trial and DMD. - Returns the updated trial table and a boolean keep-mask. Corresponds to - verifyFiles.m in GIAnT-MATLAB. - - Parameters - ---------- - fn : str or Path - Filename of the trial table file (relative to *dr*). - dr : str or Path - Directory containing the trial table and associated data. - params : dict - Parameter dictionary, as returned by - :func:`giant_python.models.params.set_params`. - - Returns - ------- - trial_table : dict - Loaded (and possibly path-corrected) trial table. - keep_trials : ndarray of bool, shape (n_dmds, n_trials) - ``True`` for each trial that has all required files present. - """ - raise NotImplementedError - - -class TrialTableBuilder(Stage): - """Stage 1: build the trial table for a microscope's raw data. - - Dispatches to :func:`build_trial_table` (Bergamo) or - :func:`build_trial_table_slap2` (SLAP2) based on *microscope*. - - Parameters - ---------- - microscope : str - ``"slap2"`` or ``"bergamo"``. - """ - - def __init__(self, microscope: str = "slap2") -> None: - self.microscope = microscope - - def run( - self, - data_dir: Union[str, Path], - save_dir: Union[str, Path], - ) -> "object": - """Organize trials and write trial_table.h5. - - Parameters - ---------- - data_dir : str or Path - Directory containing the raw recording files. - save_dir : str or Path - Directory in which to write trial_table.h5. - - Returns - ------- - TrialTable - The organized trial table (also persisted to disk). - """ - raise NotImplementedError diff --git a/src/giant_python/pipeline/pipeline.py b/src/giant_python/pipeline/pipeline.py index 230bf1b..ac5751b 100644 --- a/src/giant_python/pipeline/pipeline.py +++ b/src/giant_python/pipeline/pipeline.py @@ -1,42 +1,43 @@ -"""The :class:`Pipeline` facade: the full-pipeline orchestration object. - -This is a thin *coordinator*, deliberately not a god-object. It holds only -lightweight session state (config, paths, and the ``TrialTable`` metadata), -delegates all real work to the stage classes (which remain independently -usable for partial re-runs), and owns the cross-cutting concerns that belong -to no single stage: ``run_all``, resume/checkpointing, logging, and config -loading. - -Crucially it does **not** cache heavy pixel arrays, mean images, or source -matrices in memory; those stay on disk in HDF5/TIFF and are loaded lazily by -each stage. ``self.summary`` references an ``ExperimentSummary`` whose heavy -arrays are themselves disk-backed. +"""Chainable coordinator over the same services as standalone functions. + +Band annotation/extraction starts from existing registered trial metadata. +Returned summaries currently contain eager arrays. """ from __future__ import annotations +from dataclasses import replace from pathlib import Path -from typing import Optional, Union +from typing import Optional -from ..models.params import AlignParams, SiloParams -from .extract import SourceExtractor -from .organize import TrialTableBuilder -from .register import MotionCorrector +from ..models.experiment_summary import ExperimentSummary +from ..models.params import ( + AnnotationInput, + BandParamsInput, + BandSiloParams, + ExecutionInput, + resolve_band_options, +) +from ..models.trial_table import TrialTable +from .annotate import annotate_rois +from .extract import TrialTableInput, extract_sources class Pipeline: - """Coordinator for the full organize -> register -> extract pipeline. + """Coordinator for annotation and extraction from a registered trial table. Parameters ---------- microscope : str - ``"slap2"`` or ``"bergamo"``. - save_dir : str or Path - Directory in which all results are written. - align_params : AlignParams, optional - Motion-correction parameters. - silo_params : SiloParams, optional - Source-extraction parameters. + Acquisition type; only ``"slap2"`` band extraction is implemented. + params : BandSiloParams, dict or None + Scientific settings only; defaults to band. + execution : ExecutionOptions, dict or None + Run policy, separate from science. + annotations : AnnotationOptions, dict or None + ROI inclusion and GUI policy. + scan_mode : str or None + Optional explicit routing override; unsupported modes fail honestly. Attributes ---------- @@ -44,150 +45,123 @@ class Pipeline: The session's trial table (lightweight metadata), threaded between stages. summary : ExperimentSummary or None - The extraction result (heavy arrays disk-backed). + The extraction result (eager arrays, not a disk-backed proxy). + trial_table_path : Path or None + Exact source filename, when known. Never inferred from ``savedr``. """ def __init__( self, - microscope: str, - save_dir: Union[str, Path], - align_params: Optional[AlignParams] = None, - silo_params: Optional[SiloParams] = None, + microscope: str = "slap2", + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, + scan_mode: Optional[str] = "band", ) -> None: + """Store session config; ``tt``/``summary`` start empty.""" self.microscope = microscope - self.save_dir = Path(save_dir) - self.align_params = align_params or AlignParams() - self.silo_params = silo_params or SiloParams(microscope=microscope) - self.tt: Optional[object] = None - self.summary: Optional[object] = None + self.params = BandSiloParams() if params is None else params + self.execution = execution + self.annotations = annotations + self.scan_mode = scan_mode + self.tt: Optional[TrialTable] = None + self.summary: Optional[ExperimentSummary] = None + self.trial_table_path: Optional[Path] = None + self.annotation_path: Optional[str] = None @classmethod def from_trial_table( cls, - path: Union[str, Path], - microscope: str, - **kwargs: object, + path: TrialTableInput, + microscope: str = "slap2", + **kwargs, ) -> "Pipeline": - """Resume a session from an existing trial_table.h5. + """Start a session from a trial-table path or loaded metadata. Parameters ---------- - path : str or Path - Path to an existing trial_table.h5 file. + path : str, Path or TrialTable + Exact filename or an already loaded table. A path is read once; + in-memory inputs are not copied, serialized or reloaded. microscope : str - ``"slap2"`` or ``"bergamo"``. + ``"slap2"``; other acquisition types are unsupported. **kwargs - Forwarded to :class:`Pipeline` (e.g. ``align_params``). - - Returns - ------- - Pipeline - A pipeline with ``tt`` loaded from disk. - """ - raise NotImplementedError - - @classmethod - def from_config(cls, path: Union[str, Path]) -> "Pipeline": - """Construct a pipeline from a declarative config file. - - Hook for the future config-driven (YAML/TOML) run mode. - - Parameters - ---------- - path : str or Path - Path to the config file. - - Returns - ------- - Pipeline - A configured pipeline. - """ - raise NotImplementedError - - def organize(self, data_dir: Union[str, Path]) -> "Pipeline": - """Run stage 1 (trial organization), storing the result in ``self.tt``. - - Parameters - ---------- - data_dir : str or Path - Directory containing the raw recording files. + Forwarded to :class:`Pipeline` (e.g. ``params`` or ``execution``). Returns ------- Pipeline - ``self`` (supports method chaining). + A pipeline retaining ``tt`` and its known source filename. """ - self.tt = TrialTableBuilder(self.microscope).run( - data_dir, self.save_dir + if isinstance(path, TrialTable): + table = path + source_path = table.source_path + elif isinstance(path, (str, Path)): + source_path = Path(path) + table = TrialTable.from_h5(source_path) + # Keep the supplied filename even when savedr points elsewhere. + table.source_path = source_path + else: + raise TypeError("Expected a trial-table path or TrialTable") + pipeline = cls(microscope=microscope, **kwargs) + pipeline.tt = table + pipeline.trial_table_path = ( + Path(source_path) if source_path is not None else None ) - return self + return pipeline - def register(self) -> "Pipeline": - """Run stage 2 (motion correction), updating ``self.tt``. - - Returns - ------- - Pipeline - ``self`` (supports method chaining). - """ - corrector = MotionCorrector.for_microscope( - self.microscope, self.align_params - ) - self.tt = corrector.run(self.tt) - return self + def _require_trial_table(self) -> TrialTable: + """Return loaded metadata or explain how to initialize a session.""" + if self.tt is None: + raise ValueError( + "No trial table is loaded; use Pipeline.from_trial_table() " + "before annotation or extraction" + ) + return self.tt def annotate(self) -> "Pipeline": - """Run the optional ROI-annotation GUI step. + """Load or draw band ROIs, retaining the same trial-table object. - Imported lazily so headless installs never import the GUI extra. + Uses the public annotation service and enables ROI inclusion for a + subsequent ``extract()`` after success. Drawing is still subject to + explicit/headless/automatic GUI policy. Caller options are not edited. Returns ------- Pipeline ``self`` (supports method chaining). """ - from ..gui import annotate_rois - - annotate_rois(self.tt) + table = self._require_trial_table() + self.annotation_path = annotate_rois( + table, + self.params, + microscope=self.microscope, + scan_mode=self.scan_mode, + execution=self.execution, + annotations=self.annotations, + ) + _, _, policy = resolve_band_options( + self.params, self.execution, self.annotations + ) + self.annotations = replace(policy, enabled=True) return self def extract(self) -> "Pipeline": - """Run stage 3 (source extraction), storing ``self.summary``. + """Run source extraction, storing ``self.summary``. Returns ------- Pipeline ``self`` (supports method chaining). """ - extractor = SourceExtractor.for_scan_mode( - self.silo_params.scan_mode, self.silo_params + table = self._require_trial_table() + self.summary = extract_sources( + table, + self.params, + microscope=self.microscope, + scan_mode=self.scan_mode, + execution=self.execution, + annotations=self.annotations, ) - self.summary = extractor.run(self.tt) return self - - def run_all( - self, - data_dir: Union[str, Path], - *, - annotate: bool = False, - resume: bool = True, - ) -> "Pipeline": - """Run the whole pipeline end-to-end. - - Parameters - ---------- - data_dir : str or Path - Directory containing the raw recording files. - annotate : bool - Whether to run the GUI annotation step between registration and - extraction. - resume : bool - Skip stages whose HDF5 outputs already exist (mirrors the - ``overwriteExisting`` logic in GIAnT-MATLAB). - - Returns - ------- - Pipeline - ``self`` (with ``tt`` and ``summary`` populated). - """ - raise NotImplementedError diff --git a/src/giant_python/pipeline/register.py b/src/giant_python/pipeline/register.py deleted file mode 100644 index 70d1a72..0000000 --- a/src/giant_python/pipeline/register.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Stage 2 - motion correction (ports of *Registration.m). - -Exposes the porting-faithful registration functions plus a -:class:`MotionCorrector` ABC with one backend per MATLAB registration variant -(MultiRoi / Strip / Band). Band-mode registration is exposed as -:func:`integration_registration` / :class:`IntegrationRegistration` (the -Python API name) and corresponds to ``BandRegistration.m``, writing -``bandRegLookupTable.h5``. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Optional, Union - -from ..models.params import AlignParams -from .base import Stage - - -def multi_roi_registration( - full_path_to_trial_table: Optional[Union[str, Path]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform motion correction for SLAP2 multi-ROI recordings. - - Registers each trial to a common template computed from initial frames, - producing downsampled registered TIFF movies and alignment metadata for - each DMD. Corresponds to MultiRoiRegistration.m in GIAnT-MATLAB. - - Parameters - ---------- - full_path_to_trial_table : str or Path, optional - Full path to the trial table file. - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params` under - ``"MultiRoiRegistration"``. - """ - raise NotImplementedError - - -def strip_registration( - dr: Optional[Union[str, Path]] = None, - fns: Optional[Union[str, list]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform strip-based motion correction for Bergamo recordings. - - Registers each trial using DFT-based rigid registration, producing - downsampled and full-resolution registered TIFF or HDF5 movies together - with per-trial alignment metadata. Corresponds to StripRegistration.m in - GIAnT-MATLAB. - - Parameters - ---------- - dr : str or Path, optional - Directory containing TIFF or HDF5 files, or the path to a trial - table file. - fns : str or list, optional - Filename(s) to register, or path to a trial table file. - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params` under - ``"StripRegistration"``. - """ - raise NotImplementedError - - -def integration_registration( - full_path_to_trial_table: Optional[Union[str, Path]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform band / integration-mode motion correction for SLAP2 recordings. - - Builds the cached ``motion_correction/bandRegLookupTable.h5`` lookup table - (on first run) and writes per-trial ``*_ALIGNMENTDATA.h5`` files including - BandRegistration-only fields ``brightnessDS`` and ``logLikelihoodDS``. - Corresponds to ``BandRegistration.m`` in GIAnT-MATLAB. - - Parameters - ---------- - full_path_to_trial_table : str or Path, optional - Full path to the trial table file. - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params` under - ``"BandRegistration"``. - """ - raise NotImplementedError - - -class MotionCorrector(Stage): - """Stage 2 base class: register all trials in a trial table. - - Use :meth:`for_microscope` to obtain the appropriate backend. Concrete - subclasses wrap the corresponding registration function. - - Parameters - ---------- - params : AlignParams, optional - Alignment parameters; defaults to :class:`AlignParams` defaults. - """ - - def __init__(self, params: Optional[AlignParams] = None) -> None: - self.params = params or AlignParams() - - @staticmethod - def for_microscope( - microscope: str, - params: Optional[AlignParams] = None, - ) -> "MotionCorrector": - """Return the registration backend for the given microscope. - - Parameters - ---------- - microscope : str - ``"slap2"`` (-> :class:`MultiRoiRegistration`) or ``"bergamo"`` - (-> :class:`StripRegistration`). - params : AlignParams, optional - Alignment parameters. - - Returns - ------- - MotionCorrector - The appropriate backend instance. - """ - raise NotImplementedError - - def run(self, trial_table: "object") -> "object": - """Register every trial and update the trial table in place. - - Parameters - ---------- - trial_table : TrialTable - The organized trial table. - - Returns - ------- - TrialTable - The trial table with its ``motion_correction`` group populated. - """ - raise NotImplementedError - - -class MultiRoiRegistration(MotionCorrector): - """SLAP2 multi-ROI backend (wraps ``multi_roi_registration``).""" - - def run(self, trial_table: "object") -> "object": - """Register SLAP2 multi-ROI trials. See :meth:`MotionCorrector.run`.""" - raise NotImplementedError - - -class StripRegistration(MotionCorrector): - """Bergamo strip backend (wraps ``strip_registration``).""" - - def run(self, trial_table: "object") -> "object": - """Register Bergamo strip trials. See :meth:`MotionCorrector.run`.""" - raise NotImplementedError - - -class IntegrationRegistration(MotionCorrector): - """SLAP2 BandRegistration backend (wraps ``integration_registration``). - - Corresponds to ``BandRegistration.m``; writes ``bandRegLookupTable.h5``. - """ - - def run(self, trial_table: "object") -> "object": - """Register SLAP2 band/integration trials. - - See ``MotionCorrector.run``. - """ - raise NotImplementedError diff --git a/src/giant_python/progress.py b/src/giant_python/progress.py new file mode 100644 index 0000000..e61444b --- /dev/null +++ b/src/giant_python/progress.py @@ -0,0 +1,63 @@ +"""Shared opt-in verbose logging and progress bars. + +The BandSILo port stays quiet by default (headless / batch friendly); the +reference's unconditional ``print``s and ``tqdm`` bars were dropped. These +helpers restore that feedback opt-in: when ``verbose`` is set +(``ExecutionOptions.verbose`` / ``giant ... --verbose``) messages are printed +and long-running loops show a progress bar. ``tqdm`` is imported lazily, so a +non-verbose run neither prints nor imports it. +""" + +from __future__ import annotations + +from typing import Iterable, Optional + + +def log(message: str, verbose: bool) -> None: + """Print ``message`` when ``verbose`` is set; otherwise do nothing. + + Parameters + ---------- + message : str + The status message to display. + verbose : bool + Whether verbose output is enabled. + """ + if verbose: + print(message) + + +def progress( + iterable: Iterable, + *, + desc: str, + verbose: bool, + total: Optional[int] = None, +) -> Iterable: + """Wrap ``iterable`` in a tqdm progress bar when ``verbose`` is set. + + Returns the iterable unchanged when ``verbose`` is ``False`` so callers can + wrap any loop unconditionally at zero cost (and without importing + ``tqdm``). + + Parameters + ---------- + iterable : iterable + The loop iterable to (optionally) wrap. + desc : str + Progress-bar label. + verbose : bool + Whether to show the bar. + total : int, optional + Item count, for iterables without a ``len`` (e.g. a generator). + + Returns + ------- + iterable + The tqdm-wrapped iterable, or the original iterable. + """ + if not verbose: + return iterable + from tqdm import tqdm + + return tqdm(iterable, desc=desc, total=total) diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 816e430..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Testing library""" diff --git a/tests/test_example.py b/tests/test_example.py deleted file mode 100644 index e2f68ae..0000000 --- a/tests/test_example.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Tests for the giant_python package.""" - -import unittest - - -class PackageTest(unittest.TestCase): - """Tests for the giant_python package.""" - - def test_import(self): - """Test that the giant_python package can be imported.""" - import giant_python - - self.assertIsNotNone(giant_python.__version__) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_gui.py b/tests/test_gui.py deleted file mode 100644 index ac94e77..0000000 --- a/tests/test_gui.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Tests for the gui subpackage.""" - -import unittest - -from giant_python.gui import DrawROIs, annotate_rois - - -class TestAnnotateRois(unittest.TestCase): - """Tests for the ROI annotation entry point.""" - - def test_annotate_rois(self): - """annotate_rois raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - annotate_rois(None) - - -class TestDrawROIs(unittest.TestCase): - """Tests for the DrawROIs class.""" - - def test_show(self): - """DrawROIs.show raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - DrawROIs(None).show() - - def test_save_rois(self): - """DrawROIs.save_rois raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - DrawROIs(None).save_rois("annotations.h5") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_io.py b/tests/test_io.py deleted file mode 100644 index cbc56e7..0000000 --- a/tests/test_io.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Tests for the io subpackage.""" - -import unittest - -import numpy as np - -from giant_python.io import ( - get_online_motion, - load_struct_h5, - read_integration_trial_data, - ref_pixs_to_drc, - save_struct_h5, - scanimagetiff_data_wrapper, - scanimagetiff_wrapper, -) - - -class TestHdf5(unittest.TestCase): - """Tests for the generic struct <-> HDF5 helpers.""" - - def test_load_struct_h5(self): - """load_struct_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - load_struct_h5("dummy.h5") - - def test_save_struct_h5(self): - """save_struct_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - save_struct_h5({}, "dummy.h5") - - -class TestTiff(unittest.TestCase): - """Tests for the ScanImage TIFF readers.""" - - def test_scanimagetiff_wrapper(self): - """scanimagetiff_wrapper raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - scanimagetiff_wrapper("dummy.tif") - - def test_scanimagetiff_data_wrapper(self): - """scanimagetiff_data_wrapper raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - scanimagetiff_data_wrapper(None, "dummy.tif") - - -class TestSlap2(unittest.TestCase): - """Tests for SLAP2 readers (online motion + integration).""" - - def test_get_online_motion(self): - """get_online_motion raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - get_online_motion(None, np.arange(10)) - - def test_ref_pixs_to_drc(self): - """ref_pixs_to_drc raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - ref_pixs_to_drc(np.arange(10), 4, 4) - - def test_read_integration_trial_data(self): - """read_integration_trial_data raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - read_integration_trial_data(None, np.arange(5), np.arange(3)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_math.py b/tests/test_math.py deleted file mode 100644 index d11829c..0000000 --- a/tests/test_math.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Tests for the math subpackage.""" - -import unittest - -import numpy as np - -from giant_python.math import ( - dft_register_clipped, - interp_frame, - xcorr2_nans, - xcorr2_nans3d, - xcorr2_nans_weighted, -) -from giant_python.math import ( - compute_f0, - detect_peaks_2d, - gaussian_peaks_integrated, - get_act_im_peaks, -) -from giant_python.math.deconv import deconvolve_trace -from giant_python.math.nmf import nmf_decompose -from giant_python.math.variance import activity_image - - -class TestRegistration(unittest.TestCase): - """Tests for registration kernels.""" - - def test_xcorr2_nans_weighted(self): - """xcorr2_nans_weighted raises NotImplementedError.""" - img = np.zeros((4, 4)) - with self.assertRaises(NotImplementedError): - xcorr2_nans_weighted(img, img, img, np.zeros(2), 3.0) - - def test_xcorr2_nans(self): - """xcorr2_nans raises NotImplementedError.""" - img = np.zeros((4, 4)) - with self.assertRaises(NotImplementedError): - xcorr2_nans(img, img) - - def test_xcorr2_nans3d(self): - """xcorr2_nans3d raises NotImplementedError.""" - vol = np.zeros((2, 4, 4)) - with self.assertRaises(NotImplementedError): - xcorr2_nans3d(vol, vol) - - def test_dft_register_clipped(self): - """dft_register_clipped raises NotImplementedError.""" - img = np.zeros((4, 4)) - with self.assertRaises(NotImplementedError): - dft_register_clipped(img, img) - - -class TestInterpolation(unittest.TestCase): - """Tests for interp_frame.""" - - def test_interp_frame(self): - """interp_frame raises NotImplementedError.""" - img = np.zeros((4, 4)) - with self.assertRaises(NotImplementedError): - interp_frame(img, img, img, img) - - -class TestExtractionKernels(unittest.TestCase): - """Tests for nmf/deconv/variance kernels.""" - - def test_nmf_decompose(self): - """nmf_decompose raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - nmf_decompose(np.zeros((16, 10)), 2) - - def test_deconvolve_trace(self): - """deconvolve_trace raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - deconvolve_trace(np.zeros(10), 0.5, 30.0) - - def test_activity_image(self): - """activity_image raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - activity_image(np.zeros((10, 4, 4)), 1.0) - - -class TestPeaks(unittest.TestCase): - """Tests for the shared peak-detection kernels.""" - - def test_gaussian_peaks_integrated(self): - """gaussian_peaks_integrated raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - gaussian_peaks_integrated(np.zeros((1, 4)), np.zeros((4, 2))) - - def test_detect_peaks_2d(self): - """detect_peaks_2d raises NotImplementedError.""" - img = np.zeros((8, 8)) - with self.assertRaises(NotImplementedError): - detect_peaks_2d(img, np.zeros((8, 8), bool), 0.0, 1.0, 3.0, 3.0) - - def test_get_act_im_peaks(self): - """get_act_im_peaks raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - get_act_im_peaks(np.zeros((2, 8, 8))) - - -class TestBaseline(unittest.TestCase): - """Tests for the shared baseline kernel.""" - - def test_compute_f0(self): - """compute_f0 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - compute_f0(np.zeros((20, 3)), 5, 8) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_models.py b/tests/test_models.py deleted file mode 100644 index 0a00881..0000000 --- a/tests/test_models.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Tests for the models subpackage.""" - -import unittest - -from giant_python.models import ( - AlignmentData, - AlignParams, - ExperimentSummary, - SiloParams, - Slap2Info, - Source, - TrialTable, - UserRoi, - Visualizations, - set_params, -) - - -class TestDataclasses(unittest.TestCase): - """Dataclasses should construct with sensible defaults.""" - - def test_trial_table_defaults(self): - """TrialTable constructs empty with no motion/source groups.""" - tt = TrialTable() - self.assertIsNone(tt.motion_correction) - self.assertIsNone(tt.source_extraction) - - def test_slap2_info_defaults(self): - """Slap2Info constructs with empty collections.""" - self.assertEqual(Slap2Info().ref_stack, {}) - - def test_alignment_data_defaults(self): - """AlignmentData defaults to a non-failed registration.""" - self.assertFalse(AlignmentData().registration_failed) - - def test_experiment_summary_defaults(self): - """ExperimentSummary nests a Visualizations instance.""" - summary = ExperimentSummary() - self.assertIsInstance(summary.visualizations, Visualizations) - self.assertEqual(summary.sources, []) - - def test_source_and_user_roi(self): - """Source and UserRoi construct with defaults.""" - self.assertIsNone(Source().snr) - self.assertEqual(UserRoi().type, "polygon") - - def test_param_models(self): - """Param models expose typed defaults.""" - self.assertEqual(AlignParams().align_hz, 80.0) - self.assertEqual(SiloParams().microscope, "slap2") - self.assertEqual(SiloParams().scan_mode, "standard") - - -class TestH5RoundTrip(unittest.TestCase): - """from_h5/to_h5 are not implemented yet.""" - - def test_trial_table_from_h5(self): - """TrialTable.from_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - TrialTable.from_h5("trial_table.h5") - - def test_trial_table_to_h5(self): - """TrialTable.to_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - TrialTable().to_h5("trial_table.h5") - - def test_alignment_data_from_h5(self): - """AlignmentData.from_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - AlignmentData.from_h5("a.h5") - - def test_alignment_data_to_h5(self): - """AlignmentData.to_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - AlignmentData().to_h5("a.h5") - - def test_experiment_summary_from_h5(self): - """ExperimentSummary.from_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - ExperimentSummary.from_h5("experiment_summary.h5") - - def test_experiment_summary_to_h5(self): - """ExperimentSummary.to_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - ExperimentSummary().to_h5("experiment_summary.h5") - - -class TestSetParams(unittest.TestCase): - """Tests for the set_params compatibility shim.""" - - def test_not_implemented(self): - """set_params raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - set_params("SILo") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_package_metadata.py b/tests/test_package_metadata.py deleted file mode 100644 index b4fc5c7..0000000 --- a/tests/test_package_metadata.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Tests for package metadata declarations.""" - -import ast -import unittest -from pathlib import Path - - -PACKAGE_INIT = ( - Path(__file__).resolve().parents[1] / "src" / "giant_python" / "__init__.py" -) - - -class PackageMetadataTest(unittest.TestCase): - """Tests for package metadata declarations.""" - - def test_init_module_is_valid_and_has_single_version_line(self): - """Only one ``__version__`` line may exist so CI sed bumps stay safe. - - The shared AIND ``release-bump-version`` workflow rewrites every line - that contains ``__version__`` with:: - - sed 's|__version__.*|__version__ = "X.Y.Z"|' - - Listing ``"__version__"`` in ``__all__`` therefore produces invalid - Python on the next bump. Keep a single assignment line instead. - """ - source = PACKAGE_INIT.read_text(encoding="utf-8") - module = ast.parse(source) - - version_lines = [ - line for line in source.splitlines() if "__version__" in line - ] - self.assertEqual(len(version_lines), 1) - self.assertRegex( - version_lines[0].strip(), - r'^__version__ = "[0-9]+\.[0-9]+\.[0-9]+"$', - ) - - version_assignment = next( - node - for node in module.body - if isinstance(node, ast.Assign) - and any( - isinstance(target, ast.Name) and target.id == "__version__" - for target in node.targets - ) - ) - self.assertIsInstance(version_assignment.value, ast.Constant) - self.assertIsInstance(version_assignment.value.value, str) - self.assertRegex( - version_assignment.value.value, r"^[0-9]+\.[0-9]+\.[0-9]+$" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_parallel.py b/tests/test_parallel.py deleted file mode 100644 index 9932695..0000000 --- a/tests/test_parallel.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Tests for the parallel helper and the CLI entry point.""" - -import unittest - -from giant_python.cli import main -from giant_python.parallel import map_trials - - -class TestMapTrials(unittest.TestCase): - """Tests for the per-trial parallelism helper.""" - - def test_not_implemented(self): - """map_trials raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - map_trials(lambda x: x, [1, 2, 3]) - - -class TestCli(unittest.TestCase): - """Tests for the CLI entry point.""" - - def test_not_implemented(self): - """main raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - main([]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py deleted file mode 100644 index b3b7289..0000000 --- a/tests/test_pipeline.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Tests for the pipeline subpackage (stage functions, classes, facade).""" - -import unittest - -from giant_python.models import AlignParams, SiloParams -from giant_python.pipeline import ( - IntegrationRegistration, - IntegrationSourceExtractor, - MotionCorrector, - MultiRoiRegistration, - Pipeline, - SourceExtractor, - StandardSourceExtractor, - StripRegistration, - TrialTableBuilder, - build_trial_table, - build_trial_table_slap2, - extract_integration_sources, - integration_registration, - multi_roi_registration, - silo, - strip_registration, - verify_files, -) - - -class TestStageFunctions(unittest.TestCase): - """The porting-faithful stage functions raise NotImplementedError.""" - - def test_build_trial_table(self): - """build_trial_table raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - build_trial_table() - - def test_build_trial_table_slap2(self): - """build_trial_table_slap2 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - build_trial_table_slap2() - - def test_verify_files(self): - """verify_files raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - verify_files("trial_table.h5", "/some/dir", {}) - - def test_multi_roi_registration(self): - """multi_roi_registration raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - multi_roi_registration() - - def test_strip_registration(self): - """strip_registration raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - strip_registration() - - def test_integration_registration(self): - """integration_registration raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - integration_registration() - - def test_silo(self): - """silo raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - silo() - - def test_extract_integration_sources(self): - """extract_integration_sources raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - extract_integration_sources() - - -class TestStageClasses(unittest.TestCase): - """Stage classes construct; their run methods raise NotImplementedError.""" - - def test_trial_table_builder(self): - """TrialTableBuilder.run raises NotImplementedError.""" - builder = TrialTableBuilder("slap2") - with self.assertRaises(NotImplementedError): - builder.run("/data", "/results") - - def test_motion_corrector_for_microscope(self): - """MotionCorrector.for_microscope raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - MotionCorrector.for_microscope("slap2", AlignParams()) - - def test_motion_corrector_run(self): - """MotionCorrector.run raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - MotionCorrector().run(None) - - def test_registration_backends_run(self): - """Each registration backend's run raises NotImplementedError.""" - for backend in ( - MultiRoiRegistration, - StripRegistration, - IntegrationRegistration, - ): - with self.assertRaises(NotImplementedError): - backend().run(None) - - def test_source_extractor(self): - """SourceExtractor.run raises NotImplementedError.""" - extractor = SourceExtractor(SiloParams()) - with self.assertRaises(NotImplementedError): - extractor.run(None) - - def test_source_extractor_for_scan_mode(self): - """SourceExtractor.for_scan_mode raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - SourceExtractor.for_scan_mode("integration", SiloParams()) - - def test_extractor_backends_run(self): - """Each extraction backend's run raises NotImplementedError.""" - for backend in ( - StandardSourceExtractor, - IntegrationSourceExtractor, - ): - with self.assertRaises(NotImplementedError): - backend().run(None) - - -class TestPipelineFacade(unittest.TestCase): - """Tests for the Pipeline facade.""" - - def setUp(self): - """Construct a pipeline for each test.""" - self.pipe = Pipeline(microscope="slap2", save_dir="/results") - - def test_holds_session_state(self): - """The facade holds config and starts with no data.""" - self.assertEqual(self.pipe.microscope, "slap2") - self.assertIsNone(self.pipe.tt) - self.assertIsNone(self.pipe.summary) - self.assertEqual(self.pipe.silo_params.microscope, "slap2") - - def test_organize_delegates(self): - """organize delegates to the (unimplemented) builder.""" - with self.assertRaises(NotImplementedError): - self.pipe.organize("/data") - - def test_register_delegates(self): - """register delegates to the (unimplemented) corrector.""" - with self.assertRaises(NotImplementedError): - self.pipe.register() - - def test_extract_delegates(self): - """extract delegates to the (unimplemented) extractor.""" - with self.assertRaises(NotImplementedError): - self.pipe.extract() - - def test_run_all(self): - """run_all raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - self.pipe.run_all("/data") - - def test_from_trial_table(self): - """from_trial_table raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - Pipeline.from_trial_table("trial_table.h5", "slap2") - - def test_from_config(self): - """from_config raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - Pipeline.from_config("config.yaml") - - -if __name__ == "__main__": - unittest.main()