From 88e8779eaa5fcd828b5b47d0bae6687f77b1c215 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:03:32 +0100 Subject: [PATCH 01/52] First commit for lib_channels (ref #211) --- fractal_tasks_core/lib_channels.py | 55 ++++++++++++++++++++++++++ tests/data/omero/channels_list.json | 10 +++++ tests/test_unit_channels_addressing.py | 30 ++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 fractal_tasks_core/lib_channels.py create mode 100644 tests/data/omero/channels_list.json create mode 100644 tests/test_unit_channels_addressing.py diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py new file mode 100644 index 000000000..cdac46a4f --- /dev/null +++ b/fractal_tasks_core/lib_channels.py @@ -0,0 +1,55 @@ +from typing import Dict +from typing import Sequence + + +def get_channel( + *, channels: Sequence[Dict], label: str = None, wavelength_id: str = None +): + """ + Find matching channel in a list + + Find the channel that has the required values of ``label`` and/or + ``wavelength_id``, and identify its positional index (which also + corresponds to its index in the zarr array). + + :param channels: A list of channel dictionary, where each channel includes + (at least) the ``label`` and ``wavelength_id`` keys + :param label: The label to look for in the list of channels. + :param wavelength_id: The wavelength_id to look for in the list of + channels. + + """ + if label: + if wavelength_id: + matching_channels = [ + c + for c in channels + if ( + c["label"] == label and c["wavelength_id"] == wavelength_id + ) + ] + else: + matching_channels = [c for c in channels if c["label"] == label] + else: + if wavelength_id: + matching_channels = [ + c for c in channels if c["wavelength_id"] == wavelength_id + ] + else: + raise ValueError( + "get_channel requires at least one in {label,wavelength_id} " + "arguments" + ) + + if len(matching_channels) > 1: + raise ValueError(f"Inconsistent set of channels: {channels}") + elif len(matching_channels) == 0: + raise ValueError( + f"No channel found in {channels} for {label=} and {wavelength_id=}" + ) + + channel = matching_channels[0] + label = channel["label"] + wavelength_id = channel["wavelength_id"] + array_index = channels.index(channel) + return label, wavelength_id, array_index diff --git a/tests/data/omero/channels_list.json b/tests/data/omero/channels_list.json new file mode 100644 index 000000000..1d1f50da2 --- /dev/null +++ b/tests/data/omero/channels_list.json @@ -0,0 +1,10 @@ +[ + { + "label": "label_1", + "wavelength_id": "wavelength_id_1" + }, + { + "label": "label_2", + "wavelength_id": "wavelength_id_2" + } +] diff --git a/tests/test_unit_channels_addressing.py b/tests/test_unit_channels_addressing.py new file mode 100644 index 000000000..ff175c03d --- /dev/null +++ b/tests/test_unit_channels_addressing.py @@ -0,0 +1,30 @@ +import json +from pathlib import Path + +from devtools import debug + +from fractal_tasks_core.lib_channels import get_channel + + +def test_get_channel(testdata_path: Path): + with (testdata_path / "omero/channels_list.json").open("r") as f: + omero_channels = json.load(f) + debug(omero_channels) + + label, wl_id, index = get_channel(channels=omero_channels, label="label_1") + assert wl_id == "wavelength_id_1" + assert index == 0 + label, wl_id, index = get_channel( + channels=omero_channels, wavelength_id="wavelength_id_2" + ) + assert label == "label_2" + assert index == 1 + + label, wl_id, index = get_channel( + channels=omero_channels, + label="label_2", + wavelength_id="wavelength_id_2", + ) + assert label == "label_2" + assert wl_id == "wavelength_id_2" + assert index == 1 From 4d077bc95c06bd53e647f0522aed750fae26fccd Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:04:54 +0100 Subject: [PATCH 02/52] Rename helper function --- fractal_tasks_core/lib_channels.py | 2 +- tests/test_unit_channels_addressing.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index cdac46a4f..4b3c4c49c 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -2,7 +2,7 @@ from typing import Sequence -def get_channel( +def _get_channel_from_list( *, channels: Sequence[Dict], label: str = None, wavelength_id: str = None ): """ diff --git a/tests/test_unit_channels_addressing.py b/tests/test_unit_channels_addressing.py index ff175c03d..917a8722f 100644 --- a/tests/test_unit_channels_addressing.py +++ b/tests/test_unit_channels_addressing.py @@ -3,7 +3,7 @@ from devtools import debug -from fractal_tasks_core.lib_channels import get_channel +from fractal_tasks_core.lib_channels import _get_channel_from_list def test_get_channel(testdata_path: Path): @@ -11,16 +11,18 @@ def test_get_channel(testdata_path: Path): omero_channels = json.load(f) debug(omero_channels) - label, wl_id, index = get_channel(channels=omero_channels, label="label_1") + label, wl_id, index = _get_channel_from_list( + channels=omero_channels, label="label_1" + ) assert wl_id == "wavelength_id_1" assert index == 0 - label, wl_id, index = get_channel( + label, wl_id, index = _get_channel_from_list( channels=omero_channels, wavelength_id="wavelength_id_2" ) assert label == "label_2" assert index == 1 - label, wl_id, index = get_channel( + label, wl_id, index = _get_channel_from_list( channels=omero_channels, label="label_2", wavelength_id="wavelength_id_2", From 6a282956f097da56b449fbd36b9edac8e8f36a96 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:08:43 +0100 Subject: [PATCH 03/52] Add check in lib_omero --- fractal_tasks_core/lib_omero.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fractal_tasks_core/lib_omero.py b/fractal_tasks_core/lib_omero.py index 1cb4d8c87..0252f17fe 100644 --- a/fractal_tasks_core/lib_omero.py +++ b/fractal_tasks_core/lib_omero.py @@ -37,6 +37,11 @@ def define_omero_channels( default_colormaps = ["00FFFF", "FF00FF", "FFFF00"] for channel in actual_channels: + if channel not in channel_parameters.keys(): + raise ValueError( + f"{channel=} is not part of {channel_parameters=}" + ) + # Set colormap. If missing, use the default ones (for the first three # channels) or gray colormap = channel_parameters[channel].get("colormap", None) From 3ebf43349bd604562cddf914d35fd1012d8576c2 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:10:19 +0100 Subject: [PATCH 04/52] Add test_create_ome_zarr --- tests/test_workflows.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index b3c4e3f82..7a9aa6b12 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -57,6 +57,26 @@ coarsening_xy = 2 +def test_create_ome_zarr(tmp_path: Path, zenodo_images: Path): + + # Init + img_path = zenodo_images / "*.png" + zarr_path = tmp_path / "tmp_out/*.zarr" + metadata = {} + + # Create zarr structure + metadata_update = create_ome_zarr( + input_paths=[img_path], + output_path=zarr_path, + channel_parameters=channel_parameters, + num_levels=num_levels, + coarsening_xy=coarsening_xy, + metadata_table="mrf_mlf", + ) + metadata.update(metadata_update) + debug(metadata) + + def test_workflow_yokogawa_to_ome_zarr(tmp_path: Path, zenodo_images: Path): # Init From 01bade2ad33f6a971bb767599bd6b441b4cb2d18 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:11:12 +0100 Subject: [PATCH 05/52] Rename some tests --- tests/test_workflows.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 7a9aa6b12..556ce8422 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -76,8 +76,10 @@ def test_create_ome_zarr(tmp_path: Path, zenodo_images: Path): metadata.update(metadata_update) debug(metadata) + # FIXME: assert something (e.g. about channels) -def test_workflow_yokogawa_to_ome_zarr(tmp_path: Path, zenodo_images: Path): + +def test_yokogawa_to_ome_zarr(tmp_path: Path, zenodo_images: Path): # Init img_path = zenodo_images / "*.png" @@ -117,7 +119,7 @@ def test_workflow_yokogawa_to_ome_zarr(tmp_path: Path, zenodo_images: Path): check_file_number(zarr_path=image_zarr) -def test_workflow_MIP( +def test_MIP( tmp_path: Path, zenodo_zarr: List[Path], zenodo_zarr_metadata: List[Dict[str, Any]], @@ -165,7 +167,7 @@ def test_workflow_MIP( validate_schema(path=str(plate_zarr), type="plate") -def test_workflow_illumination_correction( +def test_illumination_correction( tmp_path: Path, testdata_path: Path, zenodo_images: Path, From 8cb77bdbbe5841f85d1cb5abe30f8b45ec3451b9 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:12:08 +0100 Subject: [PATCH 06/52] Rename some tests --- tests/test_workflows_napari_workflows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_workflows_napari_workflows.py b/tests/test_workflows_napari_workflows.py index ce1dc8604..a20c01e39 100644 --- a/tests/test_workflows_napari_workflows.py +++ b/tests/test_workflows_napari_workflows.py @@ -65,7 +65,7 @@ def prepare_2D_zarr( return metadata -def test_workflow_napari_worfklow( +def test_napari_worfklow( tmp_path: Path, testdata_path: Path, zenodo_zarr: List[Path], @@ -153,7 +153,7 @@ def test_workflow_napari_worfklow( assert "bbox_area" in meas.var_names -def test_workflow_napari_worfklow_label_input_only( +def test_napari_worfklow_label_input_only( tmp_path: Path, testdata_path: Path, zenodo_zarr: List[Path], From 27f0b96e9c0beb355d8dc3989dc8d0cb41719165 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:24:55 +0100 Subject: [PATCH 07/52] Change return of _get_channel_from_list --- fractal_tasks_core/lib_channels.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 4b3c4c49c..37aa43a44 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -49,7 +49,5 @@ def _get_channel_from_list( ) channel = matching_channels[0] - label = channel["label"] - wavelength_id = channel["wavelength_id"] - array_index = channels.index(channel) - return label, wavelength_id, array_index + channel["index"] = channels.index(channel) + return channel From 7bb37aff21af9b97f719e261190799d0002a8b0b Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:25:20 +0100 Subject: [PATCH 08/52] Adapt create_ome_zarr.py and lib_omero.py to new helper function --- fractal_tasks_core/create_ome_zarr.py | 11 +++++++---- fractal_tasks_core/lib_omero.py | 26 +++++++++++++------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index 477e203cd..992b23e54 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -46,7 +46,7 @@ def create_ome_zarr( input_paths: Sequence[Path], output_path: Path, metadata: Dict[str, Any] = None, - channel_parameters: Dict[str, Any], + channel_parameters: Sequence[Dict[str, str]], num_levels: int = 2, coarsening_xy: int = 2, metadata_table: str = "mrf_mlf", @@ -90,7 +90,7 @@ def create_ome_zarr( ) if channel_parameters is None: raise Exception( - "Missing channel_parameters argument in " "create_ome_zarr" + "Missing channel_parameters argument in create_ome_zarr" ) # Identify all plates and all channels, across all input folders @@ -168,10 +168,13 @@ def create_ome_zarr( dict_plate_paths[plate] = in_path.parent # Check that all channels are in the allowed_channels - if not set(channels).issubset(set(channel_parameters.keys())): + allowed_wavelength_ids = [ + channel["wavelength_id"] for channel in channel_parameters + ] + if not set(channels).issubset(set(allowed_wavelength_ids)): msg = "ERROR in create_ome_zarr\n" msg += f"channels: {channels}\n" - msg += f"allowed_channels: {channel_parameters.keys()}\n" + msg += f"allowed_channels: {allowed_wavelength_ids}\n" raise Exception(msg) # Create actual_channels, i.e. a list of entries like "A01_C01" diff --git a/fractal_tasks_core/lib_omero.py b/fractal_tasks_core/lib_omero.py index 0252f17fe..f5e8c342c 100644 --- a/fractal_tasks_core/lib_omero.py +++ b/fractal_tasks_core/lib_omero.py @@ -18,6 +18,8 @@ from typing import List from typing import Sequence +from fractal_tasks_core.lib_channels import _get_channel_from_list + def define_omero_channels( actual_channels: Sequence[str], @@ -35,16 +37,18 @@ def define_omero_channels( omero_channels = [] default_colormaps = ["00FFFF", "FF00FF", "FFFF00"] - for channel in actual_channels: + for wavelength_id in actual_channels: + + channel = _get_channel_from_list( + channels=channel_parameters, wavelength_id=wavelength_id + ) - if channel not in channel_parameters.keys(): - raise ValueError( - f"{channel=} is not part of {channel_parameters=}" - ) + # FIXME handle missing label + label = channel["label"] # Set colormap. If missing, use the default ones (for the first three # channels) or gray - colormap = channel_parameters[channel].get("colormap", None) + colormap = channel.get("colormap", None) if colormap is None: try: colormap = default_colormaps.pop() @@ -58,7 +62,7 @@ def define_omero_channels( "color": colormap, "family": "linear", "inverted": False, - "label": channel_parameters[channel].get("label", channel), + "label": label, "window": { "min": 0, "max": 2**bit_depth - 1, @@ -67,12 +71,8 @@ def define_omero_channels( ) try: - omero_channels[-1]["window"]["start"] = channel_parameters[ - channel - ]["start"] - omero_channels[-1]["window"]["end"] = channel_parameters[channel][ - "end" - ] + omero_channels[-1]["window"]["start"] = channel["start"] + omero_channels[-1]["window"]["end"] = channel["end"] except KeyError: pass From 55360b1503d227972569393f4f80779ed4fc1ed3 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:27:48 +0100 Subject: [PATCH 09/52] Rename parameter of create_ome_zarr --- fractal_tasks_core/create_ome_zarr.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index 992b23e54..b8366a707 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -46,7 +46,7 @@ def create_ome_zarr( input_paths: Sequence[Path], output_path: Path, metadata: Dict[str, Any] = None, - channel_parameters: Sequence[Dict[str, str]], + allowed_channels: Sequence[Dict[str, str]], num_levels: int = 2, coarsening_xy: int = 2, metadata_table: str = "mrf_mlf", @@ -68,7 +68,7 @@ def create_ome_zarr( :param input_paths: TBD (common to all tasks) :param output_path: TBD (common to all tasks) :param metadata: TBD (common to all tasks) - :param channel_parameters: TBD + :param allowed_channels: TBD :param num_levels: number of resolution-pyramid levels :param coarsening_xy: linear coarsening factor between subsequent levels :param metadata_table: TBD @@ -88,10 +88,6 @@ def create_ome_zarr( 'metadata_table="mrf_mlf", ' f"and not {metadata_table}" ) - if channel_parameters is None: - raise Exception( - "Missing channel_parameters argument in create_ome_zarr" - ) # Identify all plates and all channels, across all input folders plates = [] @@ -169,7 +165,7 @@ def create_ome_zarr( # Check that all channels are in the allowed_channels allowed_wavelength_ids = [ - channel["wavelength_id"] for channel in channel_parameters + channel["wavelength_id"] for channel in allowed_channels ] if not set(channels).issubset(set(allowed_wavelength_ids)): msg = "ERROR in create_ome_zarr\n" @@ -183,8 +179,6 @@ def create_ome_zarr( actual_channels.append(ch) logger.info(f"actual_channels: {actual_channels}") - # Clean up dictionary channel_parameters - zarrurls: Dict[str, List[str]] = {"plate": [], "well": [], "image": []} ################################################################ @@ -342,7 +336,7 @@ def create_ome_zarr( "name": "TBD", "version": __OME_NGFF_VERSION__, "channels": define_omero_channels( - actual_channels, channel_parameters, bit_depth + actual_channels, allowed_channels, bit_depth ), } @@ -382,7 +376,7 @@ class TaskArguments(BaseModel): input_paths: Sequence[Path] output_path: Path metadata: Optional[Dict[str, Any]] - channel_parameters: Dict[str, Any] + allowed_channels: Sequence[Dict[str, str]] num_levels: int = 2 coarsening_xy: int = 2 metadata_table: str = "mrf_mlf" From 0ad4c2d2a38cfc748af49566cfee10bc97e69c84 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 12:28:21 +0100 Subject: [PATCH 10/52] Align tests --- tests/test_unit_channels_addressing.py | 28 +++++++++++++++----------- tests/test_workflows.py | 19 +++++++++-------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/tests/test_unit_channels_addressing.py b/tests/test_unit_channels_addressing.py index 917a8722f..43dac4bd7 100644 --- a/tests/test_unit_channels_addressing.py +++ b/tests/test_unit_channels_addressing.py @@ -11,22 +11,26 @@ def test_get_channel(testdata_path: Path): omero_channels = json.load(f) debug(omero_channels) - label, wl_id, index = _get_channel_from_list( - channels=omero_channels, label="label_1" - ) - assert wl_id == "wavelength_id_1" - assert index == 0 - label, wl_id, index = _get_channel_from_list( + channel = _get_channel_from_list(channels=omero_channels, label="label_1") + debug(channel) + assert channel["label"] == "label_1" + assert channel["wavelength_id"] == "wavelength_id_1" + assert channel["index"] == 0 + + channel = _get_channel_from_list( channels=omero_channels, wavelength_id="wavelength_id_2" ) - assert label == "label_2" - assert index == 1 + debug(channel) + assert channel["label"] == "label_2" + assert channel["wavelength_id"] == "wavelength_id_2" + assert channel["index"] == 1 - label, wl_id, index = _get_channel_from_list( + channel = _get_channel_from_list( channels=omero_channels, label="label_2", wavelength_id="wavelength_id_2", ) - assert label == "label_2" - assert wl_id == "wavelength_id_2" - assert index == 1 + debug(channel) + assert channel["label"] == "label_2" + assert channel["wavelength_id"] == "wavelength_id_2" + assert channel["index"] == 1 diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 556ce8422..370dfd19c 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -32,26 +32,29 @@ from fractal_tasks_core.yokogawa_to_ome_zarr import yokogawa_to_ome_zarr -channel_parameters = { - "A01_C01": { +allowed_channels = [ + { "label": "DAPI", + "wavelength_id": "A01_C01", "colormap": "00FFFF", "start": 0, "end": 700, }, - "A01_C02": { + { + "wavelength_id": "A01_C02", "label": "nanog", "colormap": "FF00FF", "start": 0, "end": 180, }, - "A02_C03": { + { + "wavelength_id": "A02_C03", "label": "Lamin B1", "colormap": "FFFF00", "start": 0, "end": 1500, }, -} +] num_levels = 6 coarsening_xy = 2 @@ -68,7 +71,7 @@ def test_create_ome_zarr(tmp_path: Path, zenodo_images: Path): metadata_update = create_ome_zarr( input_paths=[img_path], output_path=zarr_path, - channel_parameters=channel_parameters, + allowed_channels=allowed_channels, num_levels=num_levels, coarsening_xy=coarsening_xy, metadata_table="mrf_mlf", @@ -90,7 +93,7 @@ def test_yokogawa_to_ome_zarr(tmp_path: Path, zenodo_images: Path): metadata_update = create_ome_zarr( input_paths=[img_path], output_path=zarr_path, - channel_parameters=channel_parameters, + allowed_channels=allowed_channels, num_levels=num_levels, coarsening_xy=coarsening_xy, metadata_table="mrf_mlf", @@ -193,7 +196,7 @@ def test_illumination_correction( metadata_update = create_ome_zarr( input_paths=[img_path], output_path=zarr_path, - channel_parameters=channel_parameters, + allowed_channels=allowed_channels, num_levels=num_levels, coarsening_xy=coarsening_xy, metadata_table="mrf_mlf", From 81a966ac2b1842cc9813f126396ca536c98abbd5 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 13:32:00 +0100 Subject: [PATCH 11/52] Basic handling of missing label --- fractal_tasks_core/lib_omero.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fractal_tasks_core/lib_omero.py b/fractal_tasks_core/lib_omero.py index f5e8c342c..5791c9209 100644 --- a/fractal_tasks_core/lib_omero.py +++ b/fractal_tasks_core/lib_omero.py @@ -13,6 +13,7 @@ Handle OMERO-related metadata """ +import logging from typing import Any from typing import Dict from typing import List @@ -43,8 +44,15 @@ def define_omero_channels( channels=channel_parameters, wavelength_id=wavelength_id ) - # FIXME handle missing label - label = channel["label"] + try: + label = channel["label"] + except KeyError: + # FIXME better handling of missing label + default_label = wavelength_id + logging.warning( + f"Missing label for {channel=}, using {default_label=}" + ) + label = default_label # Set colormap. If missing, use the default ones (for the first three # channels) or gray From bfbe5136a7240d9eb27e2901a9f734cf8486101c Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 13:32:06 +0100 Subject: [PATCH 12/52] Fix test --- tests/test_unit_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit_task.py b/tests/test_unit_task.py index 6c70d3904..f6b5d8b49 100644 --- a/tests/test_unit_task.py +++ b/tests/test_unit_task.py @@ -24,7 +24,7 @@ def test_create_ome_zarr(tmp_path, testdata_path): input_paths = [testdata_path / "png/*.png"] output_path = tmp_path / "*.zarr" default_args = create_ome_zarr_manifest["default_args"] - default_args["channel_parameters"] = {"A01_C01": {}} + default_args["allowed_channels"] = [{"wavelength_id": "A01_C01"}] for key in ["executor", "parallelization_level"]: if key in default_args.keys(): From 66261f8ea4620a6e77146f24cd1b089e5297eecb Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 13:48:37 +0100 Subject: [PATCH 13/52] Rename some channel-related variables (plus small refactors) --- fractal_tasks_core/create_ome_zarr.py | 53 ++++++++++++++------------- fractal_tasks_core/lib_omero.py | 3 +- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index b8366a707..5b9111d8c 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -91,7 +91,7 @@ def create_ome_zarr( # Identify all plates and all channels, across all input folders plates = [] - channels = None + actual_wavelength_ids = None dict_plate_paths = {} dict_plate_prefixes: Dict[str, Any] = {} @@ -102,7 +102,7 @@ def create_ome_zarr( for in_path in input_paths: input_filename_iter = in_path.parent.glob(in_path.name) - tmp_channels = [] + tmp_wavelength_ids = [] tmp_plates = [] for fn in input_filename_iter: try: @@ -112,20 +112,20 @@ def create_ome_zarr( if plate not in dict_plate_prefixes.keys(): dict_plate_prefixes[plate] = plate_prefix tmp_plates.append(plate) - tmp_channels.append( - f"A{filename_metadata['A']}_C{filename_metadata['C']}" - ) + A = filename_metadata["A"] + C = filename_metadata["C"] + tmp_wavelength_ids.append(f"A{A}_C{C}") except ValueError as e: logger.warning( f'Skipping "{fn.name}". Original error: ' + str(e) ) tmp_plates = sorted(list(set(tmp_plates))) - tmp_channels = sorted(list(set(tmp_channels))) + tmp_wavelength_ids = sorted(list(set(tmp_wavelength_ids))) info = ( f"Listing all plates/channels from {in_path.as_posix()}\n" f"Plates: {tmp_plates}\n" - f"Channels: {tmp_channels}\n" + f"Channels: {tmp_wavelength_ids}\n" ) # Check that only one plate is found @@ -152,12 +152,13 @@ def create_ome_zarr( plates.append(plate) # Check that channels are the same as in previous plates - if channels is None: - channels = tmp_channels[:] + if actual_wavelength_ids is None: + actual_wavelength_ids = tmp_wavelength_ids[:] else: - if channels != tmp_channels: + if actual_wavelength_ids != tmp_wavelength_ids: raise Exception( - f"ERROR\n{info}\nERROR: expected channels {channels}" + f"ERROR\n{info}\nERROR:" + f" expected channels {actual_wavelength_ids}" ) # Update dict_plate_paths @@ -167,17 +168,19 @@ def create_ome_zarr( allowed_wavelength_ids = [ channel["wavelength_id"] for channel in allowed_channels ] - if not set(channels).issubset(set(allowed_wavelength_ids)): + if not set(actual_wavelength_ids).issubset(set(allowed_wavelength_ids)): msg = "ERROR in create_ome_zarr\n" - msg += f"channels: {channels}\n" - msg += f"allowed_channels: {allowed_wavelength_ids}\n" + msg += f"actual_wavelength_ids: {actual_wavelength_ids}\n" + msg += f"allowed_wavelength_ids: {allowed_wavelength_ids}\n" raise Exception(msg) - # Create actual_channels, i.e. a list of entries like "A01_C01" - actual_channels = [] - for ind_ch, ch in enumerate(channels): - actual_channels.append(ch) - logger.info(f"actual_channels: {actual_channels}") + # Create actual_channels, i.e. a list of the channel dictionaries which are + # present + actual_channels = [ + channel + for channel in allowed_channels + if channel["wavelength_id"] in actual_wavelength_ids + ] zarrurls: Dict[str, List[str]] = {"plate": [], "well": [], "image": []} @@ -232,22 +235,22 @@ def create_ome_zarr( well_image_iter = glob( f"{in_path}/{plate_prefix}_{well}{ext_glob_pattern}" ) - well_channels = [] + well_wavelength_ids = [] for fpath in well_image_iter: try: filename_metadata = parse_filename(os.path.basename(fpath)) - well_channels.append( + well_wavelength_ids.append( f"A{filename_metadata['A']}_C{filename_metadata['C']}" ) except IndexError: logger.info(f"Skipping {fpath}") - well_channels = sorted(list(set(well_channels))) - if well_channels != actual_channels: + well_wavelength_ids = sorted(list(set(well_wavelength_ids))) + if well_wavelength_ids != actual_wavelength_ids: raise Exception( f"ERROR: well {well} in plate {plate} (prefix: " f"{plate_prefix}) has missing channels.\n" f"Expected: {actual_channels}\n" - f"Found: {well_channels}.\n" + f"Found: {well_wavelength_ids}.\n" ) well_rows_columns = [ @@ -362,7 +365,7 @@ def create_ome_zarr( image=zarrurls["image"], num_levels=num_levels, coarsening_xy=coarsening_xy, - channel_list=actual_channels, + channel_list=actual_wavelength_ids, # FIXME: remove this original_paths=[str(p) for p in input_paths], ) return metadata_update diff --git a/fractal_tasks_core/lib_omero.py b/fractal_tasks_core/lib_omero.py index 5791c9209..97f32693a 100644 --- a/fractal_tasks_core/lib_omero.py +++ b/fractal_tasks_core/lib_omero.py @@ -38,7 +38,8 @@ def define_omero_channels( omero_channels = [] default_colormaps = ["00FFFF", "FF00FF", "FFFF00"] - for wavelength_id in actual_channels: + for channel in actual_channels: + wavelength_id = channel["wavelength_id"] channel = _get_channel_from_list( channels=channel_parameters, wavelength_id=wavelength_id From 7076d7685a604d4cb50bd78e6ca42f431ee9d43b Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 13:49:04 +0100 Subject: [PATCH 14/52] Update some tests --- tests/test_workflow_executable.py | 16 ++++++++++------ tests/test_workflows_labeling.py | 16 ++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/test_workflow_executable.py b/tests/test_workflow_executable.py index 8121636d9..2ab5ddce2 100644 --- a/tests/test_workflow_executable.py +++ b/tests/test_workflow_executable.py @@ -8,26 +8,30 @@ import fractal_tasks_core -channel_parameters = { - "A01_C01": { +allowed_channels = [ + { "label": "DAPI", + "wavelength_id": "A01_C01", "colormap": "00FFFF", "start": 0, "end": 700, }, - "A01_C02": { + { + "wavelength_id": "A01_C02", "label": "nanog", "colormap": "FF00FF", "start": 0, "end": 180, }, - "A02_C03": { + { + "wavelength_id": "A02_C03", "label": "Lamin B1", "colormap": "FFFF00", "start": 0, "end": 1500, }, -} +] + num_levels = 6 coarsening_xy = 2 @@ -61,7 +65,7 @@ def test_workflow_yokogawa_to_ome_zarr(tmp_path: Path, zenodo_images: Path): args_create_zarr = dict( input_paths=[img_path], output_path=zarr_path, - channel_parameters=channel_parameters, + allowed_channels=allowed_channels, metadata={}, num_levels=num_levels, coarsening_xy=coarsening_xy, diff --git a/tests/test_workflows_labeling.py b/tests/test_workflows_labeling.py index 29fa16873..40f6d9336 100644 --- a/tests/test_workflows_labeling.py +++ b/tests/test_workflows_labeling.py @@ -37,26 +37,30 @@ from fractal_tasks_core.yokogawa_to_ome_zarr import yokogawa_to_ome_zarr -channel_parameters = { - "A01_C01": { +allowed_channels = [ + { "label": "DAPI", + "wavelength_id": "A01_C01", "colormap": "00FFFF", "start": 0, "end": 700, }, - "A01_C02": { + { + "wavelength_id": "A01_C02", "label": "nanog", "colormap": "FF00FF", "start": 0, "end": 180, }, - "A02_C03": { + { + "wavelength_id": "A02_C03", "label": "Lamin B1", "colormap": "FFFF00", "start": 0, "end": 1500, }, -} +] + num_levels = 6 coarsening_xy = 2 @@ -280,7 +284,7 @@ def test_workflow_with_per_well_labeling_2D( metadata_update = create_ome_zarr( input_paths=[img_path], output_path=zarr_path, - channel_parameters=channel_parameters, + allowed_channels=allowed_channels, num_levels=num_levels, coarsening_xy=coarsening_xy, metadata_table="mrf_mlf", From fb9e7ef3da42868f3258b15a5bea5a1ac5a542d6 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 13:53:36 +0100 Subject: [PATCH 15/52] Rename group_FOV to group_image in parsing task --- fractal_tasks_core/create_ome_zarr.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index 5b9111d8c..796d53e2c 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -288,11 +288,11 @@ def create_ome_zarr( "version": __OME_NGFF_VERSION__, } - group_FOV = group_well.create_group("0/") # noqa: F841 + group_image = group_well.create_group("0/") # noqa: F841 zarrurls["well"].append(f"{plate}.zarr/{row}/{column}/") zarrurls["image"].append(f"{plate}.zarr/{row}/{column}/0/") - group_FOV.attrs["multiscales"] = [ + group_image.attrs["multiscales"] = [ { "version": __OME_NGFF_VERSION__, "axes": [ @@ -334,7 +334,7 @@ def create_ome_zarr( } ] - group_FOV.attrs["omero"] = { + group_image.attrs["omero"] = { "id": 1, # FIXME does this depend on the plate number? "name": "TBD", "version": __OME_NGFF_VERSION__, @@ -345,7 +345,9 @@ def create_ome_zarr( # FIXME if has_mrf_mlf_metadata: - group_tables = group_FOV.create_group("tables/") # noqa: F841 + group_tables = group_image.create_group( + "tables/" + ) # noqa: F841 # Prepare FOV/well tables FOV_ROIs_table = prepare_FOV_ROI_table( From f262e29b50d6087ae9f1c2c97a35a8d2e0a17fb5 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 13:57:56 +0100 Subject: [PATCH 16/52] Add get_omero_channel_list function and always include wavelength_id in define_omero_channels --- fractal_tasks_core/lib_omero.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fractal_tasks_core/lib_omero.py b/fractal_tasks_core/lib_omero.py index 97f32693a..d1b4e19ae 100644 --- a/fractal_tasks_core/lib_omero.py +++ b/fractal_tasks_core/lib_omero.py @@ -19,6 +19,8 @@ from typing import List from typing import Sequence +import zarr + from fractal_tasks_core.lib_channels import _get_channel_from_list @@ -66,12 +68,13 @@ def define_omero_channels( omero_channels.append( { + "label": label, + "wavelength_id": wavelength_id, "active": True, "coefficient": 1, "color": colormap, "family": "linear", "inverted": False, - "label": label, "window": { "min": 0, "max": 2**bit_depth - 1, @@ -86,3 +89,8 @@ def define_omero_channels( pass return omero_channels + + +def get_omero_channel_list(*, image_zarr_path: str) -> List[Dict[str, str]]: + group = zarr.open_group(image_zarr_path, mode="r") + return group.attrs["omero"]["channels"] From e7b1e9f66058d52c5bd3307628a8a9a4bb4405e4 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 13:58:21 +0100 Subject: [PATCH 17/52] Do not use metadata channel list, in yokogawa_to_ome_zarr --- fractal_tasks_core/yokogawa_to_ome_zarr.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fractal_tasks_core/yokogawa_to_ome_zarr.py b/fractal_tasks_core/yokogawa_to_ome_zarr.py index 9c33a4c34..cc3c3b3d1 100644 --- a/fractal_tasks_core/yokogawa_to_ome_zarr.py +++ b/fractal_tasks_core/yokogawa_to_ome_zarr.py @@ -27,6 +27,7 @@ from anndata import read_zarr from dask.array.image import imread +from fractal_tasks_core.lib_omero import get_omero_channel_list from fractal_tasks_core.lib_parse_filename_metadata import parse_filename from fractal_tasks_core.lib_pyramid_creation import build_pyramid from fractal_tasks_core.lib_read_fractal_metadata import ( @@ -82,17 +83,20 @@ def yokogawa_to_ome_zarr( # Preliminary checks if len(input_paths) > 1: raise NotImplementedError + zarrurl = input_paths[0].parent.as_posix() + f"/{component}" parameters = get_parameters_from_metadata( - keys=["channel_list", "original_paths", "num_levels", "coarsening_xy"], + keys=["original_paths", "num_levels", "coarsening_xy"], metadata=metadata, image_zarr_path=(output_path.parent / component), ) - chl_list = parameters["channel_list"] original_path_list = parameters["original_paths"] num_levels = parameters["num_levels"] coarsening_xy = parameters["coarsening_xy"] + channels = get_omero_channel_list(image_zarr_path=zarrurl) + wavelength_ids = [c["wavelength_id"] for c in channels] + in_path = Path(original_path_list[0]).parent ext = Path(original_path_list[0]).name @@ -103,7 +107,6 @@ def yokogawa_to_ome_zarr( well_ID = well_row + well_column # Read useful information from ROI table and .zattrs - zarrurl = input_paths[0].parent.as_posix() + f"/{component}" adata = read_zarr(f"{zarrurl}/tables/FOV_ROI_table") pxl_size = extract_zyx_pixel_sizes(f"{zarrurl}/.zattrs") fov_indices = convert_ROI_table_to_indices( @@ -128,7 +131,7 @@ def yokogawa_to_ome_zarr( # Initialize zarr chunksize = (1, 1, sample.shape[1], sample.shape[2]) canvas_zarr = zarr.create( - shape=(len(chl_list), max_z, max_y, max_x), + shape=(len(wavelength_ids), max_z, max_y, max_x), chunks=chunksize, dtype=sample.dtype, store=da.core.get_mapper(zarrurl + "/0"), @@ -137,8 +140,8 @@ def yokogawa_to_ome_zarr( ) # Loop over channels - for i_c, chl in enumerate(chl_list): - A, C = chl.split("_") + for i_c, wavelength_id in enumerate(wavelength_ids): + A, C = wavelength_id.split("_") glob_path = f"{in_path}/*_{well_ID}_*{A}*{C}{ext}" logger.info(f"glob path: {glob_path}") @@ -149,7 +152,7 @@ def yokogawa_to_ome_zarr( f" in_path: {in_path}\n" f" ext: {ext}\n" f" well_ID: {well_ID}\n" - f" channel: {chl},\n" + f" wavelength_id: {wavelength_id},\n" f" glob_path: {glob_path}" ) # Loop over 3D FOV ROIs From 6af193d01c805770a3e868345362330c79915004 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:00:49 +0100 Subject: [PATCH 18/52] Move all channel-related functions to lib_channels --- fractal_tasks_core/create_ome_zarr.py | 2 +- .../create_ome_zarr_multiplex.py | 2 +- fractal_tasks_core/lib_channels.py | 92 ++++++++++++++++++ fractal_tasks_core/lib_omero.py | 96 ------------------- fractal_tasks_core/yokogawa_to_ome_zarr.py | 2 +- 5 files changed, 95 insertions(+), 99 deletions(-) delete mode 100644 fractal_tasks_core/lib_omero.py diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index 796d53e2c..207c50c2c 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -27,8 +27,8 @@ from anndata.experimental import write_elem import fractal_tasks_core +from fractal_tasks_core.lib_channels import define_omero_channels from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata -from fractal_tasks_core.lib_omero import define_omero_channels from fractal_tasks_core.lib_parse_filename_metadata import parse_filename from fractal_tasks_core.lib_regions_of_interest import prepare_FOV_ROI_table from fractal_tasks_core.lib_regions_of_interest import prepare_well_ROI_table diff --git a/fractal_tasks_core/create_ome_zarr_multiplex.py b/fractal_tasks_core/create_ome_zarr_multiplex.py index 739564ffd..14893bf3c 100644 --- a/fractal_tasks_core/create_ome_zarr_multiplex.py +++ b/fractal_tasks_core/create_ome_zarr_multiplex.py @@ -27,8 +27,8 @@ from anndata.experimental import write_elem import fractal_tasks_core +from fractal_tasks_core.lib_channels import define_omero_channels from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata -from fractal_tasks_core.lib_omero import define_omero_channels from fractal_tasks_core.lib_parse_filename_metadata import parse_filename from fractal_tasks_core.lib_regions_of_interest import prepare_FOV_ROI_table from fractal_tasks_core.lib_regions_of_interest import prepare_well_ROI_table diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 37aa43a44..32d6e4b06 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -1,6 +1,26 @@ +""" +Copyright 2022 (C) + Friedrich Miescher Institute for Biomedical Research and + University of Zurich + + Original authors: + Tommaso Comparin + + This file is part of Fractal and was originally developed by eXact lab + S.r.l. under contract with Liberali Lab from the Friedrich + Miescher Institute for Biomedical Research and Pelkmans Lab from the + University of Zurich. + +Helper functions to address channels +""" +import logging +from typing import Any from typing import Dict +from typing import List from typing import Sequence +import zarr + def _get_channel_from_list( *, channels: Sequence[Dict], label: str = None, wavelength_id: str = None @@ -51,3 +71,75 @@ def _get_channel_from_list( channel = matching_channels[0] channel["index"] = channels.index(channel) return channel + + +def define_omero_channels( + actual_channels: Sequence[str], + channel_parameters: Dict[str, Any], + bit_depth: int, +) -> List[Dict[str, Any]]: + """ + Prepare the .attrs["omero"]["channels"] attribute of an image group + + :param actual_channels: TBD + :param channel_parameters: TBD + :param bit_depth: TBD + :returns: omero_channels + """ + + omero_channels = [] + default_colormaps = ["00FFFF", "FF00FF", "FFFF00"] + for channel in actual_channels: + wavelength_id = channel["wavelength_id"] + + channel = _get_channel_from_list( + channels=channel_parameters, wavelength_id=wavelength_id + ) + + try: + label = channel["label"] + except KeyError: + # FIXME better handling of missing label + default_label = wavelength_id + logging.warning( + f"Missing label for {channel=}, using {default_label=}" + ) + label = default_label + + # Set colormap. If missing, use the default ones (for the first three + # channels) or gray + colormap = channel.get("colormap", None) + if colormap is None: + try: + colormap = default_colormaps.pop() + except IndexError: + colormap = "808080" + + omero_channels.append( + { + "label": label, + "wavelength_id": wavelength_id, + "active": True, + "coefficient": 1, + "color": colormap, + "family": "linear", + "inverted": False, + "window": { + "min": 0, + "max": 2**bit_depth - 1, + }, + } + ) + + try: + omero_channels[-1]["window"]["start"] = channel["start"] + omero_channels[-1]["window"]["end"] = channel["end"] + except KeyError: + pass + + return omero_channels + + +def get_omero_channel_list(*, image_zarr_path: str) -> List[Dict[str, str]]: + group = zarr.open_group(image_zarr_path, mode="r") + return group.attrs["omero"]["channels"] diff --git a/fractal_tasks_core/lib_omero.py b/fractal_tasks_core/lib_omero.py deleted file mode 100644 index d1b4e19ae..000000000 --- a/fractal_tasks_core/lib_omero.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -Copyright 2022 (C) - Friedrich Miescher Institute for Biomedical Research and - University of Zurich - - Original authors: - Tommaso Comparin - - This file is part of Fractal and was originally developed by eXact lab - S.r.l. under contract with Liberali Lab from the Friedrich - Miescher Institute for Biomedical Research and Pelkmans Lab from the - University of Zurich. - -Handle OMERO-related metadata -""" -import logging -from typing import Any -from typing import Dict -from typing import List -from typing import Sequence - -import zarr - -from fractal_tasks_core.lib_channels import _get_channel_from_list - - -def define_omero_channels( - actual_channels: Sequence[str], - channel_parameters: Dict[str, Any], - bit_depth: int, -) -> List[Dict[str, Any]]: - """ - Prepare the .attrs["omero"]["channels"] attribute of an image group - - :param actual_channels: TBD - :param channel_parameters: TBD - :param bit_depth: TBD - :returns: omero_channels - """ - - omero_channels = [] - default_colormaps = ["00FFFF", "FF00FF", "FFFF00"] - for channel in actual_channels: - wavelength_id = channel["wavelength_id"] - - channel = _get_channel_from_list( - channels=channel_parameters, wavelength_id=wavelength_id - ) - - try: - label = channel["label"] - except KeyError: - # FIXME better handling of missing label - default_label = wavelength_id - logging.warning( - f"Missing label for {channel=}, using {default_label=}" - ) - label = default_label - - # Set colormap. If missing, use the default ones (for the first three - # channels) or gray - colormap = channel.get("colormap", None) - if colormap is None: - try: - colormap = default_colormaps.pop() - except IndexError: - colormap = "808080" - - omero_channels.append( - { - "label": label, - "wavelength_id": wavelength_id, - "active": True, - "coefficient": 1, - "color": colormap, - "family": "linear", - "inverted": False, - "window": { - "min": 0, - "max": 2**bit_depth - 1, - }, - } - ) - - try: - omero_channels[-1]["window"]["start"] = channel["start"] - omero_channels[-1]["window"]["end"] = channel["end"] - except KeyError: - pass - - return omero_channels - - -def get_omero_channel_list(*, image_zarr_path: str) -> List[Dict[str, str]]: - group = zarr.open_group(image_zarr_path, mode="r") - return group.attrs["omero"]["channels"] diff --git a/fractal_tasks_core/yokogawa_to_ome_zarr.py b/fractal_tasks_core/yokogawa_to_ome_zarr.py index cc3c3b3d1..d2bb116c2 100644 --- a/fractal_tasks_core/yokogawa_to_ome_zarr.py +++ b/fractal_tasks_core/yokogawa_to_ome_zarr.py @@ -27,7 +27,7 @@ from anndata import read_zarr from dask.array.image import imread -from fractal_tasks_core.lib_omero import get_omero_channel_list +from fractal_tasks_core.lib_channels import get_omero_channel_list from fractal_tasks_core.lib_parse_filename_metadata import parse_filename from fractal_tasks_core.lib_pyramid_creation import build_pyramid from fractal_tasks_core.lib_read_fractal_metadata import ( From 9fcad3c56106cfb674c7c612f32f0ca4a089c90e Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:06:23 +0100 Subject: [PATCH 19/52] Adapt cellpose_segmentation --- fractal_tasks_core/cellpose_segmentation.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fractal_tasks_core/cellpose_segmentation.py b/fractal_tasks_core/cellpose_segmentation.py index e85a227e7..446d37b21 100644 --- a/fractal_tasks_core/cellpose_segmentation.py +++ b/fractal_tasks_core/cellpose_segmentation.py @@ -36,6 +36,8 @@ from cellpose.core import use_gpu import fractal_tasks_core +from fractal_tasks_core.lib_channels import _get_channel_from_list +from fractal_tasks_core.lib_channels import get_omero_channel_list from fractal_tasks_core.lib_pyramid_creation import build_pyramid from fractal_tasks_core.lib_regions_of_interest import ( array_to_bounding_box_table, @@ -172,16 +174,18 @@ def cellpose_segmentation( # Read useful parameters from metadata num_levels = metadata["num_levels"] coarsening_xy = metadata["coarsening_xy"] - chl_list = metadata["channel_list"] + plate, well = component.split(".zarr/") # Find well ID well_id = well.replace("/", "_")[:-1] # Find channel index - if labeling_channel not in chl_list: - raise Exception(f"ERROR: {labeling_channel} not in {chl_list}") - ind_channel = chl_list.index(labeling_channel) + channels = get_omero_channel_list(image_zarr_path=zarrurl) + channel = _get_channel_from_list( + channels=channels, wavelength_id=labeling_channel + ) + ind_channel = channel["index"] # Load ZYX data data_zyx = da.from_zarr(f"{zarrurl}{labeling_level}")[ind_channel] From 7c62e017b5889bd7edd92730f7221b6f87f7bd34 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:12:46 +0100 Subject: [PATCH 20/52] Add get_channel_from_image_zarr helper function, rename _get.. function into get.., update napari_workflows_wrapper task --- fractal_tasks_core/cellpose_segmentation.py | 8 +++--- fractal_tasks_core/lib_channels.py | 25 +++++++++++++------ .../napari_workflows_wrapper.py | 11 +++++--- tests/test_unit_channels_addressing.py | 8 +++--- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/fractal_tasks_core/cellpose_segmentation.py b/fractal_tasks_core/cellpose_segmentation.py index 446d37b21..8e913ee27 100644 --- a/fractal_tasks_core/cellpose_segmentation.py +++ b/fractal_tasks_core/cellpose_segmentation.py @@ -36,8 +36,7 @@ from cellpose.core import use_gpu import fractal_tasks_core -from fractal_tasks_core.lib_channels import _get_channel_from_list -from fractal_tasks_core.lib_channels import get_omero_channel_list +from fractal_tasks_core.lib_channels import get_channel_from_image_zarr from fractal_tasks_core.lib_pyramid_creation import build_pyramid from fractal_tasks_core.lib_regions_of_interest import ( array_to_bounding_box_table, @@ -181,9 +180,8 @@ def cellpose_segmentation( well_id = well.replace("/", "_")[:-1] # Find channel index - channels = get_omero_channel_list(image_zarr_path=zarrurl) - channel = _get_channel_from_list( - channels=channels, wavelength_id=labeling_channel + channel = get_channel_from_image_zarr( + image_zarr_path=zarrurl, wavelength_id=labeling_channel ) ind_channel = channel["index"] diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 32d6e4b06..e8519dfe8 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -22,9 +22,23 @@ import zarr -def _get_channel_from_list( +def get_channel_from_image_zarr( + *, image_zarr_path: str, label: str = None, wavelength_id: str = None +) -> Dict[str, Any]: + omero_channels = get_omero_channel_list(image_zarr_path=image_zarr_path) + get_omero_channel_list( + channels=omero_channels, label=label, wavelength_id=wavelength_id + ) + + +def get_omero_channel_list(*, image_zarr_path: str) -> List[Dict[str, Any]]: + group = zarr.open_group(image_zarr_path, mode="r") + return group.attrs["omero"]["channels"] + + +def get_channel_from_list( *, channels: Sequence[Dict], label: str = None, wavelength_id: str = None -): +) -> Dict[str, Any]: """ Find matching channel in a list @@ -92,7 +106,7 @@ def define_omero_channels( for channel in actual_channels: wavelength_id = channel["wavelength_id"] - channel = _get_channel_from_list( + channel = get_channel_from_list( channels=channel_parameters, wavelength_id=wavelength_id ) @@ -138,8 +152,3 @@ def define_omero_channels( pass return omero_channels - - -def get_omero_channel_list(*, image_zarr_path: str) -> List[Dict[str, str]]: - group = zarr.open_group(image_zarr_path, mode="r") - return group.attrs["omero"]["channels"] diff --git a/fractal_tasks_core/napari_workflows_wrapper.py b/fractal_tasks_core/napari_workflows_wrapper.py index 22f78d46c..b909cd685 100644 --- a/fractal_tasks_core/napari_workflows_wrapper.py +++ b/fractal_tasks_core/napari_workflows_wrapper.py @@ -32,6 +32,7 @@ from napari_workflows._io_yaml_v1 import load_workflow import fractal_tasks_core +from fractal_tasks_core.lib_channels import get_channel_from_image_zarr from fractal_tasks_core.lib_pyramid_creation import build_pyramid from fractal_tasks_core.lib_regions_of_interest import ( convert_ROI_table_to_indices, @@ -40,6 +41,7 @@ from fractal_tasks_core.lib_zattrs_utils import extract_zyx_pixel_sizes from fractal_tasks_core.lib_zattrs_utils import rescale_datasets + __OME_NGFF_VERSION__ = fractal_tasks_core.__OME_NGFF_VERSION__ @@ -135,7 +137,6 @@ def napari_workflows_wrapper( in_path = input_paths[0].parent.as_posix() num_levels = metadata["num_levels"] coarsening_xy = metadata["coarsening_xy"] - chl_list = metadata["channel_list"] label_dtype = np.uint32 # Load zattrs file and multiscales @@ -185,9 +186,11 @@ def napari_workflows_wrapper( # Loop over image inputs and assign corresponding channel of the image for (name, params) in image_inputs: channel_name = params["channel"] - if channel_name not in chl_list: - raise ValueError(f"{channel_name=} not in {chl_list}") - channel_index = chl_list.index(channel_name) + channel = get_channel_from_image_zarr( + image_zarr_path=f"{in_path}/{component}", + wavelength_id=channel_name, + ) + channel_index = channel["index"] input_image_arrays[name] = img_array[channel_index] # Handle dimensions diff --git a/tests/test_unit_channels_addressing.py b/tests/test_unit_channels_addressing.py index 43dac4bd7..fb011649a 100644 --- a/tests/test_unit_channels_addressing.py +++ b/tests/test_unit_channels_addressing.py @@ -3,7 +3,7 @@ from devtools import debug -from fractal_tasks_core.lib_channels import _get_channel_from_list +from fractal_tasks_core.lib_channels import get_channel_from_list def test_get_channel(testdata_path: Path): @@ -11,13 +11,13 @@ def test_get_channel(testdata_path: Path): omero_channels = json.load(f) debug(omero_channels) - channel = _get_channel_from_list(channels=omero_channels, label="label_1") + channel = get_channel_from_list(channels=omero_channels, label="label_1") debug(channel) assert channel["label"] == "label_1" assert channel["wavelength_id"] == "wavelength_id_1" assert channel["index"] == 0 - channel = _get_channel_from_list( + channel = get_channel_from_list( channels=omero_channels, wavelength_id="wavelength_id_2" ) debug(channel) @@ -25,7 +25,7 @@ def test_get_channel(testdata_path: Path): assert channel["wavelength_id"] == "wavelength_id_2" assert channel["index"] == 1 - channel = _get_channel_from_list( + channel = get_channel_from_list( channels=omero_channels, label="label_2", wavelength_id="wavelength_id_2", From 6abd17d3efa776a19842341084cf9b04b44d1494 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:16:34 +0100 Subject: [PATCH 21/52] Add docstrings --- fractal_tasks_core/lib_channels.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index e8519dfe8..795656b6e 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -25,6 +25,16 @@ def get_channel_from_image_zarr( *, image_zarr_path: str, label: str = None, wavelength_id: str = None ) -> Dict[str, Any]: + """ + Directly extract channel from .zattrs file + + This is a helper function that combines ``get_omero_channel_list`` with + ``get_channel_from_list``. + + :param image_zarr_path: TBD + :param label: TBD + :param wavelength_id: TBD + """ omero_channels = get_omero_channel_list(image_zarr_path=image_zarr_path) get_omero_channel_list( channels=omero_channels, label=label, wavelength_id=wavelength_id @@ -32,6 +42,11 @@ def get_channel_from_image_zarr( def get_omero_channel_list(*, image_zarr_path: str) -> List[Dict[str, Any]]: + """ + Extract the list of channels from .zattrs file + + :param image_zarr_path: TBD + """ group = zarr.open_group(image_zarr_path, mode="r") return group.attrs["omero"]["channels"] From 3f61ab69dd4d60b3b93943c418651289c8e41b1d Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:24:19 +0100 Subject: [PATCH 22/52] Fix wrong function name --- fractal_tasks_core/lib_channels.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 795656b6e..104cc4da3 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -36,9 +36,10 @@ def get_channel_from_image_zarr( :param wavelength_id: TBD """ omero_channels = get_omero_channel_list(image_zarr_path=image_zarr_path) - get_omero_channel_list( + channel = get_channel_from_list( channels=omero_channels, label=label, wavelength_id=wavelength_id ) + return channel def get_omero_channel_list(*, image_zarr_path: str) -> List[Dict[str, Any]]: From 91d4c7fa58370ae228008c78dc843101af8e148d Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:30:20 +0100 Subject: [PATCH 23/52] Remove channel_list from metadata --- fractal_tasks_core/create_ome_zarr.py | 1 - fractal_tasks_core/yokogawa_to_ome_zarr.py | 2 +- tests/conftest.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index 207c50c2c..e8c89618e 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -367,7 +367,6 @@ def create_ome_zarr( image=zarrurls["image"], num_levels=num_levels, coarsening_xy=coarsening_xy, - channel_list=actual_wavelength_ids, # FIXME: remove this original_paths=[str(p) for p in input_paths], ) return metadata_update diff --git a/fractal_tasks_core/yokogawa_to_ome_zarr.py b/fractal_tasks_core/yokogawa_to_ome_zarr.py index d2bb116c2..3046af1c0 100644 --- a/fractal_tasks_core/yokogawa_to_ome_zarr.py +++ b/fractal_tasks_core/yokogawa_to_ome_zarr.py @@ -70,7 +70,7 @@ def yokogawa_to_ome_zarr( Example arguments: input_paths[0] = /tmp/output/*.zarr (Path) output_path = /tmp/output/*.zarr (Path) - metadata = {"channel_list": [...], "num_levels": ..., } + metadata = {"num_levels": ..., } component = plate.zarr/B/03/0/ :param input_paths: TBD diff --git a/tests/conftest.py b/tests/conftest.py index c93b6bc4e..01529354e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -103,7 +103,6 @@ def zenodo_zarr_metadata(testdata_path): "image": ["plate.zarr/B/03/0/"], "num_levels": 6, "coarsening_xy": 2, - "channel_list": ["A01_C01"], "original_paths": [ str(testdata_path / "10_5281_zenodo_7059515/*.png") ], @@ -115,7 +114,6 @@ def zenodo_zarr_metadata(testdata_path): "image": ["plate_mip.zarr/B/03/0/"], "num_levels": 6, "coarsening_xy": 2, - "channel_list": ["A01_C01"], "original_paths": [ str(testdata_path / "10_5281_zenodo_7059515/*.png") ], From 57924a15eeba42063f8534d4b1b0f2c114f55525 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:35:11 +0100 Subject: [PATCH 24/52] Update channel addressing in illumination_correction task --- fractal_tasks_core/illumination_correction.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/fractal_tasks_core/illumination_correction.py b/fractal_tasks_core/illumination_correction.py index 3d444470c..a90609980 100644 --- a/fractal_tasks_core/illumination_correction.py +++ b/fractal_tasks_core/illumination_correction.py @@ -29,6 +29,7 @@ import zarr from skimage.io import imread +from fractal_tasks_core.lib_channels import get_omero_channel_list from fractal_tasks_core.lib_pyramid_creation import build_pyramid from fractal_tasks_core.lib_regions_of_interest import ( convert_ROI_table_to_indices, @@ -133,7 +134,6 @@ def illumination_correction( raise NotImplementedError(msg) # Read some parameters from metadata - chl_list = metadata["channel_list"] num_levels = metadata["num_levels"] coarsening_xy = metadata["coarsening_xy"] @@ -155,6 +155,10 @@ def illumination_correction( logger.info(f" {zarrurl_old=}") logger.info(f" {zarrurl_new=}") + # Read channels from .zattrs + channels = get_omero_channel_list(image_zarr_path=zarrurl_old) + num_channels = len(channels) + # Read FOV ROIs FOV_ROI_table = ad.read_zarr(f"{zarrurl_old}/tables/FOV_ROI_table") @@ -192,9 +196,12 @@ def illumination_correction( # Assemble dictionary of matrices and check their shapes corrections = {} - for ind_ch, ch in enumerate(chl_list): - corrections[ch] = imread(root_path_corr + dict_corr[ch]) - if corrections[ch].shape != (img_size_y, img_size_x): + for channel in channels: + wavelength_id = channel["wavelength_id"] + corrections[wavelength_id] = imread( + root_path_corr + dict_corr[wavelength_id] + ) + if corrections[wavelength_id].shape != (img_size_y, img_size_x): raise Exception( "Error in illumination_correction, " "correction matrix has wrong shape." @@ -220,7 +227,7 @@ def illumination_correction( # Iterate over FOV ROIs num_ROIs = len(list_indices) - for i_c, channel in enumerate(chl_list): + for i_c, channel in enumerate(channels): for i_ROI, indices in enumerate(list_indices): # Define region s_z, e_z, s_y, e_y, s_x, e_x = indices[:] @@ -232,12 +239,12 @@ def illumination_correction( ) logger.info( f"Now processing ROI {i_ROI+1}/{num_ROIs} " - f"for channel {i_c+1}/{len(chl_list)}" + f"for channel {i_c+1}/{num_channels}" ) # Execute illumination correction corrected_fov = correct( data_czyx[region].compute(), - corrections[channel], + corrections[channel["wavelength_id"]], background=background, ) # Write to disk From 978090f07b6b58141563bff6c1b273e2100c3384 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:49:19 +0100 Subject: [PATCH 25/52] Introduce new channel logic into multiplexing parsing task --- .../create_ome_zarr_multiplex.py | 82 +++++++++---------- tests/test_workflows_multiplexing.py | 26 +++--- 2 files changed, 55 insertions(+), 53 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr_multiplex.py b/fractal_tasks_core/create_ome_zarr_multiplex.py index 14893bf3c..8eaa65a0f 100644 --- a/fractal_tasks_core/create_ome_zarr_multiplex.py +++ b/fractal_tasks_core/create_ome_zarr_multiplex.py @@ -19,7 +19,6 @@ from typing import Any from typing import Dict from typing import List -from typing import Optional from typing import Sequence import pandas as pd @@ -45,8 +44,8 @@ def create_ome_zarr_multiplex( *, input_paths: Sequence[Path], output_path: Path, - metadata: Dict[str, Any] = None, - channel_parameters: Dict[str, Dict[str, Any]], + metadata: Dict[str, Any], + allowed_channels: Dict[str, Sequence[Dict[str, str]]], num_levels: int = 2, coarsening_xy: int = 2, metadata_table: str = "mrf_mlf", @@ -86,16 +85,12 @@ def create_ome_zarr_multiplex( 'metadata_table="mrf_mlf", ' f"and not {metadata_table}" ) - if channel_parameters is None: - raise ValueError( - "Missing channel_parameters argument in create_ome_zarr" - ) else: # Note that in metadata the keys of dictionary arguments should be - # strings, so that they can be read from a JSON file - for key in channel_parameters.keys(): + # strings (and not integers), so that they can be read from a JSON file + for key in allowed_channels.keys(): if not isinstance(key, str): - raise ValueError(f"{channel_parameters=} has non-string keys") + raise ValueError(f"{allowed_channels=} has non-string keys") # Identify all plates and all channels, per input folders dict_acquisitions: Dict = {} @@ -106,7 +101,7 @@ def create_ome_zarr_multiplex( acquisition = str(ind_in_path) dict_acquisitions[acquisition] = {} - channels = [] + actual_wavelength_ids = [] plates = [] plate_prefixes = [] @@ -119,20 +114,20 @@ def create_ome_zarr_multiplex( plates.append(plate) plate_prefix = filename_metadata["plate_prefix"] plate_prefixes.append(plate_prefix) - channels.append( - f"A{filename_metadata['A']}_C{filename_metadata['C']}" - ) + A = filename_metadata["A"] + C = filename_metadata["C"] + actual_wavelength_ids.append(f"A{A}_C{C}") except ValueError as e: logger.warning( f'Skipping "{fn.name}". Original error: ' + str(e) ) plates = sorted(list(set(plates))) - channels = sorted(list(set(channels))) + actual_wavelength_ids = sorted(list(set(actual_wavelength_ids))) info = ( f"Listing all plates/channels from {in_path.as_posix()}\n" f"Plates: {plates}\n" - f"Channels: {channels}\n" + f"Actual wavelength IDs: {actual_wavelength_ids}\n" ) # Check that a folder includes a single plate @@ -152,19 +147,25 @@ def create_ome_zarr_multiplex( ) # Check that all channels are in the allowed_channels - if not set(channels).issubset( - set(channel_parameters[acquisition].keys()) + allowed_wavelength_ids = [ + c["wavelength_id"] for c in allowed_channels[acquisition] + ] + if not set(actual_wavelength_ids).issubset( + set(allowed_wavelength_ids) ): msg = "ERROR in create_ome_zarr\n" - msg += f"channels: {channels}\n" - msg += "allowed_channels: " - msg += f"{channel_parameters[acquisition].keys()}\n" - raise Exception(msg) - - # Create actual_channels, i.e. a list of entries like "A01_C01" - actual_channels = [] - for ind_ch, ch in enumerate(channels): - actual_channels.append(ch) + msg += f"actual_wavelength_ids: {actual_wavelength_ids}\n" + msg += f"allowed_wavelength_ids: {allowed_wavelength_ids}\n" + raise ValueError(msg) + + # Create actual_channels, i.e. a list of the channel dictionaries which + # are present + actual_channels = [ + channel + for channel in allowed_channels[acquisition] + if channel["wavelength_id"] in actual_wavelength_ids + ] + logger.info(f"plate: {plate}") logger.info(f"actual_channels: {actual_channels}") @@ -250,22 +251,22 @@ def create_ome_zarr_multiplex( well_image_iter = glob( f"{image_folder}/{plate_prefix}_{well}{ext_glob_pattern}" ) - well_channels = [] + well_wavelength_ids = [] for fpath in well_image_iter: try: filename_metadata = parse_filename(os.path.basename(fpath)) - well_channels.append( - f"A{filename_metadata['A']}_C{filename_metadata['C']}" - ) + A = filename_metadata["A"] + C = filename_metadata["C"] + well_wavelength_ids.append(f"A{A}_C{C}") except IndexError: logger.info(f"Skipping {fpath}") - well_channels = sorted(list(set(well_channels))) - if well_channels != actual_channels: + well_wavelength_ids = sorted(list(set(well_wavelength_ids))) + if well_wavelength_ids != actual_wavelength_ids: raise Exception( f"ERROR: well {well} in plate {plate} (prefix: " f"{plate_prefix}) has missing channels.\n" - f"Expected: {actual_channels}\n" - f"Found: {well_channels}.\n" + f"Expected: {actual_wavelength_ids}\n" + f"Found: {well_wavelength_ids}.\n" ) well_rows_columns = [ @@ -380,7 +381,7 @@ def create_ome_zarr_multiplex( "version": __OME_NGFF_VERSION__, "channels": define_omero_channels( actual_channels, - channel_parameters[acquisition], + allowed_channels[acquisition], bit_depth, ), } @@ -402,10 +403,6 @@ def create_ome_zarr_multiplex( write_elem(group_tables, "FOV_ROI_table", FOV_ROIs_table) write_elem(group_tables, "well_ROI_table", well_ROIs_table) - channel_list = { - acquisition: dict_acquisitions[acquisition]["actual_channels"] - for acquisition in acquisitions - } original_paths = { acquisition: dict_acquisitions[acquisition]["original_paths"] for acquisition in acquisitions @@ -417,7 +414,6 @@ def create_ome_zarr_multiplex( image=zarrurls["image"], num_levels=num_levels, coarsening_xy=coarsening_xy, - channel_list=channel_list, original_paths=original_paths, ) return metadata_update @@ -430,8 +426,8 @@ def create_ome_zarr_multiplex( class TaskArguments(BaseModel): input_paths: Sequence[Path] output_path: Path - metadata: Optional[Dict[str, Any]] - channel_parameters: Dict[str, Dict[str, Any]] + metadata: Dict[str, Any] + allowed_channels: Dict[str, Dict[str, Any]] num_levels: int = 2 coarsening_xy: int = 2 metadata_table: str = "mrf_mlf" diff --git a/tests/test_workflows_multiplexing.py b/tests/test_workflows_multiplexing.py index 50bd213d2..308399117 100644 --- a/tests/test_workflows_multiplexing.py +++ b/tests/test_workflows_multiplexing.py @@ -30,29 +30,33 @@ from fractal_tasks_core.yokogawa_to_ome_zarr import yokogawa_to_ome_zarr -single_cycle_channel_parameters = { - "A01_C01": { +single_cycle_allowed_channels = [ + { "label": "DAPI", + "wavelength_id": "A01_C01", "colormap": "00FFFF", "start": 0, "end": 700, }, - "A01_C02": { + { + "wavelength_id": "A01_C02", "label": "nanog", "colormap": "FF00FF", "start": 0, "end": 180, }, - "A02_C03": { + { + "wavelength_id": "A02_C03", "label": "Lamin B1", "colormap": "FFFF00", "start": 0, "end": 1500, }, -} -channel_parameters = { - "0": single_cycle_channel_parameters, - "1": single_cycle_channel_parameters, +] + +allowed_channels = { + "0": single_cycle_allowed_channels, + "1": single_cycle_allowed_channels, } num_levels = 6 @@ -75,7 +79,8 @@ def test_workflow_multiplexing( metadata_update = create_ome_zarr_multiplex( input_paths=img_paths, output_path=zarr_path, - channel_parameters=channel_parameters, + metadata=metadata, + allowed_channels=allowed_channels, num_levels=num_levels, coarsening_xy=coarsening_xy, metadata_table="mrf_mlf", @@ -124,7 +129,8 @@ def test_workflow_multiplexing_MIP( metadata_update = create_ome_zarr_multiplex( input_paths=img_paths, output_path=zarr_path, - channel_parameters=channel_parameters, + metadata=metadata, + allowed_channels=allowed_channels, num_levels=num_levels, coarsening_xy=coarsening_xy, metadata_table="mrf_mlf", From 24730ab3e523c57edf3e1566506df48be00bc978 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 15:16:57 +0100 Subject: [PATCH 26/52] Add check_unique_labels function and improve usage of define_omero_channels --- fractal_tasks_core/create_ome_zarr.py | 11 +++++++- .../create_ome_zarr_multiplex.py | 6 ++-- fractal_tasks_core/lib_channels.py | 28 ++++++++++++++----- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index e8c89618e..c8e9cca35 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -339,7 +339,7 @@ def create_ome_zarr( "name": "TBD", "version": __OME_NGFF_VERSION__, "channels": define_omero_channels( - actual_channels, allowed_channels, bit_depth + channels=actual_channels, bit_depth=bit_depth ), } @@ -361,6 +361,15 @@ def create_ome_zarr( write_elem(group_tables, "FOV_ROI_table", FOV_ROIs_table) write_elem(group_tables, "well_ROI_table", well_ROIs_table) + # FIXME: check that labels are unique within each well + # for plate in plates: + # list images + # collect paths + # call check_unique_labels + from devtools import debug + + debug("MISSING CHECK HERE") + metadata_update = dict( plate=zarrurls["plate"], well=zarrurls["well"], diff --git a/fractal_tasks_core/create_ome_zarr_multiplex.py b/fractal_tasks_core/create_ome_zarr_multiplex.py index 8eaa65a0f..698a436dd 100644 --- a/fractal_tasks_core/create_ome_zarr_multiplex.py +++ b/fractal_tasks_core/create_ome_zarr_multiplex.py @@ -380,9 +380,9 @@ def create_ome_zarr_multiplex( "name": "TBD", "version": __OME_NGFF_VERSION__, "channels": define_omero_channels( - actual_channels, - allowed_channels[acquisition], - bit_depth, + channels=actual_channels, + bit_depth=bit_depth, + label_prefix=acquisition, ), } diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 104cc4da3..b888e548f 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -22,6 +22,19 @@ import zarr +def check_unique_labels(list_channels: Sequence[Sequence[Dict[str, Any]]]): + for ind_1, channels_1 in enumerate(list_channels): + labels_1 = set([c["label"] for c in channels_1]) + for ind_2 in range(ind_1): + channels_2 = list_channels[ind_2] + labels_2 = set([c["label"] for c in channels_2]) + intersection = labels_1 & labels_2 + if intersection: + raise ValueError( + "Non-unique channel labels\n" f"{labels_1=}\n{labels_2=}" + ) + + def get_channel_from_image_zarr( *, image_zarr_path: str, label: str = None, wavelength_id: str = None ) -> Dict[str, Any]: @@ -104,33 +117,34 @@ def get_channel_from_list( def define_omero_channels( - actual_channels: Sequence[str], - channel_parameters: Dict[str, Any], + *, + channels: Sequence[Dict[str, Any]], bit_depth: int, + label_prefix: str = None, ) -> List[Dict[str, Any]]: """ Prepare the .attrs["omero"]["channels"] attribute of an image group - :param actual_channels: TBD - :param channel_parameters: TBD + :param channels: TBD :param bit_depth: TBD :returns: omero_channels """ omero_channels = [] default_colormaps = ["00FFFF", "FF00FF", "FFFF00"] - for channel in actual_channels: + for channel in channels: wavelength_id = channel["wavelength_id"] channel = get_channel_from_list( - channels=channel_parameters, wavelength_id=wavelength_id + channels=channels, wavelength_id=wavelength_id ) try: label = channel["label"] except KeyError: - # FIXME better handling of missing label default_label = wavelength_id + if label_prefix: + default_label = f"{label_prefix}_{default_label}" logging.warning( f"Missing label for {channel=}, using {default_label=}" ) From f2abef734801ff70569b51c6a184507813925c2f Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 15:18:00 +0100 Subject: [PATCH 27/52] Keep updating tests for new channel scheme --- tests/test_unit_illumination_correction.py | 3 +- tests/test_workflows.py | 31 ++++++++------- tests/test_workflows_multiplexing.py | 46 ++++++++++++++++++---- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/tests/test_unit_illumination_correction.py b/tests/test_unit_illumination_correction.py index 317026134..ed4f689f7 100644 --- a/tests/test_unit_illumination_correction.py +++ b/tests/test_unit_illumination_correction.py @@ -56,8 +56,8 @@ def test_illumination_correction( metadata: Dict = { "num_levels": num_levels, "coarsening_xy": 2, - "channel_list": ["A01_C01", "A01_C02"], } + num_channels = 2 num_levels = metadata["num_levels"] # Read FOV ROIs and create corresponding indices @@ -69,7 +69,6 @@ def test_illumination_correction( num_FOVs = len(list_indices) # Prepared expected number of calls - num_channels = len(metadata["channel_list"]) expected_tot_calls_correct = num_channels * num_FOVs # Patch correct() function, to keep track of the number of calls diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 370dfd19c..9774608b4 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -60,26 +60,29 @@ coarsening_xy = 2 -def test_create_ome_zarr(tmp_path: Path, zenodo_images: Path): +def test_create_ome_zarr_fail(tmp_path: Path, zenodo_images: Path): + + allowed_channels = [ + {"label": "repeated label", "wavelength_id": "A01_C01"}, + {"label": "repeated label", "wavelength_id": "A01_C02"}, + {"label": "repeated label", "wavelength_id": "A02_C03"}, + ] # Init img_path = zenodo_images / "*.png" zarr_path = tmp_path / "tmp_out/*.zarr" - metadata = {} # Create zarr structure - metadata_update = create_ome_zarr( - input_paths=[img_path], - output_path=zarr_path, - allowed_channels=allowed_channels, - num_levels=num_levels, - coarsening_xy=coarsening_xy, - metadata_table="mrf_mlf", - ) - metadata.update(metadata_update) - debug(metadata) - - # FIXME: assert something (e.g. about channels) + with pytest.raises(ValueError): + _ = create_ome_zarr( + input_paths=[img_path], + metadata={}, + output_path=zarr_path, + allowed_channels=allowed_channels, + num_levels=num_levels, + coarsening_xy=coarsening_xy, + metadata_table="mrf_mlf", + ) def test_yokogawa_to_ome_zarr(tmp_path: Path, zenodo_images: Path): diff --git a/tests/test_workflows_multiplexing.py b/tests/test_workflows_multiplexing.py index 308399117..ab05c1802 100644 --- a/tests/test_workflows_multiplexing.py +++ b/tests/test_workflows_multiplexing.py @@ -14,6 +14,7 @@ from pathlib import Path from typing import Sequence +import pytest from devtools import debug from .utils import check_file_number @@ -30,9 +31,8 @@ from fractal_tasks_core.yokogawa_to_ome_zarr import yokogawa_to_ome_zarr -single_cycle_allowed_channels = [ +single_cycle_allowed_channels_no_label = [ { - "label": "DAPI", "wavelength_id": "A01_C01", "colormap": "00FFFF", "start": 0, @@ -40,14 +40,12 @@ }, { "wavelength_id": "A01_C02", - "label": "nanog", "colormap": "FF00FF", "start": 0, "end": 180, }, { "wavelength_id": "A02_C03", - "label": "Lamin B1", "colormap": "FFFF00", "start": 0, "end": 1500, @@ -55,15 +53,47 @@ ] allowed_channels = { - "0": single_cycle_allowed_channels, - "1": single_cycle_allowed_channels, + "0": single_cycle_allowed_channels_no_label, + "1": single_cycle_allowed_channels_no_label, } num_levels = 6 coarsening_xy = 2 -def test_workflow_multiplexing( +def test_multiplexing_create_ome_zarr_fail( + tmp_path: Path, zenodo_images_multiplex: Sequence[Path] +): + + single_cycle_allowed_channels = [ + {"wavelength_id": "A01_C01", "label": "my label"} + ] + allowed_channels = { + "0": single_cycle_allowed_channels, + "1": single_cycle_allowed_channels, + } + + # Init + img_paths = [ + cycle_folder / "*.png" for cycle_folder in zenodo_images_multiplex + ] + zarr_path = tmp_path / "tmp_out/*.zarr" + + # Create zarr structure + debug(img_paths) + with pytest.raises(ValueError): + _ = create_ome_zarr_multiplex( + input_paths=img_paths, + output_path=zarr_path, + metadata={}, + allowed_channels=allowed_channels, + num_levels=num_levels, + coarsening_xy=coarsening_xy, + metadata_table="mrf_mlf", + ) + + +def test_multiplexing_yokogawa_to_ome_zarr( tmp_path: Path, zenodo_images_multiplex: Sequence[Path] ): @@ -112,7 +142,7 @@ def test_workflow_multiplexing( check_file_number(zarr_path=image_zarr_1) -def test_workflow_multiplexing_MIP( +def test_multiplexing_MIP( tmp_path: Path, zenodo_images_multiplex: Sequence[Path] ): From 3cff675caeef867072f90927980400a2dfc154ac Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 15:27:52 +0100 Subject: [PATCH 28/52] Check per-image channel-label unicity --- fractal_tasks_core/lib_channels.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index b888e548f..c6c169b08 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -181,4 +181,9 @@ def define_omero_channels( except KeyError: pass + # Check that channel labels are unique for this image + labels = [c["label"] for c in omero_channels] + if len(set(labels)) < len(labels): + raise ValueError(f"Non-unique labels in {omero_channels=}") + return omero_channels From ea9333def97d7af7501042bb6312db50eaaab999 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 15:28:13 +0100 Subject: [PATCH 29/52] xfail a test --- tests/test_workflows.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 9774608b4..2c2c7adf8 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -60,6 +60,7 @@ coarsening_xy = 2 +@pytest.mark.xfail("This would fail for a dataset with N>1 channels") def test_create_ome_zarr_fail(tmp_path: Path, zenodo_images: Path): allowed_channels = [ From dc07520c4e1dca6aa2c8bfc9cb879852062abc96 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 15:29:01 +0100 Subject: [PATCH 30/52] Remove unnecessary comment --- fractal_tasks_core/create_ome_zarr.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index c8e9cca35..3ac652b99 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -361,15 +361,6 @@ def create_ome_zarr( write_elem(group_tables, "FOV_ROI_table", FOV_ROIs_table) write_elem(group_tables, "well_ROI_table", well_ROIs_table) - # FIXME: check that labels are unique within each well - # for plate in plates: - # list images - # collect paths - # call check_unique_labels - from devtools import debug - - debug("MISSING CHECK HERE") - metadata_update = dict( plate=zarrurls["plate"], well=zarrurls["well"], From f62bc129400a679daaa0ac29992a706813288ae4 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 15:43:53 +0100 Subject: [PATCH 31/52] Add checks about labels unicity --- fractal_tasks_core/create_ome_zarr.py | 8 ++++++ .../create_ome_zarr_multiplex.py | 8 +++++- fractal_tasks_core/lib_channels.py | 28 ++++++++++++++++--- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index 3ac652b99..3574636a7 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -27,6 +27,7 @@ from anndata.experimental import write_elem import fractal_tasks_core +from fractal_tasks_core.lib_channels import check_well_channel_labels from fractal_tasks_core.lib_channels import define_omero_channels from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata from fractal_tasks_core.lib_parse_filename_metadata import parse_filename @@ -361,6 +362,13 @@ def create_ome_zarr( write_elem(group_tables, "FOV_ROI_table", FOV_ROIs_table) write_elem(group_tables, "well_ROI_table", well_ROIs_table) + # Check that the different images in the each well have unique labels. + # Since we currently merge all fields of view in the same image, this check + # is useless. It should remain there to catch an error in case we switch + # back to one-image-per-field-of-view mode + for well_path in zarrurls["well"]: + check_well_channel_labels(well_zarr_path=well_path) + metadata_update = dict( plate=zarrurls["plate"], well=zarrurls["well"], diff --git a/fractal_tasks_core/create_ome_zarr_multiplex.py b/fractal_tasks_core/create_ome_zarr_multiplex.py index 698a436dd..52e3fd2e3 100644 --- a/fractal_tasks_core/create_ome_zarr_multiplex.py +++ b/fractal_tasks_core/create_ome_zarr_multiplex.py @@ -26,6 +26,7 @@ from anndata.experimental import write_elem import fractal_tasks_core +from fractal_tasks_core.lib_channels import check_well_channel_labels from fractal_tasks_core.lib_channels import define_omero_channels from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata from fractal_tasks_core.lib_parse_filename_metadata import parse_filename @@ -310,7 +311,7 @@ def create_ome_zarr_multiplex( ], "version": __OME_NGFF_VERSION__, } - zarrurls["well"].append(f"{row}/{column}") + zarrurls["well"].append(f"{plate}.zarr/{row}/{column}") except ContainsGroupError: group_well = zarr.open_group( f"{full_zarrurl}/{row}/{column}/", mode="a" @@ -403,6 +404,11 @@ def create_ome_zarr_multiplex( write_elem(group_tables, "FOV_ROI_table", FOV_ROIs_table) write_elem(group_tables, "well_ROI_table", well_ROIs_table) + # Check that the different images (e.g. different cycles) in the each well + # have unique labels + for well_path in zarrurls["well"]: + check_well_channel_labels(well_zarr_path=well_path) + original_paths = { acquisition: dict_acquisitions[acquisition]["original_paths"] for acquisition in acquisitions diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index c6c169b08..1a47bcec4 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -22,16 +22,36 @@ import zarr -def check_unique_labels(list_channels: Sequence[Sequence[Dict[str, Any]]]): - for ind_1, channels_1 in enumerate(list_channels): +def check_well_channel_labels(*, well_zarr_path: str): + """ + FIXME + """ + + # Iterate over all images (multiplexing cycles, multi-FOVs, ...) + group = zarr.open_group(well_zarr_path, mode="r") + image_paths = [image["path"] for image in group.attrs["well"]["images"]] + list_of_channel_lists = [] + for image_path in image_paths: + channels = get_omero_channel_list( + image_zarr_path=f"{well_zarr_path}/{image_path}" + ) + list_of_channel_lists.append(channels[:]) + + for ind_1, channels_1 in enumerate(list_of_channel_lists): labels_1 = set([c["label"] for c in channels_1]) for ind_2 in range(ind_1): - channels_2 = list_channels[ind_2] + channels_2 = list_of_channel_lists[ind_2] labels_2 = set([c["label"] for c in channels_2]) intersection = labels_1 & labels_2 if intersection: + hint = ( + "Are you parsing fields of view into separate OME-Zarr" + " images? This could lead to non-unique channel labels" + ", and then could be the reason of the error" + ) raise ValueError( - "Non-unique channel labels\n" f"{labels_1=}\n{labels_2=}" + "Non-unique channel labels\n" + f"{labels_1=}\n{labels_2=}\n{hint}" ) From c4c4ff31164cc1ef3ba2b1fc8080bb3bf82e6446 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 16:17:13 +0100 Subject: [PATCH 32/52] Fix check_well_channel_labels + fix some task interfaces --- fractal_tasks_core/create_ome_zarr.py | 13 +++++++------ fractal_tasks_core/create_ome_zarr_multiplex.py | 8 +++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index 3574636a7..a0c6f3a5f 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -19,7 +19,6 @@ from typing import Any from typing import Dict from typing import List -from typing import Optional from typing import Sequence import pandas as pd @@ -46,8 +45,8 @@ def create_ome_zarr( *, input_paths: Sequence[Path], output_path: Path, - metadata: Dict[str, Any] = None, - allowed_channels: Sequence[Dict[str, str]], + metadata: Dict[str, Any], + allowed_channels: Sequence[Dict[str, Any]], num_levels: int = 2, coarsening_xy: int = 2, metadata_table: str = "mrf_mlf", @@ -367,7 +366,9 @@ def create_ome_zarr( # is useless. It should remain there to catch an error in case we switch # back to one-image-per-field-of-view mode for well_path in zarrurls["well"]: - check_well_channel_labels(well_zarr_path=well_path) + check_well_channel_labels( + well_zarr_path=str(output_path.parent / well_path) + ) metadata_update = dict( plate=zarrurls["plate"], @@ -387,8 +388,8 @@ def create_ome_zarr( class TaskArguments(BaseModel): input_paths: Sequence[Path] output_path: Path - metadata: Optional[Dict[str, Any]] - allowed_channels: Sequence[Dict[str, str]] + metadata: Dict[str, Any] + allowed_channels: Sequence[Dict[str, Any]] num_levels: int = 2 coarsening_xy: int = 2 metadata_table: str = "mrf_mlf" diff --git a/fractal_tasks_core/create_ome_zarr_multiplex.py b/fractal_tasks_core/create_ome_zarr_multiplex.py index 52e3fd2e3..9f2105686 100644 --- a/fractal_tasks_core/create_ome_zarr_multiplex.py +++ b/fractal_tasks_core/create_ome_zarr_multiplex.py @@ -46,7 +46,7 @@ def create_ome_zarr_multiplex( input_paths: Sequence[Path], output_path: Path, metadata: Dict[str, Any], - allowed_channels: Dict[str, Sequence[Dict[str, str]]], + allowed_channels: Dict[str, Sequence[Dict[str, Any]]], num_levels: int = 2, coarsening_xy: int = 2, metadata_table: str = "mrf_mlf", @@ -407,7 +407,9 @@ def create_ome_zarr_multiplex( # Check that the different images (e.g. different cycles) in the each well # have unique labels for well_path in zarrurls["well"]: - check_well_channel_labels(well_zarr_path=well_path) + check_well_channel_labels( + well_zarr_path=str(output_path.parent / well_path) + ) original_paths = { acquisition: dict_acquisitions[acquisition]["original_paths"] @@ -433,7 +435,7 @@ class TaskArguments(BaseModel): input_paths: Sequence[Path] output_path: Path metadata: Dict[str, Any] - allowed_channels: Dict[str, Dict[str, Any]] + allowed_channels: Dict[str, Sequence[Dict[str, Any]]] num_levels: int = 2 coarsening_xy: int = 2 metadata_table: str = "mrf_mlf" From a4ab7e7feec3dbc5df7eb967f1c547759870cec1 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 16:26:46 +0100 Subject: [PATCH 33/52] Fix tests --- tests/test_unit_task.py | 5 ++++- tests/test_workflows.py | 6 ++++-- tests/test_workflows_labeling.py | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_unit_task.py b/tests/test_unit_task.py index f6b5d8b49..0b2ef74ea 100644 --- a/tests/test_unit_task.py +++ b/tests/test_unit_task.py @@ -39,7 +39,10 @@ def test_create_ome_zarr(tmp_path, testdata_path): debug(default_args) dummy = create_ome_zarr( - input_paths=input_paths, output_path=output_path, **default_args + input_paths=input_paths, + output_path=output_path, + metadata={}, + **default_args ) debug(dummy) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 2c2c7adf8..23b9bb258 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -60,7 +60,7 @@ coarsening_xy = 2 -@pytest.mark.xfail("This would fail for a dataset with N>1 channels") +@pytest.mark.xfail(reason="This would fail for a dataset with N>1 channels") def test_create_ome_zarr_fail(tmp_path: Path, zenodo_images: Path): allowed_channels = [ @@ -91,12 +91,13 @@ def test_yokogawa_to_ome_zarr(tmp_path: Path, zenodo_images: Path): # Init img_path = zenodo_images / "*.png" zarr_path = tmp_path / "tmp_out/*.zarr" - metadata = {} # Create zarr structure + metadata = {} metadata_update = create_ome_zarr( input_paths=[img_path], output_path=zarr_path, + metadata=metadata, allowed_channels=allowed_channels, num_levels=num_levels, coarsening_xy=coarsening_xy, @@ -200,6 +201,7 @@ def test_illumination_correction( metadata_update = create_ome_zarr( input_paths=[img_path], output_path=zarr_path, + metadata=metadata, allowed_channels=allowed_channels, num_levels=num_levels, coarsening_xy=coarsening_xy, diff --git a/tests/test_workflows_labeling.py b/tests/test_workflows_labeling.py index 40f6d9336..d177ae617 100644 --- a/tests/test_workflows_labeling.py +++ b/tests/test_workflows_labeling.py @@ -284,6 +284,7 @@ def test_workflow_with_per_well_labeling_2D( metadata_update = create_ome_zarr( input_paths=[img_path], output_path=zarr_path, + metadata=metadata, allowed_channels=allowed_channels, num_levels=num_levels, coarsening_xy=coarsening_xy, From 6a7f07e04e70c21a2dc3d9b5b012129afac5533e Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 16:38:39 +0100 Subject: [PATCH 34/52] Set appropriate modes for zarr.open_group calls --- fractal_tasks_core/copy_ome_zarr.py | 6 ++++-- fractal_tasks_core/create_ome_zarr_multiplex.py | 2 +- fractal_tasks_core/lib_channels.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/fractal_tasks_core/copy_ome_zarr.py b/fractal_tasks_core/copy_ome_zarr.py index cfc55653a..2b6a5b5a2 100644 --- a/fractal_tasks_core/copy_ome_zarr.py +++ b/fractal_tasks_core/copy_ome_zarr.py @@ -129,7 +129,9 @@ def copy_ome_zarr( for well_path in well_paths: # Replicate well attrs - old_well_group = zarr.open_group(f"{zarrurl_old}/{well_path}") + old_well_group = zarr.open_group( + f"{zarrurl_old}/{well_path}", mode="r" + ) new_well_group = zarr.group(f"{zarrurl_new}/{well_path}") new_well_group.attrs.put(old_well_group.attrs.asdict()) @@ -143,7 +145,7 @@ def copy_ome_zarr( # Replicate image attrs old_image_group = zarr.open_group( - f"{zarrurl_old}/{well_path}/{image_path}" + f"{zarrurl_old}/{well_path}/{image_path}", mode="r" ) new_image_group = zarr.group( f"{zarrurl_new}/{well_path}/{image_path}" diff --git a/fractal_tasks_core/create_ome_zarr_multiplex.py b/fractal_tasks_core/create_ome_zarr_multiplex.py index 9f2105686..13d0e4321 100644 --- a/fractal_tasks_core/create_ome_zarr_multiplex.py +++ b/fractal_tasks_core/create_ome_zarr_multiplex.py @@ -314,7 +314,7 @@ def create_ome_zarr_multiplex( zarrurls["well"].append(f"{plate}.zarr/{row}/{column}") except ContainsGroupError: group_well = zarr.open_group( - f"{full_zarrurl}/{row}/{column}/", mode="a" + f"{full_zarrurl}/{row}/{column}/", mode="r+" ) logging.info( f"Loaded group_well from {full_zarrurl}/{row}/{column}" diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 1a47bcec4..2e46477a5 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -28,7 +28,7 @@ def check_well_channel_labels(*, well_zarr_path: str): """ # Iterate over all images (multiplexing cycles, multi-FOVs, ...) - group = zarr.open_group(well_zarr_path, mode="r") + group = zarr.open_group(well_zarr_path, mode="r+") image_paths = [image["path"] for image in group.attrs["well"]["images"]] list_of_channel_lists = [] for image_path in image_paths: @@ -81,7 +81,7 @@ def get_omero_channel_list(*, image_zarr_path: str) -> List[Dict[str, Any]]: :param image_zarr_path: TBD """ - group = zarr.open_group(image_zarr_path, mode="r") + group = zarr.open_group(image_zarr_path, mode="r+") return group.attrs["omero"]["channels"] From e3690ef255ca200f6833b326932f79ce0abfef0c Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 16:40:45 +0100 Subject: [PATCH 35/52] Add workaround to fixture (ref #240) --- tests/conftest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 01529354e..faceec834 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -92,6 +92,18 @@ def zenodo_zarr(testdata_path, tmpdir_factory): ) shutil.move(str(rootfolder / zarrname), str(folder)) + # FIXME: this is a workaround, and should be fixed directly in the + # zenodo dataset + import zarr + from devtools import debug + + image_path = str(folder / "B/03/0") + group = zarr.open_group(image_path, mode="r+") + attrs = group.attrs.asdict() + attrs["omero"]["channels"][0]["wavelength_id"] = "A01_C01" + group.attrs.put(attrs) + debug(f"Adding A01_C01 in omero metadata, in {image_path}") + return folders From 8a30c8c595f8076719817e1520d34de97e204d24 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 16:46:43 +0100 Subject: [PATCH 36/52] Update plate_ones.zarr (and corresponding script) to conform to new channel metadata --- tests/data/generate_zarr_ones.py | 22 +++- tests/data/plate_ones.zarr/B/03/0/.zattrs | 108 +++++++++++++++++- .../B/03/0/tables/FOV_ROI_table/X/.zarray | 4 +- .../B/03/0/tables/FOV_ROI_table/X/0.0 | Bin 64 -> 80 bytes .../B/03/0/tables/FOV_ROI_table/obs/_index/0 | Bin 38 -> 44 bytes .../0/tables/FOV_ROI_table/var/_index/.zarray | 4 +- .../B/03/0/tables/FOV_ROI_table/var/_index/0 | Bin 128 -> 134 bytes 7 files changed, 128 insertions(+), 10 deletions(-) diff --git a/tests/data/generate_zarr_ones.py b/tests/data/generate_zarr_ones.py index 56b90db96..8d61e2013 100644 --- a/tests/data/generate_zarr_ones.py +++ b/tests/data/generate_zarr_ones.py @@ -71,12 +71,24 @@ } for level in range(num_levels) ], - "version": "0.3", + "version": "0.4", } - ] + ], + "omero": { + "channels": [ + { + "wavelength_id": "A01_C01", + "label": "some-label-1", + }, + { + "wavelength_id": "A01_C02", + "label": "some-label-2", + }, + ] + }, } with open(f"{zarrurl}{component}.zattrs", "w") as jsonfile: - json.dump(zattrs, jsonfile) + json.dump(zattrs, jsonfile, indent=4) pixel_size_z = 1.0 @@ -86,8 +98,8 @@ df = pd.DataFrame(np.zeros((2, 10)), dtype=int) df.index = ["FOV1", "FOV2"] df.columns = [ - "x_micrometer", - "y_micrometer", + "x_micrometer_original", + "y_micrometer_original", "z_micrometer", "x_pixel", "y_pixel", diff --git a/tests/data/plate_ones.zarr/B/03/0/.zattrs b/tests/data/plate_ones.zarr/B/03/0/.zattrs index 91891ccd4..9b8f984c6 100644 --- a/tests/data/plate_ones.zarr/B/03/0/.zattrs +++ b/tests/data/plate_ones.zarr/B/03/0/.zattrs @@ -1 +1,107 @@ -{"multiscales": [{"axes": [{"name": "c", "type": "channel"}, {"name": "z", "type": "space", "unit": "micrometer"}, {"name": "y", "type": "space", "unit": "micrometer"}, {"name": "x", "type": "space", "unit": "micrometer"}], "datasets": [{"path": 0, "coordinateTransformations": [{"type": "scale", "scale": [1.0, 0.1625, 0.1625]}]}, {"path": 1, "coordinateTransformations": [{"type": "scale", "scale": [1.0, 0.325, 0.325]}]}, {"path": 2, "coordinateTransformations": [{"type": "scale", "scale": [1.0, 0.65, 0.65]}]}, {"path": 3, "coordinateTransformations": [{"type": "scale", "scale": [1.0, 1.3, 1.3]}]}, {"path": 4, "coordinateTransformations": [{"type": "scale", "scale": [1.0, 2.6, 2.6]}]}], "version": "0.3"}]} +{ + "multiscales": [ + { + "axes": [ + { + "name": "c", + "type": "channel" + }, + { + "name": "z", + "type": "space", + "unit": "micrometer" + }, + { + "name": "y", + "type": "space", + "unit": "micrometer" + }, + { + "name": "x", + "type": "space", + "unit": "micrometer" + } + ], + "datasets": [ + { + "path": 0, + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 1.0, + 0.1625, + 0.1625 + ] + } + ] + }, + { + "path": 1, + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 1.0, + 0.325, + 0.325 + ] + } + ] + }, + { + "path": 2, + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 1.0, + 0.65, + 0.65 + ] + } + ] + }, + { + "path": 3, + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 1.0, + 1.3, + 1.3 + ] + } + ] + }, + { + "path": 4, + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 1.0, + 2.6, + 2.6 + ] + } + ] + } + ], + "version": "0.4" + } + ], + "omero": { + "channels": [ + { + "wavelength_id": "A01_C01", + "label": "some-label-1" + }, + { + "wavelength_id": "A01_C02", + "label": "some-label-2" + } + ] + } +} diff --git a/tests/data/plate_ones.zarr/B/03/0/tables/FOV_ROI_table/X/.zarray b/tests/data/plate_ones.zarr/B/03/0/tables/FOV_ROI_table/X/.zarray index 0bdadac38..2f285c24a 100644 --- a/tests/data/plate_ones.zarr/B/03/0/tables/FOV_ROI_table/X/.zarray +++ b/tests/data/plate_ones.zarr/B/03/0/tables/FOV_ROI_table/X/.zarray @@ -1,7 +1,7 @@ { "chunks": [ 2, - 6 + 8 ], "compressor": { "blocksize": 0, @@ -16,7 +16,7 @@ "order": "C", "shape": [ 2, - 6 + 8 ], "zarr_format": 2 } diff --git a/tests/data/plate_ones.zarr/B/03/0/tables/FOV_ROI_table/X/0.0 b/tests/data/plate_ones.zarr/B/03/0/tables/FOV_ROI_table/X/0.0 index 687066550fa8b480aa08df3c9bf07de251dc6776..406f71d50e1cc2a4fc1b49c07795fe88ca5910c3 100644 GIT binary patch literal 80 lcmZQ#G-h#NU|;~@03b#M7n~Uy);j~m98km|Vn88Gc>o?&2nPTF delta 22 acmWG=;ALVoW-(x3U;trY#K6D+!fik-0>pVh{9lh_qM(gFkSkvaB*d#k7{Wz?;uSFr&*Srp SGSf5j5_3QTm8}dD-~s??F%oV7 literal 128 zcmZQ#G-fPdU|;~@1|VhwVjdu_h|kSTF3QhMElDi`@hj2zRY-gRpnOhhUOciMm>9A? Im>9BN0QB1>CIA2c From d7235e80cc0033a489ec775cf31190f51b892b07 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 17:07:33 +0100 Subject: [PATCH 37/52] Improve docstrings --- fractal_tasks_core/lib_channels.py | 37 ++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 2e46477a5..4c7724521 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -11,7 +11,7 @@ Miescher Institute for Biomedical Research and Pelkmans Lab from the University of Zurich. -Helper functions to address channels +Helper functions to address channels via OME-NGFF/OMERO metadata """ import logging from typing import Any @@ -22,9 +22,14 @@ import zarr -def check_well_channel_labels(*, well_zarr_path: str): +def check_well_channel_labels(*, well_zarr_path: str) -> None: """ - FIXME + Check that the channel labels for a well are unique + + First identify the channel-labels list for each image in the well, then + compare lists and verify their intersection is empty + + :params well_zarr_path: path to an OME-NGFF well zarr group """ # Iterate over all images (multiplexing cycles, multi-FOVs, ...) @@ -37,6 +42,7 @@ def check_well_channel_labels(*, well_zarr_path: str): ) list_of_channel_lists.append(channels[:]) + # For each pair of channel-labels lists, verify they do not overlap for ind_1, channels_1 in enumerate(list_of_channel_lists): labels_1 = set([c["label"] for c in channels_1]) for ind_2 in range(ind_1): @@ -59,14 +65,16 @@ def get_channel_from_image_zarr( *, image_zarr_path: str, label: str = None, wavelength_id: str = None ) -> Dict[str, Any]: """ - Directly extract channel from .zattrs file + Extract a channel from OME-NGFF zarr attributes This is a helper function that combines ``get_omero_channel_list`` with ``get_channel_from_list``. - :param image_zarr_path: TBD - :param label: TBD - :param wavelength_id: TBD + :param image_zarr_path: Path to an OME-NGFF image zarr group + :param label: ``label`` attribute of the channel to be extracted + :param wavelength_id: ``wavelength_id`` attribute of the channel to be + extracted + :returns: A single channel dictionary """ omero_channels = get_omero_channel_list(image_zarr_path=image_zarr_path) channel = get_channel_from_list( @@ -77,9 +85,10 @@ def get_channel_from_image_zarr( def get_omero_channel_list(*, image_zarr_path: str) -> List[Dict[str, Any]]: """ - Extract the list of channels from .zattrs file + Extract the list of channels from OME-NGFF zarr attributes - :param image_zarr_path: TBD + :param image_zarr_path: Path to an OME-NGFF image zarr group + :returns: A list of channel dictionaries """ group = zarr.open_group(image_zarr_path, mode="r+") return group.attrs["omero"]["channels"] @@ -100,8 +109,10 @@ def get_channel_from_list( :param label: The label to look for in the list of channels. :param wavelength_id: The wavelength_id to look for in the list of channels. - + :returns: A single channel dictionary """ + + # Identify matching channels if label: if wavelength_id: matching_channels = [ @@ -124,6 +135,7 @@ def get_channel_from_list( "arguments" ) + # Verify that there is one and only one matching channel if len(matching_channels) > 1: raise ValueError(f"Inconsistent set of channels: {channels}") elif len(matching_channels) == 0: @@ -145,8 +157,9 @@ def define_omero_channels( """ Prepare the .attrs["omero"]["channels"] attribute of an image group - :param channels: TBD - :param bit_depth: TBD + :param channels: A list of channel dictionaries (each one must include the + ``wavelength_id` key). + :param bit_depth: bit depth :returns: omero_channels """ From 5d7e587a60d143c76e56be66c18033fd074d4454 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 17:12:55 +0100 Subject: [PATCH 38/52] Fix docstrings --- fractal_tasks_core/lib_channels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 4c7724521..d271d05b2 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -155,10 +155,10 @@ def define_omero_channels( label_prefix: str = None, ) -> List[Dict[str, Any]]: """ - Prepare the .attrs["omero"]["channels"] attribute of an image group + Prepare the ``attrs["omero"]["channels"]`` attribute of an image group :param channels: A list of channel dictionaries (each one must include the - ``wavelength_id` key). + ``wavelength_id`` key). :param bit_depth: bit depth :returns: omero_channels """ From 143712949da20c8e6839c537b7e0c9847527d593 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Thu, 1 Dec 2022 17:13:02 +0100 Subject: [PATCH 39/52] Update changelog --- docs/source/changelog.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index d75ba9edd..93331abb1 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -6,13 +6,20 @@ Changelog Numbers like (#123) point to `closed Pull Requests on the fractal-tasks-core repository `_. +0.6.0 +----- + +New features +~~~~~~~~~~~~ +* **(major)** Refactor of how to address channels (#239). + 0.5.1 ----- New features ~~~~~~~~~~~~ -* Fix sorting of image files when number of Z planes passes 100 (#237) +* Fix sorting of image files when number of Z planes passes 100 (#237). 0.5.0 From b333785561c804d9f4440035995dce4e3c4e68bf Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 11:36:01 +0100 Subject: [PATCH 40/52] Update lib_channels (mostly docstrings and variable names) --- fractal_tasks_core/lib_channels.py | 73 ++++++++++++++++-------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index d271d05b2..36701d62f 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -155,23 +155,31 @@ def define_omero_channels( label_prefix: str = None, ) -> List[Dict[str, Any]]: """ - Prepare the ``attrs["omero"]["channels"]`` attribute of an image group + Update a channel list to use it in the OMERO/channels metadata + + Given a list of channel dictionaries, update each one of them by: + 1. Adding a label (if missing); + 2. Adding a set of OMERO-specific attributes; + 3. Discarding all other attributes. + + The ``new_channels`` output can be used in the + ``attrs["omero"]["channels"]`` attribute of an image group. :param channels: A list of channel dictionaries (each one must include the ``wavelength_id`` key). :param bit_depth: bit depth - :returns: omero_channels + :returns: ``new_channels``, a new list of consistent channel dictionaries + that can be written to OMERO metadata. + """ - omero_channels = [] + new_channels = [] default_colormaps = ["00FFFF", "FF00FF", "FFFF00"] + for channel in channels: wavelength_id = channel["wavelength_id"] - channel = get_channel_from_list( - channels=channels, wavelength_id=wavelength_id - ) - + # Always set a label try: label = channel["label"] except KeyError: @@ -183,8 +191,8 @@ def define_omero_channels( ) label = default_label - # Set colormap. If missing, use the default ones (for the first three - # channels) or gray + # Set colormap attribute. If not specificed, use the default ones (for + # the first three channels) or gray colormap = channel.get("colormap", None) if colormap is None: try: @@ -192,31 +200,30 @@ def define_omero_channels( except IndexError: colormap = "808080" - omero_channels.append( - { - "label": label, - "wavelength_id": wavelength_id, - "active": True, - "coefficient": 1, - "color": colormap, - "family": "linear", - "inverted": False, - "window": { - "min": 0, - "max": 2**bit_depth - 1, - }, - } - ) - - try: - omero_channels[-1]["window"]["start"] = channel["start"] - omero_channels[-1]["window"]["end"] = channel["end"] - except KeyError: - pass + # Set window attribute + window = { + "min": 0, + "max": 2**bit_depth - 1, + } + if "start" in channel.keys() and "end" in channel.keys(): + window["start"] = channel["start"] + window["end"] = channel["end"] + + new_channel = { + "label": label, + "wavelength_id": wavelength_id, + "active": True, + "coefficient": 1, + "color": colormap, + "family": "linear", + "inverted": False, + "window": window, + } + new_channels.append(new_channel) # Check that channel labels are unique for this image - labels = [c["label"] for c in omero_channels] + labels = [c["label"] for c in new_channels] if len(set(labels)) < len(labels): - raise ValueError(f"Non-unique labels in {omero_channels=}") + raise ValueError(f"Non-unique labels in {new_channels=}") - return omero_channels + return new_channels From 367c3d9f1236944c7e19ac6c8ad2f7c6417a41fc Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 11:36:22 +0100 Subject: [PATCH 41/52] Update wording of a comment --- fractal_tasks_core/create_ome_zarr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index a0c6f3a5f..9acc72f69 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -361,7 +361,7 @@ def create_ome_zarr( write_elem(group_tables, "FOV_ROI_table", FOV_ROIs_table) write_elem(group_tables, "well_ROI_table", well_ROIs_table) - # Check that the different images in the each well have unique labels. + # Check that the different images in each well have unique channel labels. # Since we currently merge all fields of view in the same image, this check # is useless. It should remain there to catch an error in case we switch # back to one-image-per-field-of-view mode From 19b62757b9654117b68420886b6c182c34d6c3e2 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 11:41:01 +0100 Subject: [PATCH 42/52] Add validate_allowed_channel_input function, and use it in all create-zarr tasks --- fractal_tasks_core/create_ome_zarr.py | 4 ++++ fractal_tasks_core/create_ome_zarr_multiplex.py | 15 +++++++++------ fractal_tasks_core/lib_channels.py | 8 ++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/fractal_tasks_core/create_ome_zarr.py b/fractal_tasks_core/create_ome_zarr.py index 9acc72f69..38cadc811 100644 --- a/fractal_tasks_core/create_ome_zarr.py +++ b/fractal_tasks_core/create_ome_zarr.py @@ -28,6 +28,7 @@ import fractal_tasks_core from fractal_tasks_core.lib_channels import check_well_channel_labels from fractal_tasks_core.lib_channels import define_omero_channels +from fractal_tasks_core.lib_channels import validate_allowed_channel_input from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata from fractal_tasks_core.lib_parse_filename_metadata import parse_filename from fractal_tasks_core.lib_regions_of_interest import prepare_FOV_ROI_table @@ -95,6 +96,9 @@ def create_ome_zarr( dict_plate_paths = {} dict_plate_prefixes: Dict[str, Any] = {} + # Preliminary checks on allowed_channels argument + validate_allowed_channel_input(allowed_channels) + # FIXME # find a smart way to remove it ext_glob_pattern = input_paths[0].name diff --git a/fractal_tasks_core/create_ome_zarr_multiplex.py b/fractal_tasks_core/create_ome_zarr_multiplex.py index 13d0e4321..dc0d9dfd8 100644 --- a/fractal_tasks_core/create_ome_zarr_multiplex.py +++ b/fractal_tasks_core/create_ome_zarr_multiplex.py @@ -28,6 +28,7 @@ import fractal_tasks_core from fractal_tasks_core.lib_channels import check_well_channel_labels from fractal_tasks_core.lib_channels import define_omero_channels +from fractal_tasks_core.lib_channels import validate_allowed_channel_input from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata from fractal_tasks_core.lib_parse_filename_metadata import parse_filename from fractal_tasks_core.lib_regions_of_interest import prepare_FOV_ROI_table @@ -86,12 +87,14 @@ def create_ome_zarr_multiplex( 'metadata_table="mrf_mlf", ' f"and not {metadata_table}" ) - else: - # Note that in metadata the keys of dictionary arguments should be - # strings (and not integers), so that they can be read from a JSON file - for key in allowed_channels.keys(): - if not isinstance(key, str): - raise ValueError(f"{allowed_channels=} has non-string keys") + + # Preliminary checks on allowed_channels + # Note that in metadata the keys of dictionary arguments should be + # strings (and not integers), so that they can be read from a JSON file + for key, value in allowed_channels.items(): + if not isinstance(key, str): + raise ValueError(f"{allowed_channels=} has non-string keys") + validate_allowed_channel_input(value) # Identify all plates and all channels, per input folders dict_acquisitions: Dict = {} diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 36701d62f..e104a17b5 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -22,6 +22,14 @@ import zarr +def validate_allowed_channel_input(allowed_channels: Sequence[Dict[str, Any]]): + wavelength_ids = [c["wavelength_id"] for c in allowed_channels] + if len(set(wavelength_ids)) < len(wavelength_ids): + raise ValueError( + f"Non-unique labels in {wavelength_ids}\n" f"{allowed_channels=}" + ) + + def check_well_channel_labels(*, well_zarr_path: str) -> None: """ Check that the channel labels for a well are unique From b5efadcbdde2a10a7502a2a1bdc28b6bba339ef8 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 11:43:31 +0100 Subject: [PATCH 43/52] Remove default labeling_level from cellpose_segmentation task (close #242) --- fractal_tasks_core/cellpose_segmentation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fractal_tasks_core/cellpose_segmentation.py b/fractal_tasks_core/cellpose_segmentation.py index 8e913ee27..2ef76253f 100644 --- a/fractal_tasks_core/cellpose_segmentation.py +++ b/fractal_tasks_core/cellpose_segmentation.py @@ -125,7 +125,7 @@ def cellpose_segmentation( metadata: Dict[str, Any], # Task-specific arguments labeling_channel: str, - labeling_level: int = 1, + labeling_level: int, relabeling: bool = True, anisotropy: Optional[float] = None, diameter_level0: float = 80.0, @@ -476,7 +476,7 @@ class TaskArguments(BaseModel): metadata: Dict[str, Any] # Task-specific arguments labeling_channel: str - labeling_level: int = 1 + labeling_level: int relabeling: bool = True anisotropy: Optional[float] = None diameter_level0: float = 80.0 From 586677638e40856d4b768cbd2fb33097eba7aed3 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 12:44:15 +0100 Subject: [PATCH 44/52] Rename variables in cellpose task, and add also channel_name arg --- fractal_tasks_core/cellpose_segmentation.py | 44 +++++++++++++-------- tests/test_workflows_labeling.py | 20 +++++----- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/fractal_tasks_core/cellpose_segmentation.py b/fractal_tasks_core/cellpose_segmentation.py index 2ef76253f..179d21db8 100644 --- a/fractal_tasks_core/cellpose_segmentation.py +++ b/fractal_tasks_core/cellpose_segmentation.py @@ -124,8 +124,9 @@ def cellpose_segmentation( component: str, metadata: Dict[str, Any], # Task-specific arguments - labeling_channel: str, - labeling_level: int, + wavelength_id: Optional[str] = None, + channel_name: Optional[str] = None, + level: int, relabeling: bool = True, anisotropy: Optional[float] = None, diameter_level0: float = 80.0, @@ -148,8 +149,9 @@ def cellpose_segmentation( :param output_path: TBD (fractal default arg) :param metadata: TBD (fractal default arg) :param component: TBD (fractal default arg) - :param labeling_channel: TBD - :param labeling_level: TBD + :param wavelength_id: TBD + :param channel_name: TBD + :param level: TBD :param relabeling: TBD :param anisotropy: TBD :param diameter_level0: TBD @@ -170,6 +172,15 @@ def cellpose_segmentation( zarrurl = (in_path.resolve() / component).as_posix() + "/" logger.info(zarrurl) + # Preliminary check + if (channel_name is None and wavelength_id is None) or ( + channel_name and wavelength_id + ): + raise ValueError( + f"One and only one of {channel_name=} and " + f"{wavelength_id=} arguments can be provided" + ) + # Read useful parameters from metadata num_levels = metadata["num_levels"] coarsening_xy = metadata["coarsening_xy"] @@ -181,12 +192,12 @@ def cellpose_segmentation( # Find channel index channel = get_channel_from_image_zarr( - image_zarr_path=zarrurl, wavelength_id=labeling_channel + image_zarr_path=zarrurl, wavelength_id=wavelength_id ) ind_channel = channel["index"] # Load ZYX data - data_zyx = da.from_zarr(f"{zarrurl}{labeling_level}")[ind_channel] + data_zyx = da.from_zarr(f"{zarrurl}{level}")[ind_channel] logger.info(f"[{well_id}] {data_zyx.shape=}") # Read ROI table @@ -198,12 +209,12 @@ def cellpose_segmentation( ) actual_res_pxl_sizes_zyx = extract_zyx_pixel_sizes( - f"{zarrurl}.zattrs", level=labeling_level + f"{zarrurl}.zattrs", level=level ) # Create list of indices for 3D FOVs spanning the entire Z direction list_indices = convert_ROI_table_to_indices( ROI_table, - level=labeling_level, + level=level, coarsening_xy=coarsening_xy, full_res_pxl_sizes_zyx=full_res_pxl_sizes_zyx, ) @@ -235,9 +246,7 @@ def cellpose_segmentation( if do_3D: if anisotropy is None: # Read pixel sizes from zattrs file - pxl_zyx = extract_zyx_pixel_sizes( - zarrurl + ".zattrs", level=labeling_level - ) + pxl_zyx = extract_zyx_pixel_sizes(zarrurl + ".zattrs", level=level) pixel_size_z, pixel_size_y, pixel_size_x = pxl_zyx[:] logger.info(f"[{well_id}] {pxl_zyx=}") if not np.allclose(pixel_size_x, pixel_size_y): @@ -282,11 +291,11 @@ def cellpose_segmentation( except (KeyError, IndexError): label_name = f"label_{ind_channel}" - # Rescale datasets (only relevant for labeling_level>0) + # Rescale datasets (only relevant for level>0) new_datasets = rescale_datasets( datasets=multiscales[0]["datasets"], coarsening_xy=coarsening_xy, - reference_level=labeling_level, + reference_level=level, ) # Write zattrs for labels and for specific label @@ -341,7 +350,7 @@ def cellpose_segmentation( logger.info(f"[{well_id}] relabeling: {relabeling}") logger.info(f"[{well_id}] do_3D: {do_3D}") logger.info(f"[{well_id}] use_gpu: {gpu}") - logger.info(f"[{well_id}] labeling_level: {labeling_level}") + logger.info(f"[{well_id}] level: {level}") logger.info(f"[{well_id}] model_type: {model_type}") logger.info(f"[{well_id}] pretrained_model: {pretrained_model}") logger.info(f"[{well_id}] anisotropy: {anisotropy}") @@ -376,7 +385,7 @@ def cellpose_segmentation( do_3D=do_3D, anisotropy=anisotropy, label_dtype=label_dtype, - diameter=diameter_level0 / coarsening_xy**labeling_level, + diameter=diameter_level0 / coarsening_xy**level, cellprob_threshold=cellprob_threshold, flow_threshold=flow_threshold, well_id=well_id, @@ -475,8 +484,9 @@ class TaskArguments(BaseModel): component: str metadata: Dict[str, Any] # Task-specific arguments - labeling_channel: str - labeling_level: int + channel_name: Optional[str] = None + wavelength_id: Optional[str] = None + level: int relabeling: bool = True anisotropy: Optional[float] = None diameter_level0: float = 80.0 diff --git a/tests/test_workflows_labeling.py b/tests/test_workflows_labeling.py index d177ae617..0ec6f1d66 100644 --- a/tests/test_workflows_labeling.py +++ b/tests/test_workflows_labeling.py @@ -187,8 +187,8 @@ def test_workflow_with_per_FOV_labeling( output_path=zarr_path, metadata=metadata, component=component, - labeling_channel="A01_C01", - labeling_level=3, + wavelength_id="A01_C01", + level=3, relabeling=True, diameter_level0=80.0, ) @@ -238,8 +238,8 @@ def test_workflow_with_per_FOV_labeling_2D( output_path=zarr_path_mip, metadata=metadata, component=component, - labeling_channel="A01_C01", - labeling_level=2, + wavelength_id="A01_C01", + level=2, relabeling=True, diameter_level0=80.0, ) @@ -328,8 +328,8 @@ def test_workflow_with_per_well_labeling_2D( output_path=zarr_path_mip, metadata=metadata, component=component, - labeling_channel="A01_C01", - labeling_level=2, + wavelength_id="A01_C01", + level=2, ROI_table_name="well_ROI_table", relabeling=True, diameter_level0=80.0, @@ -382,8 +382,8 @@ def test_workflow_bounding_box( output_path=zarr_path, metadata=metadata, component=component, - labeling_channel="A01_C01", - labeling_level=3, + wavelength_id="A01_C01", + level=3, relabeling=True, diameter_level0=80.0, bounding_box_ROI_table_name="bbox_table", @@ -433,8 +433,8 @@ def test_workflow_bounding_box_with_overlap( output_path=zarr_path, metadata=metadata, component=component, - labeling_channel="A01_C01", - labeling_level=3, + wavelength_id="A01_C01", + level=3, relabeling=True, diameter_level0=80.0, bounding_box_ROI_table_name="bbox_table", From fa77add96d10fa1073e0d7ca3395d265a0951938 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:08:59 +0100 Subject: [PATCH 45/52] Add ChannelNotFoundError and capture it in cellpose_segmentation task --- fractal_tasks_core/cellpose_segmentation.py | 28 ++++++---- fractal_tasks_core/lib_channels.py | 20 +++++-- tests/test_workflows_labeling.py | 60 +++++++++++++++++++++ 3 files changed, 95 insertions(+), 13 deletions(-) diff --git a/fractal_tasks_core/cellpose_segmentation.py b/fractal_tasks_core/cellpose_segmentation.py index 179d21db8..e8abcea29 100644 --- a/fractal_tasks_core/cellpose_segmentation.py +++ b/fractal_tasks_core/cellpose_segmentation.py @@ -36,6 +36,7 @@ from cellpose.core import use_gpu import fractal_tasks_core +from fractal_tasks_core.lib_channels import ChannelNotFoundError from fractal_tasks_core.lib_channels import get_channel_from_image_zarr from fractal_tasks_core.lib_pyramid_creation import build_pyramid from fractal_tasks_core.lib_regions_of_interest import ( @@ -125,7 +126,7 @@ def cellpose_segmentation( metadata: Dict[str, Any], # Task-specific arguments wavelength_id: Optional[str] = None, - channel_name: Optional[str] = None, + channel_label: Optional[str] = None, level: int, relabeling: bool = True, anisotropy: Optional[float] = None, @@ -150,7 +151,7 @@ def cellpose_segmentation( :param metadata: TBD (fractal default arg) :param component: TBD (fractal default arg) :param wavelength_id: TBD - :param channel_name: TBD + :param channel_label: TBD :param level: TBD :param relabeling: TBD :param anisotropy: TBD @@ -173,11 +174,11 @@ def cellpose_segmentation( logger.info(zarrurl) # Preliminary check - if (channel_name is None and wavelength_id is None) or ( - channel_name and wavelength_id + if (channel_label is None and wavelength_id is None) or ( + channel_label and wavelength_id ): raise ValueError( - f"One and only one of {channel_name=} and " + f"One and only one of {channel_label=} and " f"{wavelength_id=} arguments can be provided" ) @@ -191,9 +192,18 @@ def cellpose_segmentation( well_id = well.replace("/", "_")[:-1] # Find channel index - channel = get_channel_from_image_zarr( - image_zarr_path=zarrurl, wavelength_id=wavelength_id - ) + try: + channel = get_channel_from_image_zarr( + image_zarr_path=zarrurl, + wavelength_id=wavelength_id, + label=channel_label, + ) + except ChannelNotFoundError as e: + logger.warning( + "Channel not found, exit from the task.\n" + f"Original error: {str(e)}" + ) + return {} ind_channel = channel["index"] # Load ZYX data @@ -484,7 +494,7 @@ class TaskArguments(BaseModel): component: str metadata: Dict[str, Any] # Task-specific arguments - channel_name: Optional[str] = None + channel_label: Optional[str] = None wavelength_id: Optional[str] = None level: int relabeling: bool = True diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index e104a17b5..5a65e625b 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -22,6 +22,10 @@ import zarr +class ChannelNotFoundError(ValueError): + pass + + def validate_allowed_channel_input(allowed_channels: Sequence[Dict[str, Any]]): wavelength_ids = [c["wavelength_id"] for c in allowed_channels] if len(set(wavelength_ids)) < len(wavelength_ids): @@ -144,12 +148,20 @@ def get_channel_from_list( ) # Verify that there is one and only one matching channel + if len(matching_channels) == 0: + required_match = [f"{label=}", f"{wavelength_id=}"] + required_match_string = " and ".join( + [x for x in required_match if "None" not in x] + ) + raise ChannelNotFoundError( + f"ChannelNotFoundError: No channel found in {channels}" + f" for {required_match_string}" + ) + from devtools import debug + + debug("RAISE") if len(matching_channels) > 1: raise ValueError(f"Inconsistent set of channels: {channels}") - elif len(matching_channels) == 0: - raise ValueError( - f"No channel found in {channels} for {label=} and {wavelength_id=}" - ) channel = matching_channels[0] channel["index"] = channels.index(channel) diff --git a/tests/test_workflows_labeling.py b/tests/test_workflows_labeling.py index 0ec6f1d66..df5c33451 100644 --- a/tests/test_workflows_labeling.py +++ b/tests/test_workflows_labeling.py @@ -152,6 +152,66 @@ def patched_use_gpu(*args, **kwargs): return False +def test_failures( + tmp_path: Path, + testdata_path: Path, + zenodo_zarr: List[Path], + zenodo_zarr_metadata: List[Dict[str, Any]], + caplog: pytest.LogCaptureFixture, + monkeypatch: MonkeyPatch, +): + + monkeypatch.setattr( + "fractal_tasks_core.cellpose_segmentation.use_gpu", patched_use_gpu + ) + + monkeypatch.setattr( + "fractal_tasks_core.cellpose_segmentation.segment_FOV", + patched_segment_FOV, + ) + + caplog.set_level(logging.WARNING) + + # Use pre-made 3D zarr + zarr_path = tmp_path / "tmp_out/*.zarr" + metadata = prepare_3D_zarr(zarr_path, zenodo_zarr, zenodo_zarr_metadata) + debug(zarr_path) + debug(metadata) + + # A sequence of invalid attempts + for component in metadata["image"]: + + kwargs = dict( + input_paths=[zarr_path], + output_path=zarr_path, + metadata=metadata, + component=component, + level=3, + ) + # Attempt 1 + cellpose_segmentation( + **kwargs, + wavelength_id="invalid_wavelength_id", + ) + assert "ChannelNotFoundError" in caplog.records[0].msg + + # Attempt 2 + cellpose_segmentation( + **kwargs, + channel_label="invalid_channel_name", + ) + assert "ChannelNotFoundError" in caplog.records[0].msg + assert "ChannelNotFoundError" in caplog.records[1].msg + + # Attempt 3 + with pytest.raises(ValueError): + cellpose_segmentation( + **kwargs, + wavelength_id="A01_C01", + channel_label="invalid_channel_name", + ) + + def test_workflow_with_per_FOV_labeling( tmp_path: Path, testdata_path: Path, From adfc5f2a04c6f6ce959497db9354aa72eb10b090 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 15:04:05 +0100 Subject: [PATCH 46/52] Cleanup cellpose task (docstrings and some variable names) --- fractal_tasks_core/cellpose_segmentation.py | 113 ++++++++++++-------- 1 file changed, 67 insertions(+), 46 deletions(-) diff --git a/fractal_tasks_core/cellpose_segmentation.py b/fractal_tasks_core/cellpose_segmentation.py index e8abcea29..56d268c95 100644 --- a/fractal_tasks_core/cellpose_segmentation.py +++ b/fractal_tasks_core/cellpose_segmentation.py @@ -13,7 +13,7 @@ Miescher Institute for Biomedical Research and Pelkmans Lab from the University of Zurich. -Image segmentation via cellpose library +Image segmentation via Cellpose library """ import json import logging @@ -58,21 +58,28 @@ def segment_FOV( - column, + column: np.ndarray, model=None, - do_3D=True, + do_3D: bool = True, anisotropy=None, - diameter=40.0, - cellprob_threshold=0.0, - flow_threshold=0.4, + diameter: float = 40.0, + cellprob_threshold: float = 0.0, + flow_threshold: float = 0.4, label_dtype=None, - well_id=None, + well_id: str = None, ): """ - Description + Internal function that runs Cellpose segmentation for a single ROI. - :param dummy: this is a placeholder - :param dummy: int + :param column: Three-dimensional numpy array + :param model: TBD + :param do_3D: TBD + :param anisotropy: TBD + :param diameter: TBD + :param cellprob_threshold: TBD + :param flow_threshold: TBD + :param label_dtype: TBD + :param well_id: TBD """ # Write some debugging info @@ -125,9 +132,9 @@ def cellpose_segmentation( component: str, metadata: Dict[str, Any], # Task-specific arguments + level: int, wavelength_id: Optional[str] = None, channel_label: Optional[str] = None, - level: int, relabeling: bool = True, anisotropy: Optional[float] = None, diameter_level0: float = 80.0, @@ -135,35 +142,49 @@ def cellpose_segmentation( flow_threshold: float = 0.4, ROI_table_name: str = "FOV_ROI_table", bounding_box_ROI_table_name: Optional[str] = None, - label_name: Optional[str] = None, + output_label_name: Optional[str] = None, model_type: Literal["nuclei", "cyto", "cyto2"] = "nuclei", pretrained_model: Optional[str] = None, ) -> Dict[str, Any]: """ - Example inputs: - input_paths: PosixPath('tmp_out_mip/*.zarr') - output_path: PosixPath('tmp_out_mip/*.zarr') - component: myplate.zarr/B/03/0/ - metadata: {...} - - :param input_paths: TBD (fractal default arg) - :param output_path: TBD (fractal default arg) - :param metadata: TBD (fractal default arg) - :param component: TBD (fractal default arg) - :param wavelength_id: TBD - :param channel_label: TBD - :param level: TBD - :param relabeling: TBD - :param anisotropy: TBD - :param diameter_level0: TBD - :param cellprob_threshold: TBD - :param flow_threshold: TBD + Run cellpose segmentation on a single OME-NGFF image + + Full documentation for all arguments is still TBD, especially because some + of them are standard arguments for Fractal tasks that should be documented + in a standard way. Here are some examples of valid arguments:: + + input_paths = ["/some/path/*.zarr"] + output_path = "/some/path/*.zarr" + component = "some_plate.zarr/B/03/0" + metadata = {"num_levels": 4, "coarsening_xy": 2} + + :param input_paths: TBD (default arg for Fractal tasks) + :param output_path: TBD (default arg for Fractal tasks) + :param metadata: TBD (default arg for Fractal tasks) + :param component: TBD (default arg for Fractal tasks) + :param level: Pyramid level of the image to be segmented. + :param wavelength_id: Identifier of a channel based on the + wavelength (e.g. ``A01_C01``). If not ``None``, then + ``channel_label` must be ``None``. + :param channel_label: Identifier of a channel based on its label (e.g. + ``DAPI``). If not ``None``, then ``wavelength_id`` + must be ``None``. + :param relabeling: If ``True``, apply relabeling so that label values are + unique across ROIs. + :param anisotropy: Ratio of the pixel sizes along Z and XY axis (ignored if + the image is not three-dimensional). If `None`, it is + inferred from the OME-NGFF metadata. + :param diameter_level0: Initial diameter to be passed to + ``CellposeModel.eval`` method (after rescaling from + full-resolution to ``level``). :param ROI_table_name: TBD :param bounding_box_ROI_table_name: TBD - :param label_name: TBD - :param model_type: TBD - :param pretrained_model: TBD. If not ``None``, this takes precedence - over ``model_type``. + :param output_label_name: TBD + :param cellprob_threshold: Parameter of ``CellposeModel.eval`` method. + :param flow_threshold: Parameter of ``CellposeModel.eval`` method. + :param model_type: Parameter of ``CellposeModel`` class. + :param pretrained_model: Parameter of ``CellposeModel`` class (takes + precedence over ``model_type``). """ # Set input path @@ -293,13 +314,13 @@ def cellpose_segmentation( "level are not currently supported" ) - # Set channel label - if label_name is None: + # Set channel label - FIXME: adapt to new channels structure + if output_label_name is None: try: omero_label = zattrs["omero"]["channels"][ind_channel]["label"] - label_name = f"label_{omero_label}" + output_label_name = f"label_{omero_label}" except (KeyError, IndexError): - label_name = f"label_{ind_channel}" + output_label_name = f"label_{ind_channel}" # Rescale datasets (only relevant for level>0) new_datasets = rescale_datasets( @@ -311,12 +332,12 @@ def cellpose_segmentation( # Write zattrs for labels and for specific label # FIXME deal with: (1) many channels, (2) overwriting labels_group = zarr.group(f"{zarrurl}labels") - labels_group.attrs["labels"] = [label_name] - label_group = labels_group.create_group(label_name) + labels_group.attrs["labels"] = [output_label_name] + label_group = labels_group.create_group(output_label_name) label_group.attrs["image-label"] = {"version": __OME_NGFF_VERSION__} label_group.attrs["multiscales"] = [ { - "name": label_name, + "name": output_label_name, "version": __OME_NGFF_VERSION__, "axes": [ ax for ax in multiscales[0]["axes"] if ax["type"] != "channel" @@ -326,10 +347,10 @@ def cellpose_segmentation( ] # Open new zarr group for mask 0-th level - logger.info(f"[{well_id}] {zarrurl}labels/{label_name}/0") + logger.info(f"[{well_id}] {zarrurl}labels/{output_label_name}/0") zarr.group(f"{zarrurl}/labels") - zarr.group(f"{zarrurl}/labels/{label_name}") - store = da.core.get_mapper(f"{zarrurl}labels/{label_name}/0") + zarr.group(f"{zarrurl}/labels/{output_label_name}") + store = da.core.get_mapper(f"{zarrurl}labels/{output_label_name}/0") label_dtype = np.uint32 mask_zarr = zarr.create( shape=data_zyx.shape, @@ -452,7 +473,7 @@ def cellpose_segmentation( # Starting from on-disk highest-resolution data, build and write to disk a # pyramid of coarser levels build_pyramid( - zarrurl=f"{zarrurl}labels/{label_name}", + zarrurl=f"{zarrurl}labels/{output_label_name}", overwrite=False, num_levels=num_levels, coarsening_xy=coarsening_xy, @@ -504,7 +525,7 @@ class TaskArguments(BaseModel): flow_threshold: float = 0.4 ROI_table_name: str = "FOV_ROI_table" bounding_box_ROI_table_name: Optional[str] = None - label_name: Optional[str] = None + output_label_name: Optional[str] = None model_type: Literal["nuclei", "cyto", "cyto2"] = "nuclei" pretrained_model: Optional[str] = None From 637c5fce4217eb1b8117206532b3e309ef7080cc Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 15:15:22 +0100 Subject: [PATCH 47/52] Fix docstring --- fractal_tasks_core/lib_channels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index 5a65e625b..fe32014f3 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -41,7 +41,7 @@ def check_well_channel_labels(*, well_zarr_path: str) -> None: First identify the channel-labels list for each image in the well, then compare lists and verify their intersection is empty - :params well_zarr_path: path to an OME-NGFF well zarr group + :param well_zarr_path: path to an OME-NGFF well zarr group """ # Iterate over all images (multiplexing cycles, multi-FOVs, ...) From 9942bf2672ff2a862c5e206c12197f3a71554b8f Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 15:15:53 +0100 Subject: [PATCH 48/52] Fix docstring --- fractal_tasks_core/cellpose_segmentation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fractal_tasks_core/cellpose_segmentation.py b/fractal_tasks_core/cellpose_segmentation.py index 56d268c95..5010c18a7 100644 --- a/fractal_tasks_core/cellpose_segmentation.py +++ b/fractal_tasks_core/cellpose_segmentation.py @@ -147,7 +147,7 @@ def cellpose_segmentation( pretrained_model: Optional[str] = None, ) -> Dict[str, Any]: """ - Run cellpose segmentation on a single OME-NGFF image + Run cellpose segmentation on the ROIs of a single OME-NGFF image Full documentation for all arguments is still TBD, especially because some of them are standard arguments for Fractal tasks that should be documented @@ -177,7 +177,8 @@ def cellpose_segmentation( :param diameter_level0: Initial diameter to be passed to ``CellposeModel.eval`` method (after rescaling from full-resolution to ``level``). - :param ROI_table_name: TBD + :param ROI_table_name: name of the table that contains ROIs to which the + task applies Cellpose segmentation :param bounding_box_ROI_table_name: TBD :param output_label_name: TBD :param cellprob_threshold: Parameter of ``CellposeModel.eval`` method. From 79780b831bf73268568ad33ad7479caad5eab08f Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 15:25:01 +0100 Subject: [PATCH 49/52] Rename channel_name to wavelength_id in napari-workflows wrapper, and improve docstring --- .../napari_workflows_wrapper.py | 54 +++++++++++++------ tests/test_unit_napari_workflows_wrapper.py | 8 +-- tests/test_workflows_napari_workflows.py | 14 ++--- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/fractal_tasks_core/napari_workflows_wrapper.py b/fractal_tasks_core/napari_workflows_wrapper.py index b909cd685..498d3afa9 100644 --- a/fractal_tasks_core/napari_workflows_wrapper.py +++ b/fractal_tasks_core/napari_workflows_wrapper.py @@ -69,23 +69,43 @@ def napari_workflows_wrapper( expected_dimensions: int = 3, ): """ - Description - - Example of some arguments:: - asd - - :param input_paths: TBD (fractal arg) - :param output_path: TBD (fractal arg) - :param component: TBD (fractal arg) - :param metadata: TBD (fractal arg) - :param workflow_file: absolute path to napari-workflows YAML file - :param input_specs: TBD - :param output_specs: TBD + Run a napari-workflow on the ROIs of a single OME-NGFF image + + Full documentation for all arguments is still TBD, especially because some + of them are standard arguments for Fractal tasks that should be documented + in a standard way. Here are some examples:: + + input_paths = ["/some/path/*.zarr"] + output_path = "/some/path/*.zarr" + component = "some_plate.zarr/B/03/0" + metadata = {"num_levels": 4, "coarsening_xy": 2} + + # Examples of allowed entries for input_specs and output_specs + input_specs = { + "in_1": {"type": "image", "wavelength_id": "A01_C02"}, + "in_2": {"type": "image", "channel_label": "DAPI"}, + "in_3": {"type": "label", "label_name": "label_DAPI"}, + } + output_specs = { + "out_1": {"type": "label", "label_name": "label_DAPI_new"}, + "out_2": {"type": "dataframe", "table_name": "measurements"}, + } + + :param input_paths: TBD (default arg for Fractal tasks) + :param output_path: TBD (default arg for Fractal tasks) + :param metadata: TBD (default arg for Fractal tasks) + :param component: TBD (default arg for Fractal tasks) + :param workflow_file: Absolute path to napari-workflows YAML file + :param input_specs: See examples above. + :param output_specs: See examples above. + + :param level: Pyramid level of the image to be segmented. + :param expected_dimensions: Expected dimensions (either 2 or 3). + + :param relabeling: If ``True``, apply relabeling so that label values are + unique across ROIs. :param ROI_table_name: name of the table that contains ROIs to which the\ task applies the napari-worfklow - :param level: TBD - :param relabeling: TBD - :param expected_dimensions: TBD """ wf: napari_workflows.Worfklow = load_workflow(workflow_file) @@ -185,10 +205,10 @@ def napari_workflows_wrapper( img_array = da.from_zarr(f"{in_path}/{component}/{level}") # Loop over image inputs and assign corresponding channel of the image for (name, params) in image_inputs: - channel_name = params["channel"] + wavelength_id = params["channel"] channel = get_channel_from_image_zarr( image_zarr_path=f"{in_path}/{component}", - wavelength_id=channel_name, + wavelength_id=wavelength_id, ) channel_index = channel["index"] input_image_arrays[name] = img_array[channel_index] diff --git a/tests/test_unit_napari_workflows_wrapper.py b/tests/test_unit_napari_workflows_wrapper.py index faad3e09c..3a71c87a5 100644 --- a/tests/test_unit_napari_workflows_wrapper.py +++ b/tests/test_unit_napari_workflows_wrapper.py @@ -45,7 +45,9 @@ def test_output_specs(tmp_path, testdata_path, caplog): workflow_file = str( testdata_path / "napari_workflows/wf_5-labeling_only.yaml" ) - input_specs = {"input_image": {"type": "image", "channel": "A01_C01"}} + input_specs = { + "input_image": {"type": "image", "wavelength_id": "A01_C01"} + } output_specs = {"asd": "asd"} try: @@ -79,8 +81,8 @@ def test_level_setting_in_non_labeling_worfklow(tmp_path, testdata_path): # napari-workflows workflow_file = str(testdata_path / "napari_workflows/wf_3.yaml") input_specs = { - "slice_img": {"type": "image", "channel": "A01_C01"}, - "slice_img_c2": {"type": "image", "channel": "A01_C01"}, + "slice_img": {"type": "image", "wavelength_id": "A01_C01"}, + "slice_img_c2": {"type": "image", "wavelength_id": "A01_C01"}, } output_specs = { "Result of Expand labels (scikit-image, nsbatwm)": { diff --git a/tests/test_workflows_napari_workflows.py b/tests/test_workflows_napari_workflows.py index a20c01e39..ccacd9f68 100644 --- a/tests/test_workflows_napari_workflows.py +++ b/tests/test_workflows_napari_workflows.py @@ -81,7 +81,7 @@ def test_napari_worfklow( # First napari-workflows task (labeling) workflow_file = str(testdata_path / "napari_workflows/wf_1.yaml") input_specs: Dict[str, Dict[str, Union[str, int]]] = { - "input": {"type": "image", "channel": "A01_C01"}, + "input": {"type": "image", "wavelength_id": "A01_C01"}, } output_specs: Dict[str, Dict[str, Union[str, int]]] = { "Result of Expand labels (scikit-image, nsbatwm)": { @@ -106,7 +106,7 @@ def test_napari_worfklow( # Second napari-workflows task (measurement) workflow_file = str(testdata_path / "napari_workflows/wf_4.yaml") input_specs = { - "dapi_img": {"type": "image", "channel": "A01_C01"}, + "dapi_img": {"type": "image", "wavelength_id": "A01_C01"}, "dapi_label_img": {"type": "label", "label_name": "label_DAPI"}, } output_specs = { @@ -169,7 +169,7 @@ def test_napari_worfklow_label_input_only( # First napari-workflows task (labeling) workflow_file = str(testdata_path / "napari_workflows/wf_1.yaml") input_specs: Dict[str, Dict[str, Union[str, int]]] = { - "input": {"type": "image", "channel": "A01_C01"}, + "input": {"type": "image", "wavelength_id": "A01_C01"}, } output_specs: Dict[str, Dict[str, Union[str, int]]] = { "Result of Expand labels (scikit-image, nsbatwm)": { @@ -235,13 +235,13 @@ def test_napari_worfklow_label_input_only( TABLE_NAME = "measurement_DAPI" # 1. Labeling-only workflow, from images to labels. workflow_file_name = "wf_relab_1-labeling_only.yaml" -input_specs = dict(input_image={"type": "image", "channel": "A01_C01"}) +input_specs = dict(input_image={"type": "image", "wavelength_id": "A01_C01"}) output_specs = dict(output_label={"type": "label", "label_name": LABEL_NAME}) RELABELING_CASE_1: List = [workflow_file_name, input_specs, output_specs] # 2. Measurement-only workflow, from images+labels to dataframes. workflow_file_name = "wf_relab_2-measurement_only.yaml" input_specs = dict( - input_image={"type": "image", "channel": "A01_C01"}, + input_image={"type": "image", "wavelength_id": "A01_C01"}, input_label={"type": "label", "label_name": LABEL_NAME}, ) output_specs = dict( @@ -250,7 +250,7 @@ def test_napari_worfklow_label_input_only( RELABELING_CASE_2: List = [workflow_file_name, input_specs, output_specs] # 3. Mixed labeling/measurement workflow. workflow_file_name = "wf_relab_3-labeling_and_measurement.yaml" -input_specs = dict(input_image={"type": "image", "channel": "A01_C01"}) +input_specs = dict(input_image={"type": "image", "wavelength_id": "A01_C01"}) output_specs = dict( output_label={"type": "label", "label_name": LABEL_NAME}, output_dataframe={"type": "dataframe", "table_name": TABLE_NAME}, @@ -419,7 +419,7 @@ def test_expected_dimensions( testdata_path / "napari_workflows/wf_5-labeling_only.yaml" ) input_specs: Dict[str, Dict[str, Union[str, int]]] = { - "input_image": {"type": "image", "channel": "A01_C01"}, + "input_image": {"type": "image", "wavelength_id": "A01_C01"}, } output_specs: Dict[str, Dict[str, Union[str, int]]] = { "output_label": { From 3b9e5e7e787ebcfb20c72aae4011ffba84da2d91 Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 15:30:41 +0100 Subject: [PATCH 50/52] can -> must in docstring --- fractal_tasks_core/cellpose_segmentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fractal_tasks_core/cellpose_segmentation.py b/fractal_tasks_core/cellpose_segmentation.py index 5010c18a7..a28bc7236 100644 --- a/fractal_tasks_core/cellpose_segmentation.py +++ b/fractal_tasks_core/cellpose_segmentation.py @@ -201,7 +201,7 @@ def cellpose_segmentation( ): raise ValueError( f"One and only one of {channel_label=} and " - f"{wavelength_id=} arguments can be provided" + f"{wavelength_id=} arguments must be provided" ) # Read useful parameters from metadata From d01b06c9ff8fddbe38949e20d95c145aa300c2cf Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 15:31:22 +0100 Subject: [PATCH 51/52] Support both wavelength_id and label specifications for channels, in napari-workflows wrapper --- fractal_tasks_core/napari_workflows_wrapper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fractal_tasks_core/napari_workflows_wrapper.py b/fractal_tasks_core/napari_workflows_wrapper.py index 498d3afa9..300c62af3 100644 --- a/fractal_tasks_core/napari_workflows_wrapper.py +++ b/fractal_tasks_core/napari_workflows_wrapper.py @@ -205,10 +205,16 @@ def napari_workflows_wrapper( img_array = da.from_zarr(f"{in_path}/{component}/{level}") # Loop over image inputs and assign corresponding channel of the image for (name, params) in image_inputs: - wavelength_id = params["channel"] + if "wavelength_id" in params and "channel_label" in params: + raise ValueError( + "One and only one among channel_label and wavelength_id" + f" attributes must be provided, but input {name} in " + f"input_specs has {params=}." + ) channel = get_channel_from_image_zarr( image_zarr_path=f"{in_path}/{component}", - wavelength_id=wavelength_id, + wavelength_id=params.get("wavelength_id", None), + label=params.get("channel_label", None), ) channel_index = channel["index"] input_image_arrays[name] = img_array[channel_index] From 56da93a4d5d783603a9ffa50e36c3fabd4884f9e Mon Sep 17 00:00:00 2001 From: Tommaso Comparin <3862206+tcompa@users.noreply.github.com> Date: Fri, 2 Dec 2022 15:32:56 +0100 Subject: [PATCH 52/52] Add custom exception docstrings --- fractal_tasks_core/lib_channels.py | 5 +++++ fractal_tasks_core/napari_workflows_wrapper.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/fractal_tasks_core/lib_channels.py b/fractal_tasks_core/lib_channels.py index fe32014f3..36d1c5f8d 100644 --- a/fractal_tasks_core/lib_channels.py +++ b/fractal_tasks_core/lib_channels.py @@ -23,6 +23,11 @@ class ChannelNotFoundError(ValueError): + """ + Custom error for when ``get_channel_from_list`` fails, that can be captured + and handled upstream if needed. + """ + pass diff --git a/fractal_tasks_core/napari_workflows_wrapper.py b/fractal_tasks_core/napari_workflows_wrapper.py index 300c62af3..2fde13965 100644 --- a/fractal_tasks_core/napari_workflows_wrapper.py +++ b/fractal_tasks_core/napari_workflows_wrapper.py @@ -49,6 +49,10 @@ class OutOfTaskScopeError(NotImplementedError): + """ + Encapsulates features that are out-of-scope for the current wrapper task + """ + pass