Skip to content

Commit

Permalink
Revamp argument validation (ref #377)
Browse files Browse the repository at this point in the history 
* Use PydanticV1 `validate_arguments` instead of creating
  `TaskArguments` models by hand.
* Update tests.
* Update `args_schema` script.
  • Loading branch information
@tcompa
tcompa committed May 30, 2023
1 parent 5ba0782 commit 87a38b5
Show file tree
Hide file tree
Showing 13 changed files with 216 additions and 205 deletions.
55 changes: 35 additions & 20 deletions fractal_tasks_core/__FRACTAL_MANIFEST__.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
},
"coarsening_xy": {
"default": 2,
"description": "TBD",
"title": "Coarsening Xy",
"type": "integer"
},
Expand All @@ -27,15 +26,13 @@
"type": "string"
},
"image_glob_patterns": {
"description": "TBD",
"items": {
"type": "string"
},
"title": "Image Glob Patterns",
"type": "array"
},
"input_paths": {
"description": "TBD",
"items": {
"type": "string"
},
Expand All @@ -48,18 +45,15 @@
},
"metadata_table": {
"default": "mrf_mlf",
"description": "TBD",
"title": "Metadata Table",
"type": "string"
},
"num_levels": {
"default": 2,
"description": "TBD",
"title": "Num Levels",
"type": "integer"
},
"output_path": {
"description": "TBD",
"title": "Output Path",
"type": "string"
}
Expand All @@ -70,7 +64,7 @@
"metadata",
"allowed_channels"
],
"title": "TaskArguments",
"title": "CreateOmeZarr",
"type": "object"
},
"default_args": {
Expand All @@ -97,6 +91,7 @@
"type": "string"
},
"delete_input": {
"default": false,
"title": "Delete Input",
"type": "boolean"
},
Expand All @@ -119,10 +114,10 @@
"required": [
"input_paths",
"output_path",
"metadata",
"component"
"component",
"metadata"
],
"title": "TaskArguments",
"title": "YokogawaToOmeZarr",
"type": "object"
},
"executable": "yokogawa_to_ome_zarr.py",
Expand Down Expand Up @@ -162,6 +157,7 @@
"type": "string"
},
"project_to_2D": {
"default": true,
"title": "Project To 2D",
"type": "boolean"
},
Expand All @@ -175,7 +171,7 @@
"output_path",
"metadata"
],
"title": "TaskArguments",
"title": "CopyOmeZarr",
"type": "object"
},
"default_args": {
Expand Down Expand Up @@ -218,10 +214,10 @@
"required": [
"input_paths",
"output_path",
"metadata",
"component"
"component",
"metadata"
],
"title": "TaskArguments",
"title": "MaximumIntensityProjection",
"type": "object"
},
"executable": "maximum_intensity_projection.py",
Expand All @@ -243,10 +239,12 @@
"type": "number"
},
"augment": {
"default": false,
"title": "Augment",
"type": "boolean"
},
"cellprob_threshold": {
"default": 0.0,
"title": "Cellprob Threshold",
"type": "number"
},
Expand All @@ -263,14 +261,17 @@
"type": "string"
},
"diameter_level0": {
"default": 30.0,
"title": "Diameter Level0",
"type": "number"
},
"flow_threshold": {
"default": 0.4,
"title": "Flow Threshold",
"type": "number"
},
"input_ROI_table": {
"default": "FOV_ROI_table",
"title": "Input Roi Table",
"type": "string"
},
Expand All @@ -290,14 +291,17 @@
"type": "object"
},
"min_size": {
"default": 15,
"title": "Min Size",
"type": "integer"
},
"model_type": {
"default": "cyto2",
"title": "Model Type",
"type": "string"
},
"net_avg": {
"default": false,
"title": "Net Avg",
"type": "boolean"
},
Expand All @@ -323,10 +327,12 @@
"type": "boolean"
},
"use_gpu": {
"default": true,
"title": "Use Gpu",
"type": "boolean"
},
"use_masks": {
"default": true,
"title": "Use Masks",
"type": "boolean"
},
Expand All @@ -346,7 +352,7 @@
"metadata",
"level"
],
"title": "TaskArguments",
"title": "CellposeSegmentation",
"type": "object"
},
"executable": "cellpose_segmentation.py",
Expand All @@ -365,6 +371,7 @@
"additionalProperties": false,
"properties": {
"background": {
"default": 100,
"title": "Background",
"type": "integer"
},
Expand Down Expand Up @@ -396,6 +403,7 @@
"type": "string"
},
"overwrite": {
"default": false,
"title": "Overwrite",
"type": "boolean"
}
Expand All @@ -407,7 +415,7 @@
"metadata",
"dict_corr"
],
"title": "TaskArguments",
"title": "IlluminationCorrection",
"type": "object"
},
"default_args": {
Expand All @@ -433,10 +441,12 @@
"type": "string"
},
"expected_dimensions": {
"default": 3,
"title": "Expected Dimensions",
"type": "integer"
},
"input_ROI_table": {
"default": "FOV_ROI_table",
"title": "Input Roi Table",
"type": "string"
},
Expand All @@ -458,6 +468,7 @@
"type": "object"
},
"level": {
"default": 0,
"title": "Level",
"type": "integer"
},
Expand All @@ -480,6 +491,7 @@
"type": "object"
},
"relabeling": {
"default": true,
"title": "Relabeling",
"type": "boolean"
},
Expand All @@ -491,13 +503,13 @@
"required": [
"input_paths",
"output_path",
"metadata",
"component",
"metadata",
"workflow_file",
"input_specs",
"output_specs"
],
"title": "TaskArguments",
"title": "NapariWorkflowsWrapper",
"type": "object"
},
"default_args": {
Expand Down Expand Up @@ -531,10 +543,12 @@
"type": "object"
},
"coarsening_xy": {
"default": 2,
"title": "Coarsening Xy",
"type": "integer"
},
"image_extension": {
"default": "tif",
"title": "Image Extension",
"type": "string"
},
Expand Down Expand Up @@ -571,9 +585,11 @@
"type": "object"
}
],
"default": "mrf_mlf",
"title": "Metadata Table"
},
"num_levels": {
"default": 2,
"title": "Num Levels",
"type": "integer"
},
Expand All @@ -586,10 +602,9 @@
"input_paths",
"output_path",
"metadata",
"image_extension",
"allowed_channels"
],
"title": "TaskArguments",
"title": "CreateOmeZarrMultiplex",
"type": "object"
},
"default_args": {
Expand Down
25 changes: 16 additions & 9 deletions fractal_tasks_core/_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@
from pathlib import Path
from typing import Callable

from pydantic import BaseModel


class TaskParameterEncoder(JSONEncoder):
"""
Expand All @@ -42,15 +40,16 @@ def default(self, value):
def run_fractal_task(
*,
task_function: Callable,
TaskArgsModel: type[BaseModel] = None,
validate: bool = False,
logger_name: str = None,
):
"""
Implement standard task interface and call task_function. If TaskArgsModel
is not None, validate arguments against given model.
Implement standard task interface and call task_function. If `validate`,
validate arguments via `pydantic.decorator.validate_arguments`.
:param task_function: the callable function that runs the task
:param TaskArgsModel: a class specifying all types for task arguments
:param validate: TBD
:logger_name: TBD
"""

# Parse `-j` and `--metadata-out` arguments
Expand Down Expand Up @@ -79,16 +78,24 @@ def run_fractal_task(
with open(args.json, "r") as f:
pars = json.load(f)

if TaskArgsModel is None:
if not validate:
# Run task without validating arguments' types
logger.info(f"START {task_function.__name__} task")
metadata_update = task_function(**pars)
logger.info(f"END {task_function.__name__} task")
else:
from pydantic.decorator import validate_arguments
from devtools import debug

debug(pars)
# Validating arguments' types and run task
task_args = TaskArgsModel(**pars)
logger.info(f"START {task_function.__name__} task")
metadata_update = task_function(**task_args.dict(exclude_unset=True))
debug(task_function)

vf = validate_arguments(task_function)
debug(vf)

metadata_update = vf(**pars)
logger.info(f"END {task_function.__name__} task")

# Write output metadata to file, with custom JSON encoder
Expand Down
34 changes: 1 addition & 33 deletions fractal_tasks_core/cellpose_segmentation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@
import zarr
from anndata.experimental import write_elem
from cellpose import models
from pydantic import BaseModel
from pydantic import Extra

import fractal_tasks_core
from fractal_tasks_core.lib_channels import ChannelNotFoundError
Expand Down Expand Up @@ -663,42 +661,12 @@ def cellpose_segmentation(
return {}


class TaskArguments(BaseModel, extra=Extra.forbid):
# Fractal arguments
input_paths: Sequence[str]
output_path: str
component: str
metadata: Dict[str, Any]
# Task-specific arguments
channel_label: Optional[str]
channel_label_c2: Optional[str]
wavelength_id: Optional[str]
wavelength_id_c2: Optional[str]
level: int
relabeling: bool = True
input_ROI_table: Optional[str]
output_ROI_table: Optional[str]
output_label_name: Optional[str]
# Cellpose-related arguments:
use_gpu: Optional[bool]
anisotropy: Optional[float]
diameter_level0: Optional[float]
cellprob_threshold: Optional[float]
flow_threshold: Optional[float]
model_type: Optional[str]
pretrained_model: Optional[str]
min_size: Optional[int]
augment: Optional[bool]
net_avg: Optional[bool]
use_masks: Optional[bool]


if __name__ == "__main__":

from fractal_tasks_core._utils import run_fractal_task

run_fractal_task(
task_function=cellpose_segmentation,
TaskArgsModel=TaskArguments,
validate=True,
logger_name=logger.name,
)
Loading

0 comments on commit 87a38b5

Please sign in to comment.