From 27902d93abc5b1d72f9e74a42caa330cf6cfe1fb Mon Sep 17 00:00:00 2001 From: Casey Schneider-Mizell Date: Fri, 28 Jul 2023 17:38:20 -1000 Subject: [PATCH 1/3] remove lists from metadata --- caveclient/tools/table_manager.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/caveclient/tools/table_manager.py b/caveclient/tools/table_manager.py index abca4976..35c2ea5f 100644 --- a/caveclient/tools/table_manager.py +++ b/caveclient/tools/table_manager.py @@ -2,8 +2,6 @@ import warnings import re from cachetools import cached, TTLCache, keys -import logging -logger = logging.getLogger(__name__) # json schema column types that can act as potential columns for looking at tables ALLOW_COLUMN_TYPES = ["integer", "boolean", "string", "float"] @@ -328,10 +326,10 @@ def __attrs_post_init__(self): filter_equal_dict = { tn: filter_empty(attrs.asdict( self, - filter=lambda a, v: is_list_like(v) == False + filter=lambda a, v: is_list_like(v) is False and v is not None - and a.metadata.get("is_bbox", False) == False - and a.metadata.get("is_meta", False) == False + and a.metadata.get("is_bbox", False) is False + and a.metadata.get("is_meta", False) is False and a.metadata.get("table") == tn, )) for tn in tables @@ -339,10 +337,10 @@ def __attrs_post_init__(self): filter_in_dict = { tn: filter_empty(attrs.asdict( self, - filter=lambda a, v: is_list_like(v) == True + filter=lambda a, v: is_list_like(v) is True and v is not None - and a.metadata.get("is_bbox", False) == False - and a.metadata.get("is_meta", False) == False + and a.metadata.get("is_bbox", False) is False + and a.metadata.get("is_meta", False) is False and a.metadata.get("table") == tn, )) for tn in tables @@ -369,8 +367,8 @@ def __attrs_post_init__(self): self.filter_kwargs_mat = self.filter_kwargs_live else: self.filter_kwargs_mat = { - k: replace_empty_with_none(self.filter_kwargs_live[k].get(list(tables)[0],[])) - for k in ["filter_equal_dict", "filter_in_dict", "filter_spatial_dict"] if self.filter_kwargs_live[k] is not None + k: replace_empty_with_none(self.filter_kwargs_live[k].get(list(tables)[0],[])) # noqa: E501 + for k in ["filter_equal_dict", "filter_in_dict", "filter_spatial_dict"] if self.filter_kwargs_live[k] is not None # noqa: E501 } pop_empty(self.filter_kwargs_live) @@ -404,9 +402,6 @@ def query( desired_resolution=None, get_counts=False, ): - logger.warning( - "The `client.materialize.tables` interface is experimental and might experience breaking changes before the feature is stabilized." - ) if self._reference_table is None: qry_table = self._base_table return client.materialize.query_table( @@ -447,9 +442,6 @@ def live_query( desired_resolution=None, allow_missing_lookups=False, ): - logger.warning( - "The `client.materialize.tables` interface is experimental and might experience breaking changes before the feature is stabilized." - ) if self._reference_table is None: qry_table = self._base_table return client.materialize.live_live_query( @@ -492,9 +484,6 @@ def query( desired_resolution=None, get_counts=False, ): - logger.warning( - "The `client.materialize.views` interface is experimental and might experience breaking changes before the feature is stabilized." - ) return client.materialize.query_view( self._base_table, metadata=metadata, @@ -523,7 +512,7 @@ def make_query_filter(table_name, meta, client): return QueryFilter def make_query_filter_view(view_name, meta, schema, client): - pts, val_cols, all_unbd_pts, table_map, rename_map, table_list, desc, live_compatible= get_view_info( + pts, val_cols, all_unbd_pts, table_map, rename_map, table_list, desc, live_compatible = get_view_info( view_name, meta, schema ) class_vals = make_class_vals( From bda3f92f63a508169f54cc0379c317e2ab532e4d Mon Sep 17 00:00:00 2001 From: Casey Schneider-Mizell Date: Fri, 28 Jul 2023 17:38:34 -1000 Subject: [PATCH 2/3] remove lists from metadata --- caveclient/materializationengine.py | 51 ++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/caveclient/materializationengine.py b/caveclient/materializationengine.py index 3377da0a..e22d275f 100644 --- a/caveclient/materializationengine.py +++ b/caveclient/materializationengine.py @@ -2,6 +2,7 @@ from urllib.error import HTTPError import warnings import pytz +import string import pandas as pd from IPython.display import HTML from .mytimer import MyTimeIt @@ -27,6 +28,14 @@ SERVER_KEY = "me_server_address" +def _ascii_letters_reord(starting_letter): + start_, end_ = string.ascii_lowercase.split(starting_letter) + return starting_letter+end_+start_ + +def _is_listlike(x): + if isinstance(x, str): + return False + return isinstance(x, Iterable) def convert_position_columns(df, given_resolution, desired_resolution): """function to take a dataframe with x,y,z position columns and convert @@ -1650,13 +1659,26 @@ def _assemble_attributes( for k, v in meta.items(): if re.match("^table", k): - attrs[k] = v + if _is_listlike(v): + for v_el, suffix in zip(v, _ascii_letters_reord('x')): + attrs[f"{k}_{suffix}"] = v_el + else: + attrs[k] = v else: - attrs[f"table_{k}"] = v + if _is_listlike(v): + for v_el, suffix in zip(v, _ascii_letters_reord('x')): + attrs[f"table_{k}_{suffix}"] = v_el + else: + attrs[f"table_{k}"] = v if desired_resolution is None: - attrs["dataframe_resolution"] = attrs["table_voxel_resolution"] + attrs["dataframe_resolution_x"] = attrs["table_voxel_resolution_x"] + attrs["dataframe_resolution_y"] = attrs["table_voxel_resolution_y"] + attrs["dataframe_resolution_z"] = attrs["table_voxel_resolution_z"] + else: - attrs["dataframe_resolution"] = desired_resolution + attrs["dataframe_resolution_x"] = desired_resolution[0] + attrs["dataframe_resolution_y"] = desired_resolution[1] + attrs["dataframe_resolution_z"] = desired_resolution[2] else: attrs["join_query"] = True attrs["tables"] = {} @@ -1671,9 +1693,17 @@ def _assemble_attributes( meta = self.fc.annotation.get_table_metadata(tname) for k, v in meta.items(): if re.match("^table", k): - table_attrs[tname][k] = v + if _is_listlike(v): + for v_el, suffix in zip(v, _ascii_letters_reord('x')): + attrs[f"{k}_{suffix}"] = v_el + else: + attrs[k] = v else: - table_attrs[tname][f"table_{k}"] = v + if _is_listlike(v): + for v_el, suffix in zip(v, _ascii_letters_reord('x')): + attrs[f"table_{k}_{suffix}"] = v_el + else: + attrs[f"table_{k}"] = v table_attrs[tname]["join_column"] = jcol table_attrs[tname]["suffix"] = s @@ -1682,12 +1712,15 @@ def _assemble_attributes( for tn in attrs["tables"]: res.append(attrs["tables"][tn]["table_voxel_resolution"]) if np.atleast_2d(np.unique(np.array(res), axis=0)).shape[0] == 1: - attrs["dataframe_resolution"] = res[0] + attrs["dataframe_resolution_x"] = res[0][0] + attrs["dataframe_resolution_y"] = res[0][1] + attrs["dataframe_resolution_z"] = res[0][2] else: attrs["dataframe_resolution"] = "mixed_resolutions" else: - attrs["dataframe_resolution"] = desired_resolution - + attrs["dataframe_resolution_x"] = desired_resolution[0] + attrs["dataframe_resolution_y"] = desired_resolution[1] + attrs["dataframe_resolution_z"] = desired_resolution[2] attrs.update(kwargs) return attrs From 11b50944d233b11401c8e96a59d5b158450a59ca Mon Sep 17 00:00:00 2001 From: Casey Schneider-Mizell Date: Fri, 4 Aug 2023 13:29:15 -1000 Subject: [PATCH 3/3] split voxel_resolution to resolve df concats --- caveclient/annotationengine.py | 8 +--- caveclient/base.py | 3 ++ caveclient/materializationengine.py | 62 ++++++++++++++--------------- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/caveclient/annotationengine.py b/caveclient/annotationengine.py index a6d6e339..4ee893e2 100644 --- a/caveclient/annotationengine.py +++ b/caveclient/annotationengine.py @@ -5,7 +5,7 @@ import pandas as pd from .auth import AuthClient -from .base import BaseEncoder, ClientBase, _api_endpoints, handle_response +from .base import BaseEncoder, ClientBase, _api_endpoints, handle_response, assemble_voxel_resolution from .endpoints import annotation_api_versions, annotation_common from .tools import stage @@ -215,10 +215,6 @@ def get_table_metadata(self, table_name: str, aligned_volume_name: str = None): url = self._endpoints["table_info"].format_map(endpoint_mapping) response = self.session.get(url) metadata_d = handle_response(response) - vx = metadata_d.pop("voxel_resolution_x") - vy = metadata_d.pop("voxel_resolution_y") - vz = metadata_d.pop("voxel_resolution_z") - metadata_d["voxel_resolution"] = [vx, vy, vz] return metadata_d def delete_table(self, table_name: str, aligned_volume_name: str = None): @@ -792,7 +788,7 @@ def stage_annotations( obj_name = table_name table_meta = self.get_table_metadata(table_name) schema_name = table_meta["schema_type"] - table_resolution = table_meta["voxel_resolution"] + table_resolution = assemble_voxel_resolution(table_meta) else: if schema_name is None: raise ValueError("Must specify either table name or schema name") diff --git a/caveclient/base.py b/caveclient/base.py index 1a4a1fc9..06f393a3 100644 --- a/caveclient/base.py +++ b/caveclient/base.py @@ -11,6 +11,9 @@ import datetime import pandas as pd +def assemble_voxel_resolution(md): + "Generate the voxel resolution list from metadata" + return [md[f"voxel_resolution_{ii}"] for ii in ["x", "y", "z"]] class BaseEncoder(json.JSONEncoder): def default(self, obj): diff --git a/caveclient/materializationengine.py b/caveclient/materializationengine.py index e22d275f..15609e0c 100644 --- a/caveclient/materializationengine.py +++ b/caveclient/materializationengine.py @@ -19,6 +19,7 @@ BaseEncoder, _api_endpoints, handle_response, + assemble_voxel_resolution, ) from cachetools import cached, TTLCache from .tools.table_manager import TableManager, ViewManager @@ -468,10 +469,6 @@ def get_table_metadata( response = self.session.get(url) metadata_d = handle_response(response, log_warning=log_warning) - vx = metadata_d.pop("voxel_resolution_x", None) - vy = metadata_d.pop("voxel_resolution_y", None) - vz = metadata_d.pop("voxel_resolution_z", None) - metadata_d["voxel_resolution"] = [vx, vy, vz] return metadata_d def _format_query_components( @@ -718,12 +715,14 @@ def query_table( raise ValueError( "desired resolution needs to be of length 3, for xyz" ) - vox_res = self.get_table_metadata( - table, - datastack_name, - materialization_version, - log_warning=False, - )["voxel_resolution"] + vox_res = assemble_voxel_resolution( + self.get_table_metadata( + table, + datastack_name, + materialization_version, + log_warning=False, + ) + ) df = convert_position_columns(df, vox_res, desired_resolution) if metadata: attrs = self._assemble_attributes( @@ -1240,11 +1239,13 @@ def live_live_query( raise ValueError( "desired resolution needs to be of length 3, for xyz" ) - vox_res = self.get_table_metadata( - table_name=table, - datastack_name=datastack_name, - log_warning=False, - )["voxel_resolution"] + vox_res = assemble_voxel_resolution( + self.get_table_metadata( + table_name=table, + datastack_name=datastack_name, + log_warning=False, + ) + ) df = convert_position_columns(df, vox_res, desired_resolution) if not split_positions: concatenate_position_columns(df, inplace=True) @@ -1474,12 +1475,14 @@ def live_query( raise ValueError( "desired resolution needs to be of length 3, for xyz" ) - vox_res = self.get_table_metadata( - table, - datastack_name, - materialization_version, - log_warning=False, - )["voxel_resolution"] + vox_res = assemble_voxel_resolution( + self.get_table_metadata( + table, + datastack_name, + materialization_version, + log_warning=False, + ) + ) df = convert_position_columns(df, vox_res, desired_resolution) if not split_positions: concatenate_position_columns(df, inplace=True) @@ -1763,11 +1766,6 @@ def get_tables_metadata( response = self.session.get(url) all_metadata = handle_response(response, log_warning=log_warning) - for metadata_d in all_metadata: - vx = metadata_d.pop("voxel_resolution_x", None) - vy = metadata_d.pop("voxel_resolution_y", None) - vz = metadata_d.pop("voxel_resolution_z", None) - metadata_d["voxel_resolution"] = [vx, vy, vz] return all_metadata def live_live_query( @@ -1919,11 +1917,13 @@ def live_live_query( raise ValueError( "desired resolution needs to be of length 3, for xyz" ) - vox_res = self.get_table_metadata( - table, - datastack_name, - log_warning=False, - )["voxel_resolution"] + vox_res = assemble_voxel_resolution( + self.get_table_metadata( + table, + datastack_name, + log_warning=False, + ) + ) df = convert_position_columns(df, vox_res, desired_resolution) if not split_positions: