Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split resolution metadata #112

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Split resolution metadata #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
27902d9
remove lists from metadata
ceesem Jul 29, 2023
bda3f92
remove lists from metadata
ceesem Jul 29, 2023
11b5094
split voxel_resolution to resolve df concats
ceesem Aug 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions caveclient/annotationengine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pandas as pd

from .auth import AuthClient
from .base import BaseEncoder, ClientBase, _api_endpoints, handle_response
from .base import BaseEncoder, ClientBase, _api_endpoints, handle_response, assemble_voxel_resolution
from .endpoints import annotation_api_versions, annotation_common
from .tools import stage

Expand Down Expand Up @@ -215,10 +215,6 @@ def get_table_metadata(self, table_name: str, aligned_volume_name: str = None):
url = self._endpoints["table_info"].format_map(endpoint_mapping)
response = self.session.get(url)
metadata_d = handle_response(response)
vx = metadata_d.pop("voxel_resolution_x")
vy = metadata_d.pop("voxel_resolution_y")
vz = metadata_d.pop("voxel_resolution_z")
metadata_d["voxel_resolution"] = [vx, vy, vz]
return metadata_d

def delete_table(self, table_name: str, aligned_volume_name: str = None):
Expand Down Expand Up @@ -792,7 +788,7 @@ def stage_annotations(
obj_name = table_name
table_meta = self.get_table_metadata(table_name)
schema_name = table_meta["schema_type"]
table_resolution = table_meta["voxel_resolution"]
table_resolution = assemble_voxel_resolution(table_meta)
else:
if schema_name is None:
raise ValueError("Must specify either table name or schema name")
Expand Down
3 changes: 3 additions & 0 deletions caveclient/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
import datetime
import pandas as pd

def assemble_voxel_resolution(md):
"Generate the voxel resolution list from metadata"
return [md[f"voxel_resolution_{ii}"] for ii in ["x", "y", "z"]]

class BaseEncoder(json.JSONEncoder):
def default(self, obj):
Expand Down
113 changes: 73 additions & 40 deletions caveclient/materializationengine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from urllib.error import HTTPError
import warnings
import pytz
import string
import pandas as pd
from IPython.display import HTML
from .mytimer import MyTimeIt
Expand All @@ -18,6 +19,7 @@
BaseEncoder,
_api_endpoints,
handle_response,
assemble_voxel_resolution,
)
from cachetools import cached, TTLCache
from .tools.table_manager import TableManager, ViewManager
Expand All @@ -27,6 +29,14 @@

SERVER_KEY = "me_server_address"

def _ascii_letters_reord(starting_letter):
start_, end_ = string.ascii_lowercase.split(starting_letter)
return starting_letter+end_+start_

def _is_listlike(x):
if isinstance(x, str):
return False
return isinstance(x, Iterable)

def convert_position_columns(df, given_resolution, desired_resolution):
"""function to take a dataframe with x,y,z position columns and convert
Expand Down Expand Up @@ -459,10 +469,6 @@ def get_table_metadata(

response = self.session.get(url)
metadata_d = handle_response(response, log_warning=log_warning)
vx = metadata_d.pop("voxel_resolution_x", None)
vy = metadata_d.pop("voxel_resolution_y", None)
vz = metadata_d.pop("voxel_resolution_z", None)
metadata_d["voxel_resolution"] = [vx, vy, vz]
return metadata_d

def _format_query_components(
Expand Down Expand Up @@ -709,12 +715,14 @@ def query_table(
raise ValueError(
"desired resolution needs to be of length 3, for xyz"
)
vox_res = self.get_table_metadata(
table,
datastack_name,
materialization_version,
log_warning=False,
)["voxel_resolution"]
vox_res = assemble_voxel_resolution(
self.get_table_metadata(
table,
datastack_name,
materialization_version,
log_warning=False,
)
)
df = convert_position_columns(df, vox_res, desired_resolution)
if metadata:
attrs = self._assemble_attributes(
Expand Down Expand Up @@ -1231,11 +1239,13 @@ def live_live_query(
raise ValueError(
"desired resolution needs to be of length 3, for xyz"
)
vox_res = self.get_table_metadata(
table_name=table,
datastack_name=datastack_name,
log_warning=False,
)["voxel_resolution"]
vox_res = assemble_voxel_resolution(
self.get_table_metadata(
table_name=table,
datastack_name=datastack_name,
log_warning=False,
)
)
df = convert_position_columns(df, vox_res, desired_resolution)
if not split_positions:
concatenate_position_columns(df, inplace=True)
Expand Down Expand Up @@ -1465,12 +1475,14 @@ def live_query(
raise ValueError(
"desired resolution needs to be of length 3, for xyz"
)
vox_res = self.get_table_metadata(
table,
datastack_name,
materialization_version,
log_warning=False,
)["voxel_resolution"]
vox_res = assemble_voxel_resolution(
self.get_table_metadata(
table,
datastack_name,
materialization_version,
log_warning=False,
)
)
df = convert_position_columns(df, vox_res, desired_resolution)
if not split_positions:
concatenate_position_columns(df, inplace=True)
Expand Down Expand Up @@ -1650,13 +1662,26 @@ def _assemble_attributes(

for k, v in meta.items():
if re.match("^table", k):
attrs[k] = v
if _is_listlike(v):
for v_el, suffix in zip(v, _ascii_letters_reord('x')):
attrs[f"{k}_{suffix}"] = v_el
else:
attrs[k] = v
else:
attrs[f"table_{k}"] = v
if _is_listlike(v):
for v_el, suffix in zip(v, _ascii_letters_reord('x')):
attrs[f"table_{k}_{suffix}"] = v_el
else:
attrs[f"table_{k}"] = v
if desired_resolution is None:
attrs["dataframe_resolution"] = attrs["table_voxel_resolution"]
attrs["dataframe_resolution_x"] = attrs["table_voxel_resolution_x"]
attrs["dataframe_resolution_y"] = attrs["table_voxel_resolution_y"]
attrs["dataframe_resolution_z"] = attrs["table_voxel_resolution_z"]

else:
attrs["dataframe_resolution"] = desired_resolution
attrs["dataframe_resolution_x"] = desired_resolution[0]
attrs["dataframe_resolution_y"] = desired_resolution[1]
attrs["dataframe_resolution_z"] = desired_resolution[2]
else:
attrs["join_query"] = True
attrs["tables"] = {}
Expand All @@ -1671,9 +1696,17 @@ def _assemble_attributes(
meta = self.fc.annotation.get_table_metadata(tname)
for k, v in meta.items():
if re.match("^table", k):
table_attrs[tname][k] = v
if _is_listlike(v):
for v_el, suffix in zip(v, _ascii_letters_reord('x')):
attrs[f"{k}_{suffix}"] = v_el
else:
attrs[k] = v
else:
table_attrs[tname][f"table_{k}"] = v
if _is_listlike(v):
for v_el, suffix in zip(v, _ascii_letters_reord('x')):
attrs[f"table_{k}_{suffix}"] = v_el
else:
attrs[f"table_{k}"] = v
table_attrs[tname]["join_column"] = jcol
table_attrs[tname]["suffix"] = s

Expand All @@ -1682,12 +1715,15 @@ def _assemble_attributes(
for tn in attrs["tables"]:
res.append(attrs["tables"][tn]["table_voxel_resolution"])
if np.atleast_2d(np.unique(np.array(res), axis=0)).shape[0] == 1:
attrs["dataframe_resolution"] = res[0]
attrs["dataframe_resolution_x"] = res[0][0]
attrs["dataframe_resolution_y"] = res[0][1]
attrs["dataframe_resolution_z"] = res[0][2]
else:
attrs["dataframe_resolution"] = "mixed_resolutions"
else:
attrs["dataframe_resolution"] = desired_resolution

attrs["dataframe_resolution_x"] = desired_resolution[0]
attrs["dataframe_resolution_y"] = desired_resolution[1]
attrs["dataframe_resolution_z"] = desired_resolution[2]
attrs.update(kwargs)
return attrs

Expand Down Expand Up @@ -1730,11 +1766,6 @@ def get_tables_metadata(

response = self.session.get(url)
all_metadata = handle_response(response, log_warning=log_warning)
for metadata_d in all_metadata:
vx = metadata_d.pop("voxel_resolution_x", None)
vy = metadata_d.pop("voxel_resolution_y", None)
vz = metadata_d.pop("voxel_resolution_z", None)
metadata_d["voxel_resolution"] = [vx, vy, vz]
return all_metadata

def live_live_query(
Expand Down Expand Up @@ -1886,11 +1917,13 @@ def live_live_query(
raise ValueError(
"desired resolution needs to be of length 3, for xyz"
)
vox_res = self.get_table_metadata(
table,
datastack_name,
log_warning=False,
)["voxel_resolution"]
vox_res = assemble_voxel_resolution(
self.get_table_metadata(
table,
datastack_name,
log_warning=False,
)
)
df = convert_position_columns(df, vox_res, desired_resolution)

if not split_positions:
Expand Down
29 changes: 9 additions & 20 deletions caveclient/tools/table_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
import warnings
import re
from cachetools import cached, TTLCache, keys
import logging
logger = logging.getLogger(__name__)

# json schema column types that can act as potential columns for looking at tables
ALLOW_COLUMN_TYPES = ["integer", "boolean", "string", "float"]
Expand Down Expand Up @@ -328,21 +326,21 @@ def __attrs_post_init__(self):
filter_equal_dict = {
tn: filter_empty(attrs.asdict(
self,
filter=lambda a, v: is_list_like(v) == False
filter=lambda a, v: is_list_like(v) is False
and v is not None
and a.metadata.get("is_bbox", False) == False
and a.metadata.get("is_meta", False) == False
and a.metadata.get("is_bbox", False) is False
and a.metadata.get("is_meta", False) is False
and a.metadata.get("table") == tn,
))
for tn in tables
}
filter_in_dict = {
tn: filter_empty(attrs.asdict(
self,
filter=lambda a, v: is_list_like(v) == True
filter=lambda a, v: is_list_like(v) is True
and v is not None
and a.metadata.get("is_bbox", False) == False
and a.metadata.get("is_meta", False) == False
and a.metadata.get("is_bbox", False) is False
and a.metadata.get("is_meta", False) is False
and a.metadata.get("table") == tn,
))
for tn in tables
Expand All @@ -369,8 +367,8 @@ def __attrs_post_init__(self):
self.filter_kwargs_mat = self.filter_kwargs_live
else:
self.filter_kwargs_mat = {
k: replace_empty_with_none(self.filter_kwargs_live[k].get(list(tables)[0],[]))
for k in ["filter_equal_dict", "filter_in_dict", "filter_spatial_dict"] if self.filter_kwargs_live[k] is not None
k: replace_empty_with_none(self.filter_kwargs_live[k].get(list(tables)[0],[])) # noqa: E501
for k in ["filter_equal_dict", "filter_in_dict", "filter_spatial_dict"] if self.filter_kwargs_live[k] is not None # noqa: E501
}

pop_empty(self.filter_kwargs_live)
Expand Down Expand Up @@ -404,9 +402,6 @@ def query(
desired_resolution=None,
get_counts=False,
):
logger.warning(
"The `client.materialize.tables` interface is experimental and might experience breaking changes before the feature is stabilized."
)
if self._reference_table is None:
qry_table = self._base_table
return client.materialize.query_table(
Expand Down Expand Up @@ -447,9 +442,6 @@ def live_query(
desired_resolution=None,
allow_missing_lookups=False,
):
logger.warning(
"The `client.materialize.tables` interface is experimental and might experience breaking changes before the feature is stabilized."
)
if self._reference_table is None:
qry_table = self._base_table
return client.materialize.live_live_query(
Expand Down Expand Up @@ -492,9 +484,6 @@ def query(
desired_resolution=None,
get_counts=False,
):
logger.warning(
"The `client.materialize.views` interface is experimental and might experience breaking changes before the feature is stabilized."
)
return client.materialize.query_view(
self._base_table,
metadata=metadata,
Expand Down Expand Up @@ -523,7 +512,7 @@ def make_query_filter(table_name, meta, client):
return QueryFilter

def make_query_filter_view(view_name, meta, schema, client):
pts, val_cols, all_unbd_pts, table_map, rename_map, table_list, desc, live_compatible= get_view_info(
pts, val_cols, all_unbd_pts, table_map, rename_map, table_list, desc, live_compatible = get_view_info(
view_name, meta, schema
)
class_vals = make_class_vals(
Expand Down