From 3ea1c4dbf998ea6c72ffa920c4965c3766aa6068 Mon Sep 17 00:00:00 2001 From: jochemb Date: Tue, 24 Apr 2018 22:56:14 +0200 Subject: [PATCH 01/13] Initial commit --- transmart/api/v2/dashboard.py | 280 ++++++++++++++++++++++++++++++++++ transmart/api/v2/hypercube.py | 80 ++++++++++ 2 files changed, 360 insertions(+) create mode 100644 transmart/api/v2/dashboard.py create mode 100644 transmart/api/v2/hypercube.py diff --git a/transmart/api/v2/dashboard.py b/transmart/api/v2/dashboard.py new file mode 100644 index 0000000..ca27397 --- /dev/null +++ b/transmart/api/v2/dashboard.py @@ -0,0 +1,280 @@ +""" +* Copyright (c) 2015-2017 The Hyve B.V. +* This code is licensed under the GNU General Public License, +* version 3. +""" + +import logging + +import bqplot as plt +import ipywidgets as widgets + +from .constraint_widgets import ConceptPicker +from .query_constraints import ObservationConstraint, Queryable +from .hypercube import Hypercube + +logger = logging.getLogger(__name__) +debug_view = widgets.Output(layout={'border': '1px solid black'}) + +_NUMERIC_VALUE = 'numericValue' +_STRING_VALUE = 'stringValue' + +ANIMATION_TIME = 400 + + +class Tile: + + def __init__(self, dash, title, concept, study=None): + self.dash = dash + self.fake_out = widgets.Output() + self.selected_subjects = None + + self.concept = concept + self.study = study + + self.title = title + self.fig = None + self.comb = None + + self._build_tile() + + @debug_view.capture() + def _build_tile(self): + self.fig = self.create_fig() + logger.info('Created figure.') + buttons, btn1, btn2, destroy_btn = self.get_buttons() + self.comb = widgets.HBox( + [self.fig, buttons], + layout={'margin': '0px -20px 0px -18px'} + ) + destroy_btn.on_click(self.destroyer(self.comb)) + + self.fig.layout.height = '350px' + self.fig.layout.width = '350px' + + self.fig.animation_duration = ANIMATION_TIME + self.fig.title_style = {'font-size': 'small', + 'width': '335px', + 'overflow-text': 'wrap'} + self.fig.title = self.title + + def destroyer(self, fig_box): + def remove_fig(btn): + with self.fake_out: + self.dash.tiles.remove(self) + self.dash.remove(fig_box) + return remove_fig + + def create_fig(self): + raise NotImplementedError + + def set_values(self, values): + raise NotImplementedError + + def _calc_selected_subjects(self): + raise NotImplementedError + + def refresh(self): + raise NotImplementedError + + @property + def value_type(self): + raise NotImplementedError + + def get_updates(self): + subset = self.dash.hypercube.query(concept=self.concept, study=self.study) + values = subset[self.value_type] + self.set_values(values) + + def get_buttons(self): + btn_layout = { + 'width': '35px', + 'height': '35px' + } + + btn1 = widgets.Button(icon='fa-check', layout=btn_layout) + btn2 = widgets.Button(icon='fa-link', layout=btn_layout) + btn3 = widgets.Button(icon='fa-window-close', layout=btn_layout) + + buttons = widgets.VBox([ + btn1, btn2, btn3 + ], layout={'margin': '60px 25px 0px -60px'}) + + btn1.layout.width = btn1.layout.height = '35px' + + btn1.on_click(self._calc_selected_subjects) + + return buttons, btn1, btn2, btn3 + + @debug_view.capture(clear_output=False) + def get_fig(self): + print('Sending figure to front-end.') + return self.comb + + +class HistogramTile(Tile): + value_type = _NUMERIC_VALUE + + @debug_view.capture(clear_output=False) + def create_fig(self): + print('Creating figure.') + scale_x = plt.LinearScale() + scale_y = plt.LinearScale() + hist = plt.Hist(sample=[], scales={'sample': scale_x, 'count': scale_y}) + + ax_x = plt.Axis(label='Value Bins', scale=scale_x) + ax_y = plt.Axis(label='Count', scale=scale_y, + orientation='vertical', grid_lines='solid') + + selector = plt.interacts.BrushIntervalSelector( + scale=scale_x, marks=[hist], color='SteelBlue') + + print('Returning figure.') + return plt.Figure(axes=[ax_x, ax_y], marks=[hist], interaction=selector) + + @debug_view.capture(clear_output=False) + def set_values(self, values): + print('Updating values.') + m = self.fig.marks[0] + with self.fig.hold_sync(): + m.sample = values + + @debug_view.capture() + def _calc_selected_subjects(self, *args): + subset = self.dash.hypercube.query(concept=self.concept, study=self.study, no_filter=True) + values = subset[self.value_type] + + if len(self.fig.interaction.selected): + min_, max_ = self.fig.interaction.selected + selected = values.index[values.between(min_, max_)] + print(values, min_, max_) + self.selected_subjects = set(self.dash.hypercube.data.loc[selected, 'patient.id']) + + else: + self.selected_subjects = None + + print(self.selected_subjects) + self.dash.hypercube.subject_mask = self.selected_subjects + self.dash.update(exclude=self) + + def refresh(self): + """ Seems to resolve problems with the brush selector. """ + values = self.fig.marks[0].sample + with self.fig.hold_sync(): + self.set_values([]) + self.set_values(values) + + +class PieTile(Tile): + value_type = _STRING_VALUE + + @debug_view.capture(clear_output=False) + def create_fig(self): + print('Creating figure.') + + tooltip_widget = plt.Tooltip(fields=['size', 'label']) + pie = plt.Pie(tooltip=tooltip_widget, + interactions={'click': 'select', 'hover': 'tooltip'}) + pie.radius = 100 + print('Returning figure.') + pie.selected_style = {"opacity": "1", "stroke": "white", "stroke-width": "4"} + pie.unselected_style = {"opacity": "0.2"} + + return plt.Figure(marks=[pie]) + + @debug_view.capture(clear_output=False) + def set_values(self, values): + print('Updating values.') + + pie = self.fig.marks[0] + counts = values.value_counts() + + with self.fig.hold_sync(): + pie.labels = pie.sizes = [] + pie.sizes = counts + pie.labels = list(counts.index) + + def _calc_selected_subjects(self): + pass + + def refresh(self): + pass + + +class Dashboard: + + def __init__(self, api, patients: Queryable=None): + self.api = api + self.tiles = list() + self.hypercube = Hypercube() + + if isinstance(patients, Queryable): + self.subject_set_ids = api.create_patient_set(repr(patients), patients).get('id') + else: + self.subject_set_ids = None + + self.out = widgets.Box() + self.out.layout.flex_flow = 'row wrap' + + self.cp = ConceptPicker(self.plotter, api) + + def get(self): + return widgets.VBox([self.out, self.cp.get()]) + + @debug_view.capture() + def plotter(self, constraints): + c = ObservationConstraint.from_tree_node(constraints) + + if self.subject_set_ids is not None: + c.subject_set_id = self.subject_set_ids + + obs = self.api.get_observations(c) + self.hypercube.add_variable(obs.dataframe) + + name = obs.dataframe['concept.name'][0] + + if _NUMERIC_VALUE in obs.dataframe.columns: + tile = HistogramTile(self, name, concept=c.concept, study=c.study) + tile.set_values(obs.dataframe[_NUMERIC_VALUE]) + + elif _STRING_VALUE in obs.dataframe.columns: + tile = PieTile(self, name, concept=c.concept, study=c.study) + tile.set_values(obs.dataframe[_STRING_VALUE]) + + else: + return + + self.register(tile) + + def register(self, tile): + self.tiles.append(tile) + with self.out.hold_sync(): + self.out.children = list(self.out.children) + [tile.get_fig()] + + self.refresh() + + def remove(self, tile): + tmp = list(self.out.children) + tmp.remove(tile) + with self.out.hold_sync(): + self.out.children = tmp + + self.refresh() + + def update(self, exclude=None): + for tile in self.tiles: + if tile is exclude: + continue + + tile.get_updates() + + def refresh(self): + with self.out.hold_sync(): + for tile in self.tiles: + tile.fig.animation_duration = 0 + tile.refresh() + tile.fig.animation_duration = ANIMATION_TIME + + @property + def debugger(self): + return debug_view diff --git a/transmart/api/v2/hypercube.py b/transmart/api/v2/hypercube.py new file mode 100644 index 0000000..aa9bdae --- /dev/null +++ b/transmart/api/v2/hypercube.py @@ -0,0 +1,80 @@ +""" +* Copyright (c) 2015-2017 The Hyve B.V. +* This code is licensed under the GNU General Public License, +* version 3. +""" +import pandas as pd + +concept_id = 'concept.conceptCode' +trial_visit_id = 'trial visit.id' +patient_id = 'patient.id' +start_time_id = 'start time' +study_id = 'study.name' + +dimensions = { + 'concept': concept_id, + 'study': study_id, + 'trial_visit': trial_visit_id, + 'start_time': start_time_id, + 'subject': patient_id +} + +value_columns = ['numericValue', 'stringValue'] + + +class HypercubeException(Exception): + pass + + +class Hypercube: + + def __init__(self): + self.dims = [] + self._cols = list(dimensions.values()) + value_columns + self.data = pd.DataFrame() + self._subject_bool_mask = None + self.__subjects_mask = None + self.study_concept_pairs = set() + + @property + def subject_mask(self): + return self.__subjects_mask + + @subject_mask.setter + def subject_mask(self, values): + self.__subjects_mask = values + if values is not None: + self._subject_bool_mask = self.data[patient_id].isin(values) + + def add_variable(self, df): + # check duplicate study-concept pairs + ns = {(v[0], v[1]) for v + in df.loc[:, [concept_id, study_id]].drop_duplicates().values} + if self.study_concept_pairs.intersection(ns): + return + self.study_concept_pairs.update(ns) + + sub_set = df.loc[:, self._cols] + self.data = self.data.append(sub_set, ignore_index=True) + + def query(self, no_filter=False, **kwargs): + expressions = [] + for kw, column in dimensions.items(): + if kw not in kwargs: + continue + parameter = kwargs.get(kw) + if isinstance(parameter, (pd.Series, list)): + expr = self.data[column].isin(parameter) + else: + expr = self.data[column] == parameter + expressions.append(expr) + + bools = True + for expr in expressions: + bools &= expr + + if not no_filter and self.subject_mask is not None: + bools &= self._subject_bool_mask + + return self.data.loc[bools, value_columns] + From ccfb55d7a777f0ebe25e969012227072728323ba Mon Sep 17 00:00:00 2001 From: jochemb Date: Sat, 28 Apr 2018 20:39:19 +0200 Subject: [PATCH 02/13] Start with subset in dashboard. --- transmart/api/commons.py | 13 ++ transmart/api/v2/concept_search.py | 4 +- transmart/api/v2/constraint_widgets.py | 11 +- transmart/api/v2/dashboard/__init__.py | 1 + transmart/api/v2/dashboard/dashboard.py | 122 +++++++++++++++++ transmart/api/v2/{ => dashboard}/hypercube.py | 2 + .../v2/{dashboard.py => dashboard/tiles.py} | 126 +++++------------- 7 files changed, 179 insertions(+), 100 deletions(-) create mode 100644 transmart/api/v2/dashboard/__init__.py create mode 100644 transmart/api/v2/dashboard/dashboard.py rename transmart/api/v2/{ => dashboard}/hypercube.py (95%) rename transmart/api/v2/{dashboard.py => dashboard/tiles.py} (64%) diff --git a/transmart/api/commons.py b/transmart/api/commons.py index bc44eb5..cf35ef5 100644 --- a/transmart/api/commons.py +++ b/transmart/api/commons.py @@ -41,3 +41,16 @@ def date_to_timestamp(date): dt = arrow.get(date, INPUT_DATE_FORMATS).datetime d = datetime(dt.year, dt.month, dt.day) return d.timestamp() * 1000 + + +def filter_tree(tree_dict, counts_per_study_and_concept): + concepts = set() + for k, v in counts_per_study_and_concept['countsPerStudy'].items(): + concepts.update(v) + + studies = counts_per_study_and_concept['countsPerStudy'].keys() + + return {k for k, v in tree_dict.items() + if v.get('conceptCode') in concepts + and v.get('studyId') in (*studies, None) + } diff --git a/transmart/api/v2/concept_search.py b/transmart/api/v2/concept_search.py index 52410d0..28f8682 100644 --- a/transmart/api/v2/concept_search.py +++ b/transmart/api/v2/concept_search.py @@ -25,10 +25,10 @@ def __init__(self, tree_dict, tree_identity): self._tree_dict = tree_dict self.get_schema() - def search(self, query_string, limit=50): + def search(self, query_string, limit=50, allowed_nodes: set=None): with self.ix.searcher() as searcher: query = self.parser.parse(query_string) - results = searcher.search(query, limit=limit,) + results = searcher.search(query, limit=limit, filter=allowed_nodes) return [r['fullname'] for r in results] def get_schema(self): diff --git a/transmart/api/v2/constraint_widgets.py b/transmart/api/v2/constraint_widgets.py index cb57261..62ec683 100644 --- a/transmart/api/v2/constraint_widgets.py +++ b/transmart/api/v2/constraint_widgets.py @@ -79,12 +79,15 @@ class ConceptPicker: """ - def __init__(self, target, api): + def __init__(self, target, api, allowed_nodes: set=None): self.target = target self.api = api + self.allowed_nodes = allowed_nodes - self.list_of_default_options = sorted(self.api.tree_dict.keys()) + nodes = allowed_nodes or self.api.tree_dict.keys() + + self.list_of_default_options = sorted(nodes) self.no_filter_len = len(self.list_of_default_options) self.result_count = widgets.HTML( @@ -121,7 +124,9 @@ def _build_search_bar(self): def search_watcher(change): x = change.get('new') if len(x) > 2: - self.concept_list.options = self.api.search_tree_node(x, limit=MAX_OPTIONS) + self.concept_list.options = self.api.search_tree_node(x, + limit=MAX_OPTIONS, + allowed_nodes=self.allowed_nodes) count = len(self.concept_list.options) if count == MAX_OPTIONS: diff --git a/transmart/api/v2/dashboard/__init__.py b/transmart/api/v2/dashboard/__init__.py new file mode 100644 index 0000000..632302c --- /dev/null +++ b/transmart/api/v2/dashboard/__init__.py @@ -0,0 +1 @@ +from .dashboard import Dashboard diff --git a/transmart/api/v2/dashboard/dashboard.py b/transmart/api/v2/dashboard/dashboard.py new file mode 100644 index 0000000..dfbffac --- /dev/null +++ b/transmart/api/v2/dashboard/dashboard.py @@ -0,0 +1,122 @@ +""" +* Copyright (c) 2015-2017 The Hyve B.V. +* This code is licensed under the GNU General Public License, +* version 3. +""" + +import logging + +import ipywidgets as widgets + +from ...commons import filter_tree +from ..constraint_widgets import ConceptPicker +from ..query_constraints import ObservationConstraint, Queryable +from .hypercube import Hypercube +from .tiles import HistogramTile, PieTile + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +debug_view = widgets.Output(layout={'border': '1px solid black'}) + +_NUMERIC_VALUE = 'numericValue' +_STRING_VALUE = 'stringValue' + +ANIMATION_TIME = 400 + + +class Dashboard: + + def __init__(self, api, patients: Queryable=None): + self.api = api + self.tiles = list() + self.hypercube = Hypercube() + + if isinstance(patients, Queryable): + self.subject_set_id = api.create_patient_set(repr(patients), patients).get('id') + counts = api.get_observations.counts_per_study_and_concept( + subject_set_id=self.subject_set_id + ) + self.nodes = filter_tree(api.tree_dict, counts) + + else: + self.subject_set_id = None + self.nodes = None + + self.out = widgets.Box() + self.out.layout.flex_flow = 'row wrap' + + self.cp = ConceptPicker(self.plotter, api, self.nodes) + self.counter = widgets.IntProgress( + value=10, min=0, max=10, step=1, + orientation='horizontal' + ) + + def get(self): + return widgets.VBox([self.out, self.counter, self.cp.get()]) + + @debug_view.capture() + def plotter(self, constraints): + c = ObservationConstraint.from_tree_node(constraints) + + if self.subject_set_id is not None: + c.subject_set_id = self.subject_set_id + + obs = self.api.get_observations(c) + self.hypercube.add_variable(obs.dataframe) + + name = obs.dataframe['concept.name'][0] + + if _NUMERIC_VALUE in obs.dataframe.columns: + tile = HistogramTile(self, name, concept=c.concept, study=c.study) + tile.set_values(obs.dataframe[_NUMERIC_VALUE]) + + elif _STRING_VALUE in obs.dataframe.columns: + tile = PieTile(self, name, concept=c.concept, study=c.study) + tile.set_values(obs.dataframe[_STRING_VALUE]) + + else: + return + + self.register(tile) + + def register(self, tile): + self.tiles.append(tile) + with self.out.hold_sync(): + self.out.children = list(self.out.children) + [tile.get_fig()] + + self.refresh() + + def remove(self, tile): + tmp = list(self.out.children) + tmp.remove(tile) + with self.out.hold_sync(): + self.out.children = tmp + + self.refresh() + + def update(self, exclude=None): + for tile in self.tiles: + if tile is exclude: + continue + + tile.get_updates() + + with self.counter.hold_sync(): + self.counter.max = self.hypercube.total_subjects + self.counter.value = self.hypercube.total_subjects + + if self.hypercube.subject_mask is not None: + self.counter.value = len(self.hypercube.subject_mask) + + self.counter.description = '{}/{}'.format(self.counter.value, self.counter.max) + + def refresh(self): + for tile in self.tiles: + tile.fig.animation_duration = 0 + tile.refresh() + tile.fig.animation_duration = ANIMATION_TIME + + @property + def debugger(self): + return debug_view diff --git a/transmart/api/v2/hypercube.py b/transmart/api/v2/dashboard/hypercube.py similarity index 95% rename from transmart/api/v2/hypercube.py rename to transmart/api/v2/dashboard/hypercube.py index aa9bdae..c7adfe7 100644 --- a/transmart/api/v2/hypercube.py +++ b/transmart/api/v2/dashboard/hypercube.py @@ -32,6 +32,7 @@ def __init__(self): self.dims = [] self._cols = list(dimensions.values()) + value_columns self.data = pd.DataFrame() + self.total_subjects = None self._subject_bool_mask = None self.__subjects_mask = None self.study_concept_pairs = set() @@ -56,6 +57,7 @@ def add_variable(self, df): sub_set = df.loc[:, self._cols] self.data = self.data.append(sub_set, ignore_index=True) + self.total_subjects = len(self.data[patient_id].unique()) def query(self, no_filter=False, **kwargs): expressions = [] diff --git a/transmart/api/v2/dashboard.py b/transmart/api/v2/dashboard/tiles.py similarity index 64% rename from transmart/api/v2/dashboard.py rename to transmart/api/v2/dashboard/tiles.py index ca27397..fd97799 100644 --- a/transmart/api/v2/dashboard.py +++ b/transmart/api/v2/dashboard/tiles.py @@ -9,11 +9,9 @@ import bqplot as plt import ipywidgets as widgets -from .constraint_widgets import ConceptPicker -from .query_constraints import ObservationConstraint, Queryable -from .hypercube import Hypercube - logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + debug_view = widgets.Output(layout={'border': '1px solid black'}) _NUMERIC_VALUE = 'numericValue' @@ -71,7 +69,7 @@ def create_fig(self): def set_values(self, values): raise NotImplementedError - def _calc_selected_subjects(self): + def _calc_selected_subjects(self, *args): raise NotImplementedError def refresh(self): @@ -88,8 +86,9 @@ def get_updates(self): def get_buttons(self): btn_layout = { - 'width': '35px', - 'height': '35px' + 'width': '25px', + 'height': '25px', + 'padding': '0px' } btn1 = widgets.Button(icon='fa-check', layout=btn_layout) @@ -100,8 +99,6 @@ def get_buttons(self): btn1, btn2, btn3 ], layout={'margin': '60px 25px 0px -60px'}) - btn1.layout.width = btn1.layout.height = '35px' - btn1.on_click(self._calc_selected_subjects) return buttons, btn1, btn2, btn3 @@ -129,12 +126,11 @@ def create_fig(self): selector = plt.interacts.BrushIntervalSelector( scale=scale_x, marks=[hist], color='SteelBlue') - print('Returning figure.') + logger.info('Returning figure.JOCHEMJOCHEM') return plt.Figure(axes=[ax_x, ax_y], marks=[hist], interaction=selector) @debug_view.capture(clear_output=False) def set_values(self, values): - print('Updating values.') m = self.fig.marks[0] with self.fig.hold_sync(): m.sample = values @@ -147,29 +143,33 @@ def _calc_selected_subjects(self, *args): if len(self.fig.interaction.selected): min_, max_ = self.fig.interaction.selected selected = values.index[values.between(min_, max_)] - print(values, min_, max_) self.selected_subjects = set(self.dash.hypercube.data.loc[selected, 'patient.id']) else: self.selected_subjects = None - print(self.selected_subjects) self.dash.hypercube.subject_mask = self.selected_subjects self.dash.update(exclude=self) - def refresh(self): - """ Seems to resolve problems with the brush selector. """ + @debug_view.capture() + def refresh(self, *args): + """ Seems to resolve problems with the brush selector. """ values = self.fig.marks[0].sample + with self.fig.hold_sync(): - self.set_values([]) + self.set_values([]) # TODO better way to make brush selector reliable. self.set_values(values) + self.fig.interaction.reset() + self.fig.interaction.selected = [] + print('Selected: ', self.fig.interaction.selected) + class PieTile(Tile): value_type = _STRING_VALUE @debug_view.capture(clear_output=False) - def create_fig(self): + def create_fig(self, *args): print('Creating figure.') tooltip_widget = plt.Tooltip(fields=['size', 'label']) @@ -184,8 +184,6 @@ def create_fig(self): @debug_view.capture(clear_output=False) def set_values(self, values): - print('Updating values.') - pie = self.fig.marks[0] counts = values.value_counts() @@ -194,87 +192,25 @@ def set_values(self, values): pie.sizes = counts pie.labels = list(counts.index) - def _calc_selected_subjects(self): - pass - - def refresh(self): - pass - - -class Dashboard: - - def __init__(self, api, patients: Queryable=None): - self.api = api - self.tiles = list() - self.hypercube = Hypercube() - - if isinstance(patients, Queryable): - self.subject_set_ids = api.create_patient_set(repr(patients), patients).get('id') - else: - self.subject_set_ids = None - - self.out = widgets.Box() - self.out.layout.flex_flow = 'row wrap' - - self.cp = ConceptPicker(self.plotter, api) - - def get(self): - return widgets.VBox([self.out, self.cp.get()]) - @debug_view.capture() - def plotter(self, constraints): - c = ObservationConstraint.from_tree_node(constraints) - - if self.subject_set_ids is not None: - c.subject_set_id = self.subject_set_ids - - obs = self.api.get_observations(c) - self.hypercube.add_variable(obs.dataframe) - - name = obs.dataframe['concept.name'][0] - - if _NUMERIC_VALUE in obs.dataframe.columns: - tile = HistogramTile(self, name, concept=c.concept, study=c.study) - tile.set_values(obs.dataframe[_NUMERIC_VALUE]) + def _calc_selected_subjects(self, *args): + subset = self.dash.hypercube.query(concept=self.concept, study=self.study, no_filter=True) + values = subset[self.value_type] - elif _STRING_VALUE in obs.dataframe.columns: - tile = PieTile(self, name, concept=c.concept, study=c.study) - tile.set_values(obs.dataframe[_STRING_VALUE]) + pie = self.fig.marks[0] + if pie.selected is not None: + labels = {pie.labels[i] for i in pie.selected} + print(labels) + selected = values.index[values.isin(labels)] + print(selected) + self.selected_subjects = set(self.dash.hypercube.data.loc[selected, 'patient.id']) else: - return - - self.register(tile) - - def register(self, tile): - self.tiles.append(tile) - with self.out.hold_sync(): - self.out.children = list(self.out.children) + [tile.get_fig()] - - self.refresh() - - def remove(self, tile): - tmp = list(self.out.children) - tmp.remove(tile) - with self.out.hold_sync(): - self.out.children = tmp - - self.refresh() - - def update(self, exclude=None): - for tile in self.tiles: - if tile is exclude: - continue + self.selected_subjects = None - tile.get_updates() + self.dash.hypercube.subject_mask = self.selected_subjects + self.dash.update(exclude=self) def refresh(self): - with self.out.hold_sync(): - for tile in self.tiles: - tile.fig.animation_duration = 0 - tile.refresh() - tile.fig.animation_duration = ANIMATION_TIME + self.fig.marks[0].selected = None - @property - def debugger(self): - return debug_view From c9e90f774b3058ede4b95d8247b5954238dbd4c8 Mon Sep 17 00:00:00 2001 From: jochemb Date: Sun, 29 Apr 2018 19:30:58 +0200 Subject: [PATCH 03/13] Start with subset in dashboard and search only for available nodes. --- transmart/api/v2/concept_search.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/transmart/api/v2/concept_search.py b/transmart/api/v2/concept_search.py index 28f8682..5907166 100644 --- a/transmart/api/v2/concept_search.py +++ b/transmart/api/v2/concept_search.py @@ -28,6 +28,14 @@ def __init__(self, tree_dict, tree_identity): def search(self, query_string, limit=50, allowed_nodes: set=None): with self.ix.searcher() as searcher: query = self.parser.parse(query_string) + + if allowed_nodes is not None: + allowed_nodes = { + doc_num for doc, doc_num + in zip(searcher.documents(), searcher.document_numbers()) + if doc.get('fullname') in allowed_nodes + } + results = searcher.search(query, limit=limit, filter=allowed_nodes) return [r['fullname'] for r in results] From e95f4293af7da84596e11d2fb2d888837fc5cc60 Mon Sep 17 00:00:00 2001 From: jochemb Date: Sun, 29 Apr 2018 22:16:46 +0200 Subject: [PATCH 04/13] scatter plot. --- transmart/api/v2/dashboard/dashboard.py | 62 ++++++++++++++++++-- transmart/api/v2/dashboard/hypercube.py | 2 +- transmart/api/v2/dashboard/tiles.py | 78 ++++++++++++++++++++++++- 3 files changed, 132 insertions(+), 10 deletions(-) diff --git a/transmart/api/v2/dashboard/dashboard.py b/transmart/api/v2/dashboard/dashboard.py index dfbffac..b127437 100644 --- a/transmart/api/v2/dashboard/dashboard.py +++ b/transmart/api/v2/dashboard/dashboard.py @@ -8,11 +8,11 @@ import ipywidgets as widgets -from ...commons import filter_tree +from .hypercube import Hypercube +from .tiles import HistogramTile, PieTile, CombinedPlot, ScatterPlot from ..constraint_widgets import ConceptPicker from ..query_constraints import ObservationConstraint, Queryable -from .hypercube import Hypercube -from .tiles import HistogramTile, PieTile +from ...commons import filter_tree logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -31,6 +31,7 @@ def __init__(self, api, patients: Queryable=None): self.api = api self.tiles = list() self.hypercube = Hypercube() + self.__linked_tile = None if isinstance(patients, Queryable): self.subject_set_id = api.create_patient_set(repr(patients), patients).get('id') @@ -46,6 +47,9 @@ def __init__(self, api, patients: Queryable=None): self.out = widgets.Box() self.out.layout.flex_flow = 'row wrap' + self.combination_out = widgets.Box() + self.combination_out.layout.flex_flow = 'row wrap' + self.cp = ConceptPicker(self.plotter, api, self.nodes) self.counter = widgets.IntProgress( value=10, min=0, max=10, step=1, @@ -53,7 +57,7 @@ def __init__(self, api, patients: Queryable=None): ) def get(self): - return widgets.VBox([self.out, self.counter, self.cp.get()]) + return widgets.VBox([self.out, self.counter, self.combination_out, self.cp.get()]) @debug_view.capture() def plotter(self, constraints): @@ -80,10 +84,56 @@ def plotter(self, constraints): self.register(tile) + @debug_view.capture() + def link_plotter(self, t1, t2): + + if isinstance(t1, HistogramTile) and isinstance(t2, HistogramTile): + tile = ScatterPlot(t1, t2, self) + + else: + return + + self.register(tile) + + @property + def linked_tile(self): + return self.__linked_tile + + @linked_tile.setter + @debug_view.capture() + def linked_tile(self, tile): + print(tile) + if isinstance(tile, (PieTile, HistogramTile)): + for t in self.tiles: + try: + t.link_btn.button_style = '' + except AttributeError: + pass + + if self.linked_tile is not None: + self.link_plotter(self.linked_tile, tile) + self.__linked_tile = None + + else: + self.__linked_tile = tile + tile.link_btn.button_style = 'info' + + elif tile is None: + self.__linked_tile = None + + else: + raise ValueError('Expected Tile object.') + def register(self, tile): self.tiles.append(tile) - with self.out.hold_sync(): - self.out.children = list(self.out.children) + [tile.get_fig()] + + if isinstance(tile, CombinedPlot): + box = self.combination_out + else: + box = self.out + + with box.hold_sync(): + box.children = list(box.children) + [tile.get_fig()] self.refresh() diff --git a/transmart/api/v2/dashboard/hypercube.py b/transmart/api/v2/dashboard/hypercube.py index c7adfe7..3bfed8a 100644 --- a/transmart/api/v2/dashboard/hypercube.py +++ b/transmart/api/v2/dashboard/hypercube.py @@ -78,5 +78,5 @@ def query(self, no_filter=False, **kwargs): if not no_filter and self.subject_mask is not None: bools &= self._subject_bool_mask - return self.data.loc[bools, value_columns] + return self.data.loc[bools, [patient_id, *value_columns]] diff --git a/transmart/api/v2/dashboard/tiles.py b/transmart/api/v2/dashboard/tiles.py index fd97799..1aeaf12 100644 --- a/transmart/api/v2/dashboard/tiles.py +++ b/transmart/api/v2/dashboard/tiles.py @@ -8,6 +8,8 @@ import bqplot as plt import ipywidgets as widgets +import pandas as pd +from IPython.display import display logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -40,11 +42,12 @@ def __init__(self, dash, title, concept, study=None): def _build_tile(self): self.fig = self.create_fig() logger.info('Created figure.') - buttons, btn1, btn2, destroy_btn = self.get_buttons() + buttons, btn1, self.link_btn, destroy_btn = self.get_buttons() self.comb = widgets.HBox( [self.fig, buttons], layout={'margin': '0px -20px 0px -18px'} ) + self.link_btn.on_click(self.linker()) destroy_btn.on_click(self.destroyer(self.comb)) self.fig.layout.height = '350px' @@ -63,6 +66,11 @@ def remove_fig(btn): self.dash.remove(fig_box) return remove_fig + def linker(self): + def callback(btn): + self.dash.linked_tile = self + return callback + def create_fig(self): raise NotImplementedError @@ -97,7 +105,7 @@ def get_buttons(self): buttons = widgets.VBox([ btn1, btn2, btn3 - ], layout={'margin': '60px 25px 0px -60px'}) + ], layout={'margin': '60px 30px 0px -60px'}) btn1.on_click(self._calc_selected_subjects) @@ -126,7 +134,7 @@ def create_fig(self): selector = plt.interacts.BrushIntervalSelector( scale=scale_x, marks=[hist], color='SteelBlue') - logger.info('Returning figure.JOCHEMJOCHEM') + logger.info('Returning figure.') return plt.Figure(axes=[ax_x, ax_y], marks=[hist], interaction=selector) @debug_view.capture(clear_output=False) @@ -214,3 +222,67 @@ def _calc_selected_subjects(self, *args): def refresh(self): self.fig.marks[0].selected = None + +class CombinedPlot: + + def __init__(self, t1, t2, dash): + self.t1 = t1 + self.t2 = t2 + self.dash = dash + self.df = None + self.mark = None + self.fig = None + + self.create_fig() + self.get_updates() + + def create_fig(self): + raise NotImplementedError + + def get_updates(self): + raise NotImplementedError + + def get_fig(self): + raise NotImplementedError + + def refresh(self): + pass + + +class ScatterPlot(CombinedPlot): + + @staticmethod + def get_values(tile): + df = tile.dash.hypercube.query(study=tile.study, concept=tile.concept) + non_empty = pd.notnull(df.numericValue) + df = df.loc[non_empty, ['patient.id', 'numericValue']] + return df + + def get_updates(self): + mask = self.dash.hypercube.subject_mask + + if mask is not None: + filter_ = self.df['patient.id'].isin(mask) + else: + filter_ = self.df.index + + with self.fig.hold_sync(): + self.mark.x = self.df.loc[filter_, 'numericValue_x'] + self.mark.y = self.df.loc[filter_, 'numericValue_y'] + + def create_fig(self): + t1_values = self.get_values(self.t1) + t2_values = self.get_values(self.t2) + self.df = t1_values.merge(t2_values, how='inner', on='patient.id') + + sc_x = plt.LinearScale() + sc_y = plt.LinearScale() + + self.mark = plt.Scatter(scales={'x': sc_x, 'y': sc_y}) + + ax_x = plt.Axis(scale=sc_x) + ax_y = plt.Axis(scale=sc_y, orientation='vertical') + self.fig = plt.Figure(marks=[self.mark], axes=[ax_x, ax_y]) + + def get_fig(self): + return self.fig From 4ec4343b49bd4922b87f160576c6f38725d0ff39 Mon Sep 17 00:00:00 2001 From: jochemb Date: Fri, 4 May 2018 09:25:33 +0200 Subject: [PATCH 05/13] deduplication --- requirements.txt | 4 +- transmart/api/v2/dashboard/dashboard.py | 19 +++++---- transmart/api/v2/dashboard/tiles.py | 53 +++++++++++++++---------- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/requirements.txt b/requirements.txt index 335107c..20d477e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,6 @@ requests pandas whoosh arrow -ipywidgets \ No newline at end of file +ipywidgets +IPython +bqplot diff --git a/transmart/api/v2/dashboard/dashboard.py b/transmart/api/v2/dashboard/dashboard.py index b127437..195ff9d 100644 --- a/transmart/api/v2/dashboard/dashboard.py +++ b/transmart/api/v2/dashboard/dashboard.py @@ -9,7 +9,11 @@ import ipywidgets as widgets from .hypercube import Hypercube -from .tiles import HistogramTile, PieTile, CombinedPlot, ScatterPlot +from .tiles import ( + HistogramTile, PieTile, CombinedPlot, ScatterPlot, ANIMATION_TIME, + NUMERIC_VALUE, STRING_VALUE +) + from ..constraint_widgets import ConceptPicker from ..query_constraints import ObservationConstraint, Queryable from ...commons import filter_tree @@ -19,11 +23,6 @@ debug_view = widgets.Output(layout={'border': '1px solid black'}) -_NUMERIC_VALUE = 'numericValue' -_STRING_VALUE = 'stringValue' - -ANIMATION_TIME = 400 - class Dashboard: @@ -71,13 +70,13 @@ def plotter(self, constraints): name = obs.dataframe['concept.name'][0] - if _NUMERIC_VALUE in obs.dataframe.columns: + if NUMERIC_VALUE in obs.dataframe.columns: tile = HistogramTile(self, name, concept=c.concept, study=c.study) - tile.set_values(obs.dataframe[_NUMERIC_VALUE]) + tile.set_values(obs.dataframe[NUMERIC_VALUE]) - elif _STRING_VALUE in obs.dataframe.columns: + elif STRING_VALUE in obs.dataframe.columns: tile = PieTile(self, name, concept=c.concept, study=c.study) - tile.set_values(obs.dataframe[_STRING_VALUE]) + tile.set_values(obs.dataframe[STRING_VALUE]) else: return diff --git a/transmart/api/v2/dashboard/tiles.py b/transmart/api/v2/dashboard/tiles.py index 1aeaf12..9d71bef 100644 --- a/transmart/api/v2/dashboard/tiles.py +++ b/transmart/api/v2/dashboard/tiles.py @@ -9,17 +9,16 @@ import bqplot as plt import ipywidgets as widgets import pandas as pd -from IPython.display import display logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) debug_view = widgets.Output(layout={'border': '1px solid black'}) -_NUMERIC_VALUE = 'numericValue' -_STRING_VALUE = 'stringValue' +NUMERIC_VALUE = 'numericValue' +STRING_VALUE = 'stringValue' -ANIMATION_TIME = 400 +ANIMATION_TIME = 1000 class Tile: @@ -118,7 +117,7 @@ def get_fig(self): class HistogramTile(Tile): - value_type = _NUMERIC_VALUE + value_type = NUMERIC_VALUE @debug_view.capture(clear_output=False) def create_fig(self): @@ -174,7 +173,7 @@ def refresh(self, *args): class PieTile(Tile): - value_type = _STRING_VALUE + value_type = STRING_VALUE @debug_view.capture(clear_output=False) def create_fig(self, *args): @@ -183,10 +182,13 @@ def create_fig(self, *args): tooltip_widget = plt.Tooltip(fields=['size', 'label']) pie = plt.Pie(tooltip=tooltip_widget, interactions={'click': 'select', 'hover': 'tooltip'}) - pie.radius = 100 - print('Returning figure.') + + pie.radius = 110 + pie.inner_radius = 65 + pie.font_weight = 'bold' pie.selected_style = {"opacity": "1", "stroke": "white", "stroke-width": "4"} - pie.unselected_style = {"opacity": "0.2"} + pie.unselected_style = {"opacity": "0.5"} + print('Returning figure.') return plt.Figure(marks=[pie]) @@ -194,11 +196,22 @@ def create_fig(self, *args): def set_values(self, values): pie = self.fig.marks[0] counts = values.value_counts() + labels = list(pie.labels) + + sizes = [] + for label in labels: + try: + sizes.append(counts.pop(label)) + except KeyError: + labels.remove(label) + + for index, element in counts.iteritems(): + sizes.append(element) + labels.append(index) with self.fig.hold_sync(): - pie.labels = pie.sizes = [] - pie.sizes = counts - pie.labels = list(counts.index) + pie.labels = labels + pie.sizes = sizes @debug_view.capture() def _calc_selected_subjects(self, *args): @@ -208,9 +221,7 @@ def _calc_selected_subjects(self, *args): pie = self.fig.marks[0] if pie.selected is not None: labels = {pie.labels[i] for i in pie.selected} - print(labels) selected = values.index[values.isin(labels)] - print(selected) self.selected_subjects = set(self.dash.hypercube.data.loc[selected, 'patient.id']) else: @@ -236,15 +247,15 @@ def __init__(self, t1, t2, dash): self.create_fig() self.get_updates() + def get_fig(self): + return self.fig + def create_fig(self): raise NotImplementedError def get_updates(self): raise NotImplementedError - def get_fig(self): - raise NotImplementedError - def refresh(self): pass @@ -278,11 +289,9 @@ def create_fig(self): sc_x = plt.LinearScale() sc_y = plt.LinearScale() - self.mark = plt.Scatter(scales={'x': sc_x, 'y': sc_y}) + self.mark = plt.Scatter(scales={'x': sc_x, 'y': sc_y}, default_size=16) - ax_x = plt.Axis(scale=sc_x) - ax_y = plt.Axis(scale=sc_y, orientation='vertical') + ax_x = plt.Axis(scale=sc_x, label=self.t1.title) + ax_y = plt.Axis(scale=sc_y, label=self.t2.title, orientation='vertical') self.fig = plt.Figure(marks=[self.mark], axes=[ax_x, ax_y]) - def get_fig(self): - return self.fig From 4212e96cc245170fed85f4610d71a1a0cbd73eb4 Mon Sep 17 00:00:00 2001 From: jochemb Date: Sun, 13 May 2018 19:11:31 +0200 Subject: [PATCH 06/13] Cleanup and docs --- transmart/api/v2/concept_search.py | 1 - transmart/api/v2/dashboard/dashboard.py | 21 ++- transmart/api/v2/dashboard/double_tiles.py | 82 +++++++++ transmart/api/v2/dashboard/hypercube.py | 23 ++- .../dashboard/{tiles.py => single_tiles.py} | 101 +++-------- transmart/api/v2/query_constraints.py | 2 +- transmart/api/v2/widgets/__init__.py | 2 + transmart/api/v2/widgets/concept_picker.py | 146 +++++++++++++++ .../constraint_details.py} | 169 +----------------- transmart/api/v2/widgets/shared.py | 36 ++++ 10 files changed, 332 insertions(+), 251 deletions(-) create mode 100644 transmart/api/v2/dashboard/double_tiles.py rename transmart/api/v2/dashboard/{tiles.py => single_tiles.py} (76%) create mode 100644 transmart/api/v2/widgets/__init__.py create mode 100644 transmart/api/v2/widgets/concept_picker.py rename transmart/api/v2/{constraint_widgets.py => widgets/constraint_details.py} (59%) create mode 100644 transmart/api/v2/widgets/shared.py diff --git a/transmart/api/v2/concept_search.py b/transmart/api/v2/concept_search.py index 5907166..f37067d 100644 --- a/transmart/api/v2/concept_search.py +++ b/transmart/api/v2/concept_search.py @@ -2,7 +2,6 @@ import os import shutil - from whoosh.fields import Schema, TEXT, NGRAM, NGRAMWORDS from whoosh.index import create_in, exists_in, open_dir from whoosh.qparser import MultifieldParser, FuzzyTermPlugin diff --git a/transmart/api/v2/dashboard/dashboard.py b/transmart/api/v2/dashboard/dashboard.py index 195ff9d..e98839d 100644 --- a/transmart/api/v2/dashboard/dashboard.py +++ b/transmart/api/v2/dashboard/dashboard.py @@ -9,12 +9,10 @@ import ipywidgets as widgets from .hypercube import Hypercube -from .tiles import ( - HistogramTile, PieTile, CombinedPlot, ScatterPlot, ANIMATION_TIME, - NUMERIC_VALUE, STRING_VALUE -) +from .single_tiles import HistogramTile, PieTile, ANIMATION_TIME, NUMERIC_VALUE, STRING_VALUE +from .double_tiles import CombinedPlot, ScatterPlot -from ..constraint_widgets import ConceptPicker +from ..widgets import ConceptPicker from ..query_constraints import ObservationConstraint, Queryable from ...commons import filter_tree @@ -60,6 +58,11 @@ def get(self): @debug_view.capture() def plotter(self, constraints): + """ + Add a new tile to dashboard by providing tree node constraints. + + :param constraints: constraints from tree node (concepts and study) + """ c = ObservationConstraint.from_tree_node(constraints) if self.subject_set_id is not None: @@ -85,12 +88,14 @@ def plotter(self, constraints): @debug_view.capture() def link_plotter(self, t1, t2): - + """ + Combine two tiles and add a new plot to the dashboard. + """ if isinstance(t1, HistogramTile) and isinstance(t2, HistogramTile): tile = ScatterPlot(t1, t2, self) else: - return + raise ValueError('Combination of tiles not supported.') self.register(tile) @@ -101,7 +106,7 @@ def linked_tile(self): @linked_tile.setter @debug_view.capture() def linked_tile(self, tile): - print(tile) + logger.info('Set tile link: {}'.format(tile)) if isinstance(tile, (PieTile, HistogramTile)): for t in self.tiles: try: diff --git a/transmart/api/v2/dashboard/double_tiles.py b/transmart/api/v2/dashboard/double_tiles.py new file mode 100644 index 0000000..e92f084 --- /dev/null +++ b/transmart/api/v2/dashboard/double_tiles.py @@ -0,0 +1,82 @@ +""" +* Copyright (c) 2015-2017 The Hyve B.V. +* This code is licensed under the GNU General Public License, +* version 3. +""" + +import logging + +import abc +import bqplot as plt +import ipywidgets as widgets +import pandas as pd + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +debug_view = widgets.Output(layout={'border': '1px solid black'}) + + +class CombinedPlot(abc.ABC): + + def __init__(self, t1, t2, dash): + self.t1 = t1 + self.t2 = t2 + self.dash = dash + self.df = None + self.mark = None + self.fig = None + + self.create_fig() + self.get_updates() + + def get_fig(self): + return self.fig + + @abc.abstractmethod + def create_fig(self): + pass + + @abc.abstractmethod + def get_updates(self): + pass + + def refresh(self): + pass + + +class ScatterPlot(CombinedPlot): + + @staticmethod + def get_values(tile): + df = tile.dash.hypercube.query(study=tile.study, concept=tile.concept) + non_empty = pd.notnull(df.numericValue) + df = df.loc[non_empty, ['patient.id', 'numericValue']] + return df + + def get_updates(self): + mask = self.dash.hypercube.subject_mask + + if mask is not None: + filter_ = self.df['patient.id'].isin(mask) + else: + filter_ = self.df.index + + with self.fig.hold_sync(): + self.mark.x = self.df.loc[filter_, 'numericValue_x'] + self.mark.y = self.df.loc[filter_, 'numericValue_y'] + + def create_fig(self): + t1_values = self.get_values(self.t1) + t2_values = self.get_values(self.t2) + self.df = t1_values.merge(t2_values, how='inner', on='patient.id') + + sc_x = plt.LinearScale() + sc_y = plt.LinearScale() + + self.mark = plt.Scatter(scales={'x': sc_x, 'y': sc_y}, default_size=16) + + ax_x = plt.Axis(scale=sc_x, label=self.t1.title) + ax_y = plt.Axis(scale=sc_y, label=self.t2.title, orientation='vertical') + self.fig = plt.Figure(marks=[self.mark], axes=[ax_x, ax_y]) + diff --git a/transmart/api/v2/dashboard/hypercube.py b/transmart/api/v2/dashboard/hypercube.py index 3bfed8a..cbe0d12 100644 --- a/transmart/api/v2/dashboard/hypercube.py +++ b/transmart/api/v2/dashboard/hypercube.py @@ -39,6 +39,10 @@ def __init__(self): @property def subject_mask(self): + """ + Controls a boolean mask on subjects. Setting this reduces the + values returned by self.query() method. + """ return self.__subjects_mask @subject_mask.setter @@ -59,13 +63,24 @@ def add_variable(self, df): self.data = self.data.append(sub_set, ignore_index=True) self.total_subjects = len(self.data[patient_id].unique()) - def query(self, no_filter=False, **kwargs): + def query(self, no_filter=False, **constraint_keywords): + """ + Query the hypercube for all values currently present based on constraints. + Constraints have to be provided as keyword arguments where the value is either + either a string that matches the value to look for, or a collection of strings + to look for. Collections can be provided as a set, list, or pd.Series. + + :param no_filter: if this hypercube has a subject + mask, set this to True to bypass it. + :param constraint_keywords: Possible keywords [concept, study, trial_visit, subject, start_time]. + :return: pd.Dataframe with values. + """ expressions = [] for kw, column in dimensions.items(): - if kw not in kwargs: + if kw not in constraint_keywords: continue - parameter = kwargs.get(kw) - if isinstance(parameter, (pd.Series, list)): + parameter = constraint_keywords.get(kw) + if isinstance(parameter, (pd.Series, list, set)): expr = self.data[column].isin(parameter) else: expr = self.data[column] == parameter diff --git a/transmart/api/v2/dashboard/tiles.py b/transmart/api/v2/dashboard/single_tiles.py similarity index 76% rename from transmart/api/v2/dashboard/tiles.py rename to transmart/api/v2/dashboard/single_tiles.py index 9d71bef..25d9d67 100644 --- a/transmart/api/v2/dashboard/tiles.py +++ b/transmart/api/v2/dashboard/single_tiles.py @@ -6,9 +6,9 @@ import logging +import abc import bqplot as plt import ipywidgets as widgets -import pandas as pd logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -21,7 +21,11 @@ ANIMATION_TIME = 1000 -class Tile: +class Tile(abc.ABC): + """ + Abstract base class that can be implemented to create tiles for + to be registered to a dashboard instance. + """ def __init__(self, dash, title, concept, study=None): self.dash = dash @@ -63,28 +67,35 @@ def remove_fig(btn): with self.fake_out: self.dash.tiles.remove(self) self.dash.remove(fig_box) + return remove_fig def linker(self): def callback(btn): self.dash.linked_tile = self + return callback + @abc.abstractmethod def create_fig(self): - raise NotImplementedError + pass + @abc.abstractmethod def set_values(self, values): - raise NotImplementedError + pass + @abc.abstractmethod def _calc_selected_subjects(self, *args): - raise NotImplementedError + pass + @abc.abstractmethod def refresh(self): - raise NotImplementedError + pass @property + @abc.abstractmethod def value_type(self): - raise NotImplementedError + pass def get_updates(self): subset = self.dash.hypercube.query(concept=self.concept, study=self.study) @@ -164,12 +175,13 @@ def refresh(self, *args): values = self.fig.marks[0].sample with self.fig.hold_sync(): - self.set_values([]) # TODO better way to make brush selector reliable. + self.set_values([]) # TODO better way to make brush selector reliable, and removable. self.set_values(values) - self.fig.interaction.reset() - self.fig.interaction.selected = [] - print('Selected: ', self.fig.interaction.selected) + self.fig.interaction.reset() + self.fig.interaction.selected = [] + + logger.info('Selected: ', self.fig.interaction.selected) class PieTile(Tile): @@ -177,7 +189,7 @@ class PieTile(Tile): @debug_view.capture(clear_output=False) def create_fig(self, *args): - print('Creating figure.') + logger.info('Creating figure.') tooltip_widget = plt.Tooltip(fields=['size', 'label']) pie = plt.Pie(tooltip=tooltip_widget, @@ -188,7 +200,7 @@ def create_fig(self, *args): pie.font_weight = 'bold' pie.selected_style = {"opacity": "1", "stroke": "white", "stroke-width": "4"} pie.unselected_style = {"opacity": "0.5"} - print('Returning figure.') + logger.info('Returning figure.') return plt.Figure(marks=[pie]) @@ -232,66 +244,3 @@ def _calc_selected_subjects(self, *args): def refresh(self): self.fig.marks[0].selected = None - - -class CombinedPlot: - - def __init__(self, t1, t2, dash): - self.t1 = t1 - self.t2 = t2 - self.dash = dash - self.df = None - self.mark = None - self.fig = None - - self.create_fig() - self.get_updates() - - def get_fig(self): - return self.fig - - def create_fig(self): - raise NotImplementedError - - def get_updates(self): - raise NotImplementedError - - def refresh(self): - pass - - -class ScatterPlot(CombinedPlot): - - @staticmethod - def get_values(tile): - df = tile.dash.hypercube.query(study=tile.study, concept=tile.concept) - non_empty = pd.notnull(df.numericValue) - df = df.loc[non_empty, ['patient.id', 'numericValue']] - return df - - def get_updates(self): - mask = self.dash.hypercube.subject_mask - - if mask is not None: - filter_ = self.df['patient.id'].isin(mask) - else: - filter_ = self.df.index - - with self.fig.hold_sync(): - self.mark.x = self.df.loc[filter_, 'numericValue_x'] - self.mark.y = self.df.loc[filter_, 'numericValue_y'] - - def create_fig(self): - t1_values = self.get_values(self.t1) - t2_values = self.get_values(self.t2) - self.df = t1_values.merge(t2_values, how='inner', on='patient.id') - - sc_x = plt.LinearScale() - sc_y = plt.LinearScale() - - self.mark = plt.Scatter(scales={'x': sc_x, 'y': sc_y}, default_size=16) - - ax_x = plt.Axis(scale=sc_x, label=self.t1.title) - ax_y = plt.Axis(scale=sc_y, label=self.t2.title, orientation='vertical') - self.fig = plt.Figure(marks=[self.mark], axes=[ax_x, ax_y]) - diff --git a/transmart/api/v2/query_constraints.py b/transmart/api/v2/query_constraints.py index 87f18fc..892d5a9 100644 --- a/transmart/api/v2/query_constraints.py +++ b/transmart/api/v2/query_constraints.py @@ -9,7 +9,7 @@ from functools import wraps from ..commons import date_to_timestamp, INPUT_DATE_FORMATS -from .constraint_widgets import ConceptPicker, ConstraintWidget +from .widgets import ConceptPicker, ConstraintWidget END_OF_DAY_FMT = 'YYYY-MM-DDT23:59:59ZZ' START_OF_DAY_FMT = 'YYYY-MM-DDT00:00:00ZZ' diff --git a/transmart/api/v2/widgets/__init__.py b/transmart/api/v2/widgets/__init__.py new file mode 100644 index 0000000..cad5dde --- /dev/null +++ b/transmart/api/v2/widgets/__init__.py @@ -0,0 +1,2 @@ +from .concept_picker import ConceptPicker +from .constraint_details import ConstraintWidget diff --git a/transmart/api/v2/widgets/concept_picker.py b/transmart/api/v2/widgets/concept_picker.py new file mode 100644 index 0000000..0a6c3a3 --- /dev/null +++ b/transmart/api/v2/widgets/concept_picker.py @@ -0,0 +1,146 @@ +""" +* Copyright (c) 2015-2017 The Hyve B.V. +* This code is licensed under the GNU General Public License, +* version 3. +""" + +import ipywidgets + +from .shared import create_toggle + +AGG_NUM = 'numericalValueAggregates' +AGG_CAT = 'categoricalValueAggregates' + +MAX_OPTIONS = 100 + + +class ConceptPicker: + + result_count_template = 'Number of entries: {}' + table_template = """ + + """ + + def __init__(self, target, api, allowed_nodes: set=None): + + self.target = target + self.api = api + self.allowed_nodes = allowed_nodes + + nodes = allowed_nodes or self.api.tree_dict.keys() + + self.list_of_default_options = sorted(nodes) + self.no_filter_len = len(self.list_of_default_options) + + self.result_count = ipywidgets.HTML( + value=self.result_count_template.format(self.no_filter_len), + layout={'width': '175px'} + ) + self.result_text = ipywidgets.HTML(layout={'width': '49%'}) + + self.search_bar = self._build_search_bar() + self.concept_list = self._build_concept_list() + + # Necessary output for Jlab + out = ipywidgets.Output() + + def confirm_tree_node(btn): + with out: + try: + node = self.api.tree_dict.get(self.concept_list.value) + self.target(node.get('constraint')) + except ValueError: + pass + + self._confirm = ipywidgets.Button(description='Confirm', icon='check') + self._confirm.on_click(confirm_tree_node) + + self.box_and_picker = ipywidgets.VBox([ + ipywidgets.HBox([self.search_bar, self.result_count, self._confirm]), + ipywidgets.HBox([self.concept_list, self.result_text]) + ]) + + self.concept_picker = ipywidgets.VBox([create_toggle(self.box_and_picker, out), self.box_and_picker]) + + def _build_search_bar(self): + def search_watcher(change): + x = change.get('new') + if len(x) > 2: + self.concept_list.options = self.api.search_tree_node( + x, + limit=MAX_OPTIONS, + allowed_nodes=self.allowed_nodes + ) + count = len(self.concept_list.options) + + if count == MAX_OPTIONS: + count = str(count) + '+' + + self.result_count.value = self.result_count_template.format(count) + + else: + self.concept_list.options = self.list_of_default_options[:MAX_OPTIONS] + self.result_count.value = self.result_count_template.format(self.no_filter_len) + + search_bar = ipywidgets.Text(placeholder='Type something') + search_bar.observe(search_watcher, 'value') + return search_bar + + def _build_concept_list(self): + def concept_list_watcher(change): + x = change.get('new') + if x: + node = self.api.tree_dict.get(x) + metadata = dict(node.get('metadata', {})) + d = { + 'path': node.get('conceptPath'), + 'type_': node.get('type'), + 'study_id': node.get('studyId'), + 'name': node.get('name'), + 'metadata': '
'.join(['{}: {}'.format(k, v) for k, v in metadata.items()]) + } + + self.result_text.value = self.table_template.format(**d) + + concept_list = ipywidgets.Select( + options=self.list_of_default_options[:MAX_OPTIONS], + rows=10, + disabled=False, + continous_update=False, + layout={'width': '50%'} + ) + + concept_list.observe(concept_list_watcher, 'value') + return concept_list + + def get(self): + return self.concept_picker diff --git a/transmart/api/v2/constraint_widgets.py b/transmart/api/v2/widgets/constraint_details.py similarity index 59% rename from transmart/api/v2/constraint_widgets.py rename to transmart/api/v2/widgets/constraint_details.py index 62ec683..ebeafda 100644 --- a/transmart/api/v2/constraint_widgets.py +++ b/transmart/api/v2/widgets/constraint_details.py @@ -1,3 +1,9 @@ +""" +* Copyright (c) 2015-2017 The Hyve B.V. +* This code is licensed under the GNU General Public License, +* version 3. +""" + import math from datetime import datetime @@ -6,173 +12,14 @@ from IPython.display import HTML, display from ipywidgets import VBox, HBox +from .shared import create_toggle, widget_off, widget_on + AGG_NUM = 'numericalValueAggregates' AGG_CAT = 'categoricalValueAggregates' -MAX_OPTIONS = 100 -CONCEPT_DELAY = 2 DEFAULT_VISIT = [{'relTimeLabel': 'General', 'id': 0}] -def widget_on(widget): - widget.disabled = False - widget.layout.visibility = 'initial' - widget.layout.max_height = None - - -def widget_off(widget): - widget.layout.visibility = 'hidden' - widget.layout.max_height = '0' - - -def toggle_visibility(widget): - if widget.layout.max_height == '0': - widget_on(widget) - else: - widget_off(widget) - - -def create_toggle(widget, out): - def toggle(btn): - btn.description = 'Show' if btn.description == 'Hide' else 'Hide' - with out: - toggle_visibility(widget) - - toggle_btn = widgets.Button(description='Hide') - toggle_btn.on_click(toggle) - return toggle_btn - - -class ConceptPicker: - - result_count_template = 'Number of entries: {}' - table_template = """ - - """ - - def __init__(self, target, api, allowed_nodes: set=None): - - self.target = target - self.api = api - self.allowed_nodes = allowed_nodes - - nodes = allowed_nodes or self.api.tree_dict.keys() - - self.list_of_default_options = sorted(nodes) - self.no_filter_len = len(self.list_of_default_options) - - self.result_count = widgets.HTML( - value=self.result_count_template.format(self.no_filter_len), - layout={'width': '175px'} - ) - self.result_text = widgets.HTML(layout={'width': '49%'}) - - self.search_bar = self._build_search_bar() - self.concept_list = self._build_concept_list() - - # Necessary output for Jlab - out = widgets.Output() - - def confirm_tree_node(btn): - with out: - try: - node = self.api.tree_dict.get(self.concept_list.value) - self.target(node.get('constraint')) - except ValueError: - pass - - self._confirm = widgets.Button(description='Confirm', icon='check') - self._confirm.on_click(confirm_tree_node) - - self.box_and_picker = VBox([ - HBox([self.search_bar, self.result_count, self._confirm]), - HBox([self.concept_list, self.result_text]) - ]) - - self.concept_picker = VBox([create_toggle(self.box_and_picker, out), self.box_and_picker]) - - def _build_search_bar(self): - def search_watcher(change): - x = change.get('new') - if len(x) > 2: - self.concept_list.options = self.api.search_tree_node(x, - limit=MAX_OPTIONS, - allowed_nodes=self.allowed_nodes) - count = len(self.concept_list.options) - - if count == MAX_OPTIONS: - count = str(count) + '+' - - self.result_count.value = self.result_count_template.format(count) - - else: - self.concept_list.options = self.list_of_default_options[:MAX_OPTIONS] - self.result_count.value = self.result_count_template.format(self.no_filter_len) - - search_bar = widgets.Text(placeholder='Type something') - search_bar.observe(search_watcher, 'value') - return search_bar - - def _build_concept_list(self): - def concept_list_watcher(change): - x = change.get('new') - if x: - node = self.api.tree_dict.get(x) - metadata = dict(node.get('metadata', {})) - d = { - 'path': node.get('conceptPath'), - 'type_': node.get('type'), - 'study_id': node.get('studyId'), - 'name': node.get('name'), - 'metadata': '
'.join(['{}: {}'.format(k, v) for k, v in metadata.items()]) - } - - self.result_text.value = self.table_template.format(**d) - - concept_list = widgets.Select( - options=self.list_of_default_options[:MAX_OPTIONS], - rows=10, - disabled=False, - continous_update=False, - layout={'width': '50%'} - ) - - concept_list.observe(concept_list_watcher, 'value') - return concept_list - - def get(self): - return self.concept_picker - - class ConstraintWidget: html_style = """ diff --git a/transmart/api/v2/widgets/shared.py b/transmart/api/v2/widgets/shared.py new file mode 100644 index 0000000..eec8f17 --- /dev/null +++ b/transmart/api/v2/widgets/shared.py @@ -0,0 +1,36 @@ +""" +* Copyright (c) 2015-2017 The Hyve B.V. +* This code is licensed under the GNU General Public License, +* version 3. +""" + +import ipywidgets + + +def widget_on(widget): + widget.disabled = False + widget.layout.visibility = 'initial' + widget.layout.max_height = None + + +def widget_off(widget): + widget.layout.visibility = 'hidden' + widget.layout.max_height = '0' + + +def toggle_visibility(widget): + if widget.layout.max_height == '0': + widget_on(widget) + else: + widget_off(widget) + + +def create_toggle(widget, out): + def toggle(btn): + btn.description = 'Show' if btn.description == 'Hide' else 'Hide' + with out: + toggle_visibility(widget) + + toggle_btn = ipywidgets.Button(description='Hide') + toggle_btn.on_click(toggle) + return toggle_btn From a1180615ee255d7d77da1c2ffc9911ab3e31495d Mon Sep 17 00:00:00 2001 From: jochemb Date: Sun, 13 May 2018 20:36:33 +0200 Subject: [PATCH 07/13] More cleanup and docs --- tests/v2_tests.py | 2 +- transmart/api/commons.py | 24 ++ transmart/api/v2/constraints/__init__.py | 2 + transmart/api/v2/constraints/atomic.py | 201 ++++++++++ .../composite.py} | 358 ++++-------------- transmart/api/v2/dashboard/dashboard.py | 4 +- transmart/api/v2/tm_api_v2.py | 29 +- 7 files changed, 327 insertions(+), 293 deletions(-) create mode 100644 transmart/api/v2/constraints/__init__.py create mode 100644 transmart/api/v2/constraints/atomic.py rename transmart/api/v2/{query_constraints.py => constraints/composite.py} (68%) diff --git a/tests/v2_tests.py b/tests/v2_tests.py index 13efa82..aa423f0 100644 --- a/tests/v2_tests.py +++ b/tests/v2_tests.py @@ -14,7 +14,7 @@ def test_get_api(self): @retry def test_get_patients(self): - patients = self.api.get_patients().json.get('patients') + patients = self.api.patients().json.get('patients') self.assertEqual(4, len(patients)) @retry diff --git a/transmart/api/commons.py b/transmart/api/commons.py index cf35ef5..652f2dd 100644 --- a/transmart/api/commons.py +++ b/transmart/api/commons.py @@ -1,6 +1,7 @@ from datetime import datetime import arrow +from functools import wraps from hashlib import sha1 INPUT_DATE_FORMATS = ['D-M-YYYY', 'YYYY-M-D'] @@ -54,3 +55,26 @@ def filter_tree(tree_dict, counts_per_study_and_concept): if v.get('conceptCode') in concepts and v.get('studyId') in (*studies, None) } + + +def input_check(types): + """ + :param types: tuple of allowed types. + :return: decorator that validates input of property setter. + """ + if not isinstance(types, tuple): + msg = 'Input check types has to be tuple, got {!r}'.format(type(types)) + raise ValueError(msg) + + def input_check_decorator(func): + @wraps(func) + def wrapper(self, value): + + if value is not None: + if type(value) not in types: + raise ValueError('Expected type {!r} for {!r}, but got {!r}'. + format(types, func.__name__, type(value))) + return func(self, value) + return wrapper + return input_check_decorator + diff --git a/transmart/api/v2/constraints/__init__.py b/transmart/api/v2/constraints/__init__.py new file mode 100644 index 0000000..3b37755 --- /dev/null +++ b/transmart/api/v2/constraints/__init__.py @@ -0,0 +1,2 @@ +from .composite import ObservationConstraint, RelationConstraint, GroupConstraint, Queryable +from .atomic import BiomarkerConstraint diff --git a/transmart/api/v2/constraints/atomic.py b/transmart/api/v2/constraints/atomic.py new file mode 100644 index 0000000..fd0aa09 --- /dev/null +++ b/transmart/api/v2/constraints/atomic.py @@ -0,0 +1,201 @@ +""" +* Copyright (c) 2015-2017 The Hyve B.V. +* This code is licensed under the GNU General Public License, +* version 3. +""" + +import json + +import arrow +import abc + +from ...commons import date_to_timestamp, input_check + +END_OF_DAY_FMT = 'YYYY-MM-DDT23:59:59ZZ' +START_OF_DAY_FMT = 'YYYY-MM-DDT00:00:00ZZ' + + +class Constraint(abc.ABC): + + def __init__(self, value): + self.value = value + + @property + @abc.abstractmethod + def type_(self): + pass + + @property + @abc.abstractmethod + def val_name(self): + pass + + def __str__(self): + return json.dumps(self.json()) + + def json(self): + return {'type': self.type_, self.val_name: self.value} + + +class StudyConstraint(Constraint): + type_ = 'study_name' + val_name = 'studyId' + + +class SubjectSetConstraint(Constraint): + type_ = 'patient_set' + val_name = 'patientSetId' + + +class ConceptCodeConstraint(Constraint): + type_ = 'concept' + val_name = 'conceptCode' + + +class BiomarkerConstraint: + + def __init__(self, biomarkers: list = None, biomarker_type='genes'): + self.type_ = biomarker_type + self.__biomarkers = None + + self.biomarkers = biomarkers + + @property + def biomarkers(self): + return self.__biomarkers + + @biomarkers.setter + @input_check((list,)) + def biomarkers(self, value): + self.__biomarkers = value + + def __str__(self): + return json.dumps(self.json()) + + def json(self): + d_ = dict(type='biomarker', + biomarkerType=self.type_) + if self.biomarkers: + d_['params'] = {"names": self.biomarkers} + + return d_ + + +class ValueConstraint(abc.ABC): + + def __init__(self, value): + self.value = value + + @property + @abc.abstractmethod + def value_type_(self): + pass + + @property + @abc.abstractmethod + def operator(self): + pass + + def json(self): + return {'type': 'value', + 'valueType': self.value_type_, + 'operator': self.operator, + 'value': self.value} + + +class MinValueConstraint(ValueConstraint): + value_type_ = 'NUMERIC' + operator = '>=' + + +class MaxValueConstraint(ValueConstraint): + value_type_ = 'NUMERIC' + operator = '<=' + + +class MinDateValueConstraint(MinValueConstraint): + modifier = 0 + + def json(self): + tmp_ = self.value + try: + self.value = date_to_timestamp(self.value) + self.value += self.modifier + return super().json() + + finally: + self.value = tmp_ + + +class MaxDateValueConstraint(MaxValueConstraint, MinDateValueConstraint): + modifier = 24 * 60 * 60 * 1000 - 1 # add 23:59:59:999 in ms + + +class ValueListConstraint(ValueConstraint): + value_type_ = 'STRING' + operator = '=' + + def json(self): + return {'type': 'or', + 'args': [ + {'type': 'value', + 'valueType': self.value_type_, + 'operator': self.operator, + 'value': value} + for value in self.value] + } + + +class TrialVisitConstraint: + + def __init__(self, values): + self.values = values if isinstance(values, list) else [values] + + def __str__(self): + return json.dumps(self.json()) + + def json(self): + return {'type': 'or', + 'args': [ + {'type': 'field', + 'field': { + 'dimension': 'trial visit', + 'fieldName': 'id', + 'type': 'NUMERIC'}, + 'operator': '=', + 'value': value} + for value in self.values] + } + + +class StartTimeConstraint: + operator = '<-->' + n_dates = 2 + date_fmt = (START_OF_DAY_FMT, END_OF_DAY_FMT) + + def __init__(self, values): + self.values = values if isinstance(values, list) else [values] + if len(self.values) != self.n_dates: + raise ValueError('Expected {} dates, but got {}, for {!r}.'. + format(self.n_dates, len(self.values), self.__class__)) + + def json(self): + return {'type': 'time', + 'field': { + 'dimension': 'start time', + 'fieldName': 'startDate', + 'type': 'DATE'}, + 'operator': self.operator, + 'values': [arrow.get(d).format(fmt) for d, fmt in zip(self.values, self.date_fmt)]} + + +class StartTimeBeforeConstraint(StartTimeConstraint): + operator = '<-' + n_dates = 1 + date_fmt = (END_OF_DAY_FMT,) + + +class StartTimeAfterConstraint(StartTimeConstraint): + operator = '->' + n_dates = 1 + date_fmt = (START_OF_DAY_FMT,) diff --git a/transmart/api/v2/query_constraints.py b/transmart/api/v2/constraints/composite.py similarity index 68% rename from transmart/api/v2/query_constraints.py rename to transmart/api/v2/constraints/composite.py index 892d5a9..08c9ac3 100644 --- a/transmart/api/v2/query_constraints.py +++ b/transmart/api/v2/constraints/composite.py @@ -5,42 +5,19 @@ """ import json +import abc import arrow - from functools import wraps -from ..commons import date_to_timestamp, INPUT_DATE_FORMATS -from .widgets import ConceptPicker, ConstraintWidget -END_OF_DAY_FMT = 'YYYY-MM-DDT23:59:59ZZ' -START_OF_DAY_FMT = 'YYYY-MM-DDT00:00:00ZZ' +from . import atomic +from ..widgets import ConceptPicker, ConstraintWidget +from ...commons import INPUT_DATE_FORMATS, input_check class InvalidConstraint(Exception): pass -def input_check(types): - """ - :param types: tuple of allowed types. - :return: decorator that validates input of property setter. - """ - if not isinstance(types, tuple): - msg = 'Input check types has to be tuple, got {!r}'.format(type(types)) - raise ValueError(msg) - - def input_check_decorator(func): - @wraps(func) - def wrapper(self, value): - - if value is not None: - if type(value) not in types: - raise ValueError('Expected type {!r} for {!r}, but got {!r}'. - format(types, func.__name__, type(value))) - return func(self, value) - return wrapper - return input_check_decorator - - def bind_widget_tuple(target, pos): """ :param target: widget to bind to. @@ -94,227 +71,102 @@ def wrapper(self, value): bind_widget_date = bind_widget_factory(lambda x: arrow.get(x, INPUT_DATE_FORMATS).date()) -class Query: - """ Utility to build queries for transmart v2 api. """ - - def __init__(self, handle=None, method='GET', params=None, hal=False, json=None): - self.handle = handle - self.method = method - self.hal = hal - self.params = params - self.json = json - - @property - def headers(self): - return {'Accept': 'application/{};charset=UTF-8'.format('hal+json' if self.hal else 'json')} - - -class Constraint: - - def __init__(self, value): - self.value = value - - @property - def type_(self): - raise NotImplementedError - - @property - def val_name(self): - raise NotImplementedError - - def __str__(self): - return json.dumps(self.json()) - - def json(self): - return {'type': self.type_, self.val_name: self.value} - - -class StudyConstraint(Constraint): - type_ = 'study_name' - val_name = 'studyId' - - -class SubjectSetConstraint(Constraint): - type_ = 'patient_set' - val_name = 'patientSetId' - - -class ConceptCodeConstraint(Constraint): - type_ = 'concept' - val_name = 'conceptCode' - - -class BiomarkerConstraint: - - def __init__(self, biomarkers: list = None, biomarker_type='genes'): - self.type_ = biomarker_type - self.__biomarkers = None - - self.biomarkers = biomarkers - - @property - def biomarkers(self): - return self.__biomarkers - - @biomarkers.setter - @input_check((list, )) - def biomarkers(self, value): - self.__biomarkers = value - - def __str__(self): - return json.dumps(self.json()) - - def json(self): - d_ = dict(type='biomarker', - biomarkerType=self.type_) - if self.biomarkers: - d_['params'] = {"names": self.biomarkers} - - return d_ - - -class ValueConstraint: - - def __init__(self, value): - self.value = value - - @property - def value_type_(self): - raise NotImplementedError - - @property - def operator(self): - raise NotImplementedError - - def json(self): - return {'type': 'value', - 'valueType': self.value_type_, - 'operator': self.operator, - 'value': self.value} - - -class MinValueConstraint(ValueConstraint): - value_type_ = 'NUMERIC' - operator = '>=' - - -class MaxValueConstraint(ValueConstraint): - value_type_ = 'NUMERIC' - operator = '<=' - - -class MinDateValueConstraint(MinValueConstraint): - modifier = 0 +class Queryable(abc.ABC): + @abc.abstractmethod def json(self): - tmp_ = self.value - try: - self.value = date_to_timestamp(self.value) - self.value += self.modifier - return super().json() + pass - finally: - self.value = tmp_ +class Grouper(abc.ABC): -class MaxDateValueConstraint(MaxValueConstraint, MinDateValueConstraint): - modifier = 24 * 60 * 60 * 1000 - 1 # add 23:59:59:999 in ms + def __and__(self, other): + return self.__grouper_logic(other, is_and=True) + def __or__(self, other): + return self.__grouper_logic(other, is_and=False) -class ValueListConstraint(ValueConstraint): - value_type_ = 'STRING' - operator = '=' + def __grouper_logic(self, other, is_and): + """ ObsConstraint """ + my_type, not_my_type = ('and', 'or') if is_and else ('or', 'and') - def json(self): - return {'type': 'or', - 'args': [ - {'type': 'value', - 'valueType': self.value_type_, - 'operator': self.operator, - 'value': value} - for value in self.value] - } + if isinstance(other, Grouper): + return GroupConstraint([self, other], my_type) + if isinstance(other, GroupConstraint): + if other.group_type == my_type: + other.items.append(self) + return other + if other.group_type == not_my_type: + return GroupConstraint([self, other], my_type) -class TrialVisitConstraint: - def __init__(self, values): - self.values = values if isinstance(values, list) else [values] +class GroupConstraint(Queryable): + """ + Group constraint combines ObservationConstraints and results in a + subject set. It operates using and/or. + """ + def __init__(self, items, group_type): + self.items = items + self.group_type = group_type def __str__(self): return json.dumps(self.json()) - def json(self): - return {'type': 'or', - 'args': [ - {'type': 'field', - 'field': { - 'dimension': 'trial visit', - 'fieldName': 'id', - 'type': 'NUMERIC'}, - 'operator': '=', - 'value': value} - for value in self.values] - } - - -class StartTimeConstraint: - operator = '<-->' - n_dates = 2 - date_fmt = (START_OF_DAY_FMT, END_OF_DAY_FMT) - - def __init__(self, values): - self.values = values if isinstance(values, list) else [values] - if len(self.values) != self.n_dates: - raise ValueError('Expected {} dates, but got {}, for {!r}.'. - format(self.n_dates, len(self.values), self.__class__)) - - def json(self): - return {'type': 'time', - 'field': { - 'dimension': 'start time', - 'fieldName': 'startDate', - 'type': 'DATE'}, - 'operator': self.operator, - 'values': [arrow.get(d).format(fmt) for d, fmt in zip(self.values, self.date_fmt)]} - + def __repr__(self): + return '{}({})'.format(self.group_type, self.items) -class StartTimeBeforeConstraint(StartTimeConstraint): - operator = '<-' - n_dates = 1 - date_fmt = (END_OF_DAY_FMT, ) + def __and__(self, other): + return self.__grouper_logic(other, is_and=True) + def __or__(self, other): + return self.__grouper_logic(other, is_and=False) -class StartTimeAfterConstraint(StartTimeConstraint): - operator = '->' - n_dates = 1 - date_fmt = (START_OF_DAY_FMT, ) + def __grouper_logic(self, other, is_and): + my_type, not_my_type = ('and', 'or') if is_and else ('or', 'and') + if isinstance(other, Grouper): + if self.group_type == my_type: + self.items.append(other) + return self + else: + return GroupConstraint([self, other], my_type) -class Queryable: + elif isinstance(other, self.__class__): + if other.group_type == my_type: + other.items += self.items + return other + if other.group_type == not_my_type and self.group_type == my_type: + self.items.append(other) + return self + else: + return GroupConstraint([self, other], my_type) def json(self): - raise NotImplementedError + return { + 'type': self.group_type, + 'args': [item.subselect() for item in self.items] + } -class ObservationConstraint(Queryable): +class ObservationConstraint(Queryable, Grouper): """ Represents constraints on observation level. This is the set of observations that adhere to all criteria specified. A patient set based on these constraints can be combined with other sets to create complex queries. """ - params = {'concept': ConceptCodeConstraint, - 'study': StudyConstraint, - 'trial_visit': TrialVisitConstraint, - 'min_value': MinValueConstraint, - 'max_value': MaxValueConstraint, - 'min_date_value': MinDateValueConstraint, - 'max_date_value': MaxDateValueConstraint, - 'value_list': ValueListConstraint, - 'min_start_date': StartTimeAfterConstraint, - 'max_start_date': StartTimeBeforeConstraint, - 'subject_set_id': SubjectSetConstraint, + params = {'concept': atomic.ConceptCodeConstraint, + 'study': atomic.StudyConstraint, + 'trial_visit': atomic.TrialVisitConstraint, + 'min_value': atomic.MinValueConstraint, + 'max_value': atomic.MaxValueConstraint, + 'min_date_value': atomic.MinDateValueConstraint, + 'max_date_value': atomic.MaxDateValueConstraint, + 'value_list': atomic.ValueListConstraint, + 'min_start_date': atomic.StartTimeAfterConstraint, + 'max_start_date': atomic.StartTimeBeforeConstraint, + 'subject_set_id': atomic.SubjectSetConstraint, } def __init__(self, @@ -408,25 +260,6 @@ def __repr__(self): def __str__(self): return json.dumps(self.json()) - def __and__(self, other): - return self.__grouper_logic(other, is_and=True) - - def __or__(self, other): - return self.__grouper_logic(other, is_and=False) - - def __grouper_logic(self, other, is_and): - my_type, not_my_type = ('and', 'or') if is_and else ('or', 'and') - - if isinstance(other, self.__class__): - return GroupConstraint([self, other], my_type) - - if isinstance(other, GroupConstraint): - if other.group_type == my_type: - other.items.append(self) - return other - if other.group_type == not_my_type: - return GroupConstraint([self, other], my_type) - def json(self): args = [] @@ -658,7 +491,7 @@ def fetch_updates(self): self._details_widget.disable_all() self._details_widget.update_obs_repr() - agg_response = self.api.get_observations.aggregates_per_concept(self) + agg_response = self.api.observations.aggregates_per_concept(self) self._aggregates = agg_response.get('aggregatesPerConcept', {}).get(self.concept, {}) self._details_widget.update_from_aggregates(self._aggregates) @@ -685,51 +518,10 @@ def interact(self): return self._details_widget.get() -class GroupConstraint(Queryable): - def __init__(self, items, group_type): - self.items = items - self.group_type = group_type - - def __str__(self): - return json.dumps(self.json()) - - def __repr__(self): - return '{}({})'.format(self.group_type, self.items) - - def __and__(self, other): - return self.__grouper_logic(other, is_and=True) - - def __or__(self, other): - return self.__grouper_logic(other, is_and=False) - - def __grouper_logic(self, other, is_and): - my_type, not_my_type = ('and', 'or') if is_and else ('or', 'and') - - if isinstance(other, ObservationConstraint): - if self.group_type == my_type: - self.items.append(other) - return self - else: - return GroupConstraint([self, other], my_type) - - elif isinstance(other, self.__class__): - if other.group_type == my_type: - other.items += self.items - return other - if other.group_type == not_my_type and self.group_type == my_type: - self.items.append(other) - return self - else: - return GroupConstraint([self, other], my_type) - - def json(self): - return { - 'type': self.group_type, - 'args': [item.subselect() for item in self.items] - } - - -class RelationConstraint(Queryable): +class RelationConstraint(Queryable, Grouper): + """ + Query that operates on relationships between subject sets. + """ def __init__(self, constraint, type_label): self.constraint = constraint diff --git a/transmart/api/v2/dashboard/dashboard.py b/transmart/api/v2/dashboard/dashboard.py index e98839d..3a89ed0 100644 --- a/transmart/api/v2/dashboard/dashboard.py +++ b/transmart/api/v2/dashboard/dashboard.py @@ -32,7 +32,7 @@ def __init__(self, api, patients: Queryable=None): if isinstance(patients, Queryable): self.subject_set_id = api.create_patient_set(repr(patients), patients).get('id') - counts = api.get_observations.counts_per_study_and_concept( + counts = api.observations.counts_per_study_and_concept( subject_set_id=self.subject_set_id ) self.nodes = filter_tree(api.tree_dict, counts) @@ -68,7 +68,7 @@ def plotter(self, constraints): if self.subject_set_id is not None: c.subject_set_id = self.subject_set_id - obs = self.api.get_observations(c) + obs = self.api.observations(c) self.hypercube.add_variable(obs.dataframe) name = obs.dataframe['concept.name'][0] diff --git a/transmart/api/v2/tm_api_v2.py b/transmart/api/v2/tm_api_v2.py index d98a206..1f9b005 100644 --- a/transmart/api/v2/tm_api_v2.py +++ b/transmart/api/v2/tm_api_v2.py @@ -12,9 +12,9 @@ from pandas.io.json import json_normalize from .concept_search import ConceptSearcher +from .constraints import ObservationConstraint, Queryable, BiomarkerConstraint from .data_structures import (ObservationSet, ObservationSetHD, TreeNodes, Patients, PatientSets, Studies, StudyList, RelationTypes) -from .query_constraints import Query, ObservationConstraint, Queryable, BiomarkerConstraint from ..tm_api_base import TransmartAPIBase logger = logging.getLogger('tm-api') @@ -30,6 +30,21 @@ def wrapper(*args, **kwargs): return wrapper +class Query: + """ Utility to build queries for transmart v2 api. """ + + def __init__(self, handle=None, method='GET', params=None, hal=False, json=None): + self.handle = handle + self.method = method + self.hal = hal + self.params = params + self.json = json + + @property + def headers(self): + return {'Accept': 'application/{};charset=UTF-8'.format('hal+json' if self.hal else 'json')} + + class TransmartV2(TransmartAPIBase): """ Connect to tranSMART using Python. """ @@ -90,7 +105,7 @@ def query(self, q): return r.json() @default_constraint - def get_observations(self, constraint=None, as_dataframe=False, **kwargs): + def observations(self, constraint=None, as_dataframe=False, **kwargs): """ Get observations, from the main table in the transmart data model. @@ -122,10 +137,10 @@ def func(constraint=None, *args, **kwargs): return self.query(q) func.__doc__ = doc - self.get_observations.__dict__[handle] = default_constraint(func) + self.observations.__dict__[handle] = default_constraint(func) @default_constraint - def get_patients(self, constraint=None, **kwargs): + def patients(self, constraint=None, **kwargs): """ Get patients. @@ -136,7 +151,7 @@ def get_patients(self, constraint=None, **kwargs): q = Query(handle='/v2/patients', method='POST', json={'constraint': constraint.json()}) return Patients(self.query(q)) - def get_patient_sets(self, patient_set_id=None): + def patient_sets(self, patient_set_id=None): q = Query(handle='/v2/patient_sets') if patient_set_id: @@ -181,7 +196,7 @@ def get_studies(self, as_dataframe=False): return studies - def get_concepts(self, **kwargs): + def concepts(self, **kwargs): q = Query(handle='/v2/concepts') return json_normalize(self.query(q).get('concepts')) @@ -248,7 +263,7 @@ def get_relation_types(self): q = Query(handle='/v2/pedigree/relation_types') return self.query(q) - def get_supported_fields(self): + def supported_fields(self): q = Query(handle='/v2/supported_fields') return self.query(q) From 5c78a3dd24928264494d42c70a968829cb37bf46 Mon Sep 17 00:00:00 2001 From: jochemb Date: Sun, 13 May 2018 20:50:15 +0200 Subject: [PATCH 08/13] Renamed main entry point for api --- tests/v2_tests.py | 2 +- transmart/api/v1/{tm_api_v1.py => api.py} | 0 transmart/api/v2/{tm_api_v2.py => api.py} | 0 transmart/api/v2/dashboard/dashboard.py | 2 +- transmart/main.py | 4 ++-- 5 files changed, 4 insertions(+), 4 deletions(-) rename transmart/api/v1/{tm_api_v1.py => api.py} (100%) rename transmart/api/v2/{tm_api_v2.py => api.py} (100%) diff --git a/tests/v2_tests.py b/tests/v2_tests.py index aa423f0..5f11c4b 100644 --- a/tests/v2_tests.py +++ b/tests/v2_tests.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import unittest -from transmart.api.v2.tm_api_v2 import TransmartV2 +from transmart.api.v2.api import TransmartV2 from tests.mock_server import TestMockServer, retry diff --git a/transmart/api/v1/tm_api_v1.py b/transmart/api/v1/api.py similarity index 100% rename from transmart/api/v1/tm_api_v1.py rename to transmart/api/v1/api.py diff --git a/transmart/api/v2/tm_api_v2.py b/transmart/api/v2/api.py similarity index 100% rename from transmart/api/v2/tm_api_v2.py rename to transmart/api/v2/api.py diff --git a/transmart/api/v2/dashboard/dashboard.py b/transmart/api/v2/dashboard/dashboard.py index 3a89ed0..5f383a4 100644 --- a/transmart/api/v2/dashboard/dashboard.py +++ b/transmart/api/v2/dashboard/dashboard.py @@ -13,7 +13,7 @@ from .double_tiles import CombinedPlot, ScatterPlot from ..widgets import ConceptPicker -from ..query_constraints import ObservationConstraint, Queryable +from ..constraints import ObservationConstraint, Queryable from ...commons import filter_tree logger = logging.getLogger(__name__) diff --git a/transmart/main.py b/transmart/main.py index 2a522df..34f0b04 100644 --- a/transmart/main.py +++ b/transmart/main.py @@ -1,5 +1,5 @@ -from .api.v1.tm_api_v1 import TransmartV1 -from .api.v2.tm_api_v2 import TransmartV2 +from .api.v1.api import TransmartV1 +from .api.v2.api import TransmartV2 def get_api(host, api_version=2, user=None, password=None, print_urls=False, **kwargs): From 10f9ac0cc3edace0757d3c48554f74f92f58a569 Mon Sep 17 00:00:00 2001 From: jochemb Date: Mon, 14 May 2018 17:03:52 -0400 Subject: [PATCH 09/13] Added children of observations call to constraint namespace --- transmart/api/v2/api.py | 16 +++++++++++- transmart/api/v2/constraints/composite.py | 32 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/transmart/api/v2/api.py b/transmart/api/v2/api.py index 1f9b005..e19901a 100644 --- a/transmart/api/v2/api.py +++ b/transmart/api/v2/api.py @@ -30,6 +30,15 @@ def wrapper(*args, **kwargs): return wrapper +def add_to_queryable(func): + """ + This decorator allows registration a method to an ConstraintsObjects, + so it can be accessed more easily. + """ + func.__query_method__ = True + return func + + class Query: """ Utility to build queries for transmart v2 api. """ @@ -105,6 +114,7 @@ def query(self, q): return r.json() @default_constraint + @add_to_queryable def observations(self, constraint=None, as_dataframe=False, **kwargs): """ Get observations, from the main table in the transmart data model. @@ -137,9 +147,10 @@ def func(constraint=None, *args, **kwargs): return self.query(q) func.__doc__ = doc - self.observations.__dict__[handle] = default_constraint(func) + self.observations.__dict__[handle] = add_to_queryable(default_constraint(func)) @default_constraint + @add_to_queryable def patients(self, constraint=None, **kwargs): """ Get patients. @@ -160,6 +171,7 @@ def patient_sets(self, patient_set_id=None): return PatientSets(self.query(q)) @default_constraint + @add_to_queryable def create_patient_set(self, name: str, constraint=None, **kwargs): """ Create a patient set that can be reused at a later stage. @@ -224,6 +236,7 @@ def tree_nodes(self, root=None, depth=0, counts=False, tags=True, hal=False): return tree_nodes @default_constraint + @add_to_queryable def get_hd_node_data(self, constraint=None, biomarker_constraint=None, biomarkers: list=None, biomarker_type='genes', projection='all_data', **kwargs): """ @@ -250,6 +263,7 @@ def get_hd_node_data(self, constraint=None, biomarker_constraint=None, biomarker return ObservationSetHD(self.query(q)) @default_constraint + @add_to_queryable def dimension_elements(self, dimension, constraint=None, **kwargs): q = Query(handle='/v2/dimensions/{}/elements'.format(dimension), method='GET', diff --git a/transmart/api/v2/constraints/composite.py b/transmart/api/v2/constraints/composite.py index 08c9ac3..40984f8 100644 --- a/transmart/api/v2/constraints/composite.py +++ b/transmart/api/v2/constraints/composite.py @@ -149,6 +149,23 @@ def json(self): } +def _find_query_methods(obj): + for method_name in dir(obj): + if method_name.startswith('__'): + continue + + method = getattr(obj, method_name) + if getattr(method, '__query_method__', False): + yield method_name, method + + +def override_defaults(method, **defaults): + @wraps(method) + def wrapper(*args, **kwargs): + return method(*args, **defaults, **kwargs) + return wrapper + + class ObservationConstraint(Queryable, Grouper): """ Represents constraints on observation level. This is the set of @@ -224,6 +241,9 @@ def __init__(self, if api is not None: self._details_widget = ConstraintWidget(self) + for name, method in _find_query_methods(api): + self.__dict__[name] = self._constraint_method_factory(method) + self.concept = concept self.trial_visit = trial_visit self.value_list = value_list @@ -303,6 +323,18 @@ def subselect(self, dimension='patient'): finally: self.subselection = current + def _constraint_method_factory(self, method): + @wraps(method) + def wrapper(*args, **kwargs): + func = override_defaults(method, api=self.api, constraint=self) + return func(*args, **kwargs) + + # Also set defaults of the observation call children + for name, child in _find_query_methods(method): + wrapper.__dict__[name] = override_defaults(child, api=self.api, constraint=self) + + return wrapper + @property def trial_visit(self): """ From d5ae0fbdb938e76d9059b1959cb738ae6f68a996 Mon Sep 17 00:00:00 2001 From: jochemb Date: Wed, 30 May 2018 20:17:15 +0200 Subject: [PATCH 10/13] Enable binder --- binder/postBuild.sh | 5 +++++ binder/requirements.txt | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 binder/postBuild.sh create mode 100644 binder/requirements.txt diff --git a/binder/postBuild.sh b/binder/postBuild.sh new file mode 100644 index 0000000..64bf03c --- /dev/null +++ b/binder/postBuild.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +jupyter labextension install @jupyter-widgets/jupyterlab-manager +jupyter labextension install bqplot-jupyterlab + diff --git a/binder/requirements.txt b/binder/requirements.txt new file mode 100644 index 0000000..f42e339 --- /dev/null +++ b/binder/requirements.txt @@ -0,0 +1,2 @@ +ipywidgets +bqplot From 2eea95d43c6a88d72108d234d3bb64c7d3aea9eb Mon Sep 17 00:00:00 2001 From: jochemb Date: Wed, 30 May 2018 20:30:34 +0200 Subject: [PATCH 11/13] all reqs --- binder/postBuild.sh | 7 +++++-- binder/requirements.txt | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/binder/postBuild.sh b/binder/postBuild.sh index 64bf03c..a54a09a 100644 --- a/binder/postBuild.sh +++ b/binder/postBuild.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash -jupyter labextension install @jupyter-widgets/jupyterlab-manager -jupyter labextension install bqplot-jupyterlab +cd .. +python setup.py install +jupyter labextension install @jupyter-widgets/jupyterlab-manager@0.35 --no-build && +jupyter labextension install bqplot@0.3.6 --no-build && +jupyter lab clean && jupyter lab build diff --git a/binder/requirements.txt b/binder/requirements.txt index f42e339..20d477e 100644 --- a/binder/requirements.txt +++ b/binder/requirements.txt @@ -1,2 +1,8 @@ +protobuf +requests +pandas +whoosh +arrow ipywidgets +IPython bqplot From 3468996ceca70349939dab1b163482e31bcf463b Mon Sep 17 00:00:00 2001 From: jochemb Date: Wed, 6 Jun 2018 19:50:23 +0200 Subject: [PATCH 12/13] use conda instead of pip for dependencies --- binder/environment.yml | 22 ++++++++++++++++++++++ binder/{postBuild.sh => postBuild} | 3 +-- binder/requirements.txt | 8 -------- 3 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 binder/environment.yml rename binder/{postBuild.sh => postBuild} (87%) delete mode 100644 binder/requirements.txt diff --git a/binder/environment.yml b/binder/environment.yml new file mode 100644 index 0000000..330b46b --- /dev/null +++ b/binder/environment.yml @@ -0,0 +1,22 @@ +channels: +- conda-forge +dependencies: +- jupyterlab=0.32.1 +- nodejs=8.9 +- notebook=5.4 +- nbconvert=5.3 +- ipykernel=4.7 +- ipywidgets=7.2 +- widgetsnbextension=3.2 +- bqplot=0.10 +- matplotlib=2.1 +- pandas=0.20 +- python=3.6 +- sympy=1.0 +- pyyaml +- traittypes==0.0.6 +- invoke=0.21 +- protobuf +- requests +- whoosh +- arrow \ No newline at end of file diff --git a/binder/postBuild.sh b/binder/postBuild similarity index 87% rename from binder/postBuild.sh rename to binder/postBuild index a54a09a..7ef5791 100644 --- a/binder/postBuild.sh +++ b/binder/postBuild @@ -1,7 +1,6 @@ #!/usr/bin/env bash -cd .. -python setup.py install +pip install . jupyter labextension install @jupyter-widgets/jupyterlab-manager@0.35 --no-build && jupyter labextension install bqplot@0.3.6 --no-build && jupyter lab clean && jupyter lab build diff --git a/binder/requirements.txt b/binder/requirements.txt deleted file mode 100644 index 20d477e..0000000 --- a/binder/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -protobuf -requests -pandas -whoosh -arrow -ipywidgets -IPython -bqplot From 0d4319cbfb6827fe0409f4b4ad8b62d5aef403e9 Mon Sep 17 00:00:00 2001 From: jochemb Date: Wed, 6 Jun 2018 20:31:15 +0200 Subject: [PATCH 13/13] Add example notebook --- .gitignore | 1 + README.md | 1 + examples/Example.ipynb | 529 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 531 insertions(+) create mode 100644 examples/Example.ipynb diff --git a/.gitignore b/.gitignore index 66f4cf9..fd60986 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ build/ dist/ transmart.egg-info/ transmart/__pycache__/ +!examples/* diff --git a/README.md b/README.md index 5286f8e..d891afc 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Build Status](https://travis-ci.org/thehyve/transmart-api-client-py.svg?branch=master)](https://travis-ci.org/thehyve/transmart-api-client-py) [![codecov](https://codecov.io/gh/thehyve/transmart-api-client-py/branch/master/graph/badge.svg)](https://codecov.io/gh/thehyve/transmart-api-client-py) +[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/thehyve/transmart-api-client-py/master?urlpath=/lab/tree/examples/Example.ipynb) This is meant to work with Python 3.x only. diff --git a/examples/Example.ipynb b/examples/Example.ipynb new file mode 100644 index 0000000..2049a86 --- /dev/null +++ b/examples/Example.ipynb @@ -0,0 +1,529 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# transmart python client with JupyterLab integration\n", + "A short demonstration on getting data from tranSMART into the Jupyter Notebook analytical environment." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import transmart as tm\n", + "from transmart.api.v2.dashboard import Dashboard" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First create a connection with a transmart instance with V2 api enabled. This could take a litlle time as some caches are built." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://transmart.thehyve.net/v2/studies\n", + "https://transmart.thehyve.net/v2/tree_nodes?counts=False&tags=True&depth=0\n", + "Existing index cache found. Loaded 684 tree nodes. Hooray!\n" + ] + } + ], + "source": [ + "api = tm.get_api(\n", + " host='https://transmart.thehyve.net', \n", + " user='demo-user', \n", + " password='demo-user', \n", + " print_urls=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The main objects to query transmart are created here. This `ObservationConstraint` object can be used specified and combined to create queries to the api." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ObservationConstraint(concept='O1KP:NUM39', study='ORACLE_1000_PATIENT')" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "c = api.new_constraint(study='ORACLE_1000_PATIENT', concept='O1KP:NUM39')\n", + "c" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An example call that gets the counts for the chosen constraints" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://transmart.thehyve.net/v2/observations/counts\n" + ] + }, + { + "data": { + "text/plain": [ + "{'observationCount': 1200, 'patientCount': 1200}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "c.observations.counts()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Further specify the constraint using intuitive attributes." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ObservationConstraint(concept='O1KP:NUM39', min_value=15, study='ORACLE_1000_PATIENT')" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "c.min_value = 15\n", + "c" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://transmart.thehyve.net/v2/observations/counts\n" + ] + }, + { + "data": { + "text/plain": [ + "{'observationCount': 202, 'patientCount': 202}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "c.observations.counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Possible attributes to specify constraints:\n", + "- concept\n", + "- max_date_value\n", + "- max_start_date\n", + "- max_value\n", + "- min_date_value\n", + "- min_start_date\n", + "- min_value\n", + "- study\n", + "- subject_set_id\n", + "- trial_visit\n", + "- value_list\n" + ] + } + ], + "source": [ + "print('Possible attributes to specify constraints:\\n- ', end='')\n", + "print('\\n- '.join(sorted(c.params.keys())))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://transmart.thehyve.net/v2/observations/aggregates_per_concept\n" + ] + }, + { + "data": { + "text/plain": [ + "{'aggregatesPerConcept': {'O1KP:NUM39': {'numericalValueAggregates': {'avg': 17.60090668316832,\n", + " 'count': 202,\n", + " 'max': 27.10521,\n", + " 'min': 15.01558,\n", + " 'stdDev': 2.4378601422759876}}}}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "api.observations.aggregates_per_concept(c)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://transmart.thehyve.net/v2/observations?constraint={\"args\": [{\"conceptCode\": \"O1KP:NUM39\", \"type\": \"concept\"}, {\"operator\": \">=\", \"value\": 15, \"type\": \"value\", \"valueType\": \"NUMERIC\"}, {\"studyId\": \"ORACLE_1000_PATIENT\", \"type\": \"study_name\"}], \"type\": \"and\"}&type=clinical\n" + ] + } + ], + "source": [ + "obs = api.observations(c)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
concept.conceptCodeconcept.conceptPathconcept.namenumericValuepatient.agepatient.birthDatepatient.deathDatepatient.idpatient.inTrialIdpatient.maritalStatuspatient.racepatient.religionpatient.sexpatient.sexCdpatient.trialstudy.name
0O1KP:NUM39\\Public Studies\\Oracle_1000_Patient\\Numerical ...numerical_3916.1373150NoneNone-1475subject_475NoneNoneNonemaleMaleORACLE_1000_PATIENTORACLE_1000_PATIENT
1O1KP:NUM39\\Public Studies\\Oracle_1000_Patient\\Numerical ...numerical_3917.3093650NoneNone-1714subject_714NoneNoneNonemaleMaleORACLE_1000_PATIENTORACLE_1000_PATIENT
2O1KP:NUM39\\Public Studies\\Oracle_1000_Patient\\Numerical ...numerical_3917.6927550NoneNone-1253subject_253NoneNoneNonemaleMaleORACLE_1000_PATIENTORACLE_1000_PATIENT
3O1KP:NUM39\\Public Studies\\Oracle_1000_Patient\\Numerical ...numerical_3915.5781750NoneNone-1787subject_787NoneNoneNonemaleMaleORACLE_1000_PATIENTORACLE_1000_PATIENT
4O1KP:NUM39\\Public Studies\\Oracle_1000_Patient\\Numerical ...numerical_3916.4198650NoneNone-1827subject_827NoneNoneNonefemaleFemaleORACLE_1000_PATIENTORACLE_1000_PATIENT
\n", + "
" + ], + "text/plain": [ + " concept.conceptCode concept.conceptPath \\\n", + "0 O1KP:NUM39 \\Public Studies\\Oracle_1000_Patient\\Numerical ... \n", + "1 O1KP:NUM39 \\Public Studies\\Oracle_1000_Patient\\Numerical ... \n", + "2 O1KP:NUM39 \\Public Studies\\Oracle_1000_Patient\\Numerical ... \n", + "3 O1KP:NUM39 \\Public Studies\\Oracle_1000_Patient\\Numerical ... \n", + "4 O1KP:NUM39 \\Public Studies\\Oracle_1000_Patient\\Numerical ... \n", + "\n", + " concept.name numericValue patient.age patient.birthDate \\\n", + "0 numerical_39 16.13731 50 None \n", + "1 numerical_39 17.30936 50 None \n", + "2 numerical_39 17.69275 50 None \n", + "3 numerical_39 15.57817 50 None \n", + "4 numerical_39 16.41986 50 None \n", + "\n", + " patient.deathDate patient.id patient.inTrialId patient.maritalStatus \\\n", + "0 None -1475 subject_475 None \n", + "1 None -1714 subject_714 None \n", + "2 None -1253 subject_253 None \n", + "3 None -1787 subject_787 None \n", + "4 None -1827 subject_827 None \n", + "\n", + " patient.race patient.religion patient.sex patient.sexCd \\\n", + "0 None None male Male \n", + "1 None None male Male \n", + "2 None None male Male \n", + "3 None None male Male \n", + "4 None None female Female \n", + "\n", + " patient.trial study.name \n", + "0 ORACLE_1000_PATIENT ORACLE_1000_PATIENT \n", + "1 ORACLE_1000_PATIENT ORACLE_1000_PATIENT \n", + "2 ORACLE_1000_PATIENT ORACLE_1000_PATIENT \n", + "3 ORACLE_1000_PATIENT ORACLE_1000_PATIENT \n", + "4 ORACLE_1000_PATIENT ORACLE_1000_PATIENT " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "obs.dataframe.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Integration with widgets and bqplot" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Methods `find_concept()` and `interact()` can be used to visually create and modify a constraint object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "c.find_concept('study:oracle')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "c.interact()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And there is dashboard for exploration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dash = Dashboard(api, patients=c)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dash.get()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}