notebooks/xgb-with-clinical-features.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Setup**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Enabling notebook extension jupyter-js-widgets/extension...\n",
      "      - Validating: ok\n"
     ]
    }
   ],
   "source": [
    "!jupyter nbextension enable --py widgetsnbextension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Anaconda3\\envs\\ecg\\lib\\site-packages\\numpy\\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:\n",
      "c:\\Anaconda3\\envs\\ecg\\lib\\site-packages\\numpy\\.libs\\libopenblas.FB5AE2TYXYH2IJRDKGDGQ3XBKLKTF43H.gfortran-win_amd64.dll\n",
      "c:\\Anaconda3\\envs\\ecg\\lib\\site-packages\\numpy\\.libs\\libopenblas64__v0.3.21-gcc_10_3_0.dll\n",
      "  warnings.warn(\"loaded more than 1 DLL from .libs:\"\n",
      "c:\\Anaconda3\\envs\\ecg\\lib\\site-packages\\pkg_resources\\__init__.py:123: PkgResourcesDeprecationWarning: llow is an invalid version and will not be supported in a future release\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import os.path as op\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "sys.path.append(\"..\")\n",
    "from mtecg.utils import load_ecg_dataframe, categorize_lvef, find_best_thresholds, apply_thresholds\n",
    "\n",
    "\n",
    "SEED = 42\n",
    "np.random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "lvef_threshold = 50\n",
    "clinical_feature_columns = [\"female_gender\", \"age\", \"smoke\", \"dlp\", \"dm\", \"ht\"]\n",
    "\n",
    "save_dir = \"../scripts/mtecg/xgb\"\n",
    "os.makedirs(save_dir, exist_ok=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Prepare the data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of images: 12788\n",
      "Unique splits: ['old_train' 'old_valid' 'old_test' 'new_train' 'new_valid']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>run_num</th>\n",
       "      <th>file_name</th>\n",
       "      <th>lvef</th>\n",
       "      <th>scar_cad</th>\n",
       "      <th>hcm</th>\n",
       "      <th>mri_date</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "      <th>cut</th>\n",
       "      <th>edit_filename</th>\n",
       "      <th>...</th>\n",
       "      <th>ua</th>\n",
       "      <th>chest pain</th>\n",
       "      <th>dyspnea</th>\n",
       "      <th>subs</th>\n",
       "      <th>trans</th>\n",
       "      <th>boths</th>\n",
       "      <th>split</th>\n",
       "      <th>conflict</th>\n",
       "      <th>new_cut</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2009_420521391</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2552-08-01 00:00:00</td>\n",
       "      <td>8</td>\n",
       "      <td>2009</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old_train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2009_472422791</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2552-08-01 00:00:00</td>\n",
       "      <td>8</td>\n",
       "      <td>2009</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old_train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2009_451191451</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2552-08-01 00:00:00</td>\n",
       "      <td>8</td>\n",
       "      <td>2009</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>old_train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2009_512029431</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2552-08-01 00:00:00</td>\n",
       "      <td>8</td>\n",
       "      <td>2009</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>old_train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2009_461543281</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2552-08-04 00:00:00</td>\n",
       "      <td>8</td>\n",
       "      <td>2009</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>old_train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   run_num       file_name  lvef  scar_cad  hcm             mri_date  month  \\\n",
       "0        1  2009_420521391     0         0    0  2552-08-01 00:00:00      8   \n",
       "1        2  2009_472422791     0         0    0  2552-08-01 00:00:00      8   \n",
       "2        3  2009_451191451     0         0    0  2552-08-01 00:00:00      8   \n",
       "3        4  2009_512029431     1         1    0  2552-08-01 00:00:00      8   \n",
       "4        5  2009_461543281     1         1    0  2552-08-04 00:00:00      8   \n",
       "\n",
       "   year  cut  edit_filename  ...  ua  chest pain  dyspnea  subs  trans  boths  \\\n",
       "0  2009  NaN            NaN  ...   0           1        0   NaN    NaN    NaN   \n",
       "1  2009  NaN            NaN  ...   0           1        0   NaN    NaN    NaN   \n",
       "2  2009  NaN            NaN  ...   0           1        1   NaN    NaN    NaN   \n",
       "3  2009  NaN            NaN  ...   0           0        1   NaN    1.0    1.0   \n",
       "4  2009  NaN            NaN  ...   0           1        1   NaN    1.0    1.0   \n",
       "\n",
       "       split  conflict  new_cut  \\\n",
       "0  old_train       NaN        0   \n",
       "1  old_train       NaN        0   \n",
       "2  old_train       NaN        0   \n",
       "3  old_train       NaN        0   \n",
       "4  old_train       NaN        0   \n",
       "\n",
       "                                                path  \n",
       "0  ../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...  \n",
       "1  ../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...  \n",
       "2  ../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...  \n",
       "3  ../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...  \n",
       "4  ../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_i...  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_dir = \"../../ecg/ecg-cnn-local/siriraj_data/ECG_MRI_images_new/\"\n",
    "csv_path = \"../datasets/all_ECG_cleared_duplicate_may23_final.csv\"\n",
    "\n",
    "df = load_ecg_dataframe(csv_path, image_dir, drop_impute=False, do_split=True)\n",
    "print(f\"Number of images: {len(df)}\")\n",
    "print(f\"Unique splits: {df['split'].unique()}\")\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set [\"dm\", \"ht\", \"smoke\", \"dlp\"] to np.nan if \"impute\" is True.\n",
    "df.loc[df[\"impute\"] == True, [\"dm\", \"ht\", \"smoke\", \"dlp\"]] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((9393, 31), (2500, 31))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Combine old train and new train.\n",
    "train_df = df[df.split.isin([\"old_train\", \"new_train\"])].reset_index()\n",
    "# Combine old valid and new valid.\n",
    "valid_df = df[df.split.isin([\"old_valid\", \"new_valid\"])].reset_index()\n",
    "\n",
    "train_df.shape, valid_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dm       0.374580\n",
       "ht       0.747920\n",
       "smoke    0.164985\n",
       "dlp      0.706143\n",
       "dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_impute_train_df = train_df[train_df[\"impute\"] == False]\n",
    "\n",
    "non_impute_train_df[[\"dm\", \"ht\", \"smoke\", \"dlp\"]].sum() / len(non_impute_train_df)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Impute Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../scripts/mtecg/xgb\\\\imputer.joblib']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer\n",
    "import joblib\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "imputer = IterativeImputer(\n",
    "    missing_values=np.nan,\n",
    "    max_iter=10,\n",
    "    sample_posterior=True,\n",
    "    random_state=42\n",
    "    )\n",
    "\n",
    "clinical_feature_columns = [\"age\", \"female_gender\", \"dm\", \"ht\", \"smoke\", \"dlp\"]\n",
    "\n",
    "# Fit the imputer on the train set.\n",
    "imputer.fit(train_df[clinical_feature_columns])\n",
    "\n",
    "# Save the imputer.\n",
    "imputer_path = op.join(save_dir, \"imputer.joblib\")\n",
    "joblib.dump(imputer, imputer_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Impute missing values in the train set.\n",
    "train_df[clinical_feature_columns] = imputer.transform(train_df[clinical_feature_columns])\n",
    "\n",
    "# Impute missing values in the valid set.\n",
    "valid_df[clinical_feature_columns] = imputer.transform(valid_df[clinical_feature_columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dm': [0.54, 0.0033188878449101344],\n",
       " 'ht': [0.42, 0.001278732110516545],\n",
       " 'smoke': [0.52, 0.003125978730067952],\n",
       " 'dlp': [0.44, 0.0027248412471461148]}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find the best thresholds for imputing missing values from the train set.\n",
    "best_threshold_dict = find_best_thresholds(train_df)\n",
    "\n",
    "joblib.dump(best_threshold_dict, op.join(save_dir, \"imputer_threshold_dict.joblib\"))\n",
    "\n",
    "# Apply the best thresholds to the train set and the valid set.\n",
    "train_df = apply_thresholds(train_df, best_threshold_dict)\n",
    "valid_df = apply_thresholds(valid_df, best_threshold_dict)\n",
    "\n",
    "best_threshold_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dm       0.373257\n",
       "ht       0.748430\n",
       "smoke    0.163739\n",
       "dlp      0.707229\n",
       "dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df[[\"dm\", \"ht\", \"smoke\", \"dlp\"]].sum() / len(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = train_df[clinical_feature_columns]\n",
    "x_valid = valid_df[clinical_feature_columns]\n",
    "\n",
    "y_train_scar = train_df[\"scar_cad\"]\n",
    "y_valid_scar = valid_df[\"scar_cad\"]\n",
    "y_train_lvef = train_df[\"lvef\"]\n",
    "y_valid_lvef = valid_df[\"lvef\"]\n",
    "# y_train_lvef = train_df[\"lvef\"].apply(lambda lvef: categorize_lvef(lvef, lvef_threshold))\n",
    "# y_valid_lvef = valid_df[\"lvef\"].apply(lambda lvef: categorize_lvef(lvef, lvef_threshold))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Train**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "scar_model = XGBClassifier(\n",
    "    booster=\"dart\",\n",
    "    tree_method=\"hist\",\n",
    "    grow_policy=\"lossguide\",\n",
    "    sample_type=\"weighted\",\n",
    "    sampling_method=\"gradient_based\",\n",
    "    normalize_type=\"forest\",\n",
    "    rate_drop=0.3,\n",
    "    random_state=SEED,\n",
    ")\n",
    "\n",
    "lvef_model = XGBClassifier(\n",
    "    booster=\"dart\",\n",
    "    tree_method=\"hist\",\n",
    "    grow_policy=\"lossguide\",\n",
    "    sample_type=\"weighted\",\n",
    "    sampling_method=\"gradient_based\",\n",
    "    normalize_type=\"forest\",\n",
    "    rate_drop=0.3,\n",
    "    random_state=SEED,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scar accuracy: 0.7564\n",
      "LVEF accuracy: 0.8276\n"
     ]
    }
   ],
   "source": [
    "scar_model.fit(x_train, y_train_scar)\n",
    "print(\"Scar accuracy:\", scar_model.score(x_valid, y_valid_scar))\n",
    "\n",
    "lvef_model.fit(x_train, y_train_lvef)\n",
    "print(\"LVEF accuracy:\", lvef_model.score(x_valid, y_valid_lvef))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scar AUC: 0.6571506753413132\n",
      "LVEF AUC: 0.6176889365767088\n"
     ]
    }
   ],
   "source": [
    "print(\"Scar AUC:\", roc_auc_score(y_valid_scar, scar_model.predict_proba(x_valid)[:, 1]))\n",
    "print(\"LVEF AUC:\", roc_auc_score(y_valid_lvef, lvef_model.predict_proba(x_valid)[:, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../scripts/mtecg/xgb\\\\lvef_model\\\\model.joblib']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import joblib\n",
    "scar_model_save_dir = op.join(save_dir, \"scar_model\")\n",
    "lvef_model_save_dir = op.join(save_dir, \"lvef_model\")\n",
    "os.makedirs(scar_model_save_dir, exist_ok=True)\n",
    "os.makedirs(lvef_model_save_dir, exist_ok=True)\n",
    "\n",
    "joblib.dump(scar_model, op.join(scar_model_save_dir, \"model.joblib\"))\n",
    "joblib.dump(lvef_model, op.join(lvef_model_save_dir, \"model.joblib\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A function to get XGBoost predictions.\n",
    "from mtecg.evaluation import calculate_metrics_per_task\n",
    "import pandas as pd\n",
    "from typing import List\n",
    "\n",
    "def evaluate_xgb_from_dataframe(\n",
    "    dataframe: pd.DataFrame,\n",
    "    model: XGBClassifier,\n",
    "    feature_columns: List[str],\n",
    "    label_column_name: str = \"scar_cad\",\n",
    "    task=\"scar\",\n",
    "    ):\n",
    "    x = dataframe[feature_columns]\n",
    "    predicted_probability_array = model.predict_proba(x)[:, 1]\n",
    "    prediction_array = model.predict(x)\n",
    "\n",
    "    prediction_dataframe = pd.DataFrame(\n",
    "        {\n",
    "            f\"{task}_label\": dataframe[label_column_name].values,\n",
    "            f\"{task}_prediction\": prediction_array,\n",
    "            f\"{task}_probability\": predicted_probability_array,\n",
    "        }\n",
    "    )\n",
    "    metrics_dataframe = calculate_metrics_per_task(prediction_dataframe, task)\n",
    "    return prediction_dataframe, metrics_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "scar_prediction_df, scar_metric_df = evaluate_xgb_from_dataframe(valid_df, scar_model, clinical_feature_columns, task=\"scar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "lvef_prediction_df, lvef_metric_df = evaluate_xgb_from_dataframe(valid_df, lvef_model, clinical_feature_columns, task=\"lvef\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accuracy</th>\n",
       "      <td>0.762400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sensitivity</th>\n",
       "      <td>0.001698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Specificity</th>\n",
       "      <td>0.996860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1</th>\n",
       "      <td>0.662090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AUC</th>\n",
       "      <td>0.630827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FPR</th>\n",
       "      <td>0.003140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FNR</th>\n",
       "      <td>0.998302</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0\n",
       "Accuracy     0.762400\n",
       "Sensitivity  0.001698\n",
       "Specificity  0.996860\n",
       "F1           0.662090\n",
       "AUC          0.630827\n",
       "FPR          0.003140\n",
       "FNR          0.998302"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lvef_metric_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function mtecg.evaluation.calculate_metrics_per_task(result_dataframe, task: str, is_control_population: bool = False, average: str = 'weighted')>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_metrics_per_task"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ecg",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "109b74d06ceef90c267188b655f808679842e5df9d924ed4ca45afc3047e2ff5"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}