0_preprocess_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7f4fd5d3",
   "metadata": {},
   "source": [
    "# Preprocess cifar10h data from github"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aba15dc7",
   "metadata": {},
   "source": [
    "As our multiannotator dataset, we use a subset of [cifar-10h](https://github.com/jcpeterson/cifar-10h) (Peterson et al., 2019) multiannotator labeling dataset which is a collection of 2751 annotators each labeling 200 examples for all 10,000 test images of the original [cifar10](https://www.cs.toronto.edu/~kriz/cifar.html).\n",
    "\n",
    "The code below shows an example of how to reach that subset.\n",
    "\n",
    "Before running this notebook, make sure cifar10h dataset is downloaded locally:\n",
    "1. Make new subfolder ``mkdir data``\n",
    "2. Download cifar-10h dataset ``cd data && git clone https://github.com/jcpeterson/cifar-10h.git``\n",
    "3. Clean directory ``cd cifar-10h && mv data/* . && rm -rf data``\n",
    "4. Unzip data ``unzip cifar10h-raw && rm cifar10h-raw.zip``"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3b226782",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from cleanlab.multiannotator import get_consensus_label\n",
    "from utils.eval import get_annotator_mask, get_labels_error_mask, plt_annotator_accuracy\n",
    "import seaborn as sns\n",
    "\n",
    "path = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "29e7c956",
   "metadata": {},
   "outputs": [],
   "source": [
    "# If this line throws an error, make sure you correctly downloaded and unzipped cifar10h-raw data\n",
    "pred_probs_multiannotator = np.load('./data/cifar-10h/cifar10h-probs.npy')\n",
    "c10h_df = pd.read_csv('./data/cifar-10h/cifar10h-raw.csv')\n",
    "c10h_df = c10h_df[c10h_df.cifar10_test_test_idx != -99999] # dropping all attention check trials"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4fe25d4",
   "metadata": {},
   "source": [
    "#### Creates dataset into following arrays:\n",
    "- ``c10h_labels: (N,M)``\n",
    "- ``c10h_true_labels: (K=10,000,)``\n",
    "- ``c10h_true_images: (K=10,000,)``"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2e959414",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize arrays\n",
    "c10h_num_datapoints = c10h_df['cifar10_test_test_idx'].max() + 1\n",
    "c10h_num_annotators = c10h_df['annotator_id'].max() + 1\n",
    "\n",
    "c10h_labels = np.full((c10h_num_datapoints, c10h_num_annotators), np.nan) # all annotator labels np.full([height, width, 9], np.nan)\n",
    "\n",
    "c10h_true_labels = np.zeros((c10h_num_datapoints, ))\n",
    "c10h_true_images = np.empty((c10h_num_datapoints, ) ,dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "854caccd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get annotator labels as numpy array (N=labels, M=annotators)\n",
    "for annotator_id in range(c10h_num_annotators):\n",
    "    adf = c10h_df[c10h_df.annotator_id == annotator_id] # 200 annotations per annotator\n",
    "    annotations_idx = adf['cifar10_test_test_idx'].values\n",
    "    annotations = adf['chosen_label'].values\n",
    "    errors = adf['correct_guess'].values\n",
    "    \n",
    "    c10h_labels[annotations_idx, annotator_id] = annotations\n",
    "\n",
    "# get true labels as numpy array (N = true labels,)\n",
    "idx_to_label = \\\n",
    "[(idx,label,image) for idx,label,image in zip(c10h_df['cifar10_test_test_idx'],c10h_df['true_label'],c10h_df['image_filename'])]\n",
    "idx_to_label = list(set(idx_to_label))\n",
    "\n",
    "idx = [idx_to_label[0] for idx_to_label in idx_to_label]\n",
    "true_label = [idx_to_label[1] for idx_to_label in idx_to_label]\n",
    "htrue_image = [idx_to_label[2] for idx_to_label in idx_to_label]\n",
    "\n",
    "c10h_true_labels[idx] = true_label\n",
    "c10h_true_images[idx] = htrue_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "57c63964",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAARP0lEQVR4nO3df2id133H8fdXsmyDktox0ZIlipv9EY9rREBFlC41rIozZvePpDBYq1BCJmFTWC6BbU5UNNokQ4y4YtDZ2USGRbIFq2T7IzEsI+BwS9FoRxS6BdkincnmRMnAau0sQUPRD5/9IdmVFNm6sq/9+J68X2Cs5zzHfr5/OB9OznPOcyKlhCSp/jUUXYAkqTYMdEnKhIEuSZkw0CUpEwa6JGViQ1EPvvXWW9Pdd99d1OMlqS699dZbv0wptax2r7BAv/vuuxkdHS3q8ZJUlyLi9KXuOeUiSZkw0CUpEwa6JGXCQJekTBjokpSJNQM9IoYi4kxEjF3ifkTEX0fEqYh4OyK+VPsyJUlrqWaE/gKw5zL39wL3LP7aD/zt1ZclSVqvNQM9pfQT4OxlujwE/H1a8DNga0T8Zq0KlCRVpxYbi+4E3l9yPbHY9j8rO0bEfhZG8Wzfvr0Gj5bWFhHX5TmeLaCiXdeXoiml51NKHSmljpaWVXeuSjWXUlrXryv5M4a5bgS1CPQPgLuWXLcutkmSrqNaBPox4JHF1S5fAf43pfSZ6RZJ0rW15hx6RAwDXwNujYgJ4PtAE0BKaRB4Dfg6cAr4P+CPrlWxkqRLWzPQU0pda9xPwB/XrCJJ0hVxp6gkZaKw76FLV2Lbtm2cO3fumj/neix1vOWWWzh79nJbPKT1MdBVV86dO5fNEsHrtT5enx8GuupK+v4X4KktRZdRE+n7Xyi6BGXGQFddiac/zmqEnp4qugrlxJeikpQJA12SMuGUi+pOLi8Tb7nllqJLUGYMdNWV6zF/HhHZzNPr88UpF0nKhIEuSZkw0CUpEwa6JGXCQJekTBjokpQJly1Ki5aub7/ws8sXVU8coUtcerNSLpuY9PngCF3Zu9pQrvbPO5pX0Qx0Za+aoL1caBvUqhdOuUhSJgx0aYUXXnih6BKkK2KgSys8+uijRZcgXREDXZIyYaBLUiYMdEnKhIEuSZkw0CUpEwa6JGXCQJekTBjokpQJA12SMmGgS1ImDHRpUUNDA01NTQA0NTXR0OB/Hqov/ouVFp0/f569e/cyOTnJ3r17OX/+fNElSevi99ClJY4dO0ZLS0vRZUhXpKoRekTsiYh3IuJURPSucn97RFQi4ucR8XZEfL32pUrXTmtr62cOuYgIWltbC6pIWr81Az0iGoHngL3ATqArInau6PbnwMsppXbgW8Df1LpQ6VqanJz8zMlEKSUmJycLqkhav2pG6F8GTqWU3k0pzQA/Ah5a0ScBX1j8eQvwYe1KlK69Tz/9FODii9ALv19ol+pBNYF+J/D+kuuJxbalngK+HRETwGtAebW/KCL2R8RoRIw68tGNZtOmTRw/fpyZmRmOHz/Opk2bii5JWpdarXLpAl5IKbUCXwf+ISI+83enlJ5PKXWklDp88SRJtVXNKpcPgLuWXLcuti3VA+wBSCn9NCI2A7cCZ2pRpHQ9fPrppzzwwAOcP3+ehoYGly2q7lQzQn8TuCcifisiNrLw0vPYij7vAbsBIqIEbAacU1HduDC9ciHEL/zutIvqyZqBnlKaAx4DXgfGWVjNciIinomIBxe7/SmwLyL+AxgGHk0rlwxIN7ANG1b/n9VLtUs3oqr+taaUXmPhZefStu8t+fkk8NXaliZdP1NTU3R1dfH2228zPj5OqVTi3nvvZXh4uOjSpKq59V9atHITkZuKVG8MdAlobGxkYGCA7u5uPvnkE7q7uxkYGKCxsbHo0qSqGegSsGXLFiKCgwcP0tzczMGDB4kItmzZUnRpUtUMdAn46KOPuP/++zlzZmGl7ZkzZ7j//vv56KOPii1MWgcDXQLuuOMOTpw4wRtvvMHMzAxvvPEGJ06c4I477ii6NKlqBrq0aLWPc0n1xECXgA8//JC2tjZ2797Nxo0b2b17N21tbXz4od+ZU/0w0CVg69atVCoVBgYGmJqaYmBggEqlwtatW4suTaqagS4BH3/8MVu2bKG9vZ2mpiba29vZsmULH3/8cdGlSVUz0CVgbm6OgYEByuUymzdvplwuMzAwwNzcXNGlSVUz0CUWPsJ17tw5xsbGmJ+fZ2xsjHPnzvlxLtUVA10C9u3bx4EDB7j99ttpaGjg9ttv58CBA+zbt6/o0qSqGegScN9999Hc3MzZs2dJKXH27Fmam5u57777ii5NqpqBLgH9/f28+uqrzMzMkFJiZmaGV199lf7+/qJLk6oWRW2e6OjoSKOjo4U8W1qpsbGR6elpmpqaLrbNzs6yefNm5ufnC6xMWi4i3kopdax2zxG6BJRKJUZGRpa1jYyMUCqVCqpIWj+PY5GAvr4+vvnNb9Lc3Mx7773H9u3bmZqa4oc//GHRpUlVc4QureA3XFSvDHSJhZei+/fvp7m5mYigubmZ/fv3+1JUdcUpFwk4efIkU1NTDA0NsWvXLkZGRuju7ub06dNFlyZVzRG6BGzcuJFyuUxnZydNTU10dnZSLpfZuHFj0aVJVTPQJWBmZobDhw9TqVSYnZ2lUqlw+PBhZmZmii5NqppTLhKwc+dOvvGNb1AulxkfH6dUKvHwww/zyiuvFF2aVDVH6BILyxaPHj3KoUOHmJ6e5tChQxw9epS+vr6iS5Oq5ghdArq6ugCWjdD7+/svtkv1wK3/klRH3PovSZ8DBrokZcJAl6RMGOiSlAkDXZIyYaBLUiYMdEnKhIEuSZmoKtAjYk9EvBMRpyKi9xJ9/jAiTkbEiYg4WtsyJUlrWXPrf0Q0As8BvwdMAG9GxLGU0sklfe4Bvgt8NaV0LiJ+41oVLElaXTUj9C8Dp1JK76aUZoAfAQ+t6LMPeC6ldA4gpXSmtmVKktZSTaDfCby/5HpisW2pHcCOiPjXiPhZROxZ7S+KiP0RMRoRo5OTk1dWsSRpVbV6KboBuAf4GtAF/F1EbF3ZKaX0fEqpI6XU0dLSUqNHS5KgukD/ALhryXXrYttSE8CxlNJsSum/gF+wEPCSpOukmkB/E7gnIn4rIjYC3wKOrejzCgujcyLiVhamYN6tXZmSpLWsGegppTngMeB1YBx4OaV0IiKeiYgHF7u9DvwqIk4CFeBASulX16poSdJnecCFJNURD7iQqjA8PExbWxuNjY20tbUxPDxcdEnSunimqMRCmPf19XHkyBF27drFyMgIPT09AJ4rqrrhlIsEtLW1cejQITo7Oy+2VSoVyuUyY2NjBVYmLXe5KRcDXQIaGxuZnp6mqanpYtvs7CybN29mfn6+wMqk5ZxDl9ZQKpUYGRlZ1jYyMkKpVCqoImn9DHQJ6Ovro6enh0qlwuzsLJVKhZ6eHvr6+oouTaqaL0Ulfv3is1wuMz4+TqlUor+/3xeiqivOoUtSHXEOXZI+Bwx0ScqEgS4tcqeo6p0vRSXcKao8+FJUwp2iqh/uFJXW4E5R1QtXuUhrcKeocmCgS7hTVHnwpaiEO0WVB+fQJamOOIcuSZ8DBrokZcJAl6RMGOiSlAkDXZIyYaBLUiYMdEnKhIEuSZkw0CUpEwa6JGXCQJekTBjokpQJA11a5Jmiqnd+PlfCM0WVBz+fK+GZoqofnikqrcEzRVUvrvp76BGxJyLeiYhTEdF7mX5/EBEpIlZ9mHSj8kxR5WDNQI+IRuA5YC+wE+iKiJ2r9LsZeBz4t1oXKV1rnimqHFTzUvTLwKmU0rsAEfEj4CHg5Ip+fwE8CxyoaYXSdeCZospBNVMudwLvL7meWGy7KCK+BNyVUvrnGtYmSVqHq162GBENwF8Bj1bRdz+wH2D79u1X+2ipZly2qBxUM0L/ALhryXXrYtsFNwNtwI8j4r+BrwDHVnsxmlJ6PqXUkVLqaGlpufKqpRrr7+/nyJEjdHZ20tTURGdnJ0eOHKG/v7/o0qSqrblsMSI2AL8AdrMQ5G8CD6eUTlyi/4+BP0spXXZNossWdSNx2aLqxVUtW0wpzQGPAa8D48DLKaUTEfFMRDxY21KlYrhsUTmoag49pfQa8NqKtu9dou/Xrr4s6fq6sGxx5Ry6Uy6qJ37LRcJli8qDW/8lqY5c9dZ/SdKNz0CXpEwY6JKUCQNdkjJhoEtSJgx0ScqEgS5JmTDQJSkTBrokZcJAl6RMGOiSlAkDXZIyYaBLUiYMdEnKhIEuSZkw0CUpEwa6JGXCQJekTBjokpQJA12SMmGgS1ImDHRJyoSBLkmZMNAlKRMGurRoeHiYtrY2GhsbaWtrY3h4uOiSpHXZUHQB0o1geHiYvr4+jhw5wq5duxgZGaGnpweArq6ugquTqhMppUIe3NHRkUZHRwt5trRSW1sbhw4dorOz82JbpVKhXC4zNjZWYGXSchHxVkqpY9V7BroEjY2NTE9P09TUdLFtdnaWzZs3Mz8/X2Bl0nKXC3Tn0CWgVCrx9NNPL5tDf/rppymVSkWXJlXNQJeAzs5Onn32Wbq7u/nkk0/o7u7m2WefXTYFI93oDHSJhfnyJ598kqGhIW6++WaGhoZ48sknqVQqRZcmVc05dAnn0FU/nEOX1uAcunJQVaBHxJ6IeCciTkVE7yr3/yQiTkbE2xHxRkR8sfalSteOc+jKwZqBHhGNwHPAXmAn0BURO1d0+znQkVK6F/gn4GCtC5WuJefQlYM159Aj4neAp1JKv794/V2AlNJfXqJ/O3A4pfTVy/29zqHrRuIcuurF1c6h3wm8v+R6YrHtUnqAf7lEIfsjYjQiRicnJ6t4tHR9lEolRkZGlrWNjIw4h666UtOXohHxbaAD+MFq91NKz6eUOlJKHS0tLbV8tHRV+vr66OnpoVKpMDs7S6VSoaenh76+vqJLk6pWzce5PgDuWnLduti2TEQ8APQBv5tS+rQ25UnXx4UPcJXLZcbHxymVSvT39/thLtWVaubQNwC/AHazEORvAg+nlE4s6dPOwsvQPSml/6zmwc6hS9L6XdUcekppDngMeB0YB15OKZ2IiGci4sHFbj8AbgL+MSL+PSKO1ah2SVKVqvoeekrpNeC1FW3fW/LzAzWuS5K0Tu4UlaRMGOiSlAkDXZIyYaBLUiYMdEnKhIEuSZkw0CUpEwa6tGh4eHjZARfDw8NFlySti4EusRDmjz/+OFNTUwBMTU3x+OOPG+qqKwa6BDzxxBNs2LCBoaEhpqenGRoaYsOGDTzxxBNFlyZVzUCXgImJCV588UU6Oztpamqis7OTF198kYmJiaJLk6pmoEtSJgx0CWhtbeWRRx5ZdsDFI488Qmtra9GlSVUz0CXg4MGDzM/P093dzaZNm+ju7mZ+fp6DBz3vXPXDQJdYOLGovb2d06dPk1Li9OnTtLe3e2KR6oqBLrFw9Nzx48e57bbbaGho4LbbbuP48eOUy+WiS5OqZqBLwODgIFu3buXo0aNMT09z9OhRtm7dyuDgYNGlSVUz0CVgbm6Ol156admyxZdeeom5ubmiS5OqZqBLi8bGxi57Ld3oqjpTVMrdtm3b6O3tpbGxke985zsMDg7S29vLtm3bii5NqpojdAk4fPgwN910E729vTQ3N9Pb28tNN93E4cOHiy5NqpqBLrGwbHFwcJAdO3bQ0NDAjh07GBwcdNmi6kqklAp5cEdHRxodHS3k2ZJUryLirZRSx2r3HKFLUiYMdEnKhIEuSZkw0CUpEwa6JGWisFUuETEJnC7k4dLl3Qr8sugipEv4YkqpZbUbhQW6dKOKiNFLLQuTbmROuUhSJgx0ScqEgS591vNFFyBdCefQJSkTjtAlKRMGuiRlwkCXFkXEUESciQiPKlJdMtClX3sB2FN0EdKVMtClRSmlnwBni65DulIGuiRlwkCXpEwY6JKUCQNdkjJhoEuLImIY+Cnw2xExERE9RdckrYdb/yUpE47QJSkTBrokZcJAl6RMGOiSlAkDXZIyYaBLUiYMdEnKxP8D1ooYyLNCjxcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.960</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   score\n",
       "0  0.920\n",
       "1  0.955\n",
       "2  0.955\n",
       "3  0.945\n",
       "4  0.960"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c10h_annotator_mask = get_annotator_mask(c10h_labels)\n",
    "c10h_labels_error_mask = get_labels_error_mask(c10h_labels, c10h_true_labels)\n",
    "plt_annotator_accuracy(c10h_labels_error_mask, c10h_annotator_mask).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "923c9fc2",
   "metadata": {},
   "source": [
    "# Drop datapoints by selecting worst annotators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "49708f36",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns sample labels/error_mask/annotator_mask where x_drop, y_drop are idxs that are dropped\n",
    "def get_sample_labels(x_drop, y_drop, labels, labels_error_mask, annotator_mask):\n",
    "    s_annotator_mask = annotator_mask.copy()\n",
    "    s_annotator_mask[(x_drop,y_drop)] = 0\n",
    "    s_labels_error_mask = s_annotator_mask & labels_error_mask\n",
    "    s_labels = labels.copy()\n",
    "    np.copyto(s_labels, np.nan, where=(s_annotator_mask==0)) \n",
    "    print('Total idxs dropped: ', annotator_mask.sum() - s_annotator_mask.sum())\n",
    "    return s_labels, s_labels_error_mask, s_annotator_mask\n",
    "\n",
    "# Returns a list of labeled indices to drop (random length per row)\n",
    "def get_random_drop_per_row(c10h_annotator_mask):\n",
    "    x,y = np.where(c10h_annotator_mask == 1)\n",
    "    idx_df = pd.DataFrame(zip(x,y),columns=['x','y'])\n",
    "    for x_idx in range(idx_df['x'].max()+1):\n",
    "        num_drop = np.random.randint(1, len(idx_df[idx_df['x'] == x_idx])+1)\n",
    "        idx_df = idx_df.drop(idx_df[idx_df['x'] == x_idx].sample(num_drop).index)\n",
    "    x_drop = idx_df['x'].values\n",
    "    y_drop = idx_df['y'].values\n",
    "    return x_drop, y_drop\n",
    "\n",
    "# Returns a list of labeled indices to drop \n",
    "# (Randomly drop until <= max_annotations per example. Try to minimize number of distinct annotators)\n",
    "def get_random_drop_per_row_min_annotators(c10h_annotator_mask, max_annotations = 5):\n",
    "    x,y = np.where(c10h_annotator_mask == 1)\n",
    "    xy = set([(x_idx,y_idx) for x_idx,y_idx in zip(x,y)])\n",
    "    idx_df = pd.DataFrame(zip(x,y),columns=['x','y'])\n",
    "    idx_keep = []\n",
    "    selected_annotators = set()\n",
    "    for x_idx in range(idx_df['x'].max()+1):\n",
    "        Y = idx_df[idx_df['x'] == x_idx]['y']\n",
    "        seen_annotators = set(Y).intersection(selected_annotators)\n",
    "        if len(seen_annotators) < max_annotations: # We need to randomly select more annotators to greedy add\n",
    "            num_to_find = max_annotations - len(seen_annotators)\n",
    "            y_keep = set(np.random.choice(list(set(Y).difference(seen_annotators)), num_to_find, replace=False))\n",
    "            selected_annotators = selected_annotators.union(y_keep)\n",
    "            y_keep = seen_annotators.union(y_keep)\n",
    "        else: # We have enough annotators and need to randomly select annotations out of the guys we have\n",
    "            y_keep = np.random.choice(list(seen_annotators), max_annotations,replace=False)\n",
    "        xy_keep = [(x_idx,y) for y in y_keep]\n",
    "        idx_keep.extend(xy_keep)\n",
    "    xy = xy.difference(set(idx_keep))\n",
    "    x_drop = [xy_idx[0] for xy_idx in xy]\n",
    "    y_drop = [xy_idx[1] for xy_idx in xy]\n",
    "    return x_drop, y_drop\n",
    "\n",
    "def get_worst_annotators(c10h_annotator_mask, annotator_idxs):\n",
    "    annotations_per_example = np.zeros(c10h_annotator_mask.shape[0])\n",
    "    annotations_per_annotator = np.zeros(c10h_annotator_mask.shape[1]) # dictionary of distinct examples\n",
    "    selected_annotators = set()\n",
    "    \n",
    "    annotator_mask = np.zeros_like(c10h_annotator_mask)\n",
    "    # add first x annotators\n",
    "    num_initial_annotators = 25\n",
    "    for idx in annotator_idxs[:num_initial_annotators]:\n",
    "        x = np.where(c10h_annotator_mask[:,idx] == 1)[0]\n",
    "        annotations_per_example[x] += 1\n",
    "        annotations_per_annotator[idx] = len(x)\n",
    "        selected_annotators.add(idx)\n",
    "        annotator_mask[x, idx] = 1\n",
    "    \n",
    "    # only add other annotators if they share overlap with first x\n",
    "    for idx in annotator_idxs[num_initial_annotators:]:\n",
    "        if annotations_per_example.min() > 0:\n",
    "            print('all examples have at least 1 annotator')\n",
    "            break # test every single ex has at least 1 annotator\n",
    "        \n",
    "        x = np.where(c10h_annotator_mask[:,idx] == 1)[0] # annotator annotations\n",
    "        \n",
    "        # only add annotator if they overlap with current annotations\n",
    "        if annotations_per_example[x].sum() > len(x):\n",
    "            annotations_per_example[x] += 1\n",
    "            annotations_per_annotator[idx] = len(x)\n",
    "            selected_annotators.add(idx)\n",
    "            annotator_mask[x, idx] = 1\n",
    "    \n",
    "    for ex in range(annotator_mask.shape[0]):\n",
    "        if annotations_per_example[ex] < 3: # ignore dropping when there are very little annotations\n",
    "            continue\n",
    "\n",
    "        annotators = np.where(annotator_mask[ex] == 1)[0] # annotators for example\n",
    "        drop_y = np.random.choice(annotators, len(annotators)-1, replace=False)\n",
    "\n",
    "        for y in drop_y:\n",
    "            if np.random.uniform() > 0.2:\n",
    "                x = np.where(annotator_mask[:,y] == 1)[0]  # annotations for annotator y for all examples\n",
    "                x = np.setdiff1d(x,np.array([ex])) # annotations for annotator y for all examples minus curent example\n",
    "                if annotations_per_example[x].max() > 2: # number of total annotations by our annotator\n",
    "                    annotator_mask[ex][y] = 0\n",
    "                    annotations_per_annotator[y] -= 1\n",
    "                    annotations_per_example[ex] -= 1\n",
    "\n",
    "    x_drop, y_drop = np.where(annotator_mask == 0)\n",
    "    \n",
    "    print('num_worst_annotators_selected', len(selected_annotators))\n",
    "    return x_drop, y_drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0491d2a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAARP0lEQVR4nO3df2id133H8fdXsmyDktox0ZIlipv9EY9rREBFlC41rIozZvePpDBYq1BCJmFTWC6BbU5UNNokQ4y4YtDZ2USGRbIFq2T7IzEsI+BwS9FoRxS6BdkincnmRMnAau0sQUPRD5/9IdmVFNm6sq/9+J68X2Cs5zzHfr5/OB9OznPOcyKlhCSp/jUUXYAkqTYMdEnKhIEuSZkw0CUpEwa6JGViQ1EPvvXWW9Pdd99d1OMlqS699dZbv0wptax2r7BAv/vuuxkdHS3q8ZJUlyLi9KXuOeUiSZkw0CUpEwa6JGXCQJekTBjokpSJNQM9IoYi4kxEjF3ifkTEX0fEqYh4OyK+VPsyJUlrqWaE/gKw5zL39wL3LP7aD/zt1ZclSVqvNQM9pfQT4OxlujwE/H1a8DNga0T8Zq0KlCRVpxYbi+4E3l9yPbHY9j8rO0bEfhZG8Wzfvr0Gj5bWFhHX5TmeLaCiXdeXoiml51NKHSmljpaWVXeuSjWXUlrXryv5M4a5bgS1CPQPgLuWXLcutkmSrqNaBPox4JHF1S5fAf43pfSZ6RZJ0rW15hx6RAwDXwNujYgJ4PtAE0BKaRB4Dfg6cAr4P+CPrlWxkqRLWzPQU0pda9xPwB/XrCJJ0hVxp6gkZaKw76FLV2Lbtm2cO3fumj/neix1vOWWWzh79nJbPKT1MdBVV86dO5fNEsHrtT5enx8GuupK+v4X4KktRZdRE+n7Xyi6BGXGQFddiac/zmqEnp4qugrlxJeikpQJA12SMuGUi+pOLi8Tb7nllqJLUGYMdNWV6zF/HhHZzNPr88UpF0nKhIEuSZkw0CUpEwa6JGXCQJekTBjokpQJly1Ki5aub7/ws8sXVU8coUtcerNSLpuY9PngCF3Zu9pQrvbPO5pX0Qx0Za+aoL1caBvUqhdOuUhSJgx0aYUXXnih6BKkK2KgSys8+uijRZcgXREDXZIyYaBLUiYMdEnKhIEuSZkw0CUpEwa6JGXCQJekTBjokpQJA12SMmGgS1ImDHRpUUNDA01NTQA0NTXR0OB/Hqov/ouVFp0/f569e/cyOTnJ3r17OX/+fNElSevi99ClJY4dO0ZLS0vRZUhXpKoRekTsiYh3IuJURPSucn97RFQi4ucR8XZEfL32pUrXTmtr62cOuYgIWltbC6pIWr81Az0iGoHngL3ATqArInau6PbnwMsppXbgW8Df1LpQ6VqanJz8zMlEKSUmJycLqkhav2pG6F8GTqWU3k0pzQA/Ah5a0ScBX1j8eQvwYe1KlK69Tz/9FODii9ALv19ol+pBNYF+J/D+kuuJxbalngK+HRETwGtAebW/KCL2R8RoRIw68tGNZtOmTRw/fpyZmRmOHz/Opk2bii5JWpdarXLpAl5IKbUCXwf+ISI+83enlJ5PKXWklDp88SRJtVXNKpcPgLuWXLcuti3VA+wBSCn9NCI2A7cCZ2pRpHQ9fPrppzzwwAOcP3+ehoYGly2q7lQzQn8TuCcifisiNrLw0vPYij7vAbsBIqIEbAacU1HduDC9ciHEL/zutIvqyZqBnlKaAx4DXgfGWVjNciIinomIBxe7/SmwLyL+AxgGHk0rlwxIN7ANG1b/n9VLtUs3oqr+taaUXmPhZefStu8t+fkk8NXaliZdP1NTU3R1dfH2228zPj5OqVTi3nvvZXh4uOjSpKq59V9atHITkZuKVG8MdAlobGxkYGCA7u5uPvnkE7q7uxkYGKCxsbHo0qSqGegSsGXLFiKCgwcP0tzczMGDB4kItmzZUnRpUtUMdAn46KOPuP/++zlzZmGl7ZkzZ7j//vv56KOPii1MWgcDXQLuuOMOTpw4wRtvvMHMzAxvvPEGJ06c4I477ii6NKlqBrq0aLWPc0n1xECXgA8//JC2tjZ2797Nxo0b2b17N21tbXz4od+ZU/0w0CVg69atVCoVBgYGmJqaYmBggEqlwtatW4suTaqagS4BH3/8MVu2bKG9vZ2mpiba29vZsmULH3/8cdGlSVUz0CVgbm6OgYEByuUymzdvplwuMzAwwNzcXNGlSVUz0CUWPsJ17tw5xsbGmJ+fZ2xsjHPnzvlxLtUVA10C9u3bx4EDB7j99ttpaGjg9ttv58CBA+zbt6/o0qSqGegScN9999Hc3MzZs2dJKXH27Fmam5u57777ii5NqpqBLgH9/f28+uqrzMzMkFJiZmaGV199lf7+/qJLk6oWRW2e6OjoSKOjo4U8W1qpsbGR6elpmpqaLrbNzs6yefNm5ufnC6xMWi4i3kopdax2zxG6BJRKJUZGRpa1jYyMUCqVCqpIWj+PY5GAvr4+vvnNb9Lc3Mx7773H9u3bmZqa4oc//GHRpUlVc4QureA3XFSvDHSJhZei+/fvp7m5mYigubmZ/fv3+1JUdcUpFwk4efIkU1NTDA0NsWvXLkZGRuju7ub06dNFlyZVzRG6BGzcuJFyuUxnZydNTU10dnZSLpfZuHFj0aVJVTPQJWBmZobDhw9TqVSYnZ2lUqlw+PBhZmZmii5NqppTLhKwc+dOvvGNb1AulxkfH6dUKvHwww/zyiuvFF2aVDVH6BILyxaPHj3KoUOHmJ6e5tChQxw9epS+vr6iS5Oq5ghdArq6ugCWjdD7+/svtkv1wK3/klRH3PovSZ8DBrokZcJAl6RMGOiSlAkDXZIyYaBLUiYMdEnKhIEuSZmoKtAjYk9EvBMRpyKi9xJ9/jAiTkbEiYg4WtsyJUlrWXPrf0Q0As8BvwdMAG9GxLGU0sklfe4Bvgt8NaV0LiJ+41oVLElaXTUj9C8Dp1JK76aUZoAfAQ+t6LMPeC6ldA4gpXSmtmVKktZSTaDfCby/5HpisW2pHcCOiPjXiPhZROxZ7S+KiP0RMRoRo5OTk1dWsSRpVbV6KboBuAf4GtAF/F1EbF3ZKaX0fEqpI6XU0dLSUqNHS5KgukD/ALhryXXrYttSE8CxlNJsSum/gF+wEPCSpOukmkB/E7gnIn4rIjYC3wKOrejzCgujcyLiVhamYN6tXZmSpLWsGegppTngMeB1YBx4OaV0IiKeiYgHF7u9DvwqIk4CFeBASulX16poSdJnecCFJNURD7iQqjA8PExbWxuNjY20tbUxPDxcdEnSunimqMRCmPf19XHkyBF27drFyMgIPT09AJ4rqrrhlIsEtLW1cejQITo7Oy+2VSoVyuUyY2NjBVYmLXe5KRcDXQIaGxuZnp6mqanpYtvs7CybN29mfn6+wMqk5ZxDl9ZQKpUYGRlZ1jYyMkKpVCqoImn9DHQJ6Ovro6enh0qlwuzsLJVKhZ6eHvr6+oouTaqaL0Ulfv3is1wuMz4+TqlUor+/3xeiqivOoUtSHXEOXZI+Bwx0ScqEgS4tcqeo6p0vRSXcKao8+FJUwp2iqh/uFJXW4E5R1QtXuUhrcKeocmCgS7hTVHnwpaiEO0WVB+fQJamOOIcuSZ8DBrokZcJAl6RMGOiSlAkDXZIyYaBLUiYMdEnKhIEuSZkw0CUpEwa6JGXCQJekTBjokpQJA11a5Jmiqnd+PlfCM0WVBz+fK+GZoqofnikqrcEzRVUvrvp76BGxJyLeiYhTEdF7mX5/EBEpIlZ9mHSj8kxR5WDNQI+IRuA5YC+wE+iKiJ2r9LsZeBz4t1oXKV1rnimqHFTzUvTLwKmU0rsAEfEj4CHg5Ip+fwE8CxyoaYXSdeCZospBNVMudwLvL7meWGy7KCK+BNyVUvrnGtYmSVqHq162GBENwF8Bj1bRdz+wH2D79u1X+2ipZly2qBxUM0L/ALhryXXrYtsFNwNtwI8j4r+BrwDHVnsxmlJ6PqXUkVLqaGlpufKqpRrr7+/nyJEjdHZ20tTURGdnJ0eOHKG/v7/o0qSqrblsMSI2AL8AdrMQ5G8CD6eUTlyi/4+BP0spXXZNossWdSNx2aLqxVUtW0wpzQGPAa8D48DLKaUTEfFMRDxY21KlYrhsUTmoag49pfQa8NqKtu9dou/Xrr4s6fq6sGxx5Ry6Uy6qJ37LRcJli8qDW/8lqY5c9dZ/SdKNz0CXpEwY6JKUCQNdkjJhoEtSJgx0ScqEgS5JmTDQJSkTBrokZcJAl6RMGOiSlAkDXZIyYaBLUiYMdEnKhIEuSZkw0CUpEwa6JGXCQJekTBjokpQJA12SMmGgS1ImDHRJyoSBLkmZMNAlKRMGurRoeHiYtrY2GhsbaWtrY3h4uOiSpHXZUHQB0o1geHiYvr4+jhw5wq5duxgZGaGnpweArq6ugquTqhMppUIe3NHRkUZHRwt5trRSW1sbhw4dorOz82JbpVKhXC4zNjZWYGXSchHxVkqpY9V7BroEjY2NTE9P09TUdLFtdnaWzZs3Mz8/X2Bl0nKXC3Tn0CWgVCrx9NNPL5tDf/rppymVSkWXJlXNQJeAzs5Onn32Wbq7u/nkk0/o7u7m2WefXTYFI93oDHSJhfnyJ598kqGhIW6++WaGhoZ48sknqVQqRZcmVc05dAnn0FU/nEOX1uAcunJQVaBHxJ6IeCciTkVE7yr3/yQiTkbE2xHxRkR8sfalSteOc+jKwZqBHhGNwHPAXmAn0BURO1d0+znQkVK6F/gn4GCtC5WuJefQlYM159Aj4neAp1JKv794/V2AlNJfXqJ/O3A4pfTVy/29zqHrRuIcuurF1c6h3wm8v+R6YrHtUnqAf7lEIfsjYjQiRicnJ6t4tHR9lEolRkZGlrWNjIw4h666UtOXohHxbaAD+MFq91NKz6eUOlJKHS0tLbV8tHRV+vr66OnpoVKpMDs7S6VSoaenh76+vqJLk6pWzce5PgDuWnLduti2TEQ8APQBv5tS+rQ25UnXx4UPcJXLZcbHxymVSvT39/thLtWVaubQNwC/AHazEORvAg+nlE4s6dPOwsvQPSml/6zmwc6hS9L6XdUcekppDngMeB0YB15OKZ2IiGci4sHFbj8AbgL+MSL+PSKO1ah2SVKVqvoeekrpNeC1FW3fW/LzAzWuS5K0Tu4UlaRMGOiSlAkDXZIyYaBLUiYMdEnKhIEuSZkw0CUpEwa6tGh4eHjZARfDw8NFlySti4EusRDmjz/+OFNTUwBMTU3x+OOPG+qqKwa6BDzxxBNs2LCBoaEhpqenGRoaYsOGDTzxxBNFlyZVzUCXgImJCV588UU6Oztpamqis7OTF198kYmJiaJLk6pmoEtSJgx0CWhtbeWRRx5ZdsDFI488Qmtra9GlSVUz0CXg4MGDzM/P093dzaZNm+ju7mZ+fp6DBz3vXPXDQJdYOLGovb2d06dPk1Li9OnTtLe3e2KR6oqBLrFw9Nzx48e57bbbaGho4LbbbuP48eOUy+WiS5OqZqBLwODgIFu3buXo0aNMT09z9OhRtm7dyuDgYNGlSVUz0CVgbm6Ol156admyxZdeeom5ubmiS5OqZqBLi8bGxi57Ld3oqjpTVMrdtm3b6O3tpbGxke985zsMDg7S29vLtm3bii5NqpojdAk4fPgwN910E729vTQ3N9Pb28tNN93E4cOHiy5NqpqBLrGwbHFwcJAdO3bQ0NDAjh07GBwcdNmi6kqklAp5cEdHRxodHS3k2ZJUryLirZRSx2r3HKFLUiYMdEnKhIEuSZkw0CUpEwa6JGWisFUuETEJnC7k4dLl3Qr8sugipEv4YkqpZbUbhQW6dKOKiNFLLQuTbmROuUhSJgx0ScqEgS591vNFFyBdCefQJSkTjtAlKRMGuiRlwkCXFkXEUESciQiPKlJdMtClX3sB2FN0EdKVMtClRSmlnwBni65DulIGuiRlwkCXpEwY6JKUCQNdkjJhoEuLImIY+Cnw2xExERE9RdckrYdb/yUpE47QJSkTBrokZcJAl6RMGOiSlAkDXZIyYaBLUiYMdEnKxP8D1ooYyLNCjxcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all examples have at least 1 annotator\n",
      "num_worst_annotators_selected 511\n",
      "Total idxs dropped:  485611\n",
      "Make sure 12 <= max_annotations and 1 > 0: \n",
      "(10000, 511) (10000, 511)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAART0lEQVR4nO3db2id53nH8e9lRbKwHddubZLZju2wpON4gs1FpIWGNGItpH2RGAZLlIyx+RDjdTpkLLT2ekabZChgRgbdIVvmYFM2kEryxhiWLYVxRtFoSxy2FcciRST2ImcQN3Gb1kaxIt97oWNFUmzrKD7S43P7+wFjnee5rXO9+vnheu4/kVJCktT+VhRdgCSpNQx0ScqEgS5JmTDQJSkTBrokZeKmor54w4YNafv27UV9vSS1pVdfffXnKaWNl7tXWKBv376dY8eOFfX1ktSWIuLUle7ZcpGkTBjokpQJA12SMmGgS1ImDHRJysSCgR4RhyPinYg4foX7ERF/FxFjEfHTiPhc68uUJC2kmSf07wH3XeX+V4E7G3/2AP9w7WVJkhZrwUBPKf0QeO8qQx4A/ilN+zGwLiJ+o1UFSpKa04qFRZuBt2Z9Hm9c+7/5AyNiD9NP8WzdurUFXy0tLCKW5Xs8W0BFW9aXoimlgyml3pRS78aNl125KrVcSmlRfz7JvzHMdT1oRaCfBm6b9XlL45okaRm1ItCPAn/UmO3yBeCXKaWPtVskSUtrwR56RAwD9wIbImIc+A7QCZBSeg54CfgaMAacB/5kqYqVJF3ZgoGeUupf4H4C/qxlFUmSPhFXikpSJgx0ScqEgS5JmTDQJSkTBrokZaKwM0WlT+LTn/40Z8+eXfLvWY7tAtavX897711tmyRpcQx0tZWzZ89ms8x+ufaY0Y3DloskZcJAl6RM2HJRW0nfWQtPfKroMloifWdt0SUoMwa62ko8+X5WPfT0RNFVKCe2XCQpEwa6JGXCQJekTNhDV9vJZf72+vXriy5BmTHQ1VaW44VoRGTz4lU3FlsukpQJA12SMmGgS1ImDHRJyoSBLkmZMNAlKRMGuiRlwkCXpEwY6JKUCQNdkjJhoEsNw8PD9PT0ANDT08Pw8HDBFUmLY6BLTIf5Y489xrlz5wA4d+4cjz32mKGuthJFbULU29ubjh07Vsh368ayXLszuqGXlkNEvJpS6r3cPZ/Qlb2U0oJ/YDr4b731VlasWMGtt9468x9BM//eMNf1wECXGtasWcPQ0BATExMMDQ2xZs2aokuSFsX90KWGqakpdu/ezalTp9i2bRtTU1NFlyQtioEuNZw/f57x8XFSSoyPj/Phhx8WXZK0KE21XCLivoh4PSLGImL/Ze5vjYh6RPxXRPw0Ir7W+lKlpXOpX37x4sU5f+dy3J1uDAsGekR0AM8CXwV2AP0RsWPesL8CXkgp7QQeAv6+1YVKS+nSS835ge7LTrWTZp7Q7wLGUkpvpJQuAN8HHpg3JgFrGz9/Cni7dSVKy2PVqlVs376dFStWsH37dlatWlV0SdKiNNND3wy8NevzOPD5eWOeAH4QERVgNfDly/2iiNgD7AHYunXrYmuVltQHH3zAyZMnATh58iQdHR3FFiQtUqumLfYD30spbQG+BvxzRHzsd6eUDqaUelNKvRs3bmzRV0utMTU1NdMzjwhnuajtNBPop4HbZn3e0rg2Wxl4ASCl9COgG9jQigKl5XSpZ27vXO2omUB/BbgzIm6PiC6mX3oenTfmf4HfA4iIEtOBfqaVhUrLYf369UQE69evL7oUadEWDPSU0ofAAPAyMMr0bJbXIuKpiLi/Mexx4NGI+B9gGPjj5COO2swdd9zBpk2biAg2bdrEHXfcUXRJ0qI01UNPKb2UUvpsSuk3U0qDjWvfTikdbfx8IqX0xZTS76SUfjel9IOlLFpaCmNjY9xzzz2899573HPPPYyNjRVdkrQo7rYoAZ2dnQBzVofedNP0JLDJyclCapIux90WpQXs3buXixcvcssttxAR3HLLLVy8eJG9e/cWXZrUNPdykYBarQbA888/T0qJX/ziF3z961+fuS61A1suktRGbLlITbh0pmhHR4dniqot2XKRmA7zarXKoUOHuPvuuxkZGaFcLgPQ399fcHVSc2y5SEBPTw+7du3iyJEjjI6OUiqVZj4fP3686PKkGVdrufiELgEnTpzg/PnzH3tCv7RZl9QO7KFLQFdXFwMDA/T19dHZ2UlfXx8DAwN0dXUVXZrUNANdAi5cuECtVqNerzM5OUm9XqdWq3HhwoWiS5OaZstFAnbs2MGuXbuoVCozPfRHHnmEI0eOFF2a1DSf0CWgWq0yNDRErVZjYmKCWq3G0NAQ1Wq16NKkpvmELvHR1MTZT+iDg4NOWVRbcdqiJLURV4pKTXClqNqdLRcJV4oqD7ZcJKZXitZqNfr6+mau1et1KpWKK0V1XbHlIi1gdHSUF198ke7ubiKC7u5uXnzxRUZHR4suTWqagS4B69at4+DBgzz99NOcO3eOp59+moMHD7Ju3bqiS5OaZqBLwPvvv8/atWvZuXMnnZ2d7Ny5k7Vr1/L+++8XXZrUNANdYvos0WeeeYZKpUJ3dzeVSoVnnnlmzhmj0vXOQJeAlStXcvbsWY4fP87U1BTHjx/n7NmzrFy5sujSpKY5bVECHn30Ufbt2wdMHxj93HPPsW/fPg+JVlsx0CU+OiT6W9/6Fo8//jgrV65k7969HhKttuI8dElqI85Dl6QbgIEuSZkw0CUpEwa6JGXCQJekTBjoUoP7oavdOQ9dwv3QlQfnoUu4H7rax9XmoRvoEtDR0cHExASdnZ0z1yYnJ+nu7mZqaqrAyqS5rnlhUUTcFxGvR8RYROy/wpg/iIgTEfFaRAxdS8HSciuVSoyMjMy5NjIyQqlUKqgiafEWDPSI6ACeBb4K7AD6I2LHvDF3An8JfDGl9NvAn7e+VGnpVKtVyuUy9XqdyclJ6vU65XKZarVadGlS05p5KXoXMJZSegMgIr4PPACcmDXmUeDZlNJZgJTSO60uVFpKl158VioVRkdHKZVKDA4O+kJUbaWZQN8MvDXr8zjw+XljPgsQEf8JdABPpJT+bf4viog9wB6ArVu3fpJ6pSXT399vgKuttWoe+k3AncC9QD/wfESsmz8opXQwpdSbUurduHFji75akgTNBfpp4LZZn7c0rs02DhxNKU2mlN4EfsZ0wEuSlkkzgf4KcGdE3B4RXcBDwNF5Y44w/XRORGxgugXzRuvKlCQtZMFATyl9CAwALwOjwAsppdci4qmIuL8x7GXg3Yg4AdSBb6SU3l2qoiVJH+fCIklqI55YJEk3AANdkjJhoEtSJgx0ScqEgS5JmTDQpQZPLFK788QiCU8sUh6chy7hiUVqH55YJC3AE4vULlxYJC3AE4uUAwNdwhOLlAdfikp4YpHyYA9dktqIPXRJugEY6JKUCQNdkjJhoEtSJgx0ScqEgS5JmTDQJSkTBrokZcJAl6RMGOiSlAkDXZIyYaBLDR5Bp3bnbosSHkGnPLjbooRH0Kl9eASdtACPoFO7cPtcaQGlUoknn3xyTg/9ySef9Ag6tRUDXQL6+vo4cOAAu3fv5le/+hW7d+/mwIEDc1ow0vXOQJeY7pfv27ePw4cPc/PNN3P48GH27dtHvV4vujSpafbQJeyhq33YQ5cWUCqVGBkZmXNtZGTEHrraSlOBHhH3RcTrETEWEfuvMu73IyJFxGX/95CuV9VqlXK5TL1eZ3Jyknq9TrlcplqtFl2a1LQFFxZFRAfwLPAVYBx4JSKOppROzBt3M/AY8JOlKFRaSpcWD1UqFUZHRymVSgwODrqoSG2lmZWidwFjKaU3ACLi+8ADwIl54/4aOAB8o6UVSsukv7/fAFdba6blshl4a9bn8ca1GRHxOeC2lNK/XO0XRcSeiDgWEcfOnDmz6GIlSVd2zS9FI2IF8LfA4wuNTSkdTCn1ppR6N27ceK1fLUmapZlAPw3cNuvzlsa1S24GeoD/iIiTwBeAo74YlaTl1UygvwLcGRG3R0QX8BBw9NLNlNIvU0obUkrbU0rbgR8D96eUnGQuSctowUBPKX0IDAAvA6PACyml1yLiqYi4f6kLlCQ1p6n90FNKLwEvzbv27SuMvffay5IkLZYrRSUpEwa6JGXCQJekTBjokpQJA12SMmGgS1ImDHRJyoSBLkmZMNAlKRMGuiRlwkCXpEwY6JKUCQNdkjJhoEtSJgx0ScqEgS5JmTDQJSkTBrokZcJAl6RMGOiSlAkDXZIyYaBLUiYMdEnKhIEuSZkw0CUpEwa6JGXCQJekTBjokpQJA12SMmGgS1ImDHRJyoSBLjUMDw/T09NDR0cHPT09DA8PF12StCg3FV2AdD0YHh6mWq1y6NAh7r77bkZGRiiXywD09/cXXJ3UnKae0CPivoh4PSLGImL/Ze7/RUSciIifRsS/R8S21pcqLZ3BwUEOHTpEX18fnZ2d9PX1cejQIQYHB4suTWpapJSuPiCiA/gZ8BVgHHgF6E8pnZg1pg/4SUrpfET8KXBvSunBq/3e3t7edOzYsWutX2qJjo4OJiYm6OzsnLk2OTlJd3c3U1NTBVYmzRURr6aUei93r5kn9LuAsZTSGymlC8D3gQdmD0gp1VNK5xsffwxsuZaCpeVWKpUYGRmZc21kZIRSqVRQRdLiNRPom4G3Zn0eb1y7kjLwr5e7ERF7IuJYRBw7c+ZM81VKS6xarVIul6nX60xOTlKv1ymXy1Sr1aJLk5rW0peiEfGHQC/wpcvdTykdBA7CdMulld8tXYtLLz4rlQqjo6OUSiUGBwd9Iaq20kygnwZum/V5S+PaHBHxZaAKfCml9EFrypMkNauZQH8FuDMibmc6yB8CHp49ICJ2Av8I3JdSeqflVUpLzGmLysGCPfSU0ofAAPAyMAq8kFJ6LSKeioj7G8P+BlgDvBgR/x0RR5esYmkJOG1ROWiqh55Segl4ad61b8/6+cstrktaVqOjo4yPj9PT0zPTQ9+3bx+jo6NFlyY1zZWiErBp0ya++c1vMjQ0NNNyefjhh9m0aVPRpUlNM9ClhomJCXbv3s2pU6fYtm0bExMTrFmzpuiypKa5OZcEnD59emaVaEQA0NnZyenTH5vQJV23DHQJ6OrqYv/+/bz55ptMTU3x5ptvsn//frq6uoouTWqagS4BFy5coFarzVkpWqvVuHDhQtGlSU2zhy4BO3bsYNeuXXNWij7yyCMcOXKk6NKkpvmELjG9l8vQ0BC1Wo2JiQlqtRpDQ0Pu5aK24hO6hHu5KA8L7oe+VNwPXZIW71r3Q5cktQEDXWrwkGi1O3voEu62qDzYQ5eAnp4earUafX19M9fq9TqVSoXjx48XWJk019V66Aa6hIdEq334UlRagIdEKwf20CWmFxY9+OCDrF69ema3xXPnzvHd73636NKkpvmELs1zabdFqd0Y6BLTR9Dt2bOH1atXA7B69Wr27NnjEXRqK7ZcJODEiROcP3/+Y9MWT548WXRpUtN8QpeY3g99YGBgziHRAwMD7oeutuK0RQlYsWIFn/nMZ1izZs3MS9Ff//rXvPvuu1y8eLHo8qQZTluUFrB582YmJyeBj16KTk5Osnnz5iLLkhbFQJcaVq1axeHDh5mYmODw4cOsWrWq6JKkRTHQJeDtt9/mwIEDVCoVuru7qVQqHDhwgLfffrvo0qSmOctFYnql6JYtW+bs21Kv110pqrbiE7rE9ErRcrk855DocrnsEXRqKz6hS3gEnfLgtEVJaiNOW5SkG4CBLkmZMNAlKRMGuiRlwkCXpEwUNsslIs4Apwr5cunqNgA/L7oI6Qq2pZQ2Xu5GYYEuXa8i4tiVpoVJ1zNbLpKUCQNdkjJhoEsfd7DoAqRPwh66JGXCJ3RJyoSBLkmZMNClhog4HBHvRMTxhUdL1x8DXfrI94D7ii5C+qQMdKkhpfRD4L2i65A+KQNdkjJhoEtSJgx0ScqEgS5JmTDQpYaIGAZ+BPxWRIxHRLnomqTFcOm/JGXCJ3RJyoSBLkmZMNAlKRMGuiRlwkCXpEwY6JKUCQNdkjLx/379xTlQkGKcAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Consensus label accuracy:  0.9379\n"
     ]
    }
   ],
   "source": [
    "# get annotator accuracy\n",
    "annotator_accuracy_df = plt_annotator_accuracy(c10h_labels_error_mask, c10h_annotator_mask).sort_values(by='score')\n",
    "annotator_idxs = annotator_accuracy_df.index.values.tolist()\n",
    "\n",
    "x_drop, y_drop = get_worst_annotators(c10h_annotator_mask, annotator_idxs)\n",
    "c10h_labels, c10h_labels_error_mask, c10h_annotator_mask = \\\n",
    "                    get_sample_labels(x_drop, y_drop, c10h_labels, c10h_labels_error_mask, c10h_annotator_mask)\n",
    "\n",
    "print(f'Make sure {c10h_annotator_mask.sum(axis=1).max()} <= max_annotations and { c10h_annotator_mask.sum(axis=1).min()} > 0: ')\n",
    "\n",
    "# drop all empty annotators\n",
    "drop_axis = c10h_labels.copy()\n",
    "c10h_labels = c10h_labels[:, ~np.isnan(drop_axis).all(axis=0)]\n",
    "c10h_labels_error_mask = c10h_labels_error_mask[:, ~np.isnan(drop_axis).all(axis=0)]\n",
    "c10h_annotator_mask = c10h_annotator_mask[:, ~np.isnan(drop_axis).all(axis=0)]\n",
    "print(c10h_labels.shape, c10h_annotator_mask.shape)\n",
    "\n",
    "# get new annotator accuracy\n",
    "annotator_accuracy_df = plt_annotator_accuracy(c10h_labels_error_mask,  c10h_annotator_mask).sort_values(by='score')\n",
    "annotator_accuracy_df.describe()\n",
    "consensus_labels = get_consensus_label(c10h_labels, pred_probs=None)\n",
    "print('Consensus label accuracy: ', (c10h_true_labels == consensus_labels).sum() / 10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f5ec8d33",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if dataset creation conditions are met\n",
    "for col in range(c10h_annotator_mask.shape[1]):\n",
    "    annotator = np.where(c10h_annotator_mask[:,col] == 1)[0]\n",
    "    intersects = False\n",
    "    for j in range(c10h_annotator_mask.shape[1]):\n",
    "        if j == col:\n",
    "            continue\n",
    "        annotator2 = np.where(c10h_annotator_mask[:,j] == 1)[0]\n",
    "        if len(np.intersect1d(annotator, annotator2)) > 0:\n",
    "            intersects = True\n",
    "    if not intersects:\n",
    "        print(f'annotator {col} does not intersect with any other annotator')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "70b2bdf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2     2929\n",
      "3     2431\n",
      "1     1716\n",
      "4     1614\n",
      "5      830\n",
      "6      307\n",
      "7      127\n",
      "8       34\n",
      "9        9\n",
      "10       2\n",
      "12       1\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[<AxesSubplot:title={'center':'0'}>]], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPYklEQVR4nO3df4xl5V3H8fe3bAl0p2X5lQnuokMD0iAbaJkgFdPMQGtoIcAfpILYLAayibYRlUa3/tfERhqt2BhisinVjakMiBAIWJVQxsbEoruldUuRgrggG8q2Ydl2sGld/frHPYuT6Qz3zsz9Md973q9kM/c899x5vk/uvZ959jnn3BuZiSSpnreMugBJ0toY4JJUlAEuSUUZ4JJUlAEuSUUZ4JJUlAEuSUUZ4Gq1iDglIh6IiNcj4oWI+KVR1yT1atOoC5BG7E7gR8AkcCHwSER8PTOfGmlVUg/CKzHVVhGxGTgMnJ+Z32ra/gI4mJm7Rlqc1AOXUNRmPw0cPRbeja8DPzOieqRVMcDVZhPA95a0HQHePoJapFUzwNVmC8A7lrS9A/j+CGqRVs0AV5t9C9gUEecsarsA8ACmSvAgplotIuaABG6hcxbK3wA/51koqsAZuNru14ATgUPA3cCvGt6qwhm4JBXlDFySijLAJakoA1ySijLAJamooX6Y1WmnnZZTU1ND6+/1119n8+bNQ+tvI2nz2KHd43fs4zf2ffv2fTczT1/aPtQAn5qaYu/evUPrb35+npmZmaH1t5G0eezQ7vE79plRl9F3EfHCcu0uoUhSUQa4JBVlgEtSUQa4JBVlgEtSUQa4JBVlgEtSUQa4JBVlgEtSUUO9ElO1Te16ZMX7Dtx+5RArkQTOwCWpLANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpqJ4DPCKOi4gnI+LhZvusiHgiIp6LiHsi4vjBlSlJWmo1M/BbgacXbX8auCMzzwYOAzf3szBJ0pvrKcAjYhtwJfC5ZjuAy4D7ml32ANcOoD5J0goiM7vvFHEf8PvA24GPAzcBX2lm30TEmcAXM/P8ZR67E9gJMDk5edHc3Fzfiu9mYWGBiYmJofW3kQxi7PsPHlnxvu1bT+prX+vlc+/Yx8ns7Oy+zJxe2r6p2wMj4irgUGbui4iZ1XacmbuB3QDT09M5M7PqX7Fm8/PzDLO/jWQQY79p1yMr3nfgxv72tV4+9zOjLmMk2jb2rgEOXApcHREfAk4A3gF8FtgSEZsy8yiwDTg4uDIlSUt1XQPPzE9k5rbMnAKuB76UmTcCjwPXNbvtAB4cWJWSpB+znvPAfwf4rYh4DjgVuKs/JUmSetHLEsobMnMemG9uPw9c3P+SJEm98EpMSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSpq06gLkBab2vXIsu0Hbr9yyJVIG58zcEkqqmuAR8QJEfHPEfH1iHgqIj7ZtJ8VEU9ExHMRcU9EHD/4ciVJx/QyA/8hcFlmXgBcCFwREZcAnwbuyMyzgcPAzQOrUpL0Y7oGeHYsNJtvbf4lcBlwX9O+B7h2EAVKkpYXmdl9p4jjgH3A2cCdwB8AX2lm30TEmcAXM/P8ZR67E9gJMDk5edHc3Fz/qu9iYWGBiYmJofW3kQxi7PsPHlnxvu1bTxpoH6v9/T73jn2czM7O7svM6aXtPZ2Fkpn/A1wYEVuAB4B39dpxZu4GdgNMT0/nzMxMrw9dt/n5eYbZ30YyiLHftMIZIgAHbuxPXyv1sdrf73M/M+oyRqJtY1/VWSiZ+RrwOPBeYEtEHPsDsA042N/SJElvppezUE5vZt5ExInAB4Cn6QT5dc1uO4AHB1SjJGkZvSyhnAHsadbB3wLcm5kPR8Q3gbmI+D3gSeCuAdapIVrpYppqfUjjrmuAZ+a/Au9epv154OJBFCVJ6s4rMSWpKANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpKANckorq6QsdVMPiT/i7bfvRN74c4cDtV46qJEkD5AxckooywCWpKANckooywCWpKA9itphfaybV5gxckooywCWpKANckopyDVwD5Tq7NDjOwCWpKANckooywCWpKANckoryIKZKWOlg6EqftLj/4JE3Po2xl/2lipyBS1JRBrgkFWWAS1JRBrgkFWWAS1JRBrgkFWWAS1JRBrgkFWWAS1JRBrgkFdU1wCPizIh4PCK+GRFPRcStTfspEfFoRDzb/Dx58OVKko7pZQZ+FLgtM88DLgE+GhHnAbuAxzLzHOCxZluSNCRdAzwzX87Mrza3vw88DWwFrgH2NLvtAa4dUI2SpGVEZva+c8QU8GXgfODFzNzStAdw+Nj2ksfsBHYCTE5OXjQ3N7fuonu1sLDAxMTE0Pobtf0Hj7xxe/JEeOUHndvbt57Udf/1GkYfq+n30KtH3hh/L/uPk7a97hcb17HPzs7uy8zppe09B3hETAD/AHwqM++PiNcWB3ZEHM7MN10Hn56ezr17966u8nWYn59nZmZmaP2N2uKPXL1t+1E+s7/zacErfYRqP7+vchh9rKbfP/nCg2+Mv5f9x0nbXveLjevYI2LZAO/pLJSIeCvw18AXMvP+pvmViDijuf8M4FC/ipUkddfLWSgB3AU8nZl/tOiuh4Adze0dwIP9L0+StJJevpHnUuAjwP6I+FrT9rvA7cC9EXEz8ALw4YFUqBIGvVQyKqv9JiBpmLoGeGb+IxAr3H15f8uRJPXKKzElqSgDXJKKMsAlqaheDmJqg1ntAcNxPcAotZ0zcEkqygCXpKIMcEkqygCXpKI8iKnSVjpAe9v2IRfSA6/qVL85A5ekogxwSSrKAJekolwDHyLXQCX1kzNwSSrKAJekogxwSSrKAJekojyIuQH4aYGj16/nwOdSw+QMXJKKMsAlqSgDXJKKcg1cWgPXurUROAOXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKK6BnhEfD4iDkXENxa1nRIRj0bEs83PkwdbpiRpqV5m4H8OXLGkbRfwWGaeAzzWbEuShqhrgGfml4FXlzRfA+xpbu8Bru1vWZKkbiIzu+8UMQU8nJnnN9uvZeaW5nYAh49tL/PYncBOgMnJyYvm5ub6UngvFhYWmJiYGFp/3ew/eGRofU2eCK/8YGjdbTgrjX/71pOW3X+Yz81SK9W0VhvtdT9M4zr22dnZfZk5vbR93d+JmZkZESv+FcjM3cBugOnp6ZyZmVlvlz2bn59nmP11c9MQv0fxtu1H+cz+9n7l6UrjP3DjzLL7D/O5WWqlmtZqo73uh6ltY1/rWSivRMQZAM3PQ/0rSZLUi7UG+EPAjub2DuDB/pQjSepVL6cR3g38E3BuRLwUETcDtwMfiIhngfc325KkIeq6SJqZN6xw1+V9rkWStApeiSlJRRngklSUAS5JRbX3ROE+mFrh3OEDt1855EoktZEzcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKK8kEcaMS8I01o5A5ekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSqqdRfyDOOiiZX6kFZjta/VY/vftv0oN/XwGvRCofqcgUtSUQa4JBVlgEtSUQa4JBVV/iDmmx0w7PVgzlp/vySNkjNwSSrKAJekogxwSSrKAJekosocxPRgotQx6PfCWq8A7XV/9Y8zcEkqygCXpKIMcEkqqswa+KC5xq62We1rvp/vkUGvs49qXX7Y/ToDl6Si1hXgEXFFRDwTEc9FxK5+FSVJ6m7NAR4RxwF3Ah8EzgNuiIjz+lWYJOnNrWcGfjHwXGY+n5k/AuaAa/pTliSpm8jMtT0w4jrgisy8pdn+CPCzmfmxJfvtBHY2m+cCz6y93FU7DfjuEPvbSNo8dmj3+B37+PmpzDx9aePAz0LJzN3A7kH3s5yI2JuZ06Poe9TaPHZo9/gde3vGvp4llIPAmYu2tzVtkqQhWE+A/wtwTkScFRHHA9cDD/WnLElSN2teQsnMoxHxMeDvgOOAz2fmU32rrD9GsnSzQbR57NDu8Tv2lljzQUxJ0mh5JaYkFWWAS1JRYxXgEXFcRDwZEQ8322dFxBPNpf73NAdbx1JEHIiI/RHxtYjY27SdEhGPRsSzzc+TR13nIETEloi4LyL+LSKejoj3tmHsEXFu83wf+/e9iPiNNoz9mIj4zYh4KiK+ERF3R8QJbXrfj1WAA7cCTy/a/jRwR2aeDRwGbh5JVcMzm5kXLjoPdhfwWGaeAzzWbI+jzwJ/m5nvAi6g8xoY+7Fn5jPN830hcBHwX8ADtGDsABGxFfh1YDozz6dzMsX1tOh9PzYBHhHbgCuBzzXbAVwG3Nfssge4diTFjc41dMYNYzr+iDgJeB9wF0Bm/igzX6MFY1/icuDfM/MF2jX2TcCJEbEJeBvwMi16349NgAN/DPw28L/N9qnAa5l5tNl+Cdg6grqGJYG/j4h9zccXAExm5svN7W8Dk6MpbaDOAr4D/FmzfPa5iNhMO8a+2PXA3c3tVow9Mw8Cfwi8SCe4jwD7aNH7fiwCPCKuAg5l5r5R1zJCP5+Z76Hz6ZAfjYj3Lb4zO+eLjuM5o5uA9wB/mpnvBl5nyZLBGI8dgGaN92rgr5beN85jb9b2r6HzR/wngM3AFSMtasjGIsCBS4GrI+IAnU9FvIzOuuiW5r9WMOaX+jezETLzEJ110IuBVyLiDIDm56HRVTgwLwEvZeYTzfZ9dAK9DWM/5oPAVzPzlWa7LWN/P/AfmfmdzPxv4H46WdCa9/1YBHhmfiIzt2XmFJ3/Sn4pM28EHgeua3bbATw4ohIHKiI2R8Tbj90GfgH4Bp2PNtjR7DaW48/MbwP/GRHnNk2XA9+kBWNf5Ab+f/kE2jP2F4FLIuJtzTGvY899K973MIZXYkbEDPDxzLwqIt5JZ0Z+CvAk8MuZ+cMRljcQzTgfaDY3AX+ZmZ+KiFOBe4GfBF4APpyZr46ozIGJiAvpHLw+Hnge+BU6k5M2jH0znSB7Z2Yeadpa8bwDRMQngV8EjtJ5j99CZ8177N/3MIYBLkltMRZLKJLURga4JBVlgEtSUQa4JBVlgEtSUQa4JBVlgEtSUf8H5elEwlUHJgAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get stats\n",
    "print(pd.DataFrame(c10h_annotator_mask.sum(axis=1)).value_counts())\n",
    "pd.DataFrame(c10h_annotator_mask.sum(axis=0)).hist(bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "236c6d3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(511, 511)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATQAAAEGCAYAAADrM2k5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABcT0lEQVR4nO19ebglRZHvL6ABFQQFHBZBGxE3dERAwOeG4NIwDqiDCE9xeTrIQ3BhfArjfMpjhic6ozxcQFEYZWRTQW2RRQdRcB4giMgq0rSo7LKICyh033h/VBYk0RGRkVV17jn39vndr757KpfIyC0yIrKqkpgZU0wxxRTzAauMm4EppphiiqEwFWhTTDHFvMFUoE0xxRTzBlOBNsUUU8wbTAXaFFNMMW8wFWhTTDHFvMHIBBoRLSKi64hoCREdPKpypphiiilajESgEdGqAD4LYBcAzwKwNxE9axRlTTHFFHMTRHQ8Ed1BRFcZ8UREn0pK0RVEtHWJ5qg0tO0ALGHmpcz8AIBTAOw+orKmmGKKuYkvAVjkxO8CYIt07QvgmBLBUQm0JwL4TXZ/UwqbYooppgAAMPP5AO52kuwO4ARucBGAxxHRRh7NBUMy2BULVn/ivHr/6v5bLgAAPHrjF+P+Wy7Aozd+8SPi8nuZp81n0bXiorS88tvwUjnRtG2c/G/Fefxa8SUetXxa/0Tpe/1Z0z+1/VLb9xr91dZ/ChUJFPDgnUtDc3X1J2z+TjRaVYtjmfnYyuIsxehWK8OoNLSbAWya3W+Swh4CEe1LRJcS0aUzM38aERuzg/tvuWAFIdIOKDmw2sHZppcDUE6WnG4UefkRnq0JZuWLCgBZJy2f1U6y/LxOLe+RttHaIr/Pyy8tNBqfUcEu82hjxuJf0vLq3dYjSr8aM8tDFzMfy8zbZletMOuEUQm0SwBsQUSbEdHqAPYCsDhPkFd4lVXWfCi8ywQeNzwBIuujTWitzu0E0SafBTnRtfTaBJY85JMi5zmftPlEtuqqwRPYlqCR6SOT0xKQsm0iPHddWDx6ER7z8Jpx0KYZTIjl4JnYNQyKipHESAQaMy8DcACAcwBcC+CrzHx1KV90sI4b2kS3ICdOScjIMK9cLU4zrbw6eOXlcRHBZfHspdcEp0ejpFF5dcj/R03MtpxIumicJUwjPNa28eCYmYldw2AxgDen3c4dANzLzKa5CYzwOTRmPpOZn8bMmzPz4ZE8oxJmtZ1dmmAl7crLEy1To2tpN55PxqIXMa+s9CWfl5dXS2tpY3k9PDoWXYsfWYdaSAGitXHJLJZC0hPmJdRom33BPBO6IiCikwFcCODpRHQTEb2diPYjov1SkjMBLAWwBMAXAOxfpDkJ30Mb16bA0Bqh54saopwIv5rDuyZ/lIdSvSyNUbuvdXyXeCuFefGR/or6pmrq47VXTsdrwza/lX7ZAzf33hR44Dc/i20KbPrc3mV1wZx/9anPqjSUMPP8MpaTNkozR1SrkEKipSf5y/+X6Er/laeZWRPZMr2iQjoS5/nlIlpvxNSXGrrGg0ajpH1b7aOV7dVFM1EH096CmwLjQmeBRkSbEtF5RHQNEV1NRO9J4YcS0c1EdHm6dh2O3RUxap9bZDJFV3MvjTbgrUmgmTYtvEnRppcDP+qP8miXBFjUPxZtL4sH6Zfz8pZ8WFoZEtqCZbW55qoo/c/LKZmwbb7axbMKs7spUI3OJmd6wG0jZr6MiB4L4CcAXgNgTwB/ZOZ/i9KaT8+hSc0IsP1Vlhkh6UT+1/BWW5chEDUHI6ZqlM+S5twVEXPbc+yX+q5kBteYyXmdh3gO7YGlP46ZnE/Zbiwm52A+NCL6FoDPAHghZkGgDe3/6oqaSVIaeJEJUqJvCYKh26sLT0PU0Uvb1Rc3VPq+bez1n5cHsAVmHjeED+0vN1wUmqtrbL7D3PWhEdFCAM8DcHEKOiC9THo8ET1+iDIkopOpFl3y5OaEZtJoQq/kcyqZYpo5qplKEf+Oh4hvTfJjpZOTLkLbakMtvzQZc/Mryqesl1bHUh9q9GvbJ6+HzO+Zt22YNHMHw+w+tlGN3hoaEa0F4IcADmfm04loAwB3AmAA/4zGLP0fSr59kV6NoFXX2SZ/uHYSEdEuSqZBiX6LUWkXlrZUMqE0GpqAsWiXyoyYcDK+tq27aD+1+YZyA/TR9DweBtHQfv7DmIb2jJfOPQ2NiFYDcBqAE5n5dABg5tuZeTk3D6N8Ac2XN1aA9abApEJORG217bMiRhzaHl8tLI1ME0QaHanR5HTySWI50GVYpEzZbprmpQmVmra2hLCnTXnaoUyfx3nausaXRseDp21aPHSxPFRM+KZAn11OAnAcgGuZ+ZNZeP42/GsBqN86qsVgHdITmllTStvCm0BevlpEeLL4zwVrHh8ROtEytfJL5lgJkTbTTDorX94OJdpavbzfJTpem2gaWE1/98aEm5x9NLQXAtgHwE7iEY2PE9GVRHQFgJcBeN8QjA7uCxgQmq8jsiJbYVEfjFeG5d+R+UpllyZ8CZ7GaGlMni8pv7e0qC7Q2sTqxy5jMaKtSfpWn8k26mJKd8Z81dCY+UfMTMz818y8VbrOZOZ9mPk5KXy30rtXHiZFK7PQ1S9TgmYOaQIkIhxLficrv2d2yjCJiD8wn7RW+pKwytuk1u9o8ZMvSJYLwFuwLL6lwI4uTjKNx5PHw2BzacI1tHnx6tPIV6VA2RFncSm9Z05oeUqaVlfndVRQe+nkxLX499JH6in5KaXxUBpHXjt4/Fn1rhm30THkYYhNgT9ffkZorj5qq1fPzefQiOhGAH8AsBzAMmbelojWBXAqgIUAbgSwJzPfY9GYrQdruw7YUlpvMnq0NY2p5CSu9bd1rVMt71oeLb5W6JSEiCfo+whrLY/V9tpY0PJGBbxl/nqLgZe2DRviwdo/X7Y4JtC23m3u7XJmeFkyObdN9wcDOJeZtwBwbrofO2qcuxFa2sQpaRaaGWmZCzK+ZIJ6pk4b3sX0sLQ/z/9mxctwy7cU9TNZE1trb22iy3RWG+b/NV6koJJ583vN3C6ZwTKtHHPSpB0ZJtyHNpSGti0z35mFXQdgR2a+Ne16/oCZn27RaDW0vibDuFA7iGpNhlotC6gz8aKajKQb7a+INlerOXsaYKRta83ELmVEaGk8Rdqpi1tgEJPzktNiGtrz/27OamgM4LtE9JP0sCwAbJBtBtwGYIMIIW1Vn2Roqn4pbZ7eiperdr76lviRGkAOTZOUk8KaVFrf5PWOan5aPTWhFPWbWbQszUvy72mZHj+yHlY+K72WNqKx5zxYcbKOXbRyExOuoQ1xSMqLmPlmIvorAN8jop/nkczMRDT+nYcBERkgNaupZjbJOG0Aa6aHpc1E/URWPb0JU4I3EbWyI74ib0JHtCurrTSBqNGq0VBlGqnlSkGm1cXqi4imOKiSMMYdzAh6a2jMfHP6fweAb6B5M+D29gHb9P8Oma/LISmDrjQ9yrA0SW2gaXQ1s83S9rxyLJOwvc81B6te1iTSypH0aujLeEuT1NK14ZYGE9FErTqX+lv2UbS+FjzeLS1YGzu15QyG5cti15jQ99WnNdOng0BEawJ4JZo3AxYDeEtK9hYA35J5u7z6VGN6dEXNaqYNLm3Ay4GqxckV2jKHPF9SXnZOozSpLWiaUy6ENKGkTVhPA63hJ6ep/c7pWBpgiY9SeRqvJe0pKvxKpmhEo8vzWDR7YcKfQ+uroW0A4EdE9DMAPwbwHWY+G8ARAF5BRNcDeHm6D6NPJwyhXpdW4TwuH1yaGWHlifKQw9PoZDqtzJIfyMtv0Y/UoySANBNcpvHuZVipTCmIo+VZWmVpwZCCTRsfnmmr3UfcCBZvfcC8PHSNC70EGjMvZebnpmvL9jAUZr6LmXdm5i2Y+eXM7J2OvAKG7oRaaIM2ksebJDLcSuOZgJI3SxvK80V8K5r5awlljVdZniUcZVpLkFkCyNOMZNoSzxqPJbNf49mCFGIR89rjV4ZZC0OXMqowzzW0icEQHdfVDPE0uohGZcXXaHQ5X3LieXSkVhktT9KPCDHJiya4PG3J0mg8U1xCLgge3552FEVJkyvFeeakDLMWuUEx4buc80agDdFxURqeX0TSksLCKk+aJhFNQE7wkoYR4VOjbeWzhFNEKHlllfjw2ibSdp5QlRpuFwHh1V8uHNIULbk7SoJbjiGPVidMuIbW50yBp6N5vanFUwB8GMDjAPw9gN+m8H9k5jM9WnPxTAFr0kRNkihtTYOS6SITuEQjvwd8TTIaFynPqq9XF40/qx6RtFH07duatsvDgZhgL/XdEA/W3v/do0Nz9dGv3H9uPVjLzNe1X9kAsA2A+9A8tgEAR+Zf4BiAz4mDZyLIlV2umBosLUaagfmqa8VZvGoagZZWq4Osn8Z/abJH8pVMcitM0zAtf5ql2UbK9PKV0uc8Rs10T/ONmM0lvqqxkpicOwO4gZl/NRC94Z2ZA0MTUrm5I4WSDMvzALaTX8IyEWWYJ5AiAqQ0kfLwqKDW6EaFi8a/155dzThLSGqCN+ID1MqwoI2nPK83RqSWm9MY1I824SbnUAJtLwAnZ/e9D0kZ3Jk5IDQh1aJ0L8O0312FudTavAlm8abFaxNeK0vjJdci8vrVTLSSqZbzKye/FlaikyOq8XgLk5bHGjul8RBZgCL8dMZ8F2hEtDqA3QB8LQUdA2BzAFsBuBXAJ4x81W8KTAo8waNpCFa8pzlEBqE0OSzhWWMa9UFEmykJ2NICYOXTeIlovZoA9MqICL6aOuSavtVPnras0Y5q4Z2wEpicuwC4jJlvB+bnISneILYmjKXdaFpNaUDmPMjBHxmspckYFXC5kOg6Saw6a7x45rOm5XmTvbT4aD4uiy+v7hr/kh/Zf/kVMblLZUfM7M6Yz68+JeyNzNwc1SEpJUQ6rmvneoOoq9lk+eBkHqvMfPB7ZpE2ETUh6/mvSr46j77Fv6Sv1UWajZZQkXk1f1LJv5WXpfWVDI9q0DVxUqiWFjUZHlkse2M+m5zp/c1XADg9Cx7JISlA2VwppR+6c0uDurTK15gG1uCOpPdWbs8PpvHchZ+S0JZlW9qFZS5KRM1GS+hZQitSZ8+HJttBhnlCyNMQNX5H5VaYdJNzXpwpMG5YJlgp3FuJa+iVeIpoci20SS15HcL09OhK/mvpWWa+vNfK8Oom4yJpLV7bOneB1WYWry2G+AT3/V//l9hzaHv809w8U2AIzEWBFp18EfPFStuFl5r0nqk2apQEj5cP8B//GJKHmr4dR5rSeGrx6I1fPMyDtV89LCbQ9vzw3HqwdpQYWl2upVfrH5ErYck88Hw5pfI1X1fUrxM1M2tRMim135pPLGIyafzWCnMLHh3LVNbic3NS+ictM7WWX0tjlnEl+tVgjl1jQkigpefJ7iCiq7KwdYnoe0R0ffr/+BRORPQpIlqSnkXbupapobWEWnp9y5eDuETfMvusvFHfkeaj0SaWnHwWX1qcnNhe3UvCXTNzI34qWSetblp9rDK0NrO0bClYrLpI4WLxqtVVy+fR8XydvbFsWewaE6Ia2pcALBJh1slOuwDYIl37onkubeLRVSv0NJCSCaD9lunzASsvSdfS3PL8luM7nziWWZf/1tJ7dY+0g6VBeouDZmppbWsJAg2e71HWveRnsxYRqx+iAi6/zy+Joa2dITcFiGgREV2XlJ8VToYjoicR0XlE9NOkHO1apBn1oRHRQgBnMPOz0716shMRfT79Plmms2jX+NC6+ktmEyV/R5e80fQln4+HqA9Lm/CWLyfSFloaL77vGIi2keWbqqHbtc+7jgOrH4CBXk4/4ZCYD+3NH3XLIqJVAfwCzVMSNwG4BMDezHxNluZYAD9l5mOI6FkAzmTmhR7dPj4062SnJwL4TZbuphQ2CCZFmJVWeKlZteEaHct8y8vRzKKSdiI1OrmaR1Z2q/y2LGkSefx4dGScJ3AiNL2w0hjKNauIMJI8W31eoy3lmmmkfpLfkc2T4Xxo2wFYkj4S+wCAUwDsLksDsHb6vQ6AW0pEB9kU4EbNq/IE9nn1aXA1ugO0wdve5/EWpJDJV1ZJpyRM8vQyrya0LLNGm5zStKyZ4FZ9tfCSAIq0q6Zd5XHeYiHLyvujJNQ1lPJaglBL65nlpQVTK68XhnuwNqL4HArgTUR0E4AzARxYIjpxJudQ5kTU7CuVN0rzZq6ixozy+gGwtVZL+NSanhofkT6v4a1Ufk7LMtkjdYjw5+UbxOQ87v0hgfGYd3zinWh86C2OZeZj2xsi2gPAImZ+R7rfB8D2zHxAluYgNDLqE0T0AgDHAXh2eq1SRR8NzTrZaTGAN6fdzh0A3OsJM4moH8cKK5l1edqIL6WrydBiPgoz7be890wfa4J6iJqlloZbMuVLPFhjKMp3tCytTbV6au2hpRnamuHly2NX9q52uo4VpG4GsGl2v0kKy/F2AF8FAGa+EMCjAKzv8Rd9bONkABcCeDoR3UREb4d9stOZAJYCWILm5fT9I2W08CZJC2u1sgap9PV4PhhtRdXKXRmRm8hA2V8WoSXDPDq5eW4tapI3rZz8Xpv0eR01893KV4Ice5orQOO1JLQ0RLW4agxncl4CYAsi2ix9sWcvNMpQjl+j+dYiiOiZaATab+EgJNCYeW9m3oiZV2PmTZj5OOtkJ27wLmbenJmfw8yXRspoITvQ81XJASCFVnTQSR9NyRczDtSs6n1oeSi1p9Y/WphHy2prmTc6US0BqAkWbUH0BIalJZV8dxGecjrWnLD4KqXphYEe22DmZQAOAHAOgGsBfJWZryaiw4hot5TsHwD8fTom82QAb+WCj2xiXn3KfR2A/pyO17ERn1lNZ0d9cCsTrIUm7yutH/N0XcvUJnuJticENLNXq4dGs9Y3W2obq96ROlu0W/p5niF8aPd99oCYD+1dn5nMdzmJ6HgArwZwR7Yh8K8A/hbAAwBuAPA2Zv5d2ji4FsB1KftFzLxfiYlcoFnoI0j6TCZvcPfByiIYLcFj+dEiGlG0PC28tMB5fJWETC2vHu+egPR49QT7EC+n3/fp/WMC7cCjJ/Zdzi9hxbcEvodmt+Gv0Twcd0gWd0N2QEpRmLXw/Fp5vKXe5yaO52OxzCIt3ygFzlwWZrLNPWguAalNtPdaH0e1mRK/mkCyTE2ZT/rlPL4ipqhnoua8afxbaSwMvnAuXx67xoSiQGPm8wHcLcK+m2xgALgIzQ7FoNAGjzXgZB45OWS4JrRyv5tXRmny9MEoaQ8Jrz2B8oSVAkL6SS2BIKH5mKz+8yZ2Hq+NJyuPR0tLYwlGLVzWUWsXrV3lYjO4hTGfP/CY8D8AnJXdb5bevfohEYVa0Rq4coBbAqeN02CtbN5A0gZMRFMooUabGRX6Cs0abUL2X4mmJlC8SS7TegteycfmhWuaaB4mtfuIQNaEsQatDbw6aXNl0HE1w7FrTOj7xdoPAVgG4MQUdCuAJzHz8wAcBOAkIlrbyl+CpQGU8gArrmyeeeRpEPnvvsKgK40hNbfo4LZMds/HldPPNV5tYSjRtNwAnhlXO3E9c9hLG0nfprH6TtPkNHeIVX4tBhtDE/7F2s4CjYjeimaz4I3tVioz/4WZ70q/f4Jmw+BpRv6HXn1aY8PYF4a0lVBqB9bK55kTUZNyiJWuC41x+Nsik9qDJWgiQt3TyktmYV8hEPGTabwM3a/a2NYQFVTDmZzzUEMjokUAPgBgN2a+Lwt/QnqLHkT0FDSfEFqq0cifJP7LbZcB0E08TShFBph2H3HSeuaCpjGsTLB8N4A/6SUNSzOS8d4ktPpBmrtaWiuvrJ9lxnplW2GWlhkZe55fsUQ/53kI8MxM6BoXFpQSpLcEdgSwfnpJ9CNodjXXAPA9IgIefjzjJQAOI6IHAcwA2K994NaDNoBrnJnRdJZwq1m9xqEtTQo089EyF0uCTIZJ097yfWq/LSEZudfMPs0c7FqvvIyST81qg0i9PWHYxi97QL5Z1AFj3MGMYGIerG1/a6tRrVCzBoOWRtKIljcujJNHa/Jbv61+kGlyml15qO1rSSuyKEZoWnk8/mroeTyV0g3xHNqfDntjSGCs+eETJ/PB2tmA9YFHTfXXVqKagWiFeQMvWo5X3qRgCN6imkrfsiOCqCusfu6jcUbyRXkZxfgZ4k2BPx26d0ygHXryxD5YO1bkQib3j0SdsZ4mZpWXpxna2Ttu9BUC1r1n8sg+K9GX6T3TM0+vlWnxEqUrw7VFttY/VfKDaaZ9hOZQfjIXc31TwDgg5VAiupmILk/XrlncIekb4dcR0auijOSDTQ4ky9Hvrd4loSUHjeZnyfkYlZAaahBqdPpONAmrzfI219qxtv1k+rwcTchY9L0wq4wWlgDW8tRodVb6NjyykJb6aaTCbR48tvElrPjqEwAcmb3idCYApO9+7wVgy5Tn6HbXMwJr8NeshNKx6uXTJqQsU6I2PMpvX0QndB9epKac5yuZh6Xfpfy15lutT87T1rxxJONKrhFP2Ei/ohWmQS70NVZJFSZcQyvucjLz+eml8wh2B3AKM/8FwC+JaAmab4dfWMroDcCIeamZKFEfSS0/Q6Sfi5DaiTT9S0JHug8kXUvDk2XXQgri0oJp8WDRlHXzEG2fvAyZz0pXskqG2OXkZZO9y9nHh3ZAOlrq+PZMTvQ4IGVIFdnSImpNAI3HIQTXKMyB2TA5JSITv5RHi/N8YF00r5LAlFdJo69BrpVpvj6vDE3Ly+uhuWa6mPhVmHANratAOwbA5gC2QvO60ydqCchDUqyBZDlxNRPBc/hqsDS6UTtXo6ZTX5qzrWVqQqhLm1qTUgoHK6/2P+dR4zvKc0kLstJorpCIKVmTTpY9EsE2D3xoK4CZb2fm5emwgi+gMSuB2HfCWxorvClgDYhcxfa0Ks+skHRLGwJWulELO4snCxG/3yhgTVqL75r+sOJyM69kNlqQfetpfZ7LQo5Vb5zI8WsJNa8NNM1NarQynZa3NyZcQyv60DQQ0UbZwSevBdDugC5G80L6JwFsjObVpx9HaOYd0qVTtM4vmQ9yMJeEZsTkGcosjUIb6KNGyYyLtoFnXpXot2ksQePxrJVfa86WyrP4tzS1PMwSmhGXiTYPhhyTPEZhFUHksQ3tgJSPE9GVRHQFgJcBeB8AMPPVaE5puQbA2QDexcwhL2IuyDztqZRfGwSWgPMGuDaoIhrabAqzIRHR7rQJ1LW+3kJjaTD5hO9iurXleiarxaess6QRETZ5eF7XHCXLo0TfE+iDYNny2DUmRD7wqB2Qsk86AOWvmXm3/Jg6Zj48HZDydGY+y6PdImrytXElFd0yAzSaJb9PVAOYZNT4E6Npov5KrX1rJpfnPtAWK0tY5elyAREVbnLMeILKC48I37xdPXPSaxeZbrBxO+Em50S9+mT5GQB7W9xTpyU9K31JJS/l74vZNlP7INLeNXn6ljk0nSH62muHFhGTOk+Xp/UWYC1PGzbEq09/2G9RSGA89nNnj+XVp04+tFFA8x1YQszL763O1mCNDAhJa0hMsjCLCgFgxf7S/GH5faSMIYShVqZFO7qwafSl1uhp9zJtlIeaukRo1WISFCAPXV99OjV77elGIro8hS8kovuzuM8NwaQ3KL0VyxqsXufXmL8rA0oT3jL92t/S96T5gkYp0CVvbZnafYSWRd8T5HlYZNG0zHRpJlvmco2PsBoTbnJ2evWJmd/QvvYE4DQAp2fRg5z6lIdH/A6aiZrDi4vyFuFDKy8SPmmIThRNKNRoOSWhoglAjc+IP08r3+O3xJvHs/ytCb2SBlvy4dYI68HG3VwXaNqpTy2o+brjnmhONR4J2k7VhInV2Z7ZacVHVjsrXkNpkgyJ2sEaSR/VYPO40sS3+i9yX3KMW2adxmepjFKZ+Zi08mmCNmJK5rQ9k1TWU+aT9R1q3PGymdA1LoQ2BdK7nGe0Bw1n4S8B8Elm3jZLdzWaszp/D+CfmLk4ex68cykDj1zFah24+f0Qjl2vnJUZtf1i+ddq6OXpNA2kpJ1H6jDUeCvl93jsMm5L/se8rCE2Be7dZ+eQ+rXOf5w7uR94dATaMQCWMPMn0v0aANZi5ruIaBsA3wSwJTP/XqG5L4B9AYBWXWebVVZZc4Vyawdpnq52ImmmVB8zVeNvKKE4TgFbI6ysfJHwKC9t+SU++wjUGh67tI9Xj6Fw/y3DfLH2d2/cKSTQHnfi9+fWBx6JaAGA1wE4tQ2rOfXJOiSlxkyQAkdT2a2B5KnyXr6uvoghB+s4tUWpJUX9i1a85i9q4dHz/E15uFa2t1B1HX9aWZqPS+aXZmLEBLbKlmmHXJAfwlz3oTl4OYCfM/NNbUDNqU858k7XBoQcQJrPTBtE7X2N78sqo43rMjC6CsFRowtfnu8oqoF4wkWbhJbgkBqYJxw14VISSBaka0Srh0ZfG1uWNufxovWBV99BMRO8xoSurz4BzYcc5WbASwBckR7j+DqCpz6V0EVV1yaKdNBKQWgN5L4DY1L9b7NVL8vvlE9MbQFp0+e/NSEh81iuBE1racMsbUYTvhrffaGZx1EzVeaRdRpyQeUZDl3jwkS9KRCF9In09cPIVTGSfr6iq79IhrewNNyuvqaatrfK03i1fGu17ZHT0u4jfkaZLlKPCM0hfGh3v/alobm67jd+OLd8aKOEZZJY5qC1okboa8JMrm61PpdRYMhyPFoRgW6llWZ/VOu1zCfPpNU0kKgJrPGqhXtCUBsflhlsjWeLr65mqGVq5/97Y8JNzqKGRkSbAjgBwAYAGMCxzHwUEa2LZkNgIYAbAezJzPekZ9OOArArgPsAvJWZL/PKsM7l7ILSajYk/VHxOBT6aDRd0pS0nRo+cjpDtVFUQxoVvYgVUHKjSEGrpbesjiEe27jrb2Ma2nrfnlwNbRmAf2DmZwHYAcC70mEoBwM4l5m3AHBuugeAXdBsBmyB5rGMY6LMeDZ/jeYlVzOPXiTO00q6wNNOatFH26pNby0Qfcqs4cPyZ0WgCQGrXBlv3ZcsA8ts9OA58zV/maYF5lqZpyV3wlzX0FbIQPQtAJ9J147MfCsRbQTgB8z8dCL6fPp9ckp/XZvOoim/ttH+BmIOfw2eH8IL0/jo66ebL7A0jhYlv2bNwtDFj9RF841oTVo6ry1KY9gaXzJtZAzXYAgN7c5dYhra+mdNrob2ENIDts8DcDGADTIhdRsakxTocFCKXHlLg8zyC5QGVVSb0ML7alLzAbJdNE1A9o3W5lqcpXFZ7S01b/nb8svJ31EBITUny5TUfGilMq20+b0nxOXvUY5RXha7IiCiRen83iVEdLCRZk8iuoaIriaik0o0wwKNiNZC8yL6e+WT/9yoeb23S/NO9cxFGV/ju7EGtnZvlbsyQ/aJpV3k8dpvSVMKQkt4tr8100wTpp52nQuKPia/xYvWTppbJGK6WgJaa98SvT4Y6oyU9LzqZ9G4qJ4FYO/kysrTbAHgEAAvZOYtAby3RDck0IhoNTTC7ERmbr+scXsyNZH+35HCQwel5Kc+ffGEkx/RUfkg9DQjbWB7K3pJGJUm5FCDwxOqQ9AcJQ1PkFg0SnRrFomoL9USZlJ4ygW0tkyPd61cS2B7+bS4nN/SeBpSqA146NN2aF6bXMrMDwA4Bc25vjn+HsBnmfkeAGDmO1BA5MFaAnAcgGuZ+ZNZ1GIAb0m/3wLgW1n4m6nBDgDu1fxn+atPBx589AodZa1wwMMDVt636SQ0DUDS1Ojm4UMiorX0oTlKGrVmjezLqLnXV3vyND1P65c8RhZCTfOSv2V6i0ZpsZALv0xTWpR7gyl05QpLuvYVlCKuqacBeBoR/RcRXUREi1BAREN7IYB9AOyUfbhxVwBHAHgFEV2P5jWoI1L6M9G87rQEzRF3+wfKeARKZoflqyj5SaxJJYWhtaLPFibZT1cSMNZCpJn+JTpaXqv8mj4aIq1cRC1e27SRcko+MitNF/67Iqqh5QpLuo7tUNwCNE9L7AhgbwBfIKLHeRki30P7ETNTOhCl/XDjmcx8FzPvzMxbMPPL21ecuMG70kEpz2HmS2troWle2m/ZwaXBFPXPWAJvtjBuP51nttSY7vmE7yqEclqeFm7RtcZQzl8pv/ydt0GEL08oWvD4LC0Qo7QweIZCVwAR19RNABYz84PM/Es0nyXbwiM6kW8KSB+a9Ruwd0hbOtIZK/PIMCvdODS1ccEzW7TJHjG3PJdCdCKWNB6NnqU95bxo+TUhJXmP+MMsQZ7T0cawxbNVXk09+2BmOYWuAC4BsAURbUZEq6N5N3yxSPNNNNoZiGh9NCao+7GLiRBomvkhTcr8t+eH8To7v7fKy39HBqxVn5rwuQbLDJRh3kSKCIYa7dujm+eRgtSiZWlpnrCwBKpXt+j41eqpYdSL7lCbAsy8DMABAM4BcC2ArzLz1UR0GBHtlpKdA+AuIroGwHkA/lf7eTILfV59+lcAfwvgATTfPXsbM/8uPat2LYDrEomLSmcLWC+nl7SiPlqTNpg131nJHzRbmptXzri0R9lukf7K08s4beGK0JamrWaOeXF53haehmTxrdWxpt/6jK2IOT7Eg7W/eX7si7WbXjKhX6xNj2RsxMyXEdFjAfwEwGvQ2LzfZ+ZlRPQxAGDmD1pft/WwYPUncknNbhH1o+Tpaya7NahWJpOzD0oTXKaNxnmCyiq3y71cyLwyLUSFbyRPSQhH6tZiCIH2621jAu1Jl45HoEU2BW5tXy5n5j+g0b6eyMzfTWojAFyERsD1hqXq5/EyjWfKWWaPlc/T2rpiFKbmOM3XUnvLe29CeqadZ2Z6JqLGr2ZqaqZkyc1g1aUETfvTwrTy8rRdBPWQY2XATYGRoOpdzqR9nQ/g2fnbAkT0bQCnMvNXuhyUUvs9NKDbSlmz+kZNzpUNXTTovubykK6HaNqS6QqUtcc2TUSD6hMW4R0YRkP75XNfEZqrm/3se5OpobWwXn0iog+h+SLHiSnoVgBPYubnATgIwElEtLZC76EH72Zm/vRQuLUCR1fk6CqsrYiaRtCmLa3cNcjLHWr1tLTQvjRq80ed2lpcrfaltaPUSDRaVj3bvJ7Z3MZpY0Wm1cZMTR01rTQy7q24ITDpGlqfV59ARG8F8GoAb0zvc4YPSrEOSckHgSaMSp2jmTzRQeTRzgd7X1iTocRDlKYXVksjh9RgLVOoi9D38luTVC46Xj0i7aqZxxY9Dd4i7JnTXt5Su0ht0VqshwIzha5xofOrT+k1hA8A2I2Z78vCqw9KkR3m+a+8FcpayTRnbwmaj2MoU7OkQfSl6YX1Qe7P0TTXPhphVIuzwmoWulIbR3jRFreIYPV8hnl4RKtr8+WLS0lD7YsB3+UcCSK7nC8CcAGAK/Hwp9v+EcCnAKwB4K4UdhEz70dEfwfgMAAPpvQfYeZve2Xku5w5av0Fnv+mrx+nC1YWf1utDwvwF6aob0pLr9GOjIESL11g8TrEWOzC2xA+tF88c1HIh/a0a8+ezMc2ZgP5yekR1HR6S7c0UYYob5IxxOTs6ozO44H+JmnXfBYNTdB4vHqukJr80baS5UXaWeNjCIF23TN2CQmMp//8rJVXoFkamgVvkHqDFog/BFoqt2vaUZc7SpQmqTXZuggyq49L/dhXA9Tq09LzyooKfI/3Uvo+uP+WYU59unaLXUMC45nXnznZu5yjhvQF5KjxM8mBL30+Ht1SeM3AstJ2GZyTIMyAR/qIrD6xzD2vf2V6Lb8sK+9XzXfq9WFEUGi+MK0sGSfHnVUPjXfZBl57a2XI/JLWEJj0Xc4+rz4diuYDbL9NSf+Rmc9MeQ4B8HYAywG8m5nP8cqIPodWaz4A9c+OeRrfbGtKk6KZlVDjE+qqwWlm3hD81fJRW25Or3Ys1fj+ImmGMDmvesqrQ3P12UvPmFgNzTr1CQCOzD8pBAApbi8AWwJYBODodtezBG/lBuydqnylslbWSHk15c8G5oIwA/xHajTtqoXld5LItaFSWiuvDNPSWfxpaTXNyqMl7y2eNMEdqfNsjZU5/9iG9eqTk2V3AKek59F+ieZDj9tFmJEO0IhK3ebzBo9XXoSON3jnCmp4rzHFPc2pZFrKCV4yZ3OTVYZZkP1nmaVa+Z4mKceqVi/LrIzwq5munm+4FDYUmGPXuNDn1CcAOICIriCi44no8Sms+tSnFt4qVcpTG6el9fwZLS9anDeoLL9RV6FRyuv5jjx49WzjoyZhpMwucVJLy8M85JqOpyVZ4Z7GFamH9d/Lp/Fc08ajEmozTKFrXOjz6tMxADYHsBWa150+UVOwfPVJmhUlH5kMq4GmBcpB1EI6tK3V1+JR49UTDp7mkefVVnFt8Je0gnyydjFpopN0LqOmbn3aQVsES4tsaVEdWrDNzFDoGhdCj22kV5/OAHCOOCiljV+I9MmgtCEAZv5oijsHwKHMfKFF/8E7l/IQjndLyEQEo1euljanm5frre5euhIPtdCEr/d76PJHDc8Ms9IPUddRtFEX3qwx6GGITYFLN3lNyKDc9qZvTuZzaOnVpy8DuJuZ35uFb9Se5kRE7wOwPTPvRURbAjgJjd9sYwDnAtiCmZdbZcjn0LTBqplEpUEQTWPRtPIMAY+3mjJrTcGo0K7lY1SYBB4mAUO0wxAC7ZInvjYk0J5/8zcmdpfTOvXp40R0JRFdAeBlAN4HAMx8NYCvArgGwNkA3uUJsxaab6PktK2Bpr15pqbHi8VDjT9P8lIjwPL/JfNb+p1KZkgXnkaJWh76uB8mAdY4sxb3rnS7YtJ9aBPxpkBrcpbMCKtzLc2idlXzyo+YpH1X0a5aZh7XoqS1eSa0R7tGw5svGFJjjubTxlSUttVXQ2hoF238upDA2OGW0yfT5JwNdPnAo4eojypirtUKqqj/Y+hyI3m7+O7mg6AaZR3G1T5dfMBDCLT/2nCP0Fx94W1fn0yTk4g2JaLziOgaIrqaiN6Twk/NTNAbiejyFL6QiO7P4j7XlTlN/a4xmWpMQC1cOvIjtCzTLr+v3VGM1smjUSq3q6kzaSabhr4Cp2tfDVWGTKcJL6+OQwrcmeA1LnQ+JIWZr8nSfALAvcx8WNdDUoC45uLFA7EHEKMmV0mrGUKLGtrU7MqzFT8bWsgkaILzUZPLyx/i5fTzN3x9SEN7yW1fm0wNrfSmQNoF3RPAyX2ZKTn9SwPCmqS58z93kOf5NAEn/Rf5/3wjIbJhoPFZWlWjPjHJi6UlajyX4ktlRxFtk2j6UcDTRkv8eOlrXRb57xLdKI9DCdMZjl3jQt83BQDgxQBuZ+brs7DNiOinRPRDIgq3ZMQ5XYKczJqQzE1XbRD03WTwyrTurTAPJc3Mm6CW8MrbLeenr4Cp7dOINqrde6hJW2PSleJLC7VM6y0meXwtj0NgBhS6xoXwpkB6U+CHAA4X5wocA2AJM38i3a8BYC1mvouItkFznPuW+cEqEtqmQK2K3lWl72LmaoLS2qGScaPGfHDwz1bfzxa9IdC3X4cyOc/d4A0hgbHz7adOpskJuIekLADwOgCntmHRQ1LyV58+fcT+armjMjty7azLyhsd7K0pW6t55XzWhHu89W3L2TQBh9LmZqv8LoiMCU0D1/JFNMCh6rQcFLrGhc6HpCS8HMDPmfmmLH3okJT81KcDDz4awIq+q4hzP9eCvEEiB4dGP2oCWiaBp6113TiwyozUVatzF4xKY5kNITkOX1wEkfHnWQFavtkxOSd7l7PPmwJA890zuRnwEgBXpMc4vg5gP2a+2ytACqWuzvJcI9Jo5/cy3qId0dA8Z3sXX1RUM5MCzxrYnlCOLgCjQBdH+ajKmARE/bNdfGdDCfY5L9CY+UfMTMz81/Jjjsz8Vmb+nEh/GjNvmdJtXTrxCdBfe2qhCYOaidDFuVu7k6RNfk3QWuVrgsraifUcwpYDWdbJ4tOqXw2GmjizIVCj5Wv3Nelr8+ZhpY0lr2xvkesKBoWucWEizhTwOjSfyFFTyxMeeVpLG6zZlbIEhsef/B3xe3n8eoNea4uSr6WmTlp+i7dR0xnSvOy7EFqLdGmxzPtZG88aXWu89nEzWJih2DUuTIRA8zQIeZ93ktTetI71VrRSejkgPHPYmoCWSVsjMEqmpDf58klSol8bF8FQEypqjo0TNTxq46Lk29X6UhOKcgNhSEE/5x/bIKJHATgfzaHCCwB8nZk/QkSbATgFwHpo3h7Yh5kfSI9tnABgGzSHEL+BmW/0yhjiXc6ujuuSphTxS+WDNCoASzxZNEu0vTI1QV7SYsctJLpiLvFeGjelvtY0Oi3PEO9ynr7hfw/N1dfddtLEPrbxFwA7MfNz0XyddhER7QDgY2gOSXkqgHvQnPKE9P+eFH5kSueiRlPRfGpA2USx/A2R/NqKp6n7JU3J488K9zRCjYY18D3+IqbwkKv8bGAoYdal3rV5ouatZZFo1suohPkMUegaFyKbAszMf0y3q6WLAeyEZhcTaD4A+Zr0e/d0jxS/c3r0w4ScWCW1ujTRSj6iPJ2l4stVTpoD1qoYKVe7lz5Dr06lCaOZGrI+EQ0zpzdXtB2JcZjMNXm0BTrq48xpzBY4eI0L0U9wr4rGrHwqgM8C+FcAFyUtrD2786z0Ce6rACxqn00johvQfM32Tot+/j00S7hIaKZgzYTsOnmjk7vWvIuUF9XeIvQsx/MUkwGt3z3NLdJ/Q5icp270xpC8esOtJxbLIqJFAI4CsCqALzLzEUa6v0OjHD2fmS/1aIY2BZh5OTNvBWATNJ/WfkYkn4f8TYE1NtwagP4cl4TUZKImkqWuyzQR535OT9OcLM1Po+tB8pNriaUBbGmunpmi5ZtrpuYkIdrH2hjSNHrP5aI5/yMaXy2G2uVMStJnAewC4FkA9s7O+83TPRbAe/DI98dNVO1yMvPvAJwH4AUAHpdefQIaQXdz+n0zgE0TMwsArINmc0DSeuhNgb/cdtlD4dqEA+IdL+nkwqXGvyDpawNMM0FlWst/ZZVn1cETNFqcVp6l6Vlhkbgu6SYVQ/MfMQvlguL1mzWOrUUujx+qbgO++rQdmnfAlzLzA2g2GHdX0v0zGj/8nyNEI68+PYGIHpd+PxrAK9B8Qug8AHukZG8B8K30e3G6R4r/PgfsWs2PILUv2ZneipXns+K0MvLfJd9GSdDkfMs0nnZp1aEUVorTzGCZJ+qgril3rkD2SVch4I3DLnknaaEY8Dm04vm9RLQ1gE2Z+TtR/iIa2kYAzkuHoVwC4HvMfAaADwI4iIiWoHl047iU/jgA66XwgwAcXCrAW3U0oSHzar8Bf6dSwhqE2sZApOwS3RK/Vh6ZtmYCWMLKmsQro38t0qc1NCKIuk0i+YfiyUL01afcpZSufWvKIaJVAHwSwD9U5ZsLZwpYuzpRJ6l0sEZ8W13SeOV6tIbYOCjx7PE5ijxd69IFQ5c1Dt69MvuM5xxDbAr8+xPfFBIYb7v5K25ZRPQCNOf1virdy/N810HzpZ72CYsNAdwNYDdvY2Ai3hQAfJ+Q5b/yaFj5S50v/WIabat8b9CVkA/sWhND+tpKZlPEp+KZ46V8WvpRmk1DCx+tfSLtZd1HNG9vzHlugC4aXR8MaHJeAmALItqMiFZH86GLxW0kM9/LzOsz80JmXgjgIhSEGRDzoT2KiH5MRD9Lh6T87xR+IhFdR0RXEdHx6ZtpIKIdieje7MscH47Urqv5lk+8Gqe/RdvzqWk81DrrLVhmbSSv3PgotV/UROnSjlaeuWa21voSo0LHg/StSjeJ5i+2FrAhfIEahvraBjMvA3AAgHPQ+OS/ysxXE9FhRLRbV/6iJ6evycx/TELrR2i2UdcFcFZKdhKA85n5GCLaEcD7mfnVUSYevHMpA/0e8NQc3iWTsEQjCmsgenXw+NPoeeVa9SjxEq1XbdwUdYi08xDtPYTJ+blNYybnfr/xTc5RofObAsx8ZopjAD9G8+hGJ2jOdk/11kwraSp21S4iPi8tn/Zby5drYnI1lfXWNkYiZmCuqZU0xhqtTZYxRRxdN3wiVkdEGxtKS5vz30MDmofg0gcb70Czy3lxFrcamg9Anp1leUEyUc8ioi1L9GWHSIEkBVnJtLIEn/Q3RNXyiFlp8WzdW4NVq1eNoLYEvdXGGg+WgO2CUZk+pbLGgYjQirgkPAHn0dbylsJrMekCrWqXMz2P9g0ABzLzVSnsCwD+xMzvTfdrA5hJJuquAI5i5i08uvmrT8AwjS8nf1QYRMqOmJbRcqLmpZY/YmZrZXbhdWpizn0MYXJ+OmhyHjipJmeO7E2BRQBARB8B8AQ0z5u1aX7fmqjpy7arEdH6kpb16lONQLHCI74lDVG1XubxtD3NeasJoCi/mpkqebH49OphxUWE2bg1oxJGyd9QGmxXszGyITDopsBc/8Cj8abAz4noHQBeBWBvZp7J0m/Yfl2DiLZLZdwl6eavPq2yypohIaWF55DCQQo4uWvkwUsjzThNEJfMB41Pqz553tzcLvnr5H1JMGl+PRke5XOS0MUvOhTtaBkROloazdzUXBhTH9rDsN4U+ByADQBcKB7P2APAVUT0MwCfArBX5NUnoCykNN+PlldzwLfhljCQZddsMpTyRp27Fr2crpfe4tMqu9RmFt0hJucQqJmk3oJYw29XwRDxbdXSLmlskfJrsTx4jQuRXc4rmPl56ZCUZzPzYSl8ATNvnh2c0oZ/Jh2S8lxm3oGZ/9+omI9obXlcafDUaEp5eHRQlgZ1X/OuRlhqQrgP3dnGfPLpdR1PHo1RYdJNzol+9SkyuS3Hf7SDI+n6Ov7z+5LjXuNFbgK0abpsfPTlf9zow8e469Cn74coe4iT0494cmxT4OBfzYFNgdlGjY9J63htcHhmVh6vmVuWX83yO2m8lQZnyWTMfSK1wlvyK3/LMvv4XUaVt8/kHsrfNWT5nh+2lEf+ri27C+b8F2udQ1K+BOClAO5NSd/KzJenDYGjAOwK4L4UftmKlB9GzSEp+QCwVi/LvIz6xPogqk15+TwNrYtZWjKZo3SnGA1K47hmTHmL2xCPbRz+5NgXaz/0q/IXa0eBPq8+7QfgDGb+uki/K4AD0Qi07dE8h7a9V4Ym0LoIoRpNpYa+1IZKmlFfwRAV1BZ/Gk+luBo+amC1XR86k4paHruYoH0whEA7LCjQPjwmgdbnkBQLuwM4IeW7CM2XbTeqZaxkPlp5as3MCP08Tk5OzRz1TOXcPNVMXA01u4qW6RLhS7ZTdAGp4a0rRqlRD4VaHruYoOPGfHhsw3v16XAiuoKIjkzncQKBL1GWULMDacVHVjhLXY9AE4hamFWX3B8W5bPky8vp5+mkP9Dy1dS2xyROuFpMWh00V0mOPgvfEJj0Xc5Oh6QQ0bMBHILmsJTno/nyxge7MiEnaD6Buzq8LRMxpyknfo0DtnbweOapRMTXZuWV9ZBCU9a5Cw99MRsTby7BWow1eH3fdyMnghlw6BoXqh/bSA/Q3sfM/5aF7Yj0ySAi+jyAHzDzySnuOgA7MvOtgs6+APYFAFp1nW1WWWVNs8yhndYlv07Un2aFyY2L9vdQPHdNX+ubW5kw19sgwv8QPrQPLYydnH74jRN6crrz6tNGKYzQHDJ8VcqyGMCbqcEOAO6VwgywT30CyiaVhdzEiuTXNDZLOLXQNDyNZq4V1exIWnEl7VHjIeoTa9Na9RrVqj9JmtokCDNv7JbG9GzxPx98aNarTycS0ZUArgSwPoB/SenPBLAUwBIAXwCwfy1TUghEOyv3S5VUc+0+6gezttItrcjio4tWoG0+SJS0WY0ny5+n1dOiV4tJECJ9MYRQ9hau0iZByYXQVTmwsBwcusaFiXpToDTBo+ZbSbOx8kTMTMtHZ5nFNUKrq3la4sGqRx8+xy2M+pryXeswdN1rXR5aumhbDGFyvn/h3iGB8W83njyZz6HNBrxXn1pEJl9fP1OJbkTgRvjSfGztvZZG0pJ5a3nz6uOVO24hpvExKXxJ9BFUo8IQAu2ghXuFBMYnbzxlMn1oswnNtNPiS0KnvZeqt4QnULTyuggzLV9rPuR5NA1QM5stczziY9Pqo7X5bOyWdYXlLuiC2jp6PksJja8aP+jQGKqcSX/1qc+pTxdkJzvdQkTfTOGdTn0CfJ9QJK838XN4fihJQ3PUSqFZ2kiI8K79rsmXQ6ubxV+NRjnfUCsM+4xPj2ZpbEVQ2tAZSjOcD5sCfwGwEzM/F8BWABYR0Q7M/OL200EALgRwepbnAvlZIQ+WQxNYcVKWdnqsAaKls2ho9xYfpZXY4yW6Ytf4DL30nmbm5ZftNQkCrtbZPRTPo6573kddNLohNVgNk74p0PvVp3SGwE4AvtmVibzzSgJJm4xemugk1rQwTWhpJqHHs+WE18qXtEr8yjKs+Pze8t9F81vlzTa8RclKD/QTSJ6PtA+8ekQFVO2i1xWT/mBt71Of0DyDdi4z/z4Lqzr1yYJm+lhOYa3DpJDxJrGm3UWc9BYtTeuMamuaoIxqYRosM9qa5LOhgY1Ly6sVxhFB7rktaqFpnl3o9RkvHibdhzbEqU9nAfgiM5+W7kOnPtW8KQD08wVEdpxygeeltX5b5UT48uh4NC1ege4bGEP4XGrbYdLojwLj5vn+W4b5wOM7F74+JDA+f+PXJn+XUzn1aX0A2wH4TpYmdOqT96aABs3EiK5knp8rspKVzAxpxlnaYsQs9fiJrNSlXcqopirz1GDUEzdKfxJ8fS2GbpNaLW66KZBgvfqUovdA8020P2fpQ6c+RdDFAepNZIuGNslLwjEXHFKbipiqluD0nMEejRpfY0lDi7SDlbcrRiF8JkmLG7p+o3b+W+Dg37jQ59UnANgLwMkiffWpT9EJrCHqJ9M0styXJPPK9JbmZWllmiDr6vcq5bXaytutrNHQonz3mVSlTYpxoCs/3vibTT6GpgFM/i7nxL4p4PlypIkX2eGJCkdL69KEZdTnZtWjhrcaXmW6nNecdy1Nqbxa/rpgNssaN/q2ExAXlkO8KfCWhX8XEhhfvvG0yfehzSZKvhxP0HXZ4ckHVo1WpWkWpfyRnbMSorzKdpQ+Nmkmt9pm110ya5LVaAi12tpsCLMSP55Py8tbavdS3pLGPbTWO8McusaFsEBLj278lIjOSPebEdHFRLSEiE4lotVT+BrpfkmKXzgUs5YmJCdRRNhZE9Zy6OfakBxIVt7oJoW3iWDxGynHKtfzpWltOIRwqXUfjEvjivaZFy/bNVIXy3fa19wfuh0n/bGNGg3tPQCuze4/BuBIZn4qgHsAvD2Fvx3APSn8yJSuCMtHpaWTKGlBWpqIr00zbTWfnfwtyygJBxmvCc6S6W1BE4iRzQdZ/6ExtJ9pKIybB2tcRhcV6fMdXEObJw/WbgLgbwB8Md0TmrcD2hOfvozmAVugOSTly+n31wHs3O56eogIJW3l0zQvKXyslV/TlPJJrGl/JZ41Ohr/NU55y3wpaXqaGV2aLKX2sfLWTpxxC45RoYv/T8JyY0TMSs29YJXTBfNhlxMA/i+AD+DhR0zWA/A7Zl6W7vODUB46JCXF35vSVyNiDskwL05CmnSWf0njIaIB5uVYvFm8WLS8ySIHfVQIDxE2tC9naM2iFl351doh4kOLoMa36I3HPlgGDl3jQuQ5tFcDuIOZfzJqZrxJ3t5rkOk0s1HSkc7wWl+cFx4xbzWeS8KhRMNDraCpcU7nGMqXMyoNbhSCxlv4SrSsMa4tiJb2V+uf7IP5oKG9EMBuRHQjgFPQmJpHoTlvc0FKswmAm9PvmwFsCgApfh0oD9YS0b5EdCkRXfrpI/SvdJfMrzydFafllXRzAefla/OWTGGrTMlzxPws3UdXYs3/l6cvmTglP+Bso9Y/pNV3KKHZh44cB9p41xbdNk77PTSPOYZ8U4CIFhHRdWkD8WAl/iAiuiYdlXkuET25SLPyXc4d8fDpTl8DcBozn0JEnwNwBTMfTUTvAvAcZt6PiPYC8Dpm3tOjK59Dsyabdi/9ZZYAtNKUHN99B37NRMrr4g1sSbulH6FdCi/5/qYYBlbfST9rqc+iYwUY5jm01z7pb0MC4xu//rZbFhGtCuAXaN48ugnNQ/t7M/M1WZqXAbiYme8jov+J5vS4N3h0+zyH9kEABxHREjQ+suNS+HEA1kvhBwFYQfJGYTnXNc1Ey6OZkJpQlCZqxNFtmbJWPk0LlOWVhJJGS7aLVXYbV6rTpGli8xVt33nuCW8By8eA5XIZBQbc5dwOwBJmXsrMD6Cx/nbPEzDzecx8X7q9CI0l6GKi3xSIrjxyxcrRRWMp5Y3S6DKwPA2piwZWw4/XdlMtbTh4Gpbsa0tzy6FZKhKP3vjFg2hor37S34QExnd+c+Y7kb6mk3AsMx/b3hDRHgAWMfM70v0+ALZn5gM0ekT0GQC3MfO/aPEtJuJNAcsvFhVOmvbmaXElWlbe0oQv0c7TWU53S4v0youYo1rZmiYb9VvWxI8bk8afp1VbPtr8f07DmifReVCLqIaWf00nXceWqesgojcB2BbAvxbTToqGpq0wUT9S1w6zJn1JM/FMgqh/rqsvLcKHpNGmqfXJdOV/SMx3zXA26zeEhrbLpruEBMZZvzmr5EN7AYBDmflV6f4QAGDmj4p0LwfwaQAvZeY7SuX2efXpxLRDcRURHU9Eq6Xwzoek5H4hS72WyCdtxPcl8+b/LXpaPk9wSR9XhB/Nv5X7RzyNTtKQk0SrpyxHK9Nb3UvmbTTcQh+zfQhY7d0lv5fXG+fRMku+2yEx4C7nJQC2SK9Qro7myz2L8wRE9DwAnwewW0SYARUaGhEdhEbtWzvtcu4K4KwUfRKA85n5mHwnNEQYwIN3LmVLK/J8CV76EmrStYhOrlrzs4tfzKPZpX28Nu+DLn1TotMnzSShZA3U+nPztFpbDKGhvXLTRSGB8d3fnF0sK8mQ/wtgVQDHM/PhRHQYgEuZeTER/SeA5wC4NWX5NTPv5tHs9OoT0HyNNh2gwgB+jMAORBTSVJL/oytRrtnk2ks+YUs+rRK0ldjzQ9VokZJHrU6SljbwpdallR81T7U6lerg8RZFJG9UG84xTv9aybQv+b9kv+fpa/2gUQz5LmeSIU9j5s2Z+fAU9mFmXpx+v5yZN8hOkHOFGdD91aeHkEzNfQCcnQVXHZLiTUIAKwgly9zTTK28gzXhKHmQ9PK8mumX82H5AHO+JD8RAa3xYtXDglYHTahpvz3TuytGZR7V8uRNfK+/h4KnMWv3tRbD0Pwv55nQNS4M8erT0WjMzbblLgPw5HSO56dhHG+XvykwM/OnR8R52pm3ikX8Olp+eW8JEy+9x0NptZTCu0RPoka7zAWpJaTl5KkRElEfWqmdhvLF1eaVi5Dn44rSjvjQvHyRMV7CUKb4vHz1iYi+AgBE9BEAT0DzAC2AboekaCc+5eahFqf9tvLmEzcyUSy/hqVNyUFfKsObzNYk0srQtEmpKUaFXSk8KkisiRM1nfrQKaEmb2nhq6FdsyhpQsxzM2j94y3IfTHnP/DIzIcw8ybMvBDNTsT3mflNRPQOAK9C87rCQzrmUIekeCaZZyrmK6umXUUmimfaer+l4JPpNE0shzc4NVqef0pb0T2zWKOlmcyjwChpzwVYGnlEQ/P6qouGXcJ8+sCjxOcAbADgQvF4RvUhKcCKGkceHskbXUGjZoem9Ug6moCQNGT5UhPT6q1pa57ZXfKrWPRl3SJtUcJQpuKofVe15deanF34Ly1kpTKB0W0GtJj0DzxOzIO1WrimJeU+K0vjsPJptL20Xr4orHK0+ByShxp+S4LWymuVPVRdI3mHTltDb9SaaJQXQBdMfcbi/bcMc9DwC574spDAuPDm88ZySMqcFGhd0gCxiWqZmrmAkIO/ZkJovOT3Gh8afYsvrx1qUBKk40Kkv6N5Jw19+CuZl0MJtO02fmlIYPz4lh9OT32SsPwFGjRtRDPn8gmaX3kZmj+ujfO0qchg1GjIMmUdNLol07OEvN6aP0+W3Xei1fJn0akZExI1bofZRO6+yO/z+Dxc9l0LWT/NZdIXk77LWfOmwKoALgVwc3pT4EsAXormE9sA8FZmvjxtCBwFYFcA96Xwyzza7ZsCJRNTwjIPLS0LiD3a4aWT6UtaTC3vknaElpfGqrtnwnYte2XDKNrE6z/A3jTw0rc0h3hTYNuNXhwSGJfeesFkm5zKq09fAnAGM39dpNsVwIFoBNr2AI5i5u092u3L6S1qTApPYGj0ZHyt2RgRAJ5giwg4LY9VjyifFu+lhaS23Foe5iKGrENpYfNcHRF6LYYSaFtv9KKQwLjs1h9NrsmpvfrkYHcAJ6S3oi5C86nujSLlWKaVZyLk5iPwSAEjB0eJjgXNPPQEpmeWWrRr8kRQ8slFeNHM2loMbXL2TTNKdCk/b1+rraXbQ5ZpuS9GAWYOXeNC31efDk/f+z6SiNZIYQ+d+pSQnwhlwuswT6Vu83hOc5kmv7doarxF4jTNz0tvDUA5SKOCxRLcmsZl/ff4ivKg8a0tMvl/GS75b+Os9JFFqbSwefyU+NQWu2gZGiJ9ro0NuSANqREvx0zoGhf6vPp0CIBnAHg+gHXRfJI7DO/VpxwRJ7A2oLt2oiWI8oFqDRJpIkg+rEGsmRIeH9591PcXQdTfGMmnadEW7ZJPSFu0vHbJacqFLcK/dW9p655l0MeMj/bdKM36SX9ToOhDI6KPonn5fBmARwFYG8DpzPymLM2OePjwlM8D+AEzn5zirkNzuMGtknYL7fNBgP14RA4tvuTnsnwV0fsuGGqQRXyDEd5lW0V8eFq5Wn6vrpF2qGn/UU3eUplA3G8apduFrxp6Q/jQttxg+5C0uvr2iyfTh+a8+rQR8NAp6q8BcFXKshjAm6nBDgDu9YQZ4D+TpU2SiE8r1wbkql7SfiKmi/bbutfKjJhZMs4yI2pNQ0+YaQLTM10sc1XjLeI+yPNHNTAPURMwCjmeWniC3xsHpbJkWq9cbwwOhUnX0Po8h3YiEV0J4EoA6wNoDy84E8BSAEsAfAGAfuimAs/k8szJyIQrletpMho9zUT07rWyLCEc5c8zZdrB7Gmq0faqMaFrTGBPyJc0PqsOGk/RBaoUJ/utVNcaS0OD108aj5Y5PiTmzXNoo0Tp1CdvZY+q/drqGBkoXcwoi/eSUCqZch6iZnWU11Ido3FdIAWHp12XFqOaNszzebQjfRrJU8NXaXHS0so0Q5icT3vCtiGB8YvfXjrZz6GNEtarT0B8skRXpK6TrzShZdldVmStPG9y9ZkQXnx0YajhoSR0hsDQgjWnC5R3tK3wvmMhWmZJSA4h0DZff+uQwLjhzssm04fWQjkk5YLsIJRbiOibKbzzISnAw6aSNihkOokaNT6nofkeZFntYLHiugrJ0m/P1C7lzZHzr/lZLPOvtFCUzGApkKNmmkXHQm36Gsj+zeskTfw2XOMrj7PGUpQXWV9vbAzZFpNuctb40N4D4Nr2hplf3H7rG8CFAE7P0l6QfQf8sAhxObHasBaWDyUyiLSy8t9yYHr+GjmYIhO9NJkt4RzVOkt+JK1tvfx5+ZaJFxXAllaW86XxIvujVMfSomIJao2WV44l/GX5Vv9FxnQJlvkqy62hGQXzTOgaF3q/KUBEawPYCcantqOwNJHSCuMJstJAtSayRaNE31p1NaEp+ZADUeOlxGsprCS85b3GUxse0Rxzut5iJQWcFBQ1mp3GX55e/pe/vXKs9rG0W41PrYxSuITFQ2nhGgKT/j203oekoHlk41xm/n0WVnVIitYR2sDzBEie35qIOW2Nfl5GRHvIy/FWbq18a0Ut1VlDjani5bf8MnmcNmlLPMn2zq+ctgz3eNPqV+IjgpKg9Gh74y5Sj1pBJMeQtmgOLeDm/KtPgUNS9gZwcnYfOiQlhzXZ5Aqoxcu0UWhptUkYmUj5CmlN1BLvsg6l1TYXuJZgteqplauljWgeFn3ZH5rJJ80lj8caAVpCTVrZtjmN0rjU+rhkhkZhKQB53Ci0tfmgoXmHpKwPYDsA32kTRw9Jka8+SY3IMnXyeM1/INNbNLWBGhFgHl815oUFqa14AkPT9KICoE2jCUbPXLPiopqbljeSrmZilky+2sVPLlo5XW/MRYW9xk+Jv8hiMwrTc/nMTOgaFzq/KZCi90DzCaE/t+mjh6Ropz5FtJb2v7X6eRPcm6g5fQ8tXc0k1WhYQs4zbSUdK97T5kp1y7XJGk23ZtHxwruaZ13bPMpnKb3sf6tdS2VYY3EI1ArHGsynXU4Ne+GR5ibQ8ZAUb+Jbk8EztSSd6ICRgkLG5TxowiIyCXPtSgoUS8BJSK3JM/Eszc2qZ2kiSoFjta8X5k18b4GxaMrwqFCOpPd48ASTt/BFyogIJs+8HQUm3Yc28Q/WaqgVUNpg9/J38T109VtIjTL/XUtPpq3N25Zdyl+zeGj16MOnxe+QfRqpX2kR68NrLU9avNbeQzxY+4R1nh6aq7+997rJfrB2tiFXN221slR+b7BpA1LC84No4Za2UTLP2vTR1dWjp2lh0YlWU3ZpYkXM+pK5ZbW9zNNVmPUxwaRwjrSZpc17GrVWbiR+1JhqaAHkGpqlUbWIrvKWxtGGdaFphXWFp7V4aaP5o5pUV+3SQ0RjKPXBuNpay5vzmNOo0dQ8nrRyavultJANoaE9fq2nhgTGPX9cMrkaGhHdSERXpleZLk1h6xLR94jo+vT/8SmciOhTRLQkfc1261qmNO3L0mSimk87KOTKatGMCEgZZ9XFCotoUBbtkoCQ9C2NssRvSQPW0siy5X3e9pF0VlklTcvS9CL+KWvsaeW38bVtp9VdE7rSYvDq1aaNaLldMOmPbYQ0tPTIxrbMfGcW9nEAdzPzEUR0MIDHM/MHux6Skt9HJ7xEjUYi6WurobViWvSjPEdW3hptsVaztModQiOSWrAlVCP8ynQ1PJS096FNtAi/XbXwPnTzuCE0tLXXfEpIWv3+T0snV0MzsDuAL6ffX0bzxkAbXnVIirXSaHFyhZSrnqexlVZHGR7x0+TanLZKW7xodbJW1doJqLVdzpuMH3IVl1pwxCTTwiPaq3Uf7esIStp3SeBYZnVEu+zbL0NrZ8D8+cAjA/guEf2EiPZNYRtkX6K9DcAG6XenQ1IkrAGbq9Tt5PG0HTnopKaiTWotLDoZLWHlhXmmcVRAWqaQpCvvtXCrTaLmplY32WcReOlKNCzNULuv5cEaF55WLPNKM1KjlV95uogJbfHbF/PlObQXMfPWAHYB8C4iekkemZ4zq6pF/qbAGhtu7U7y9l6bFN4KWZpAJbNLG3QybSmsq8lUS6PkeypppHk7WtpqH2j+nRL6Ch4rrlZDi/ZHJJ1sU88sl7S1PvPaqGbxiGJeaGjMfHP6fweAb6B53en27FyBjQDckZLfDGDTLPsmKUzSXOFNAQ3S5KsZnDWCQAqxkhal5Y3C8yvJOkYGeykuylON1uOFafC0Xc9NMArUtlUfjdJyieSLs8eXptHl5ZWE3ND+whmeCV0RENEiIroubSAerMSvQUSnpviLiWhhiWbk5fQ1ieix7W8Ar0RzIMpiAG9Jyd4C4Fvpd/UhKYA/yEt+NA+eWm/RqVXx83S1vp9Iui4T3jIRtTJK/0eNUQmuFl3qUZtHM88BfydTg+UmKI0tyx1QMkdrMdRzaES0KoDPorH6ngVgbyJ6lkj2dgD3MPNTARwJ4GNFuoFj7J6CRisDgAUATmLmw4loPQBfBfAkAL8CsCcz353e4/wMgEUA7gPwNma+1CvDew4t0pGW76IkxLw0kUlm+eYkDzUO40j9SjyVTOkojSitLrS7pJUTHIjvKkfTlvgoCZnIeM0RWQBL4zunI/sur/sQu5yrBd/qebBQFhG9AMChzPyqdH8IADDzR7M056Q0FxLRAjS++id4r1JOxIO18lzOFtaE1wa2lm4U8CZ5Xr42sCJCoDRJ+kxOK2+XiamlsRaSGs21Nl0fga/1TUmAdBXwfYXqEBhCoEVfUyyVRUR7AFjEzO9I9/sA2J6ZD8jSXJXS3JTub0hp7tRoAoirkKO+AOzbJ35KY0pjSqOOxigvAPsCuDS79hXxewD4Yna/D4DPiDRXAdgku78BwPpuueOstGD+0j7xUxpTGlMadTTGeQF4AYBzsvtDABwi0pwD4AXp9wIAdyJZldY1sS+nTzHFFPMalwDYgog2I6LV0XyKbLFIk2887oHmW4yuybtgcDanmGKKKQpg5mVEdAAaLWxVAMcz89VEdBga7XIxgOMA/AcRLQFwNxqh52KSBNqxPeOnNKY0pjTqaIwV3Hyi/0wR9uHs958BvL6G5kTsck4xxRRTDIGpD22KKaaYN5gKtCmmmGLeYCrQpphiinmDsWwKENEz0Hw3rf2s0M0AFjPztePgZ76AiDZA1qbMfLuIJzQfFsjb/cf5Vvhs0MjSrQsAzHx3bV1mi0YwvljOpNCY75j1TQEi+iCa09ZPQfOtNKD5IsdeAE7h5gu466B50O41AP4KzaeJ7kDzAvwRAP6I5sXV1wLYONG4OcUfx8wPDkEj43miBQURbQXgcwDWwcNfNtkEwO8A7M/MlxHRKwEcDeB6keapAPZPbTMbNJ4E4OMAdk7hBGBtAN8HcDCAx00IjRkvnplvLLU7mgdBx06DmS/DyoIxPCH8CwCrKeGrA7g+e0L4gwA2zOI3TGHfRXMW6DEAdkDTcZuk38cAOHVAGlsBuAjAtQD+M10/T2FbpzSvBLAEwFkAvpius1PYK2eDBoDL0bzjJtt0BwA/S7+vBbBQSbNZipstGhcCeAOAVbP4VdEsaBdNEA03Pt275UwKjdme4+O8Zr/AZiI+WQl/MoDr0u/rnPzXAfiFE/+LAWnM1iTvRQNpITDqsiT9vx7AAiV+dTSCc9ZoOGmunyM0rg/QKLXHrNGw4ubjNQ4f2nsBnEtE1+PhT3U/CY3Z0r5p/ysi+gCAL/PDZtUGAN6a8qxFRK8HcBpz8zU5IloFzUN49wxIY01mvlhWgJkvSt+GAxo/5E0yDRrVfzU0AmDUNM4kou8AOAEPt+mmAN6MRtMDgOMBXEJEp4g0e6F5InujjjSehEaDiNL4CREdjeYcijzNWwD8FMBtE0JjphAPAGcVyll/QmisNBjLg7VJcEh/0SXMvDzFPx6NH2N3NGcVMIDb0bzb9TE0PoSPAdgJjfAhNP6D89D4FX5ZQeNlaHwNQON7yWl8CsDm0AfKL5n5gPQdpz3R+ASloPgqgI060mgFRZTGLtA3Wh56Ejt9QG83Jc01KT5C45lGmpbGrkYZZ6b41dH4LleggcZ3+ZcSH7NBA814ccsotVmEz9mggZUIE/umQNoJ3QSNn+CPWfgiZj47u18v/TyKmd+UhW8P4OfMfC8RPQaNcNsawNUA/g+A+9FsTtwC4DI0H6R8YYo/ltOmwFwRFKMAEf0VN59d99Ksx8x3jYqHKaaowrhtXu0C8G40fq5vArgRwO5Z3GVoVid5/bH9ndJdjeTvQfNe25EAXgTgIwBOB3AigFNTnv9IYfsA+BIaM3WU9furQJr1Kuitg2bn9lo0L/HelX4fAeBxKc3aAD6a6rq3yH80gHWV60YAjwewbkp3BNL3qABsA2ApGn/TrwC8FMDz0Wi4X0GjQX4PjfZ7CYDnpXwLALwTzQbIFek6C8B+aMzrVVP8PwP4b4LPf3La4BfZ7wMyPjcHcD4aTf5iAM9J4U9BY0L/M4C1AHwBzfe3vgZgYYnPRKOaVwjfbYnXEp/R/l9ZrrEzYHT6lQDWSr8XovlA3HvS/U/RCLWvANgxTaQdAdyafr80pbs2o3eZoH85gCvS7wVoTNFV0z1lcXNCUMDe0T0YwHfT/WmpnNegEeKnAVijbR80jyn8UlwPpv9L237J6J8H4Pnp99NSH/0YzTfi90ZjGu+R4ncGcGH67e4uo9nhPQmNr/UnAD4p+xHAHwD8Pv1vr+VZ+NVZnu8AeG36vSOA/0q/zwfwP1MbXQXg/alt347mkYjILrjLa4nPlM7ltcRnSlvs/5XlGjsDKlNZJ6f7tdA4Nz+JRhitAuB9aCb2VinNUpHna2jOMwCAf0dz8ns7+S5Jg2N1NILlD3hYuDwKSRhGBgomQFCgsKOb/l8uwj+UJsx6ic9/SG38nCzNL0Wea/Gw1nuRiLsSwE+z+1+L+J+m/+7uMtJiku4XoNGuTwewRkbjU2j8iRtovObtgcY3m5dxRc6PxWuJz5yWxWuJzwivwTYt9v/Kco2dAaMTvo8kqMSAOQHA8ixsEzSC6zNKZ6+Dxny8AY36/iAazeeHAJ6LRiAuRaMFvRvAuWjU+SsBfCQ6UCZBUKB5ru4DYuJsgEYY/2dWxioi71vRmOa/Eu35SQCPxYqLxIGprJ0AHArgKDQa5P9Go6FeiOa5udendn1NyvdSpC+oonnG6/U5L2gWqDekfvq50tYfSW16fRa2TRon7075l2Zxh6e+fwqAf0SjQT0ZwNsAnJHS/ATNgrEdmodX2wXvqWgEictnui/y6vEZ4TXj8/kan+l3sf9XlmvsDKhMNRNrQyPuhUrY3wD4P0b6tdEIsG3yDk9xGwPYOP1+HJqvYm6Xxc8JQYFGy/wYmmf87kFjHl+bwlrN8+MAXq60zyKI55jQbD5cBOA2Jf2OaEzDn6IRtmei+X78aqmdz0Hja3pGqsfvUlv8t5R/Ycp/BxqN7Bfp96lonrv7CpqDMWS57wDwoAhbBY2guADALUofXIxGCPwBwDVoNoPWSfE7o/HTXovGt3oaGjP/DjSbMy2fv008tnGnAtgs0Qjx6vGZ4t9m8Vrgsx0Hxf5fWa6xMzDJlxgod4uB8viUZtSCYkFQUDwDwMuRfI85H9nvZ6QJItPsIuMBPBrAsytoLEr/n+nFp9/bo9GM1kOzs/x+ALtm8dvhYbP7WQAOyuOVNC8G8GGHxpZoNGVJY3uR5v0yTYpbL11fCYyZE5y4jQDcFaDxH4X4MyAWURH/4lTfV457Ds32NbGPbUw6iOhtzPzvtWmI6NEANmfmq7rSkPFotL53oRG2W6HZQPlWir+MmbcmogPR7KipadCYPSUa7/bSJBr7o1kALBofQeMPXIDGB7odgB8AeAUaob1AxG+Pxq/4CjSHahzegcYj4oM0tleaeyc05iOYeTcikt/AJzTPNX5fyRul8Yg0QRo/ZubtAICI3oGmj76JRqv/NjMf4dCaXxi3RJ2rF4Qvq0uaoWigsCuc/pd2jmeTxqoAHoNmB3DtFP5oNL4rN34WaUR20n/qpQnScNOUysjbNv2+BM1hvACwJrINp5XhmqQzBSYORHSFFYXGl1ZMM0s0/sDp4WNuvr6wI4CvE9GTUxqgMVG8NKX4oWgs4+aNkPuI6AZm/n1Kfz8RzTQ/3fjZorEtgPeg2eT5X8x8ORHdz8w/zNp/Gy8NEUVouGmIyC2j7Zf0ZswqaB6W/22qy5+IaBlWJoxbok7yheb5tK3Q7Drl10Ik524pzWzQQGBXuJRmFmlcDOAx6Xe+g7gOGm3FjZ8tGtm9uZMeTTNqGmiea1yK9CgQgI1S+FoQu/Dz/Ro7A5N8oXmn70VG3EmRNLNBA4Fd4VKaWaSxhhG/Ppon49342aKhhJs76dE0s0UjS/sYpB3ZleWabgpMMcUU8wbTMwWmmGKKeYOpQJtiiinmDaYCbYopppg3mAq0KaaYYt5gKtCmmGKKeYP/D8P5ExU4QiqJAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Check annotator similarity\n",
    "similarity = np.matmul(c10h_annotator_mask.T,c10h_annotator_mask)\n",
    "np.fill_diagonal(similarity, 0)\n",
    "print(similarity.shape)\n",
    "ax = sns.heatmap(similarity, square=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ec9e3280",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save labels and true images/labels as npy\n",
    "np.save('./data/benchmark_data/c10h_labels', c10h_labels)\n",
    "np.save('./data/benchmark_data/c10h_true_labels', c10h_true_labels)\n",
    "np.save('./data/benchmark_data/c10h_true_images', c10h_true_images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "dcce99e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save labels as csv with image paths\n",
    "mini_df = c10h_df[['image_filename','true_category','cifar10_test_test_idx']].drop_duplicates()\n",
    "image_locs = [path + \n",
    "              '/data/cifar10/test/' +\n",
    "              mini_df[mini_df['image_filename'] == c10h_true_images[i]]['true_category'].item() +\n",
    "              '/test_batch_index_' +\n",
    "              str(mini_df[mini_df['image_filename'] == c10h_true_images[i]]['cifar10_test_test_idx'].item()).zfill(4) +\n",
    "              '.png'\n",
    "              for i in range(len(c10h_true_images))]\n",
    "consensus_df = pd.DataFrame(zip(image_locs, consensus_labels), columns=['image', 'label'])\n",
    "consensus_df.to_csv('./data/benchmark_data/cifar10_test_consensus_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9201596e",
   "metadata": {},
   "source": [
    "#### Cifar-10h dataset attribution\n",
    "Peterson, J.*, Battleday, R.*, Griffiths, T., & Russakovsky, O. (2019).\n",
    "    Human uncertainty makes classification more robust.\n",
    "    In Proceedings of the IEEE International Conference on Computer Vision."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}