KTA6 - MAML.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data - Sine Regression Task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "\n",
    "def uniform(shape, min_value, max_value):\n",
    "    return torch.rand(*shape) * (max_value - min_value) + min_value\n",
    "\n",
    "\n",
    "class MetaLoader:\n",
    "    def __init__(self, num_tasks, samples_per_task):\n",
    "        self.num_tasks = num_tasks\n",
    "        self.samples_per_task = samples_per_task\n",
    "\n",
    "    def __iter__(self):\n",
    "        for ep in range(self.num_tasks):\n",
    "            A = 0.1 + torch.rand(1) * (5.0 - 0.1)\n",
    "            p = torch.rand(1) * np.pi\n",
    "            \n",
    "            x_supp = -5.0 + torch.rand(self.samples_per_task) * 10.0\n",
    "            y_supp = A * torch.sin(x_supp - p)\n",
    "\n",
    "            x_query = -5.0 + torch.rand(self.samples_per_task) * 10.0\n",
    "            y_query = A * torch.sin(x_query - p)\n",
    "\n",
    "            yield (x_supp.unsqueeze(1), y_supp.unsqueeze(1)), (x_query.unsqueeze(1), y_query.unsqueeze(1))\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.num_tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAB7CAYAAACLgBAMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxO0lEQVR4nO2deVxU5f7H3+cMsumggAqiuKSmueSGIGi5kaiVmWtppea1Mk1LM7VrLnXLyiXTLK1fapbeLPNqiyFiZZYLKpq5lqVIIptswzpyzvn9MQw6MMCMMrI979eLe51znmfme5rnfOY53+f7fL+SpmkaAoFAIKgWyBVtgEAgEAjKDyHqAoFAUI0Qoi4QCATVCCHqAoFAUI0Qoi4QCATVCCHqAoFAUI0Qoi4QCATVCCHqAoFAUI0Qoi4QCATVCCHqAoFAUI1wcvQHrF69miVLlhAfH0+nTp1YtWoVgYGBVttu2LCBCRMmWBxzcXEhNzfX5s9TVZW4uDj0ej2SJN2S7QKBQFAZ0DQNg8GAn58fslz6XNyhor5lyxZmzJjBmjVrCAoKYsWKFYSFhXHu3DkaNmxotY+Hhwfnzp0rfG2vMMfFxeHv739LdgsEAkFlJDY2liZNmpTaxqGivnz5ciZNmlQ4+16zZg3fffcd69atY86cOVb7SJKEr6/vTX+mXq8HTBfv4eFx0+9THdl9Op43vz/LmKzPeNIpnKK/l5qG1WNmSvp9zZddkdo9iG7QW+DkXL5GCwSlcXYnGd/+G49rSYWHzGPW1vlg0XGfgp7YgH/T6b7HytHQWyMjIwN/f/9CfSsNh4m60Wjk6NGjzJ07t/CYLMuEhoZy4MCBEvtlZmbSrFkzVFWla9euvPHGG7Rv377E9nl5eeTl5RW+NhgMgGnGL0QdFFUj6kIKu0/Hs+7XiwySj/B87V2AdNODvjh58OdWtD+/Qgp5Dga8Vg6WCwTWMY9p3blv6B41nToSyC7XB2jZ47V0PMik2Ym5XMg7zx1jlpeDxeWHLZ4Lh4l6cnIyiqLg4+NjcdzHx4ezZ89a7dOmTRvWrVvH3XffTXp6OkuXLiUkJIRTp06V+MixePFiFi1aVO72VwfCT15h0TenuZJuWpOQUflPrXV2D3ib22sa2v6VSCCEXeAQzGM6IT2bX1wWoQFykfFZXktpLc59zP730kltGsZ9Ax/G2blW+byxg3H4Qqk9BAcHExwcXPg6JCSEu+66i7Vr1/Laa9ZFYu7cucyYMaPwtfkxpaZSdGZ+I93l0/ztZuSAzp1UnYynolJfUZBUMCS7kJcjo8+W0Dvn08opjzoNjEh2xEdJEmiAduA9vq7/JA3rehDYwgtd0btOILgJwk9eYfJn0WhACCdxOZdHQpYHzrUVPFtlITuBAhx2deGwqysAgbm5BOTmobPzs8w/DCHJWyF5K1eOzmZ/61kMf+zZ8rwkh+AwUa9fvz46nY6EhASL4wkJCTb7zGvVqkWXLl04f/58iW1cXFxwcXG5JVurCztPXGHejpOkZBkBkDWV9sl/45WXjqHhef5s9ztP1rJ8cgo8pzJ+t0pjg+V7/YMexVklv10ucV2MNNAUutpwc0gAmkrathfZo7bl09oNGTJkOAM7lr64IxCURv61fLas2crM87tpk3wFv8xMEqlbeD7xuAeGDrnMGOxOuu76KP2QutRVFBYmphAco5Cfq8PJVcHdzgmLDykM+3MuP3/0N/dOfAtke38mbh+SIysfBQUFERgYyKpVqwBTuGHTpk2ZOnVqiQulN6IoCu3bt2fw4MEsX26bbysjI4O6deuSnp5eo3zqi3eeZu3PFwqFvEf8KfpdiqbetazCNsl62HCfTFQb02gOPKcyc5sKFIhxCRjc4MMwGdVV5f6kXAaomXbN4uM0L6Lvms2gUU+LWbvANlQFYvZDZgIZv10mZuk6nHKNJTbX0ACJrwMlNvW3FNzAsyrjI1Xq3zBxya2tcq6nkRZ+WXbP5LU6jZAGvw3thth3TbeAPbrmUFHfsmUL48aNY+3atQQGBrJixQq++OILzp49i4+PD0888QSNGzdm8eLFALz66qv06NGDVq1akZaWxpIlS9i+fTtHjx6lXbt2Nn1mTRN1RdVYtfsMe77YbRLy2GjqGbOstlUxifeyYTKHW0usfl/B21C6oIPJpQKW7fLdVZp1ScPDv+w9BGrBG7wovUizXo8wtV8rIe4Cqyj5+Vz++jV8Tq3HGJfD3wm1qXXaFZBsGqeqBGNflFGdSp+43HgvnGulsTA5hdDsHJts1MzWjNp424TdHl1zqE999OjRJCUlMX/+fOLj4+ncuTPh4eGFi6eXLl2yCKRPTU1l0qRJxMfH4+npSbdu3di/f7/Ngl6TUFSN9344z++fb+eJI18RlpteZh8Z02Aev1sly0W2mLmUhsR1YS98r2yZf371pEnPVAthtxZ5IEsmYX9Z+5DXfsjm6V8bMmLYSOGSERSi5OcTtWE2Hf/5hOQrrqQcqoNLbm3sCZCVAJ0GA6M1dgaCpGqM3239SfTGe2FKK5kXGtZnaGYWbqqKf34+ozMyS/xs03tpsH0yXDkOLXpD816VxiXj0Jl6RVATZuo7T8Qx58tonj65mb5njgFlz7aLsjVEYsT+W/vqNcDJPZ/0Eekk1dLRQLHN7w4Qp3mSELKILmHjbskGQdXn2K5PaP3rLKINzsSf09Phb/vH841831VifZiOdjEqCzerZbZfOEbmdFMJSYO7YjU8MyG9NnT3MDAjvezJEgBuXvDguw6buVeambqg/Hn9u5Nc276Q9dEncc+qWBeGBCjZTizNb8DpxqYnLjdVJSwzm1euppQ6y2pEKo0OTENpUg9d+4dui72CysfRbz7Gd/UiYmI9aaDINCiH90zwNP2/Z6Zt7T0zIfAP06z+xqfXZH1t3uvlzOS6SWVOVLScFPjicdSRGyt8PIuEXlWIt3/+kpifxzJq3yncblHQTzU1LZyWPY8pG/PNI6kaLWIhKdadJ6QmLK9bt8Q+UoFPR9v2FBht82UKqhcHZs3HfdYSMi7WRlZuXYo0QJEgvKvp3kitY1s/31SNmdtUvIu4I70M0O/7Wqw878eSvPoccnZBKeE9pAID0r+cSviJmJu8gvJBzNSrCBEXd/PZ+UWs/sE0rG5W0lUgRQ+nm8lsuM80mLVbeD8w3Tzm0MgbZzp5utrs9XOhV2AiOiv7NiQJnJRceMMXgqdC2Ou3YIWgqmDMN7Jj+jja7zl+y2PPjNmR+G2gVLhIesZfIllvEmdrPxnme6H/MVNva353gIFHgCPOZDt7syQAAu5MJTS3+EREksCLDIK/6sGxuDfoMnBCsTa3AzFTr+QoqsKhK4d45dcF3BWrUd+GaJWSMK/4b7hPRpMlotrILBsmc7WMdBIled5VTLN9fY71mY6LAg1jnTj3VSNi9nmW/hkH3kPZ/IiNVyKoqiw5+BbjX+tMuz3HAVtiWmxDkmUyR/Tn20Fehcc0WWLDfTISxZ9IzffCns4S9TNtu6fcjfDQfvDc5Mm+xJL92nWlbDoffB5l17ybuJJbR8zUKzGRMZEsjnqTxGzTBi5P65GKNpNSEKd+tkM9hjbtR1CjIOrfVx/pWY3Lh6PIi4/D+1QcHofOIBksP0zFcgZgvik+6S8zLrKsWHeJrMtuRP8q07XnVetNNJD/+J5j339Ml0ETb+k6BZWTRYsH0W/bRR6wMeqqLFQgv2UTmox8DM8xjyI7O7NXVTgcf5jDCYdBg8ABgfj1TiX2tUXIyWmFfc33Qq18KHnaYh23a+D2Qx329YV7fDKsN9JAPrAKxa8Luo7Db/YSbwoR/VIJUVSFj35by4/frMYz0+TeOOMvcVesZtNq/o3ovLzQP3A/V7o0Ib6VFw30PnRt2BVdKeFXmqKQfeQoxsQEwjMOceBMOCN3ZRVZRDLdFJmu2GSTeZCld8smqEUacgnTiXTNnQMjDotwx2rGxtXPErDqR+DmnjTN4+d0E/Bs2BTvoBDaP/0StVzdbOuvKOz77kO2H1rHZZdszvhLaLJkc4SMNRQJ0vpk0qskYQcUJE70eOeWXTGVZvNRRVDVRT0yJpLvP36Fkd+mUTf7+vFkfcGseI9aoo/QjKL3oP6woej79cc9oBuS7tbiZxVVIfrKEQyHD+GZBXe06MaI2PnE5ybS85TK9K/tuyk0ScOzTSaNOlufsj1b61VWzZ0mNihVAzRFIf3Afv6Y+hR1cm/edagBe3rW4a75iwltFnrT9ihFZvKX0i4yZPbOMu+p0uzyC0mhXtOSN+GpGvwWsvKWwneFqFdRUY+MieTI/GkMOaQVG/zmL+nrIIkhh0ybomUr571G9cdnwbu3LOS22PrCTy/c1EzHbKuhQw5BHVKLnZ9mnMqjE2cQ3NK7HCwVVBQZEREkvLGY/Pj4W3qfXL0r2syJdBo1udQnzJtl0wfP0eXdyGL3lK1okkZu70w6+Rqshj6qGiRK3jSY9wc6p5vzeNuja2KhtJKgqAr/+3AWQw5Z/401i3zP0xrLH5ZIKbK4eVUPKZN74/vqew4XdIDQZqG80+cdYlrUtjs00nwttU+5sVxfPOwxkXokGmwvYSiofKR9v5N/pk3n2k0KuqLTMN7XgSYb1tH54BG6PjLVIYIOMHbyKnzfWUaOp22unKJImoTrT3o+OelHpHPx95Al8OUqZw/tulVTbUIslFYCNEVhy+Z5PPl1bqmPpxJQ3wAGN4kpz8qFu99yPVx57NHXCW058HaZDJiEve/YvnyQPI5+a4/aFZ5m3tLt/WNtIkKMDDDmoGlwBW+i1LbcZ8hjx/HLNNS7ivS9VYx9G9/Cc/EGu9PdmtBw9Tfi//VJnNzcy9mykvEaNBjPAWHs++5Djn+1ltBDpsI79ozn4JOgnvQkqoMrgVaeQHNSL5efwaXZItwvFUtGRAQJr79BfpEUxaXx7hCZX9vLGNM6oc8PYf/0p3C+yce68iLl+53EvfQSumslbc8oGRXwutOATxcDk689z24tsDAJGECjuq4seLAdAzs0Kj+DBQ5h0+qpdFm1xy7fuQbk1oK/uuUxrMlVTvRaUWEx3mB6ao6c/QT+30Tf9KJu7cY5NLvHUtg/brmSGI9uNPNy5/Hg5jg72e4oEe6XKkJ6eDiXp03nmh2CDpBSG9R8d/KujOY/YQ9XuKCDaabT/vhv5LZrYWeAmGkQpv5Rh+/2dmOXainoAFfSc3nms2jCT14pL3MFDmDfxrfp9J59gm52230TpjCi+VVi2k2sUEEH0Mk6wpZsotE7y7jmenPVjrIuu5IWYyrUoWoQp3nz+ikvNh6I4bXvztD2le9ZvPN0eZpdiBD1CiJ9507iXjBVbLLnJkh3h7P+ElLySNY8FlCpZq+STkeXbTuJ276MH7u6oGJ7BLCERMv4K8w5+j6ylm+1zZxtv6MUVXxBpSBtVzjeb6xHZ+fXk6KHI2G5vFwvC3XEhkpVE9Rz0GA6Hj1G+mOD7JqoSJjGc/yRuigFD66Lrj2OeoPcqhqs/fmCQ4RduF8qgIQlS0j5eJ1dfcxf0qoHa7Gv/uNEz5xp1+Pb7caYb+T9p3owYL9pO7U9P1wGV4lVQX3Y53l/sXPT+rVixoA25WSl4FbRFIWsqChipj2HZMiy6XvW0DC6aFzsn0OnVl1oEfYCtLin0qSutUba9zuJm/kikp2TinM9FHY2GkWEFmz1vCzB2dcGlXkvC/dLJSZt5067Bd1MzN1GdusWsfT+xyq1oAM4OznTYcHbfB0kodrpmKyTqzF374/cm/J1sXMrfzjP69+dKicrBbdCRkQEf/btR+yEJ5FtFHQzdwSkMdQjnexOT0HLPpVa0AHqDRpM44Lqa/bIepuDOv71/Vc8dmk1slY8RkzV4NMDF8vHyAIqtzJUIzRFIXHVe1yZMdPuvulucGxALsY7W7NyTFClcrmURmizUAJeXcmkOXWIam17P6ng76V9P3Pv1eLC/tG+i7z+nWP8kQLbyIiI4J9p08lPTLSrnyKBX4ipsEo67rQNCnOQheVP3YEDabzyXRQv+zwA+lwYG32B/4bPJSTut2LnY1KyrfS6eYSo3wYyIiL4I6QnV1evtrmPCmS4wvJRkD4qlSGe2dQa+X8MvtvPcYY6gNBmofzy+EGyFky2e8au02DOvp+5J/W7Yuc+2neBHcduT4iYwBJNUbg060XAPreaBqT2yaSufy6aBn8GvHbTm3EqCo8BA+iwbz9Z93ZBw75Zuz5PYV7Up8XGczOv8g3dFKLuYDIiIrg8bTqqrRVUCpDQqBVsYI0cx4C8HM6HvM3gTlUzH4pO1vFswDS8xo+zOzIGYNqBH3FWTJuRZFR6yKcZIu/nv19u5s3vfi9fYwVlcnzEA+jyrtkl6Ipketo0J8A6XjuEgAf+5RgDHYyk0xHw4WZq9+trXz9MPwJPRf+Ic+0TgMmn/nhw8/K1TyyUOg5NUfijZy/UtDR7e+IXkkrdprmmYrrD1932TG+O4tLkZ8n68Ue7+6nA0VZ3EtrtBH5SSuHxOM2LKz0W0G3Q+PIzUlAih15+Fv22H20WdLO4XO2byT0+GWgaHPB5hJBn1zrKxNtK2s6dxM9fgJZpY5mlAhaM8iDKOI+n723J3MFl12AWC6WVhKyow3YKuumBzi/4uqBLwc9VG0EHaPrB+9QbZ39iIxkIOH+O/F8s5yC+pNDl4HSUUzvKyUJBSUT8sZM6dgg6gOau4NczpVDQj3ZfVm0EHaDe4MG0OXSQSxPsSzLmlZtJcNdomwTdXoSoOwBzJrgTuzbZ3derbSZ1m5l8jkqPqRD2HwdYWLE0mjuHxiveQXF3tbOnROZlV9JjrvczZw/I3jELVPt3swpsI+JiBHvefdEuwcgLyKTdAwnU889F0SSig96psi6X0pB0Ovo8/zZX9bb72FPrwMmcrSw/Uv5x+ULUy5nImEjCvgrjyV1PsufSHjt6ajQKTsWnswFNg/+1fA2ngdW3vJvHwIG0P3yE9LEDbU4GZt7UcfmgJ/k3dJIl0BsTUC7+6ghTazwRFyOY9fMsfIqnMymRHGfocEcGkgyaBscDl9Ft8JOOM7KCcXFxI+ZJ02y9NGE3Vws742+ajaw/tZ6IixHlaosQ9XIkMiaSGT/NIKGgUtGpZrZ7HhuHpFKvmWkx8Lv6Exj2xDQHWVl5kHQ6erzyDikvT7ArkkDSJKYrfkS6W2bE++vvv8rdxppOZEwkM/fORNVUEkqvSFiIBmT3zMRJNv37wp3j6XZ/9a9mNXbyKo5NDyWzhGSPRctJmnn90Oso5fiUKUS9nFBUhTej3kS7QZpON5UwuJUlViZB9yhIsp+CnkGTlzrU1srGPU+8RMorE8l2saPPEZjRsL6FsCdq9crfuBqMeUybCe8qoUilj2cNMLTPoZdPhmmRv/VA7hj7rqNNrTSMnbyKTgej+W1IOzKKeBdT9LBsmExUG0vZTclNIToxutxsEKJeTnx04qPCGboZTZZYO8j0n7j4jVCwKFog6JpmekyN6VH1YnfLg15jXyR/21qbZ+u1c03/BRfV9+JaQcIkXfOejjSxxhGdGG0xplUnmW8DTTNMa9+TBuT45xHU0eSnkXpMQTd2y22wtHLh4uLGI29/xbH1z7FwjMy7Q2QWjpGZ8qyumKCbScpOKrfPd7ior169mubNm+Pq6kpQUBBRUVGltv/yyy9p27Ytrq6udOzYkZ07dzraxFsm4mIEq3+zvrEoqo3MsmEyV4sUtXByU2jc0xTlAoAEF9pUfIa6iqR7i57E+ti2Xfy8HyBJpOl0fFTPg5W1JqIis+P4ZQ78dVUk/ioHrAnNpn46q6kfVEyVrLqGXDUJ/oj1MPCN22FmpWVSp6eJa+3Fr+1lTjezdLkUpYF7g3L7XIfGqW/ZsoUnnniCNWvWEBQUxIoVK/jyyy85d+4cDRs2LNZ+//793HvvvSxevJgHHniAzZs389ZbbxEdHU2HDh1s+szbHaduXkRSreR1uBFJ1bgrVmPOP+ncqcvDvYER6YafVOXe2ej6vexgays/kWd34jfUlErB2i1gHqxjZkoozqYfAF1+LbTLb5Cefd0vKXKw3zqH4w/z5C7ri5tyvsrAaA2fVEioB/2bpDDAmEOc5o2hz2u06Tv29hpbSYm4GMHMvaWnBvF19yV8eHiplZ0qTY3SoKAgunfvznvvvQeAqqr4+/vz3HPPMWfOnGLtR48eTVZWFt9++23hsR49etC5c2fWrFlj02feTlG35QsrRNPwVRTCY+OKVYTR3LyRZv1Z6ZMa3S6iJ4zC9YBpp+iNwm4eqIdbw9IRli6q7JhJKNkti73X+2O6VLnUCpUFRVUI+/xeEo3paJL1WaasaSxJTGZAdg4rrw1lQ61HODx/oKhUdQPLjyxn/an1Vs9JSCzvs7zMYtqVYvOR0Wjk6NGjhIZeN1aWZUJDQzlw4IDVPgcOHLBoDxAWFlZie4C8vDwyMjIs/m4H5hm6Pcy+mlpc0AHp/uVC0G+g6/ovStyCbU3QJVWjY/Kf9P7nGB2Tzltkw5v632PsPCGKa5SGoiocjDvIqmOrWBW9ikNXDqGoCjpgZkGNUano3K9gEcgs6AD7tQ6M69VSCHoRZgTMYFnvZXi6WIYP+br72iTo9uKwFbnk5GQURcHHx8fiuI+PD2fPnrXaJz4+3mr7+FKK1y5evJhFixbdusF2EH72W35c9RLjUiHB0xQVoJaRCndKWjqhBYPfjKaBGjwFXYehDrS2atLs/fdRcnL4cfZ4kv44QbwXfNL3usvFTOA5lfERKvUzIwuPJbvo+aNLM3RN4JLWkGmb85HlQOGKsUJkTCSLflmA319peGaaNsV85L8WD9d6vKK/m0EZydTKd+NNb08SbljA91UUZl9NJTQ7B1WDeLz506UDm/rZkY6zBjGg+QD6N+1PdGI0SdlJNHBvQNeGXR1STLvKh1nMnTuXGTNmFL7OyMjA39/fYZ/3zQsjaRF+kvE3TFwe/0Hj20CNTf2sfEGaho+iMCkto+hhLrSZwB01fDGpNHRubvRdsZneW3qTbiyeEC3wnMrMbcXXMrzzDAQf/J0mPU0pXv/ttIlPtz6I0m6jmEXeQGRMJP9dO503dqvUN1w/nqyHDfel8uKdP6NzdyM0O4e+2TlEu7qQpNPRQFHomptn8dT56rXHeX1MZ/HftxR0so7uvt0d/jkOc7/Ur18fnU5HQpH6mwkJCfj6+lrt4+vra1d7ABcXFzw8PCz+HMXmKQNp+f1J5CJPorIGQw5pjP3B+gaCOUXcLhm4czx4BXeMWeEwW6sLOlnHwpCFxY5LqsaUb02CXlRGpIL/jTnoiaaCjMY47Wv+XD1SpBIoQFEVtn30EjO3qXgbLM95GWDmNpXAcxpvenuiADqge24eg7Oy6V5E0D+QRjF0zDPiSaiS4DBRd3Z2plu3buzZc32rvKqq7Nmzh+Bg66WdgoODLdoD7N69u8T2t5O969+g856YwgION2J+/UCUhnzD/nVZ01iWmGzhdsnT1ab2yxdrdOiivYQ2C+WdPu/g437dNdf+ooqbseR83hLgpEhs/8sbSQJJgrZXd6MtawenixfeqGmsObySJ7+xXmpQxrTWMz5SJVHWEe1a8q6wVK02ncf8Rwh6JcKhceozZszgo48+4pNPPuHMmTNMnjyZrKwsJkwwCdoTTzzB3LlzC9tPnz6d8PBwli1bxtmzZ1m4cCFHjhxh6tSpjjSzTFK/30mDtz4tNTudhKmow8Dogml8kUUkMwntJ6FztmPrpAAwCfuu4btYF7aOt+55i2evtLepX/PfXFhat27hay0rHu2Lx2u0sB+aN4Xe4z6kdl7JP4oyUN8Ad8VqJOlK9vt+qXuQHq2KhycLKg6H+tRHjx5NUlIS8+fPJz4+ns6dOxMeHl64GHrp0iVk+frvSkhICJs3b2bevHm8/PLLtG7dmu3bt9sco+4IUr77lisvvmTzr59PqmWYlxlNgzSpDo2HvOIYQ2sAN/okD1+zHiJWFNd8OJyhJ9zdyMDsHNMsVIO8bVNwaXt/jYs6ipo3Ff3WH2xu75kJDZyLu6w0DVKpQ9OHXxF+9EqGKJJRCps+eI4u70balT96Qz+Jvs1SGJhzXdDNmxt/C1lJlzD7c4kLinP83Vdx+eC/NrXdGiJxuT48ei2De/WGwk1fSpMgdP8q3wx5lZnc7EzOd+uOrNlehm7pI7BWi8Pphg6qBkjwW7AYz7eLShGnXtVZHrWUpusiy25YgIapZFefJincl5NncS5R8haCXs60f3qWzSl7R+zXmP61RsPv9Zz81peMWFOmJfmfQ7BrnuOMrERExkTy5tw+6GwUdHOK2NGuqRaCDqbxrIzYKMZzJaXKhzQ6AmO+kcO7PmGQoey2N3Khk5EHjaYZ+oVu87gq1cXNszFtg8LwrYFJuhxJLVc3Mof1Rb/NVBqvtJQCN57TZctc/tUTCsIdtQPvIfWfD07Ojja5QtAUhZ+/XcPWX9+nx9+2/QwWTMSJCcnlsdzrT5x7lY58oDzE+EcfZWCHqlkvtyYglKYImqKwc9vbdD9je+ibBhzvks+YNsmFx1o0b0GLjiMcYKHATNAb7/PjxTB8oi8VO2dN0M2vNSDhmAf6xrlIssbpHctoN3wu1Y20XeFcWPhvGqZmM92OflkucK5PLo95plgcPyB3Zfzox0SkSyVHuF9uICMigvP9Q2kzfxODom1batCApD6ZjGmTaHmijo/V9oLype/mXRx5tjeZRQKKrIWeXj8nkZ/tRHaSaXZ++Fh0tUsl8MumpcRNfwGX1GyL46UVI9EABbg6MrWYoKsa9Bw9Wwh6FUCIegHp4eFcnjad/CIpCUotTSVpNApJobfv9d2iqgbZbr7QLMRBlgqK8sS0Nez9cEJh7uqtIbYtAyb+XgfDFWdi1AY8uzma8JPVQ9gj/46g1pKPgZL3VBQd1+bXWR1yGJifQ1EipGBC2orEaFUBIepARng4cTOsZ1s0P67fiHm249MjFU9zPnSuR7nEBs6vcaFyFc2LQS8x4fFl7G+v42Rz20Q9N9mVf/bW5/6vfyUk7ncWfXO6yudhN+Yb2fLfV/DILX1jVtFzqgSZHXII6mBZiFTTIEerBcP/T4QuVhFqvKhnRERw+fkXQC15EanoUL6qh/X3NcbY1N3ieDzevFzrJVr1HuMASwVlMbDFQJb2XsoZf4lkPTZHx7jlG5kX9Qktzhwk6kJK2R0qKZExkYRuDaXpX7at8B9vDt93hS/6QqvhcQRaEXQkOBuyjIEdxcJoVaFGL5RqikLCG4ttbv99V4nDbSS66Q0YknvSK286gfJZGpJGIvU4rLZl9cgAMaOpQAY0H8DyfivYHrOAiZ+noFL2zMX8NDb59GcsOWhgFrMIbOFVpb5Hc9FzzeaCgHDeT+KLe2XeSUzGNbv4eaNTHZyGvU+X9g+Vo6UCR1OjRT37yNFiPvTSuMMvk6laOk7pkOD8KRG5ARxU2wGmSjurRaWdSkFos1D6zu/Lb+3WUmvlRuTk4hkeiyIB9TPB6fw3jIs34G0cVmUqJymqwsIDCwsF/VQziRH7yxb3000plpsIClJCSxJOs/5E5+peQm9BZaVGu1/yk2wt9qrh5J7PQ+7pOAOyBI24yo4HJN59pDP/ndSDX2b3qxICUFPQyTq6jnqWDnt/pU7//jb388wCZ+99xKdnMPmzyr94qqgKC/cvJD3v+g/X6aYSBrfSo1wy3GCca0qx3ETm/eUXWz8pBL2KUqNF3amB7cVefbpkWNQUBXDJTeKhzo0JbuldpR7VaxKSTod7d9tzWKfVAVlTCXR7hYczl/HZB1+Ql3fNgRbePJExkYR9Fcb2v7ZbHNdkibWDTIO1pCiX8/fmMjC3eJSLhsTBRmNpOXZ5+RssuC3UaPeLe0A3nHx9yU9IuD5FKYqk4Rds2n1YlEStHnc62EbBreM55lES33671MVwDbhaB/TZsPp9paBoxBVgDdEh/4dx8jx6/2v0bbK4ZJScHBLfXkL8uWh+5w+S+wLOxSOtotrILBsG44sUwEjRg9Y9k7Ge1ss+RvX6iOD7RjrIesHtoMYn9MqIiOCfadPRKPrYYvrP0jgkFY+mloJuLt8V89hBgluLtKNVgYQlS0j5eF3hFvgbMd8AXwdJDDlkemWt4HXayHEEL5yFVEoqWkcSM3kyWT/+VMw2a3VbzUiqxl2xGg0MGk8Y0+haNwtrlRdVQPJojPT87yIctxIiEnrZwS++HXgjcBxXXetaHDe4uePXM5U6/sUFHWBlrYkEtrTdfSOoWHxmzcJr4pNIUnE3WY4zLH9Youfp4oJufi0Bnl9+wonArhz5/D2U21hBKf+akSN9g4sJupnuf8KLW/Ot9tVkidPNZDo0NhDoWYKga6ZdttLAN4WgVwNq9Ew9/OQVnvksGjD5Udsn/41XnoEUFz2n6t/BfbojLKi1ET/peuxynObNomuP87Ao31UlUY1G/rf4KQxHDpHrDHs7SJxqLnNXrMbCzWVHtptvlg9G1eGhiYvLvRJ8UfZ++jZuS9ejzyu5jdmmMTOLF+ZG06inqvx06TIlybXRqQ7Owz6AdkPKw2SBA7BH12qsqCuqRq+3fuBKenFf+Y3IqBax6Ee0tqx8tBuD7xZbpqsyy48sZ8OpDYVhgD1PqUz/2vYshlfrwJQpOpb3W1Guwq6oSmHF+egv3mfYhr8A29LlhneFdWE3uGEKbu13rIQtQsGPgSQjzY0DZ7dbN17gMOwR9Rq7UBp1IaVMQQdQkQtj0QHeH9OVwXeLGXpVZ0bADCbfPZl+W/uReS2T1Dq29zXHtA86rPLdmZcIGraaOt0Db9rXrikK2UeOcvzMD3ya+B37G5h2dq7+Sin8PFvwLbIZ1kdRmHM11bqgFywuSMFThaBXM2qsqCcayhb0G5EleO9RIejVCTdnN17r+Rov/PQCZ5qYikJ4G2wX0fE/aEAO/3z9JGoDL5KeepDa/fpyZ0w+f184ylV3jSut6uHp7o1PbR+6NuyKTtaZZuP/RJH3xf+o++tJnC5cgdxcvIDpwFg97OksWUSt2EK8FwXTb40paelMSsso0eWiSRJyyHMw4DX7PkRQ6amx7pcDf13l0Y8O2vy+74/pIlwu1ZTImEj+/fNsOpzJZeY21a7yhWZUTD8GBlfwuGG+kO4Gv7SHpLoS+R61adS8A7qDxxhwIK/EKAXze9ljh4bJp35N8mLB1XhGZCdhbeuEpkG+s55as/4UM/QqhHC/2EBgCy8a1XUlPj231GwZ9dxr8eawjmJRtBoT2iyUvmOjmPPt4yxXj/P8Dg1dKYPCWlikXHBcX+QBsG4O3H/E3CsTOFhmdhbze9mCud3pJvUwXHkUJbsFe+QjjKi1AlXDQthVQJIkaj38vhD0akyNDWnUyRILHjT5ykuaEU3v34qj8+4Tgl4D0Mk6lgzZzPuvHuXvf/VAKyU1lj0pbUuirHbm87Ys3Rq9rzGs52lCc68CMrvUQCZfe554vCzf06Mx0qiNIsqlmlNj3S9mwk9eYdE3py0WTRvVda0yyZwEjiFj3RtcevcTdHkVO+8x5+63ZoVRBsM9mfRqlFG4Ia5X3ruoBa1lVDb2z6eXr2KqxNUsRMShV1GE+8UOBnZoxH3tfIm6kEKiIZeGetcql3ZVUP54PPky7bu3Y9/GuRxMhjxJIstVY9wPt9eOL+6R6H9cs1g0NbhCdGeFfzVNKNxMJEvgx1UC5bOF0VoqMro77oWW3rfXaEGFUuNFHUyumGAx8AVFkDoO5d63HkT9+CVckz8lytmV3ENuuGRJSDe1nGo7GqZiLNuCJbaFmDZHeWZCWm0I8DDwYrr1dMINSSv8t1ftWgS28LLaTlB9cdizZUpKCmPHjsXDw4N69eoxceJEMjMzS+3Tp08fJEmy+HvmmWccZaJAUDayjj6TlnHZfyEjM3S07JxWIOeWXsvSCjrbi/l9Ntwn00BTeSgri071Munpm8ZG7Z8SBR0gkXqF//7PQx3EE2cNxGEz9bFjx3LlyhV2797NtWvXmDBhAk899RSbN28utd+kSZN49dVXC1+7u4uczoKKZ/hjz2I0TmL3zq10ZTaZ0a7k51z3TxvcQJ+DTZWWwHoETeE5F5WkXjk845FL19i8EmPNLfpocAVvotS2ADx9bwsRgltDcYionzlzhvDwcA4fPkxAQAAAq1atYvDgwSxduhQ/v5IHm7u7O76+vo4wSyC4JZyda3H/0Ec55mbkbr9p5CQ785vqyqe+ddjfXEf3P7ViqW6tUeKMXlap3y6T+u0yTbn77dsfx6Jrj+NZ25XXHuogNsnVYBwS/bJu3TpmzpxJaur1Qrb5+fm4urry5Zdf8vDDD1vt16dPH06dOoWmafj6+vLggw/yyiuvlDpbz8vLIy/verajjIwM/P39bY5+EQhuhmO7PsHvwEJ8SEEBol1dSNLpaHBN4c5YjUPJHtQ974xH8R36JOthYz+Jxuka/S7m00IzUq95NnV8jMUKsdiCJskcDliG0vYhschfTanw6Jf4+HgaNrTMM+7k5ISXlxfxpdQEHTNmDM2aNcPPz48TJ04we/Zszp07x7Zt20rss3jxYhYtWlRutgsEttAlbBxK/7Fc+vo1/E+sICA377o7xQcG+CSTfxccS6/NhWsuOOdINHEyormrGPw0ntIUurrlobuFh1JVA0kCafh6AjsMLYerElQH7BL1OXPm8NZbb5Xa5syZMzdtzFNPPVX4744dO9KoUSP69+/PX3/9RcuWLa32mTt3LjNmzCh8bZ6pCwSORufkRNNhi6BtN/hmGuSkWpx3kiGgXhYBZGGRxt3K7P1mMLg0pO7QZWIzkcACu0R95syZjB8/vtQ2d9xxB76+viQmJlocz8/PJyUlxS5/eVBQEADnz58vUdRdXFxwcXEpfG32JmVkWC/XJRCUO036wNPH4NeVcORjyE0rPJWi1aYeWYDlln2tYJZdGtYWU7Nx44Lfg9zZazi1WvQgQ9aBGOvVHrOe2eQt1xzA6dOnNUA7cuRI4bFdu3ZpkiRply9ftvl9fvnlFw3QfvvtN5v7xMbGmqPLxJ/4E3/ir1r9xcbGlqmBDksTMGjQIBISElizZk1hSGNAQEBhSOPly5fp378/GzduJDAwkL/++ovNmzczePBgvL29OXHiBC+88AJNmjRh7969Nn+uqqrExcWh1+utli6rDJhdRLGxsdV6MbcmXKe4xupDZb5OTdMwGAz4+fkhy6WvpjssTn3Tpk1MnTqV/v37I8syw4cPZ+XKlYXnr127xrlz58jOzgbA2dmZyMhIVqxYQVZWFv7+/gwfPpx58+bZ9bmyLNOkSZNyvRZH4eHhUekGjyOoCdcprrH6UFmvs27duja1c5ioe3l5lbrRqHnz5hb+IX9/f7tm5AKBQCAoTo1NvSsQCATVESHqFYCLiwsLFiywiNqpjtSE6xTXWH2oLtdZ7fKpCwQCQU1GzNQFAoGgGiFEXSAQCKoRQtQFAoGgGiFEXSAQCKoRQtQrEXl5eXTu3BlJkjh+/HhFm1NuXLx4kYkTJ9KiRQvc3Nxo2bIlCxYswGg0VrRpt8zq1atp3rw5rq6uBAUFERUVVdEmlRuLFy+me/fu6PV6GjZsyNChQzl37lxFm+VQ3nzzTSRJ4vnnn69oU24aIeqViJdeeqnUAiJVlbNnz6KqKmvXruXUqVO88847rFmzhpdffrmiTbsltmzZwowZM1iwYAHR0dF06tSJsLCwYsnsqip79+5lypQpHDx4sLCC2YABA8jKyqpo0xzC4cOHWbt2LXfffXdFm3Jr2JwpS+BQdu7cqbVt21Y7deqUBmjHjh2raJMcyttvv621aNGios24JQIDA7UpU6YUvlYURfPz89MWL15cgVY5jsTERA3Q9u7dW9GmlDsGg0Fr3bq1tnv3bq13797a9OnTK9qkm0bM1CsBCQkJTJo0iU8//bTG1GRNT0/Hy6vqVro3Go0cPXqU0NDQwmOyLBMaGsqBAwcq0DLHkV5Q8Loqf28lMWXKFO6//36L77Oq4rDcLwLb0DSN8ePH88wzzxAQEMDFixcr2iSHc/78eVatWsXSpUsr2pSbJjk5GUVR8PHxsTju4+PD2bNnK8gqx6GqKs8//zw9e/akQ4cOFW1OufL5558THR3N4cOHK9qUckHM1B3EnDlzkCSp1L+zZ8+yatUqDAYDc+fOrWiT7cbWa7yRy5cvM3DgQEaOHMmkSZMqyHKBvUyZMoWTJ0/y+eefV7Qp5UpsbCzTp09n06ZNuLq6VrQ55YJIE+AgkpKSuHr1aqlt7rjjDkaNGsU333xjkftdURR0Oh1jx47lk08+cbSpN42t1+js7AxAXFwcffr0oUePHmzYsKHMvNCVGaPRiLu7O1u3bmXo0KGFx8eNG0daWho7duyoOOPKmalTp7Jjxw5+/vlnWrRoUdHmlCvbt2/n4YcfRqfTFR5TFAVJkpBlmby8PItzVQEh6hXMpUuXLErvxcXFERYWxtatWwkKCqoyueHL4vLly/Tt25du3brx2WefVbkbxRpBQUEEBgayatUqwOSiaNq0KVOnTmXOnDkVbN2to2kazz33HP/73//46aefaN26dUWbVO4YDAZiYmIsjk2YMIG2bdsye/bsKulqEj71CqZp06YWr+vUqQNAy5Ytq5Wg9+nTh2bNmrF06VKSkpIKz9lTs7ayMWPGDMaNG0dAQACBgYGFBV4mTJhQ0aaVC1OmTGHz5s3s2LEDvV5PfHw8YCrW4ObmVsHWlQ96vb6YcNeuXRtvb+8qKeggRF1wG9i9ezfnz5/n/PnzxX6oqvKD4ujRo0lKSmL+/PnEx8fTuXNnwsPDiy2eVlU++OADAPr06WNxfP369WUWoBdUHML9IhAIBNWIqrtSJRAIBIJiCFEXCASCaoQQdYFAIKhGCFEXCASCaoQQdYFAIKhGCFEXCASCaoQQdYFAIKhGCFEXCASCaoQQdYFAIKhGCFEXCASCaoQQdYFAIKhGCFEXCASCasT/AyLLuy2UXJYwAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 400x100 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(4, 1))\n",
    "for (x, y), (w, z) in MetaLoader(num_tasks=2, samples_per_task=100):\n",
    "    plt.scatter(x.tolist(), y.tolist())\n",
    "    plt.scatter(w.tolist(), z.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.func import vmap, grad, functional_call\n",
    "import torch.nn.functional as F\n",
    "from torch import optim\n",
    "import torch\n",
    "\n",
    "\n",
    "class MAML():\n",
    "    def __init__(self, model, meta_lr, inner_lr, inner_step, ts_inner_step, iterations, path):\n",
    "        self.path = path # model checkpoint (for saving or load)\n",
    "\n",
    "        self.model = model\n",
    "        self.meta_lr = meta_lr   # beta learning rate in outer loop\n",
    "        self.inner_lr = inner_lr # alfa learning rate in inner loop\n",
    "        self.tr_inner_steps = inner_step # optimization step in inner loop during training\n",
    "        self.ts_inner_steps = ts_inner_step # inner optimization steps during evaluation\n",
    "        self.iterations = iterations  # epochs\n",
    "        self.meta_optimizer = optim.Adam(self.model.parameters(), lr=self.meta_lr)\n",
    "       \n",
    "        # Store performance history\n",
    "        self.best_loss = 100000  # trackng best loss for model checkpointing\n",
    "        self.tr_history = {\"tr_loss\" : [], \"tr_inner_loss\": []}\n",
    "        self.vl_history = {\"vl_loss\" : [], \"vl_inner_loss\": []}\n",
    "    \n",
    "    def _inner_loss(self, params, buffers, x, y):\n",
    "        \"\"\"functional call to compute loss on the support set\"\"\"\n",
    "        y_out = functional_call(self.model, (params, buffers), x)\n",
    "        return F.mse_loss(y_out, y)\n",
    "\n",
    "    def _inner_loop(self, x_support, y_support, x_query, y_query):\n",
    "        \"\"\"Single task inner loop training.\"\"\"\n",
    "        # needed for functional call\n",
    "        params = dict(self.model.named_parameters())\n",
    "        buffers = dict(self.model.named_buffers())\n",
    "\n",
    "        for _ in range(self.tr_inner_steps):  # inner loop optimization\n",
    "            grads = grad(self._inner_loss)(params, buffers, x_support, y_support)\n",
    "            # inner SGD step: params = params - alpha * grad(inner_loss, params)\n",
    "            params = {k: params[k] - g * self.inner_lr for k, g, in grads.items()}\n",
    "\n",
    "        # final losses on support and query set\n",
    "        supp_loss = self._inner_loss(params, buffers, x_support, y_support)\n",
    "        query_loss = self._inner_loss(params, buffers, x_query, y_query)\n",
    "        return supp_loss, query_loss\n",
    "\n",
    "    def train_episode(self, meta_loader):\n",
    "        self.model.train()\n",
    "        self.meta_optimizer.zero_grad()  # reset meta-gradients\n",
    "        num_tasks = len(meta_loader)\n",
    "        all_supp_loss, all_query_loss = 0, 0\n",
    "\n",
    "        # if you want, you can parallelize this loop with a vmap!\n",
    "        for (x_support, y_support), (x_query, y_query) in meta_loader:           \n",
    "            # inner loop for each task on support loss\n",
    "            res = self._inner_loop(x_support, y_support, x_query, y_query)\n",
    "            all_supp_loss += res[0]\n",
    "            all_query_loss += res[1]   \n",
    "\n",
    "        # outer loop on query loss\n",
    "        all_query_loss.sum().backward()  # meta-gradient of the meta (query) losses\n",
    "        self.meta_optimizer.step()\n",
    "\n",
    "        # track total loss and accuracy\n",
    "        all_query_loss = (all_query_loss.detach().sum() / num_tasks).item()\n",
    "        all_supp_loss = (all_supp_loss.detach().sum() / num_tasks).item()\n",
    "        return all_supp_loss, all_query_loss\n",
    "\n",
    "    def evaluation(self, meta_loader):\n",
    "        self.model.eval()\n",
    "        supp_losses, query_losses = [], []\n",
    "\n",
    "        for (x_support, y_support), (x_query, y_query) in meta_loader:\n",
    "            new_params = dict(self.model.named_parameters())\n",
    "            buffers = dict(self.model.named_buffers())\n",
    "\n",
    "            # inner loop optimization\n",
    "            for _ in range(self.ts_inner_steps):  \n",
    "                supp_logits = functional_call(self.model, (new_params, buffers), x_support)\n",
    "                supp_loss = F.mse_loss(supp_logits, y_support)\n",
    "                grads = torch.autograd.grad(supp_loss, new_params.values())\n",
    "                new_params = {k: new_params[k] - g * self.inner_lr for k, g, in zip(new_params, grads)}\n",
    "\n",
    "            # add support and query loss to history\n",
    "            supp_losses.append(supp_loss.detach())\n",
    "            qry_logits = functional_call(self.model, (new_params, buffers), x_query).detach()\n",
    "            qry_loss = F.mse_loss(qry_logits, y_query, reduction='none')\n",
    "            query_losses.append(qry_loss.detach())\n",
    "\n",
    "        # get mean inner/meta performance over metabatches\n",
    "        supp_losses = torch.mean(torch.stack(supp_losses)).item()\n",
    "        query_losses = torch.cat(query_losses).mean().item()\n",
    "        return supp_losses, query_losses\n",
    "\n",
    "    def fit_and_evaluate(self, train_loader, eval_loader):\n",
    "        for iteration in range(self.iterations):\n",
    "            tr_inner_loss, tr_loss = self.train_episode(train_loader)\n",
    "            vl_inner_loss, vl_loss = self.evaluation(eval_loader)\n",
    "\n",
    "            # add performance to history\n",
    "            self.tr_history[\"tr_loss\"].append(tr_loss)\n",
    "            self.vl_history[\"vl_loss\"].append(vl_loss)\n",
    "            self.tr_history[\"tr_inner_loss\"].append(tr_inner_loss)\n",
    "            self.vl_history[\"vl_inner_loss\"].append(vl_inner_loss)\n",
    "\n",
    "            # pretty print\n",
    "            if iteration % (self.iterations // 10) == 0 or iteration == self.iterations - 1:\n",
    "                print(\n",
    "                    f'[Epoch {iteration}] | ',\n",
    "                    f'[TR] Meta Loss: {tr_loss:.2f} -',\n",
    "                    f'Inner Loss: {tr_inner_loss:.2f} - |',\n",
    "                    f'[VL] Meta Loss: {vl_loss:.2f} -  ',\n",
    "                    f'Inner Loss: {vl_inner_loss:.2f} - |')\n",
    "\n",
    "                self.model.eval()\n",
    "                (x_support, y_support), _ = next(iter(eval_loader))\n",
    "                new_params = dict(self.model.named_parameters())\n",
    "                buffers = dict(self.model.named_buffers())\n",
    "                # inner loop optimization\n",
    "                for _ in range(self.ts_inner_steps):  \n",
    "                    supp_logits = functional_call(self.model, (new_params, buffers), x_support)\n",
    "                    supp_loss = F.mse_loss(supp_logits, y_support)\n",
    "                    grads = torch.autograd.grad(supp_loss, new_params.values())\n",
    "                    new_params = {k: new_params[k] - g * self.inner_lr for k, g, in zip(new_params, grads)}\n",
    "\n",
    "                y_out = functional_call(self.model, (new_params, buffers), x_support)\n",
    "                plt.figure(figsize=(4, 1))\n",
    "                plt.scatter(x_support.tolist(), y_support.tolist(), label=\"true\")\n",
    "                plt.scatter(x_support.tolist(), y_out.tolist(), label=\"pred\")\n",
    "                plt.legend()\n",
    "\n",
    "            # model checkpointing\n",
    "            if vl_loss <= self.best_loss:  \n",
    "                self.best_loss = vl_loss\n",
    "                torch.save({\n",
    "                    'epoch': iteration,\n",
    "                    'model_state_dict': self.model.state_dict(),\n",
    "                    'optimizer_state_dict': self.meta_optimizer.state_dict(),\n",
    "                    'loss': vl_loss,\n",
    "                }, self.path)\n",
    "\n",
    "        return self.tr_history, self.vl_history\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 0] |  [TR] Meta Loss: 5.55 - Inner Loss: 4.64 - | [VL] Meta Loss: 1.73 -   Inner Loss: 1.74 - |\n",
      "[Epoch 40] |  [TR] Meta Loss: 2.33 - Inner Loss: 1.49 - | [VL] Meta Loss: 3.38 -   Inner Loss: 2.68 - |\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[5], line 28\u001b[0m\n\u001b[0;32m     26\u001b[0m train_loader \u001b[38;5;241m=\u001b[39m MetaLoader(num_tasks\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m, samples_per_task\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m25\u001b[39m)\n\u001b[0;32m     27\u001b[0m test_loader \u001b[38;5;241m=\u001b[39m MetaLoader(num_tasks\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m, samples_per_task\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m25\u001b[39m)\n\u001b[1;32m---> 28\u001b[0m tr_history, vl_history \u001b[38;5;241m=\u001b[39m \u001b[43mmaml\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_and_evaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_loader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtest_loader\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[4], line 95\u001b[0m, in \u001b[0;36mMAML.fit_and_evaluate\u001b[1;34m(self, train_loader, eval_loader)\u001b[0m\n\u001b[0;32m     93\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit_and_evaluate\u001b[39m(\u001b[38;5;28mself\u001b[39m, train_loader, eval_loader):\n\u001b[0;32m     94\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m iteration \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miterations):\n\u001b[1;32m---> 95\u001b[0m         tr_inner_loss, tr_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_episode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_loader\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     96\u001b[0m         vl_inner_loss, vl_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mevaluation(eval_loader)\n\u001b[0;32m     98\u001b[0m         \u001b[38;5;66;03m# add performance to history\u001b[39;00m\n",
      "Cell \u001b[1;32mIn[4], line 54\u001b[0m, in \u001b[0;36mMAML.train_episode\u001b[1;34m(self, meta_loader)\u001b[0m\n\u001b[0;32m     51\u001b[0m \u001b[38;5;66;03m# if you want, you can parallelize this loop with a vmap!\u001b[39;00m\n\u001b[0;32m     52\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m (x_support, y_support), (x_query, y_query) \u001b[38;5;129;01min\u001b[39;00m meta_loader:           \n\u001b[0;32m     53\u001b[0m     \u001b[38;5;66;03m# inner loop for each task on support loss\u001b[39;00m\n\u001b[1;32m---> 54\u001b[0m     res \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inner_loop\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx_support\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_support\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mx_query\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_query\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     55\u001b[0m     all_supp_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m res[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m     56\u001b[0m     all_query_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m res[\u001b[38;5;241m1\u001b[39m]   \n",
      "Cell \u001b[1;32mIn[4], line 36\u001b[0m, in \u001b[0;36mMAML._inner_loop\u001b[1;34m(self, x_support, y_support, x_query, y_query)\u001b[0m\n\u001b[0;32m     33\u001b[0m buffers \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mnamed_buffers())\n\u001b[0;32m     35\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtr_inner_steps):  \u001b[38;5;66;03m# inner loop optimization\u001b[39;00m\n\u001b[1;32m---> 36\u001b[0m     grads \u001b[38;5;241m=\u001b[39m \u001b[43mgrad\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inner_loss\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mx_support\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_support\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     37\u001b[0m     \u001b[38;5;66;03m# inner SGD step: params = params - alpha * grad(inner_loss, params)\u001b[39;00m\n\u001b[0;32m     38\u001b[0m     params \u001b[38;5;241m=\u001b[39m {k: params[k] \u001b[38;5;241m-\u001b[39m g \u001b[38;5;241m*\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minner_lr \u001b[38;5;28;01mfor\u001b[39;00m k, g, \u001b[38;5;129;01min\u001b[39;00m grads\u001b[38;5;241m.\u001b[39mitems()}\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\_functorch\\apis.py:363\u001b[0m, in \u001b[0;36mgrad.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    361\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[0;32m    362\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m--> 363\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43meager_transforms\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgrad_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margnums\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhas_aux\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\_functorch\\eager_transforms.py:1295\u001b[0m, in \u001b[0;36mgrad_impl\u001b[1;34m(func, argnums, has_aux, args, kwargs)\u001b[0m\n\u001b[0;32m   1293\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgrad_impl\u001b[39m(func: Callable, argnums: argnums_t, has_aux: \u001b[38;5;28mbool\u001b[39m, args, kwargs):\n\u001b[0;32m   1294\u001b[0m     func \u001b[38;5;241m=\u001b[39m lazy_dynamo_disable(func)\n\u001b[1;32m-> 1295\u001b[0m     results \u001b[38;5;241m=\u001b[39m grad_and_value(func, argnums, has_aux\u001b[38;5;241m=\u001b[39mhas_aux)(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1296\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m has_aux:\n\u001b[0;32m   1297\u001b[0m         grad, (_, aux) \u001b[38;5;241m=\u001b[39m results\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\_functorch\\vmap.py:44\u001b[0m, in \u001b[0;36mdoesnt_support_saved_tensors_hooks.<locals>.fn\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     41\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(f)\n\u001b[0;32m     42\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfn\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m     43\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mgraph\u001b[38;5;241m.\u001b[39mdisable_saved_tensors_hooks(message):\n\u001b[1;32m---> 44\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m f(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\_functorch\\eager_transforms.py:1256\u001b[0m, in \u001b[0;36mgrad_and_value.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m   1253\u001b[0m diff_args \u001b[38;5;241m=\u001b[39m _slice_argnums(args, argnums, as_tuple\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m   1254\u001b[0m tree_map_(partial(_create_differentiable, level\u001b[38;5;241m=\u001b[39mlevel), diff_args)\n\u001b[1;32m-> 1256\u001b[0m output \u001b[38;5;241m=\u001b[39m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1257\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_aux:\n\u001b[0;32m   1258\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28misinstance\u001b[39m(output, \u001b[38;5;28mtuple\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(output) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m2\u001b[39m):\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\_dynamo\\eval_frame.py:489\u001b[0m, in \u001b[0;36m_TorchDynamoContext.__call__.<locals>._fn\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    487\u001b[0m     dynamo_config_ctx\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__enter__\u001b[39m()\n\u001b[0;32m    488\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 489\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m fn(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    490\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m    491\u001b[0m     set_eval_frame(prior)\n",
      "Cell \u001b[1;32mIn[4], line 26\u001b[0m, in \u001b[0;36mMAML._inner_loss\u001b[1;34m(self, params, buffers, x, y)\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_inner_loss\u001b[39m(\u001b[38;5;28mself\u001b[39m, params, buffers, x, y):\n\u001b[0;32m     25\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"functional call to compute loss on the support set\"\"\"\u001b[39;00m\n\u001b[1;32m---> 26\u001b[0m     y_out \u001b[38;5;241m=\u001b[39m \u001b[43mfunctional_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffers\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     27\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mmse_loss(y_out, y)\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\_functorch\\functional_call.py:143\u001b[0m, in \u001b[0;36mfunctional_call\u001b[1;34m(module, parameter_and_buffer_dicts, args, kwargs, tie_weights, strict)\u001b[0m\n\u001b[0;32m    137\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    138\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    139\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpected parameter_and_buffer_dicts to be a dict, or a list/tuple of dicts, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    140\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbut got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(parameter_and_buffer_dicts)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    141\u001b[0m     )\n\u001b[1;32m--> 143\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mnn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstateless\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_functional_call\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    144\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodule\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    145\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparameters_and_buffers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    146\u001b[0m \u001b[43m    \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    147\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    148\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtie_weights\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtie_weights\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    149\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstrict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstrict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    150\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\nn\\utils\\stateless.py:263\u001b[0m, in \u001b[0;36m_functional_call\u001b[1;34m(module, parameters_and_buffers, args, kwargs, tie_weights, strict)\u001b[0m\n\u001b[0;32m    259\u001b[0m     args \u001b[38;5;241m=\u001b[39m (args,)\n\u001b[0;32m    260\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _reparametrize_module(\n\u001b[0;32m    261\u001b[0m     module, parameters_and_buffers, tie_weights\u001b[38;5;241m=\u001b[39mtie_weights, strict\u001b[38;5;241m=\u001b[39mstrict\n\u001b[0;32m    262\u001b[0m ):\n\u001b[1;32m--> 263\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m module(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\nn\\modules\\module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\nn\\modules\\module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\nn\\modules\\container.py:217\u001b[0m, in \u001b[0;36mSequential.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    215\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m    216\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[1;32m--> 217\u001b[0m         \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m    218\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\nn\\modules\\module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\nn\\modules\\module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32m~\\mambaforge\\envs\\791aa\\lib\\site-packages\\torch\\nn\\modules\\linear.py:116\u001b[0m, in \u001b[0;36mLinear.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 116\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAB4CAYAAAANFGKiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgFElEQVR4nO2deVwUZ7b3f91NQ7M0jQurQUTkIyEMQ0BZzAImJjBjTJibO/Em5qK+kWgGDMRJ1GQkvOq8ohMTMZqgSSZqoo5OvEnIos5F4zJeURLRMYpg8BVBoEVNaJDdrrp/tN3S0Es1VPV6vp9Pf7S7T1Wdaqp+z/Oc59R5RCzLsiAIgiCcArGtHSAIgiD4g0SdIAjCiSBRJwiCcCJI1AmCIJwIEnWCIAgngkSdIAjCiSBRJwiCcCLcbO0A3zAMg6amJsjlcohEIlu7QxAEMWxYlkV7eztCQkIgFpvuizudqDc1NSE0NNTWbhAEQfBOQ0MD7rnnHpM2TifqcrkcgObkfX19zdqXVSmxel81rrX1DPou0NcDS38TBQBYtPtf4PLorXabx6KDLPKbIIREzbA4VfcLrt/qhr+PDAnjRkAippGso9DW1obQ0FCdvplC5GxlAtra2qBQKKBSqcyK+v5zzXhpe6VZsfbzkqK1s4/T8bW3Scnz8ciICea0DUEIhZphsfG7n7Dlf+rQ2nX3Gg5WyFA4I5quUQfBEl1z2YlSNcNi+ddVnHrfXAUdgG5/y7+ugppxqvaScDD2nm1C7PJ/YN2Bn/QEHQCUqm68tL0S+88183pMNcOi/NJNlJ5pRPmlm3QP2ACnC79wpeLyz2hWdQuybxZAs6obFZd/RkrEKEGOQRCmKNpbhc1HLxv9noVmVLn86yo8Fh3ESyhm/7lmLP+6Su++ohGB9XHZnnpLuzCCbu1jEMRA9p5tNinoWvp3PoaLNpQ5sKMk1IiAMI7L9tQDfNwR7COBkHNFAV5idHc7l7BLpVJIJBJbu0EYQc2wWFZ6zqJthtv5MBXKFGJEQJjG5USdZVkolUooelvx/6YFgGFgNq4uEQEMa95OiwiARCyC4nYrLl9uHZ7Ddoifnx+CgoLoOQA7pOLyz/i5o9eibQLksmEf01Qok8KR1sXlRF2pVKK1tRWBgQEYHSzlFFcP8dNc9C1tvbjNMJyOE+Ing1zmPixf7Q2WZdHZ2YmWlhYAQHAwxUntDUt73cEKGRLDR1rlmBSOtA4uJepqtRqtra0ICAjAqFGaHoOHTIam1m70qQeLtVQiRoifDApPjTiPVrDo6FGjj2FwW82iT82gtbNPT+gHbuNseHp6AgBaWloQEBBAoRg7w9Jed+GM6GGHRLgec7gjAoIbLiXqfX2atC4vLy/dZwpPd/jKpHpi7SYWQSoRw9tDohdiEIlE8JHp/2TBChk6etS4zTBwEw/exhnR/n59fX0k6nZGYvhIBCtkZkegCk83rHk6lpesFO0xlapugyFKEYAgHkYEBDcEzX4pKirC5MmTIZfLERAQgMzMTNTU1JjcZuvWrRCJRHovmYzfFn6g6GrFeoSXO/zlHhjh7Q4fmRsncdZu6+fFfRtHxxXO0VGRiEUonBENU3+hJ2KDUVnwOG9phtpjAhh0XO17PkYEBDcEFfUjR44gJycHJ06cQFlZGfr6+vD444+jo6PD5Ha+vr5obm7Wva5cuSKkmwThVGTEBKPk+XgEK/Q7Q6O83fH+c/HY+Fw87wKrPWbQgGMGKWT0dLWVETT8sn//fr33W7duRUBAAE6dOoWHH37Y6HYikQhBQVQ7hSCGSkZMMB6LDkLF5Z/R0t6NALkm/CFkb9kWxyQGY9WYukqlAgCMHGk6tnbr1i2EhYWBYRjEx8dj1apVuO+++wza9vT0oKfnbjGutrY2/hy2I9LS0hAXF4fi4mJbu0I4CBKxyOophLY4JqGP1Z4oZRgG+fn5eOCBBxATE2PUbuLEifj4449RWlqK7du3g2EYTJkyBVevXjVoX1RUBIVCoXtZq+yuvdW4YFkWt2/ftqkPBEHYAayVWLBgARsWFsY2NDRYtF1vby8bERHBLlu2zOD33d3drEql0r0aGhpYAKxKpRpk29XVxVZVVbFdXV1DOgct+35sYpNXHWDDlnyjeyWvOsDu+7FpWPs1xuzZs1lonuHQvbZs2cICYPfu3cvGx8ezUqmUPXToEDt79mz2qaee0ts+Ly+PTU1N1b1Xq9XsqlWr2HHjxrEymYyNjY1lP/vsM87+8PU7EgTBDZVKZVTXBmKV8Etubi6++eYbHD161GyB94FIpVLcf//9qK2tNfi9h4cHPDw8+HCTE8bK9WprXAgxKbR+/XpcvHgRMTExWLFiBQDg/PnzAIClS5di7dq1GD9+PEaMGMFpf0VFRdi+fTs2bdqEyMhIHD16FM8//zz8/f2RmprKq+8EQVgXQUWdZVksXLgQX3zxBQ4fPozw8HCL96FWq/Hjjz/it7/9rQAeWuiLjWpcKBQKuLu7w8vLSzeBXF1dDQBYsWIFHnvsMc776unpwapVq3DgwAGkpKQAAMaPH49jx45h8+bNJOoE4eAIKuo5OTnYuXMnSktLIZfLoVQqAWhESvtkYlZWFsaMGYOioiIAGpFKTk7GhAkT0NrairfeegtXrlzBvHnzhHSVE/ZY42LSpEkW2dfW1qKzs3NQQ9Db24v777+fT9cIC1EzLGWOEMNGUFEvKSkBoMnc6M+WLVswZ84cAEB9fb3eQqq//PILsrOzoVQqMWLECCQkJOD48eOIjo4W0lVO2GONC29vb733YrEY7IDFrLRP0gKazCIA+PbbbzFmzBg9O2uGsQh9qBY5wReCh1/McfjwYb3369atw7p16wTyaHjYssaFu7s71Gq1WTt/f3+cO6dfevXMmTOQSqUAgOjoaHh4eKC+vp5CLXaCLeZpCOfFZRfJGAraGhfGBsQi8FP1zhDjxo3DyZMnUVdXhxs3boAxUi3ykUcewQ8//IBPPvkEP/30EwoLC/VEXi6X49VXX8Urr7yCbdu24dKlS6isrMSGDRuwbds23v0mTGNungagpREJyyBRtwBb1rh49dVXIZFIEB0dDX9/f9TX1xu0S09PR0FBARYvXozJkyejvb0dWVlZejYrV65EQUEBioqKcO+99yIjIwPffvvtkCayieFhyTwNQXBBxHKJkTgQplbd7u7uxuXLlxEeHj6sImGuHv/k63ckgNIzjcjbdcas3fr/iMNTcWPM2hHOiSldG4hLld7lC6pxQfAF1SIn+IZEHQBYFui9Baj7AOY2IHYDJFLA3QcwUmaWalwQfEC1yAm+IVHvagVUVwGmb/B3YimguAfw9NO8NyT+YqnmzlP3mW0ICGIg2nmal7ZXQgT9dXCpFjkxFFxb1LtagV8uG/+e6bvz/Z0JRGPi35+BDQFBmEFbi3zgPE2QC83TEPzhuqLOshqR5kJrPcCazxEHoN8Q2EDYWZZFd4cKzO0+iN2kkHkraKUiB4DmaQi+cF1R771lvtethaug90d1FZAprBqK6VTdgLSjGZ64W4K3r80Nfd7B8FKMtpofxNCgeRqCD1xX1NUcBX2oMH2ahsNDLuxx7tCpugHPWw2aN/3aETf2NtxuNaATIGG3AVTPhbA2rivqEqnwxxC64bgDy7KQdjQDGDwwEIk0kSZpRzNY31HCh2IYNXDlOHDrGuATCIRNAcQS4zZeozVOdlw3bu+guPrzDIRtcF1Rd/fRTGpyDcEMBWs0HAC6O1SakIsRvRaJACluo6tDBU8fP+EcqfoK2L8EaGu6+5lvCJCxBoh+0rhNfwbaa+HSWNgRVM+FsBWuK+oikSZLxVT2i85WYnlcXXwnvdEKMLe5NUxc7caNG4f8/Hzk5+dzd6LqK+DvWcBAGWtr1nz+zCea94ZsjNmbagiMiT8XtA1Ee7NmhODtD8iDzY8qODYmtqq7TxCAFWq/vPfeexg3bhxkMhmSkpJQUVFh0v6zzz5DVFQUZDIZfvWrX2Hv3r3COefpB4wI1wiwIcRSzfd+Yy3ft+Ieq02Sit24jQi42lkMo9aIrqmyVPuXAvsWG7ExYs+o7zYWA3v2WvGv+soyX6u+AopjgG1PAJ9nA/94Q/Pvtic0n/ffX3/b/3rBsI0B+tdzEYNBsrgKT4qPI1lcBTEY4/VcGDVw+Z/Aj3s0/zJDmKAXcn+EQyBoT3337t1YtGgRNm3ahKSkJBQXFyM9PR01NTUICAgYZH/8+HE8++yzKCoqwhNPPIGdO3ciMzMTlZWVJherHhaefposFbNPlIbr56kzakB5Fuj8GfAaCQTFanpwNshTl3kr0NfmBjf2ts7d3t4+uLtrRJxlgdsiN8i8FcI4cOW48XCKxgOgrdGCHd6xrztmprEQacQ/ajq3UIyx0YSWtibzowpDI4kBaOvpp4srUCj9BCGiu+LdxI7E8r4s/INJ1K+7z/doxND+PHyBuFma38vOw1fE0BG0p/7OO+8gOzsbc+fORXR0NDZt2gQvLy98/PHHBu3Xr1+PjIwMvPbaa7j33nuxcuVKxMfHY+PGjUK6qRFuD7lGnH0CNP96yPV72p5+QOB9wKgJQMsFYNcs4JtXgO9Wav7dPQu4cVFjI4Cgp6WlITc3F7m5uVAoFBg9ejQKCgrAsixEIhEmJE/HyuIPkfVyAXwnPoQXF/8ZAPDPk6fx8L/9H/hGJGLs2LF4+eWX0dHRodtvS0sLZsyYAU9PT4SHh2PHjh2WO3frGl+nqc/lf3JrLK4cN78vk6OJAfs0OaoYMJIwQIBchnRxBUqkxQiCfm88CD+jRFqMdHHF3XouQoxGDO2vpw04WcJ5xDEkaHRgcwQT9d7eXpw6dQrTpk27ezCxGNOmTUN5ebnBbcrLy/XsAU0pWWP2gGbNzba2Nr2XYIhEwKVDwJd/AG4p9b9rvwZ8Ph+48LVgh9+2bRvc3NxQUVGB9evX45133sFHH32kcU0sxtsfbEdMdBRO/2MnCvLn4VJdA37zfC5mZD6Ns2fPYvfu3Th27Bhyc3N1+5wzZw4aGhpw6NAh7NmzB++//z5aWlosc8wnkM/TvAvX6BWXRsXsaKIfbY2aeLtRTDcmiWEKrHD/FAAwMGSufb/c/VMkhim4h664iiPXxks7KuFT2IcYriL4RTBRv3HjBtRqNQID9W/4wMBA3VqlA1EqlRbZA0BRUREUCoXuFRoaOnznjcH3DWghoaGhWLduHSZOnIhZs2Zh4cKFeqtEPfLIo3ht+VsIiX0YQdEpWLl5D56b9TwWv74MkZGRmDJlCt5991188skn6O7uxsWLF7Fv3z58+OGHSE5ORkJCAv7617+iq6vLMsfCpmhCBaaWD/Edo5mM5KTUd+zDHuR2fC6NihCjCSP7lDSUIxA3Bwm6FrEICMJNSBrKuYeuuIxGAMsaL+2ohI/rle/RBjFkHH6RjNdffx0qlUr3amhoEO5gfN+AFpKcnKyXZ56SkoKffvpJt8zdpEmTIBKJ4OnjB28/f5yvuoBt27bBx8dH90pPTwfDMLh8+TIuXLgANzc3JCQk6PYZFRUFPz8/yxwTSzSxXwBGlw/JWA385i9GbIzYhz/ErbEIm2LeRyFGE8b2ybUBuXXNMls+7bTwcb3auLMzZEyFiriEkew01CTYROno0aMhkUhw7Zr+RXbt2jUEBQUZ3CYoKMgie0CzWLLVFkwW6sbiiYGLUN+6dQvz58/Hyy+/PMh27NixuHjxIn8Hj35SM3locLJv9d3JPkM2/Rlon7HmzoSlkRqGGau5TfhpRxNtzTAbmvAdA7AM0K40YivS7MtYY8K1AbGkoRFin1qGe71a0tkJf2h4x+ILUxPTwNCet/ANAdKLAK9Rg1NgtQ3AlWOaSyr8IWDcg4JMVgsm6u7u7khISMDBgweRmZkJAGAYBgcPHtSL6fYnJSUFBw8e1MuPLisrQ0pKilBuWoaQNxYHTp48qff+xIkTiIyMhERi+MKIj49HVVUVJkyYYPD7qKgo3L59G6dOncLkyZMBADU1NWhtbR2ag9FPajIrTOV1D7Qx90Qp18bCHNrRxN+zzBiKNPsFht6YmG1ABjQKltiaI2yK5jftvMHNHhj+9WrnnZ1BmHym4j8Nb8PleYu2JuCz2fqf+YYAMf8OnP4U6Prl7uf/fAvwHAnMWD+07CYTCJrSuGjRIsyePRuTJk1CYmIiiouL0dHRgblz5wIAsrKyMGbMGBQVFQEA8vLykJqairfffhvTp0/Hrl278MMPP+CDDz4Q0k3uWHqz8kx9fT0WLVqE+fPn6xaLfvvtt43aL1myBMnJycjNzcW8efPg7e2NqqoqlJWVYePGjZg4cSIyMjIwf/58lJSUwM3NDfn5+fD09By6k2KJ+d4YF5v+cGksuO7H1EjBd4z5UQWXxkSvAeHQKPA1GtEee/rbg8XFGFzDV6awcWfHIriEigxyJ4V235I7fxaOq4C2NQHH3zX8XdfPmkbkmU95FXZBRX3mzJm4fv063nzzTSiVSsTFxWH//v26ydD6+nqIxXfD+lOmTMHOnTuxbNkyvPHGG4iMjMSXX34pXI66pVh6s/JMVlYWurq6kJiYCIlEgry8PLz44otG7WNjY3HkyBH86U9/wkMPPQSWZREREYGZM2fqbLZs2YJ58+YhNTUVgYGB+POf/4yCggJB/B8WljYExujfQJh7onQ4jYklIwy+RiNa7ssEGl82LiY6RPxcrzbu7FiERRPJA2GB9qFuawJLnrXgAC08PRQMxtPGDO0G5EhaWhri4uJQXFwsyP4tgRaetgBLygzwXd/m3JfA3kVA583B3/F9vepCGoDBzo6Jh7Wsyo97NOmW9sbsb0x2WmjhaaHhKxxAOD+WjDD4Go1oickEomdwr3MzHPgebQiFPYSADMHjfAOJ+lDh+wYkCCGw5nXqCJ0dS7KgBiHSNIgiDHF7E/DY2JCoOwiHDx+2tQsEYR577+zcmRdj/54FFvoP6jDQ6LUmYGRkzuw3d1IeDc6rDRE+Jqv74fAPHxEEQVjCfmYyXurNg5Idqfe5kh2Fl3rzcTrlXcB3QK1735C78wLaUNNAm6HCc3KFS/bUnWxu2Oq40u9Hy9E5F9pa981MIv67ZxISxdUIQCta4IcKJgosxPhXpQzHXvtRU8aB6/MWPoFAx03gv18fnEAR8/TgPHXAMfPU7Q2pVFOKtrOzc3i52C5OZ2cngLu/p7NCy9E5H/1r3TMQ4wQTPcimWdWNiisqpEQM4XkL7cT0wMZg2v91/CdK7RGJRAI/Pz9dFUIvLy/h1+x0IliWRWdnJ1paWuDn52f0SVZngJajc070atjzYDcIY3MKYgkQkaZ5CYxLiToAXR0Zi8vLEjr8/PxM1uNxdGg5OudFV8OeJzt7xOVEXSQSITg4GAEBAejrE3DRaSdFKpU6dQ8d0B+iG6L/cnQpEaOs5xgxbBLDRyJYIYNS1W3s2VcEKWRICBuB8ks3HXIuxeVEXYtEInF6cSKGhuBDdMJmSMQiFM6IxkvbK40lLeLJXwcj9a1DDjuXQimNBDEAVxiiuzIZMcEoeT4eQQr9v1+QQoYXHw7HB0cvDxqpaedS9p8ztSKWfeCyPXWCMAbXIXpi+EgD3xKOQEZMMB6LDtJLV00IG4HUtw45/FwK9dQJl0XNsCi/dBOlZxpRfukm1IzmdtYO0QGj6zihcEa0Xd/YhHkkYhFSIkbhqbgxSIkYhVNXfuE8l2LPCCLqdXV1eOGFFxAeHg5PT09ERESgsLAQvb29JrdLS0uDSCTSey1YsEAIFwkXZ/+5Zjy45js8++EJ5O06g2c/PIEH13ynG16bGqJTOqNz4ixzKYKEX6qrq8EwDDZv3owJEybg3LlzyM7ORkdHB9auXWty2+zsbKxYsUL33svLSwgXCRdm79km/GHn6UGfD8xBNzREd6QsCMIynGUuRRBRz8jIQEZGhu79+PHjUVNTg5KSErOi7uXl5dQ50IRt2Xu2Gbl/GyzogOG4qXaITjg/ls6l2GsJCatNlKpUKowcaX5iaceOHdi+fTuCgoIwY8YMFBQUmOyt9/T0oKenR/e+ra2NF3+JoWGvFzqgCbn8YWelSRvKQXdduKQ7audS7LmEhFVEvba2Fhs2bDDbS3/uuecQFhaGkJAQnD17FkuWLEFNTQ0+//xzo9sUFRVh+fLlfLtMDAF7vtC1T4lyxd7jpoQwaOdSBl7HQf2uY3svIWHRcnZLly7FmjVrTNpcuHABUVFRuveNjY1ITU1FWloaPvroI4uc++677/Doo4+itrYWERERBm0M9dRDQ0M5LftE8IexC13bw7H1hV5+6Sae/fAEZ/u/ZSdTT92FMTbiVDMsHlzzndEsGW2I5tiSR3gdoQq2nN0f//hHzJkzx6TN+PHjdf9vamrC1KlTMWXKFHzwwQeWHAoAkJSUBAAmRd3DwwMeHh4W75vgD0eolWJJzzuYctBdHmNzKVxLSKwrq8EDE/xtEn60SNT9/f3h7+/PybaxsRFTp05FQkICtmzZArHY8uzJM2fOAACCgyl9zJ5xhFoplmQsUA46YQyunYONhy5h46FLNgk/CpKn3tjYiLS0NIwdOxZr167F9evXoVQqoVQq9WyioqJQUVEBALh06RJWrlyJU6dOoa6uDl999RWysrLw8MMPIzY2Vgg3CZ6wNL/X2EM/QqLNbDAl1WIR8P5z99s8/k/YL5amM9qivIAgE6VlZWWora1FbW0t7rnnHr3vtCH8vr4+1NTU6BZccHd3x4EDB1BcXIyOjg6Ehobi6aefxrJly4RwkeARS/J7DU2mjvSW4ndxYzAtOkiw4aqpzAYtG5+Nx29jSdAJ45hLexxI//Cjt9QNJ+tuAtCEdpLHjxLkWrdootQRsGRCgeAH7eSRufzegun3ImfnaZM3g7HhKl+pkvacoUM4BtqkAGB4y077eUmx+t9+xem6s0TXSNQJXjB2oWtl973n4rHy2yqTsff+9v2zZfgWYnvOpSccA0PX5FDZxCEzjESdRN0mmBJfhac755TC/mlhZVVKu06VJFwXbefgf2pvYOOh2iHvJ5hDCqRgKY0EYQpTtVJKzzRy3o82W+bE/79p96mShOuiTXtMDB+J/6q8yjnOPhC+M8Oo9C7BKwPLmWrFdihFkMov3XSKUqiEc2OqVDNX+HyCmUSdsApcUgoHw63fQ4/0E7bGWKlmrvBZ+ZFEnbAK/Xsz5hBBE2dMGT+ak729l0IlXIOMmGAcW/II/padjPX/EYcd85IQKHc3ux3fTzA7XUxdO+9L1RrtjyljvbE2MxKr91XjWluPQRttDvmrUyMR7S+Fv4caLW09RlMlA3w9EDXKjf7ehN1wn78U9/lLAQCLHw3DK7v/ZdL+1amR6LjVbtJGe31zyWtxuuyXq1evIjQ01NZuEARB8E5DQ8OgBzoH4nSizjAMmpqaIJfLIRLZPiNCWzWyoaHB6VMsXelcAdc6X1c6V8D+zpdlWbS3tyMkJMRsHS2nC7+IxWKzLZkt8PX1tYuLwxq40rkCrnW+rnSugH2dr0Kh4GRHE6UEQRBOBIk6QRCEE0GiLjAeHh4oLCx0iYU8XOlcAdc6X1c6V8Cxz9fpJkoJgiBcGeqpEwRBOBEk6gRBEE4EiTpBEIQTQaJOEAThRJCoEwRBOBEk6jagp6cHcXFxEIlEOHPmjK3dEYS6ujq88MILCA8Ph6enJyIiIlBYWIje3l5bu8YL7733HsaNGweZTIakpCRUVFTY2iVBKCoqwuTJkyGXyxEQEIDMzEzU1NTY2i2rsHr1aohEIuTn59vaFYsgUbcBixcvRkhIiK3dEJTq6mowDIPNmzfj/PnzWLduHTZt2oQ33njD1q4Nm927d2PRokUoLCxEZWUlfv3rXyM9PR0tLS22do13jhw5gpycHJw4cQJlZWXo6+vD448/jo6ODlu7Jijff/89Nm/ejNjYWFu7YjksYVX27t3LRkVFsefPn2cBsKdPn7a1S1bjL3/5CxseHm5rN4ZNYmIim5OTo3uvVqvZkJAQtqioyIZeWYeWlhYWAHvkyBFbuyIY7e3tbGRkJFtWVsampqayeXl5tnbJIqinbkWuXbuG7OxsfPrpp/Dy8rK1O1ZHpVJh5Ej+FgOwBb29vTh16hSmTZum+0wsFmPatGkoLy+3oWfWQaVSAYDD/x1NkZOTg+nTp+v9jR0Jp6vSaK+wLIs5c+ZgwYIFmDRpEurq6mztklWpra3Fhg0bsHbtWlu7Mixu3LgBtVqNwMBAvc8DAwNRXV1tI6+sA8MwyM/PxwMPPICYmBhbuyMIu3btQmVlJb7//ntbuzJkqKc+TJYuXQqRSGTyVV1djQ0bNqC9vR2vv/66rV0eFlzPtz+NjY3IyMjA73//e2RnZ9vIc2K45OTk4Ny5c9i1a5etXRGEhoYG5OXlYceOHZDJHHeJRKr9MkyuX7+OmzdvmrQZP348nnnmGXz99dd6C3eo1WpIJBLMmjUL27ZtE9pVXuB6vu7umrUZm5qakJaWhuTkZGzdutVsgX97p7e3F15eXtizZw8yMzN1n8+ePRutra0oLS21nXMCkpubi9LSUhw9ehTh4eG2dkcQvvzyS/zud7+DRCLRfaZWqyESiSAWi9HT06P3nb1Com4l6uvr9dbRbGpqQnp6Ovbs2YOkpCS7XNhjuDQ2NmLq1KlISEjA9u3bHeKG4EJSUhISExOxYcMGAJqwxNixY5Gbm4ulS5fa2Dt+YVkWCxcuxBdffIHDhw8jMjLS1i4JRnt7O65cuaL32dy5cxEVFYUlS5Y4TMiJYupWYuzYsXrvfXx8AAARERFOK+hpaWkICwvD2rVrcf36dd13QUFBNvRs+CxatAizZ8/GpEmTkJiYiOLiYnR0dGDu3Lm2do13cnJysHPnTpSWlkIul0OpVALQrMLj6elpY+/4RS6XDxJub29vjBo1ymEEHSBRJwSirKwMtbW1qK2tHdRoOfrgcObMmbh+/TrefPNNKJVKxMXFYf/+/YMmT52BkpISAEBaWpre51u2bMGcOXOs7xBhFgq/EARBOBGOPWtFEARB6EGiThAE4USQqBMEQTgRJOoEQRBOBIk6QRCEE0GiThAE4USQqBMEQTgRJOoEQRBOBIk6QRCEE0GiThAE4USQqBMEQTgR/wtu4TA6tpljHAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 400x100 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAB8CAYAAABE1SrsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAfBElEQVR4nO2de1xU55nHf+cMdxlACAgYlNFSKaWiqCiNVuJqpGZNTbNN1saI1pCYQhSJGzWfKqLdYorrJTZeNrUaY7xkzTboVu2apMEQQbIiUSR4IaBEICCGq3KbOfvHOCNzPzOcM3Nm5vl+PvPROfOe932OnnnOM8/7XBiO4zgQBEEQkoN1tAAEQRCEcUhBEwRBSBRS0ARBEBKFFDRBEIREIQVNEAQhUUhBEwRBSBRS0ARBEBKFFDRBEIREIQVNEAQhUUhBEwRBSBRRFfT69evBMIzOKzY2VswlCYIgXAYPsRf48Y9/jI8//vjhgh78l1SpVKivr4dcLgfDMGKIRxAEYVc4jkNHRwciIyPBsuZtZNEVtIeHB8LDw206t76+HlFRUQJLRBAE4Xjq6urw6KOPmh0juoK+fv06IiMj4ePjg+TkZOTl5WHEiBFGx/b09KCnp0f7XlNor66uDgEBAYLLdqayEdlHv4J+OT+Nrb7luQTMirPt4UIQ7sCZykZsOlWF79offm+HBXhj9c9j6btjgvb2dkRFRUEul1scy4hZbvTUqVPo7OzEmDFj0NDQgNzcXNy+fRsVFRVGhVu/fj1yc3MNjre1tfFW0EoVh9Kau2jq6EaY3AdJimDIWEP3iFLFYeqbn6KhrdvoPAyA8EAfFK2aYfR8gnBG+H4/+HC6ogGvHCwzaeDsWpCI1PiIQcnrirS3tyMwMJCXXhNVQevT2tqKkSNHYsuWLViyZInB5/oWtOZJw1dBn65oQO6JSh2lGxHog5y5cQY3SnF1C+a/U2JxzsPpU5A8OsTiOCFvfIIQA2u+H5YgA8d2rFHQors4BhIUFIQf/vCHuHHjhtHPvb294e3tbdPcpp7mjW3deOVgmcHTvKnD+I2lD59xQt74BCEG1n4/LFFac9ekcgYADkBDWzdKa+7yMnAI49g1DrqzsxPV1dWIiBBWaSlVHHJPVBrcfAC0x3JPVEKpejgiTO7Da25L4zQ3vv7NqrnxT1c08FqHIMTClu+HJYQ0cAjTiKqgV65cicLCQtTW1uLcuXN4+umnIZPJMH/+fEHXseZpriFJEYyIQB+Y+vHFQG0FJymCTc4rxo1PEEJjy/fDEkIZOIR5RHVxfPvtt5g/fz5aWloQGhqKqVOnoqSkBKGhoYKuY8vTXMYyyJkbh1cOloEBdJSsRmnnzI0z6z/je+OXVLeAZRnyTxMOQQxrV2PgNLZ1GzVQND5oSwYO7duYR1QFfeTIETGn12Lr0zw1PgK7FiQa+I/DefqP+d7QGYfK0Hq/T/ue/NOEPRHD2h2sgUP7Nvyw6yahWAx8mjNQIYmtQhha0YQglKpiwYE1+TRPjY/ArLhwm57kfG/ogcoZsH1jhiBsQQhr1xi2GjhCb1i6Mi6hoDVP848O7cY6zwOIZB760uq5YGzoW4h5c5eaVLoylrFpp9nSjW8KDuovRe6JSsyKC6efdYRN8HURCOHOM4W1Bg6ffZvVH16G3McTU0aFuP13w65x0NZiTbwgKo+D+2AhOHA6O58qAAwYMM8eAOKeElxGjTUAwColrWHtkz/CoscUbn8jEtZx8lIDfldQgbtdvdpjllwEUnAr8M0/AFzX5SHZRBVr4X0hKiWwLR5orzcxgAECIoGsywArM37+zXNA53eA/zBg5E+NjzOBsRs/yNfTwLVhiiBfTyx+LBqZM2JIURMWyTtZiT1na4x+xsB8Bp+jN+YKym9j+ZFyXmNdNSPR/RR0zefAu/9secK0/wEU03SPVR4HTq/SVe4BkUDqm1ZZ3Po3vorj8Pyfz/M+HwCC/Dyx6Zc/cambkRCWk5fq8dtDF82OiZBwBp81FrSGkCFeKF7zT/DycI3y9dYoaNe44s7vbBtXeRz4YKGh5d3eoD5eeZy3CBo/9i/GDUfy6BBMGRViNs7aGK33+ii5hTCJUsXhdwUVFsdZG9NsTyzlHxijpasXU/I+dsvvhWsoaP9h1o9TKdWWs7ntitOr1eP4oFKqLfnLx4CazyGDCjlz4wDAqpuRAyW3EMYprbmLu1383GZSzeDTbFgC1n0v7na5p/HiGgp65E/VbglzeYEBw9XjNNw8Z8ZnDQAc0H5bPc4SlcfVPvB3/xn4cIn6z23xSGW/xK4FiYgM8MQUthJPsecwha0EC5XZ6aRsARGOwxqlK+UMPk14Xnig9TK6m/HiEmF2YGVqn/EHCwFTgUSpm3Q3/mx1i+ijcZPoW+IP3CSpP30Vs32Ogel9+DCo54bicP8M3OQitLHaKr1npVQtIMJx8FW6IUO8rI5ptjea8LyS6haDRC5TuGMBJtewoAH1ht6zB4AAvQ22gEj1cf0NP1vcIvpYdJNwwLm3wOhZ6hH4Hq95foi3vP6EI16/xwXvl/Gq7EMdy1rKFhDhGDT+W0ts/EW8JDcI9ZGxDB6LeQSbnvmJVe4OdzJeXEdBA2olnFWhjtZ4Zq/6z6zLxqMxbHGL6GPRTWJiZr0lhzJdeM3zQ/yf91KksqUWizQR7onGf2tOmb38MwXmjHWuKCCNyyN4iBev8e5kvLiWggbUbgzFNOAn/6L+01Q8s8YtAsBQSZtwi+jD103Ck6HoxE7PbdiZ+K1TWECE/dEoM31LOniIJ3b+ejzWzIlzkGSDIzU+AiVr/gnBQzxNjuFTYdLVcI046MFgNA56uFo5W4qD5ht/bQUcACZguOmkGoKA4xNOxMJUZq4rJa24X6LKYLE1k1CbwdgA2xK9zWAsqYYg3AAppKSLiWRbXomFUqlEXx+/+FCTREx6+PfePgA853viPx5sFAKCKumOu0C3uJshnp6ekMnISiekxWAqTLoaTq2gOY5DY2MjWltbHSeETAE8sR+43wqo+h8eZz0ATz+gp922eZkwoMZ4vQUhCQoKQnh4OBj9nUuCcCC2Vph0NZxaQWuUc1hYGPz8/ByrZDgO6LsHKPsAmadaOTMM0N0OdDQCXL/lOTQwHsAjMYbhHoKKy+HevXtoamoCAESEhgBfvgN8XwsMjQYmpQMe/HbVCYIQB6dV0EqlUqucQ0Ik8qT19TU85uMDBIYCvZ1q5a3sATqbAc5MCvnQEcbnEhjfB2s01VYibO8EyPq7Hn74v78DkjOBJzaKLgdBEMZxWgWt8Tn7+fk5WBIeMAzgLX/43j8c6Gw0VNSsJxD4KOAbZDfR/JQdQO899HkP1VXQnAo495b67wOV9CBLsxIEwR+nVdAanNJ3yjCAPEKtqDWWtcwT8PIX1a1hAKcC0333oUzGKH4bmLFW7e4QqDQr4WIMJgpq4HlRk4G68+r3Q0LVbsN7d2wzBFRKoLZIHQrLABg51XxeBN/r0XzW0QB0NavllEeIZqg4vYJ2avQta3vTdcfyGE6p9k0HRpmtOWI0nZ6QHkL/ArL1oW3sPIZV/3IzhjWGQOVxcCeWg7k/sOBYPuA7FJj7lvVyadYGDD+zRT4rcNo46O7ubtTU1EChUMDHx31SPwWlrQ7dbc2oud0MxRevwaezzvi4SS8CV0/a3rGGkAZC/wIyVShMk1Zi6qFt8jxzWJhzwNzcBy8MPEOLphconn3PSrn0C7CZkZGHoeJ+BfudjJSUFGRlZTlaDEDmzX+sUKVZCWFQKYHqz4BPfw988nvgm0LztcsFbE6hXd+WeupmzzMHjxrtKiXun1gJcMYr7GjV7KlVVsrFV1bOuhryPHB7Ba1UcSiubkFB+W0UV7dIotYsx3Ho77ciLM9WhjxieQwjA4ZPsjwOELw2CWGCyuNA/mjgvV8AZ/OBz/OBA08B+T8wrmiFbk4B2F5P3cYCY2bnfICy9gv43v/O7DYOAwAd9QLLNQCBDRW3VtCnKxow9c1PMf+dEiw/Uo7575Rg6pufitq1YdGiRSgsLMT27dvBMAwYhsH+/fvBMAxOnTqFCRMmwNvbG0VFRVi0aBHmzZunc35WVhZSUlK071UqFfLy8qBQKODr64uEhAQcO3aMnzAMC/haCFFMzgACh/Obj28JV1vR61ojpKUiJoIaAZXHgQ9eAO5/b/jZ/bvqz/SVtJDNKTTYWk9diIe4iTmqv6nmPYWqo5HXnDYh4Fxuu0moKcqi/1VpbOvGKwfLRCvKsn37dly7dg3x8fHYsGEDAODKlSsAgNWrV2Pz5s0YNWoUhg4dymu+vLw8HDx4ELt370ZMTAzOnj2LBQsWIDQ0FNOnT7c8gXwY4F0Pg2c1I1Mr5yc2qhVhQKSZmiMPfNDmSrMOFieNIDld0YCNxy8jqvMrhKEVTQhCnX8C1j5lQ3NglRI49TqPRVcDsU8+3A8QqjnFQGytpy7EQ9zEHE1cEH7Ic4qvO/zwYx5z2oSAc4muoN9++23k5+ejsbERCQkJ2LFjB5KSksRe1ixKFYfcE5Umf/AxULfWmRUXLnj+f2BgILy8vODn54fw8HAAQFVVFQBgw4YNmDVrFu+5enp68Ic//AEff/wxkpOTAQCjRo1CUVER9uzZw09BA+q466VFwOX3jGcS2tKxRkgsdK2RTASJXgjWV997oPKL8/gvj08R6fXQ4q3vCcaGQwuBXy+1Tklr5raExhrWFNsSojmFPpp66tY+tC2eZw7zhoAs+jE0FAUjHHdNujk4DmhAMG74/URXQQ9KrgFYqiFvJaK6OI4ePYrs7Gzk5OSgrKwMCQkJmD17tja92FGU1tzVqZSlz8DWOvZk4sSJVo2/ceMG7t27h1mzZsHf31/7OnDgAKqr+f/cA6BWxskZwJx89Z/6ad7WdqwRCjH8p2IwsC/lf6cDf38DCaWvI9vzQ0RA1x0RjrvY6bkNn330F+vcHdZYuAPHCtGcQh9b66mbPc8clg2BpNGh+A/ZbwCoFbE+HPegKXPfQoQFDLFCLr5yMoIbKqIq6C1btiA9PR2LFy9GXFwcdu/eDT8/P/zlL38Rc1mL8G2ZY+/WOkOG6N40LMtCPwpyYNW+zs5OAMDf/vY3lJeXa1+VlZX8/dDWYE3HGqEQw38qNKYiJB6gb81pfpQt69uL0upm/utYY+EOHCtEcwpj2PrQNnUeY0Yd8TAEZCyDGfNexNK+LHwPf4PPv4c/XunLwiX5z4wX/Td7Pe+pXwGRJuQbLoqhIpqLo7e3FxcuXMCaNWu0x1iWxcyZM1FcXGz0nJ6eHvT09Gjft7fbWAnOAnxb5ojVWsfLywtKpWWLLzQ0FBUVFTrHysvL4emp7joRFxcHb29v3Lp1i787Y7BoOtbYCzH8p0JiY9gYywCRaME3tV8AMU/zO2nkT9VZa5bcHMasYY3yMerH59GcwhRxT6n93dYmvxg7T4BMwjljI/DVt89h4tmJmMxWIpmtBDigmIvDeVUcOLDYNTfOtOvS0vVoPnP2TMI7d+5AqVRi2DDdp/6wYcO0Pld98vLykJubK5ZIWjTNNxvbuk15zxAuYmud6OhonD9/HrW1tfD394dKZTx7asaMGcjPz8eBAweQnJyMgwcPoqKiAuPHjwcAyOVyrFy5EitWrIBKpcLUqVPR1taGL774AgEBAUhLSxNFfrsihv9USAYZnhXGtPIfzMqAn/9RHalhDlPWsK3KlI9ctjy0jZ0nwMN/zZw4JDwahN8VeKO4K157nHfRf3PXY2cDRVJRHGvWrEF2drb2fXt7O6KiogRfR9N885WDZaa2vJBj7ik7SFauXIm0tDTExcXh/v372Ldvn9Fxs2fPxtq1a/H666+ju7sbv/nNb7Bw4UJcvnxZO2bjxo0IDQ1FXl4evvnmGwQFBSExMRFvvPGGKLLbHVs3o+zFIC330aNGW3dC3FPqn9onlhmG2vkGA3O3m7eG7f0LyEHMGRuJ2fERTl/0X7RU797eXvj5+eHYsWM6sbxpaWlobW1FQUGBxTnETvV29dY6fHCKlHltFAdg9HHqyCgOG/tSqjigxy8cvv9WaZsFq4kJv1mk/idRTAOip1KqvRMgiZZXXl5emDBhAj755BOtglapVPjkk0+QmZkp1rJWQa11nASx/KcCoIxKxh2EIJRrAd/bRgV1FUbfufm2K1RWBoxOUb8Il0VUF0d2djbS0tIwceJEJCUlYdu2bejq6sLixYvFXNYqqLWOkyCW/3SQlN5sw/7eF7DLcxtUHIwqaY7TjeRgAoaDcfCDhXAORFXQzz33HJqbm7Fu3To0NjZi3LhxOH36tMHGIUHwQoL+06aObvxdlYRX+rKQ43kAkTCMnW9AMA73zUCbbxRyfj0DsujHHP5gIZwD0TcJMzMzJePSIAih0YRi/l2VhDM9E5HEVmEY7iKEaUcLF4DvEIxSVSxUYLF7XiJko9xjb8OZUao4ybg9JRXFQRDOxsCQTRVYlKjiDMawDLBzvji1XQhhkVrggFtXsyOIwaIJ2QRMJwT/af54zBlLylnqaAqo6ZeB0BRQE7PKpSlIQRPEIEmNj8CuBYkID9QNU4wI9MHuBYmYM9ZEejAhGSwVUAPUBdTsXS+eXBwEAKC3X4X3P/8GN+/ew8hgP7yQHA0vD3p+84VCNp0bawqo2TPqixQ0gbb7fUh/6yy+bX9YH+TfT36N9GkKrJlj6FMljEMhm86LVAuokYnk4kRHR2Pbtm0mP2/q6EZHd79BeUYVB+w5W4O8k5XiCkgQEsDRBdRMQQrajVFxHFq7es2OeefzGvT2Gy/mRBCugiYax0zFbEQE+mDCyKF27WFKLg5NFwwJZafp09vbCy8vL8sDraSls9dikUwVB7xXXIsl00YJvj5BSAU+BdSeSojA9Px/2DUEz70t6IFdMD5cov5zW7z1LeitJCUlRZvAExgYiEceeQRr167VFuePjo7Gxo0bsXDhQgQEBOCll14CABQVFWHatGnw9fVFVFQUli1bhq6uLu28TU1NmDt3Lnx9faFQKPD++++blaNXyc8yvnn3no1XShDOg6lonPBAH7z0MwX+82yNwUZiQ1s3lh4sw8lL4oTgua+CNtUFQ9PnTmQl/e6778LDwwOlpaXYvn07tmzZgj//+c/azzdv3oyEhARcvHgRa9euRXV1NVJTU/HMM8/g0qVLOHr0KIqKinSyNBctWoS6ujr84x//wLFjx7Bz506z7cW8ZPz++0cG+9l+oQThRKTGR6Bo1QwcTp+C7f86DofTp6Dw3x7H8a8azP7azDxchpOXbK8Lbgr3dHFY7HPHGHZGFpioqChs3boVDMNgzJgxuHz5MrZu3Yr09HQA6mL9r732mnb8iy++iOeffx5ZWVkAgJiYGLz11luYPn06du3ahVu3buHUqVMoLS3FpEmTAAB79+7Fj370I5MyhPh7oaHFvJwsA7yQHD2oayUIZ0I/Gqe4usVsCB6gdgX+9tBF7GYZQd0d7mlBS6DP3ZQpU8AMKHGWnJyM69eva1th6TeQ/eqrr7B//36d5rCzZ8+GSqVCTU0Nvv76a3h4eGDChAnac2JjYxEUFGRSBpZhEDTEvG87fZqC4qEJt8aa0Dqhk1nc04KWep87GDaQ7ezsxMsvv4xly5YZjB0xYgSuXbtm0zphch/U+3gYbWxqLA5aSoVkCMIeWBNaJ3Qyi3sqaAn0uTt//rzO+5KSEsTExEAmM+5SSUxMRGVlJX7wgx8Y/Tw2Nhb9/f24cOGC1sVx9epVtLa2WpQl0NcTJ5f9DP91sdFsJqHUCskQhD3QhOBZcnNoEDKZxT1/u2r63JmLejTWGVlAbt26hezsbFy9ehWHDx/Gjh07sHz5cpPjV61ahXPnziEzMxPl5eW4fv06CgoKtJuEY8aMQWpqKl5++WWcP38eFy5cwIsvvghfX19e8nh5sFgybRQ2/CIeS6aNMqqcpVZIhiDswcCCWHwQMpnFPRU0KwNS33zwRl9JP3hvqjOyQCxcuBD3799HUlISMjIysHz5cm04nTHGjh2LwsJCXLt2DdOmTcP48eOxbt06REY+LMSzb98+REZGYvr06fjlL3+Jl156CWFhYYOWVaqFZAjCXqTGR2DnrxPNtjXTJLMkKYIFW1e0prFCIHbTWFQeN9Lnbrjofe5SUlIwbtw4synY9oLPv2NxdQvmv1Nica7D6VOoFgXh0py8VI/fHrpocFyjt3ctsFz3WxJNY50Cifa5kxpSLSRDEPZmzthI7GYZg72YcJH2YtxbQQOS7HMnNaRaSIYgHIE9S8uSgnYAn332maNFsIqBbZ2M+cMYqC0IIX1vBCFl7FVa1j03CQmrMNfWSfM+Z24cxUMThMCQgiZ4Ya6QDJ+NEYIgrMfpXRwqFdUqHgzW/PtRWyeCsC9Oq6C9vLzAsizq6+sRGhoKLy8vndoWhHk4jkNvby+am5vBsizvetOObOtEaeaEu+G0CpplWSgUCjQ0NKC+Xvgyf+6Cn58fRowYAZaVtreL0swJd8RpE1U0cByH/v5+bRU4gj8ymQweHh6S/+WhSTPXv1GtSQ4gCKkgiUSV6Oho3Lx5U+dYXl4eVq9eLeg6DMPA09MTnp6egs5LSANLaeYM1Gnms+LCyd1BuByiujg2bNigLUAPAHK5XMzlCBektOau2SpiHIQv8UgQUkFUBS2XyxEeHi7mEoSLQ2nmhDsj6s7Qpk2bEBISgvHjxyM/Px/9/f1mx/f09KC9vV3nRbg3lGZOuDOiWdDLli1DYmIigoODce7cOaxZswYNDQ3YsmWLyXPy8vKQm5trcJwUtfsSG+KBUG8lmtp7TKaZhwV4IzbEg+4TwinQ3Ke84jM4K1i1ahUHtdvP5Ovrr782eu7evXs5Dw8Prru72+T83d3dXFtbm/ZVWVlpcT160Yte9HLGV11dnUWda1WYXXNzM1pazLeBHjVqlNGkhytXriA+Ph5VVVUYM2YMr/VUKhXq6+shl8slFwrW3t6OqKgo1NXVWQyVcTXo2una3e3aAeGun+M4dHR0IDIy0mL+gVUujtDQUISGhtokVHl5OViWtarDB8uyePTRR21az14EBAS45c0K0LXTtbsnQlx/YGAgr3Gi+KCLi4tx/vx5PP7445DL5SguLsaKFSuwYMECDB06VIwlCYIgXA5RFLS3tzeOHDmC9evXo6enBwqFAitWrEB2drYYyxEEQbgkoijoxMRElJRY7mHnzHh7eyMnJwfe3t6OFsXu0LXTtbsjjrh+SdfiIAiCcGekXcKMIAjCjSEFTRAEIVFIQRMEQUgUUtAEQRAShRS0gPT09GDcuHFgGAbl5eWOFkd0amtrsWTJEigUCvj6+mL06NHIyclBb2+vo0UTjbfffhvR0dHw8fHB5MmTUVpa6miRRCcvLw+TJk2CXC5HWFgY5s2bh6tXrzpaLIewadMmMAyDrKwsu6xHClpAXn/9dURGRjpaDLtRVVUFlUqFPXv24MqVK9i6dSt2796NN954w9GiicLRo0eRnZ2NnJwclJWVISEhAbNnz0ZTU5OjRROVwsJCZGRkoKSkBGfOnEFfXx+eeOIJdHV1OVo0u/Lll19iz549GDt2rP0WtaZYEmGakydPcrGxsdyVK1c4ANzFixcdLZJD+OMf/8gpFApHiyEKSUlJXEZGhva9UqnkIiMjuby8PAdKZX+ampo4AFxhYaGjRbEbHR0dXExMDHfmzBlu+vTp3PLly+2yLlnQAvDdd98hPT0d7733Hvz8/BwtjkNpa2tDcHCwo8UQnN7eXly4cAEzZ87UHmNZFjNnzkRxcbEDJbM/bW1tAOCS/8+myMjIwJNPPqnz/28PnLart1TgOA6LFi3C0qVLMXHiRNTW1jpaJIdx48YN7NixA5s3b3a0KIJz584dKJVKDBs2TOf4sGHDUFVV5SCp7I9KpUJWVhYee+wxxMfHO1ocu3DkyBGUlZXhyy+/tPvaZEGbYPXq1WAYxuyrqqoKO3bsQEdHB9asWeNokQWD77UP5Pbt20hNTcWvfvUrnT6UhGuRkZGBiooKHDlyxNGi2IW6ujosX74c77//Pnx87N+1h1K9TcC39vWzzz6LEydO6NSrViqVkMlkeP755/Huu++KLargWFv3u76+HikpKZgyZQr2799vscatM9Lb2ws/Pz8cO3YM8+bN0x5PS0tDa2srCgoKHCecncjMzERBQQHOnj0LhULhaHHswkcffYSnn34aMplMe0ypVIJhGLAsi56eHp3PhIYU9CC5deuWTqul+vp6zJ49G8eOHcPkyZMlX896sNy+fRuPP/44JkyYgIMHD4p6szqayZMnIykpCTt27ACg/rk/YsQIZGZmYvXq1Q6WTjw4jsOrr76Kv/71r/jss88QExPjaJHsRkdHB27evKlzbPHixYiNjcWqVatEd/OQD3qQjBgxQue9v78/AGD06NFuoZxTUlIwcuRIbN68Gc3NzdrPXLGbe3Z2NtLS0jBx4kQkJSVh27Zt6OrqwuLFix0tmqhkZGTg0KFDKCgogFwuR2NjIwB10XlfX18HSycucrncQAkPGTIEISEhdvHBk4ImbObMmTO4ceMGbty4YfAwcsUfZs899xyam5uxbt06NDY2Yty4cTh9+rTBxqGrsWvXLgBASkqKzvF9+/Zh0aJF9hfIjSAXB0EQhERxvd0cgiAIF4EUNEEQhEQhBU0QBCFRSEETBEFIFFLQBEEQEoUUNEEQhEQhBU0QBCFRSEETBEFIFFLQBEEQEoUUNEEQhEQhBU0QBCFRSEETBEFIlP8HSNP/sgNj0JAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 400x100 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from types import SimpleNamespace\n",
    "data_root = './logs'\n",
    "exp_root = './logs'\n",
    "os.makedirs(exp_root, exist_ok=True)\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(1, 40),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(40, 40),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(40, 40),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(40, 1)\n",
    ").to('cpu')\n",
    "\n",
    "maml = MAML(\n",
    "    model=model,\n",
    "    meta_lr=1e-3, \n",
    "    inner_lr=0.01, \n",
    "    inner_step=10, \n",
    "    ts_inner_step=10, \n",
    "    iterations=400, \n",
    "    path=data_root + \"/model.pth\"\n",
    ")\n",
    "\n",
    "train_loader = MetaLoader(num_tasks=10, samples_per_task=25)\n",
    "test_loader = MetaLoader(num_tasks=10, samples_per_task=25)\n",
    "tr_history, vl_history = maml.fit_and_evaluate(train_loader, test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computational Graph\n",
    "\n",
    "It's big and messy, but you can see that there is the whole computational graph in `all_query_loss`. Going from the leaf () towards the roots:\n",
    "- We have two MSELossBackward: these are the losses computed on the finetuned model on the query sets\n",
    "- The SGD updates for each task (SubBackward0): We have two because each task trained a different model\n",
    "- The MSELossBackwardBackward: these are the inner loss computed on the support set for each task\n",
    "\n",
    "Try to experiment with it!\n",
    "- if you change the the number of SGD steps in the inner loop (`inner_steps`) you see that the computational graph grows deeper\n",
    "    - each SGD step adds operations to the computational graph\n",
    "- if you add more tasks (`num_tasks` in the `MetaLoader`) the graph grows \"wider\": each task trains a separate model, starting from the same init (the blue rectangles are the model's initialization parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 10.0.1 (20240210.2158)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"213pt\" height=\"1771pt\"\n",
       " viewBox=\"0.00 0.00 213.04 1771.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(0.595094 0.595094) rotate(0) translate(4 2972)\">\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-2972 354,-2972 354,4 -4,4\"/>\n",
       "<!-- 1419523198224 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>1419523198224</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"195,-32 141,-32 141,0 195,0 195,-32\"/>\n",
       "<text text-anchor=\"middle\" x=\"168\" y=\"-6.5\" font-family=\"monospace\" font-size=\"10.00\"> ()</text>\n",
       "</g>\n",
       "<!-- 1420016004320 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1420016004320</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"212,-88 124,-88 124,-68 212,-68 212,-88\"/>\n",
       "<text text-anchor=\"middle\" x=\"168\" y=\"-74.5\" font-family=\"monospace\" font-size=\"10.00\">SumBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016004320&#45;&gt;1419523198224 -->\n",
       "<g id=\"edge89\" class=\"edge\">\n",
       "<title>1420016004320&#45;&gt;1419523198224</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M168,-67.62C168,-61.1 168,-52.05 168,-43.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"171.5,-43.65 168,-33.65 164.5,-43.65 171.5,-43.65\"/>\n",
       "</g>\n",
       "<!-- 1420016006480 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1420016006480</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"212,-144 124,-144 124,-124 212,-124 212,-144\"/>\n",
       "<text text-anchor=\"middle\" x=\"168\" y=\"-130.5\" font-family=\"monospace\" font-size=\"10.00\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016006480&#45;&gt;1420016004320 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>1420016006480&#45;&gt;1420016004320</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M168,-123.59C168,-117.01 168,-107.96 168,-99.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"171.5,-99.81 168,-89.81 164.5,-99.81 171.5,-99.81\"/>\n",
       "</g>\n",
       "<!-- 1420016006432 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>1420016006432</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"224,-200 112,-200 112,-180 224,-180 224,-200\"/>\n",
       "<text text-anchor=\"middle\" x=\"168\" y=\"-186.5\" font-family=\"monospace\" font-size=\"10.00\">MseLossBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016006432&#45;&gt;1420016006480 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1420016006432&#45;&gt;1420016006480</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M168,-179.59C168,-173.01 168,-163.96 168,-155.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"171.5,-155.81 168,-145.81 164.5,-155.81 171.5,-155.81\"/>\n",
       "</g>\n",
       "<!-- 1420013803488 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>1420013803488</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"218,-256 118,-256 118,-236 218,-236 218,-256\"/>\n",
       "<text text-anchor=\"middle\" x=\"168\" y=\"-242.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420013803488&#45;&gt;1420016006432 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1420013803488&#45;&gt;1420016006432</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M168,-235.59C168,-229.01 168,-219.96 168,-211.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"171.5,-211.81 168,-201.81 164.5,-211.81 171.5,-211.81\"/>\n",
       "</g>\n",
       "<!-- 1420013972128 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>1420013972128</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"264,-480 176,-480 176,-460 264,-460 264,-480\"/>\n",
       "<text text-anchor=\"middle\" x=\"220\" y=\"-466.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420013972128&#45;&gt;1420013803488 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1420013972128&#45;&gt;1420013803488</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M218.31,-459.72C213.45,-433.3 198.67,-355.54 182,-292 179.85,-283.79 177.14,-274.85 174.68,-267.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"178.03,-266.1 171.61,-257.68 171.38,-268.27 178.03,-266.1\"/>\n",
       "</g>\n",
       "<!-- 1420016189696 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>1420016189696</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"321,-928 233,-928 233,-908 321,-908 321,-928\"/>\n",
       "<text text-anchor=\"middle\" x=\"277\" y=\"-914.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016189696&#45;&gt;1420013972128 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>1420016189696&#45;&gt;1420013972128</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M279.58,-907.67C284.56,-888.81 295,-844.73 295,-807 295,-807 295,-807 295,-581 295,-550.5 290.14,-541.23 273,-516 265.39,-504.8 254.28,-494.78 244.19,-487.05\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"246.36,-484.3 236.21,-481.27 242.25,-489.97 246.36,-484.3\"/>\n",
       "</g>\n",
       "<!-- 1420016192432 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>1420016192432</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"220,-760 120,-760 120,-740 220,-740 220,-760\"/>\n",
       "<text text-anchor=\"middle\" x=\"170\" y=\"-746.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016189696&#45;&gt;1420016192432 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>1420016189696&#45;&gt;1420016192432</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M270.94,-907.6C253.87,-881.12 205.15,-805.54 182.22,-769.96\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"185.17,-768.08 176.81,-761.57 179.29,-771.87 185.17,-768.08\"/>\n",
       "</g>\n",
       "<!-- 1420016190032 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>1420016190032</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"330,-1432 242,-1432 242,-1412 330,-1412 330,-1432\"/>\n",
       "<text text-anchor=\"middle\" x=\"286\" y=\"-1418.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190032&#45;&gt;1420016189696 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>1420016190032&#45;&gt;1420016189696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M288.58,-1411.67C293.56,-1392.81 304,-1348.73 304,-1311 304,-1311 304,-1311 304,-1029 304,-996.65 292.85,-960.44 284.88,-938.7\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"288.25,-937.73 281.41,-929.64 281.72,-940.23 288.25,-937.73\"/>\n",
       "</g>\n",
       "<!-- 1420016191952 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>1420016191952</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"229,-1264 129,-1264 129,-1244 229,-1244 229,-1264\"/>\n",
       "<text text-anchor=\"middle\" x=\"179\" y=\"-1250.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190032&#45;&gt;1420016191952 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>1420016190032&#45;&gt;1420016191952</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M279.94,-1411.6C262.87,-1385.12 214.15,-1309.54 191.22,-1273.96\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"194.17,-1272.08 185.81,-1265.57 188.29,-1275.87 194.17,-1272.08\"/>\n",
       "</g>\n",
       "<!-- 1420016190176 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>1420016190176</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"339,-1936 251,-1936 251,-1916 339,-1916 339,-1936\"/>\n",
       "<text text-anchor=\"middle\" x=\"295\" y=\"-1922.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190176&#45;&gt;1420016190032 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>1420016190176&#45;&gt;1420016190032</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M297.58,-1915.67C302.56,-1896.81 313,-1852.73 313,-1815 313,-1815 313,-1815 313,-1533 313,-1500.65 301.85,-1464.44 293.88,-1442.7\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"297.25,-1441.73 290.41,-1433.64 290.72,-1444.23 297.25,-1441.73\"/>\n",
       "</g>\n",
       "<!-- 1420016191472 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>1420016191472</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"238,-1768 138,-1768 138,-1748 238,-1748 238,-1768\"/>\n",
       "<text text-anchor=\"middle\" x=\"188\" y=\"-1754.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190176&#45;&gt;1420016191472 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>1420016190176&#45;&gt;1420016191472</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M288.94,-1915.6C271.87,-1889.12 223.15,-1813.54 200.22,-1777.96\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"203.17,-1776.08 194.81,-1769.57 197.29,-1779.87 203.17,-1776.08\"/>\n",
       "</g>\n",
       "<!-- 1420016190320 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>1420016190320</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"341,-2440 253,-2440 253,-2420 341,-2420 341,-2440\"/>\n",
       "<text text-anchor=\"middle\" x=\"297\" y=\"-2426.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190320&#45;&gt;1420016190176 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>1420016190320&#45;&gt;1420016190176</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M300.58,-2419.74C307.49,-2401.02 322,-2357.19 322,-2319 322,-2319 322,-2319 322,-2037 322,-2004.65 310.85,-1968.44 302.88,-1946.7\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"306.25,-1945.73 299.41,-1937.64 299.72,-1948.23 306.25,-1945.73\"/>\n",
       "</g>\n",
       "<!-- 1420016191136 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>1420016191136</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"251,-2272 151,-2272 151,-2252 251,-2252 251,-2272\"/>\n",
       "<text text-anchor=\"middle\" x=\"201\" y=\"-2258.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190320&#45;&gt;1420016191136 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>1420016190320&#45;&gt;1420016191136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M293.78,-2419.72C286.28,-2398.79 266.37,-2346.72 241,-2308 234.59,-2298.22 226.14,-2288.39 218.66,-2280.43\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"221.32,-2278.14 211.84,-2273.41 216.3,-2283.02 221.32,-2278.14\"/>\n",
       "</g>\n",
       "<!-- 1420016190464 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>1420016190464</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"350,-2832 250,-2832 250,-2812 350,-2812 350,-2832\"/>\n",
       "<text text-anchor=\"middle\" x=\"300\" y=\"-2818.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 1420016190464&#45;&gt;1420016190320 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>1420016190464&#45;&gt;1420016190320</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M299.71,-2811.58C299.16,-2792.57 298,-2748.25 298,-2711 298,-2711 298,-2711 298,-2541 298,-2509.87 297.59,-2473.78 297.3,-2451.68\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"300.8,-2451.88 297.16,-2441.93 293.8,-2451.98 300.8,-2451.88\"/>\n",
       "</g>\n",
       "<!-- 1420016190896 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>1420016190896</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"194,-2776 94,-2776 94,-2756 194,-2756 194,-2776\"/>\n",
       "<text text-anchor=\"middle\" x=\"144\" y=\"-2762.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190464&#45;&gt;1420016190896 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>1420016190464&#45;&gt;1420016190896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M272.83,-2811.59C247.7,-2802.9 210.12,-2789.89 182.02,-2780.16\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"183.41,-2776.94 172.82,-2776.97 181.12,-2783.55 183.41,-2776.94\"/>\n",
       "</g>\n",
       "<!-- 1420016090016 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>1420016090016</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"327,-2900 273,-2900 273,-2868 327,-2868 327,-2900\"/>\n",
       "<text text-anchor=\"middle\" x=\"300\" y=\"-2886.5\" font-family=\"monospace\" font-size=\"10.00\">0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"300\" y=\"-2874.5\" font-family=\"monospace\" font-size=\"10.00\"> (1)</text>\n",
       "</g>\n",
       "<!-- 1420016090016&#45;&gt;1420016190464 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>1420016090016&#45;&gt;1420016190464</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M300,-2867.55C300,-2860.34 300,-2851.66 300,-2843.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"303.5,-2843.92 300,-2833.92 296.5,-2843.92 303.5,-2843.92\"/>\n",
       "</g>\n",
       "<!-- 1420016190416 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>1420016190416</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"193,-2552 105,-2552 105,-2532 193,-2532 193,-2552\"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-2538.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190416&#45;&gt;1420016190320 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>1420016190416&#45;&gt;1420016190320</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M161.62,-2531.62C186.85,-2512.87 243.79,-2470.55 275.19,-2447.21\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"277.26,-2450.03 283.2,-2441.26 273.08,-2444.41 277.26,-2450.03\"/>\n",
       "</g>\n",
       "<!-- 1420016190512 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>1420016190512</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"194,-2608 100,-2608 100,-2588 194,-2588 194,-2608\"/>\n",
       "<text text-anchor=\"middle\" x=\"147\" y=\"-2594.5\" font-family=\"monospace\" font-size=\"10.00\">ViewBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190512&#45;&gt;1420016190416 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>1420016190512&#45;&gt;1420016190416</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M147.35,-2587.59C147.59,-2581.01 147.93,-2571.96 148.23,-2563.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"151.73,-2563.93 148.6,-2553.81 144.73,-2563.67 151.73,-2563.93\"/>\n",
       "</g>\n",
       "<!-- 1420016190704 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>1420016190704</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"190,-2664 102,-2664 102,-2644 190,-2644 190,-2664\"/>\n",
       "<text text-anchor=\"middle\" x=\"146\" y=\"-2650.5\" font-family=\"monospace\" font-size=\"10.00\">SumBackward1</text>\n",
       "</g>\n",
       "<!-- 1420016190704&#45;&gt;1420016190512 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>1420016190704&#45;&gt;1420016190512</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M146.17,-2643.59C146.3,-2637.01 146.46,-2627.96 146.62,-2619.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"150.11,-2619.87 146.8,-2609.81 143.12,-2619.75 150.11,-2619.87\"/>\n",
       "</g>\n",
       "<!-- 1420016190800 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>1420016190800</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"194,-2720 34,-2720 34,-2700 194,-2700 194,-2720\"/>\n",
       "<text text-anchor=\"middle\" x=\"114\" y=\"-2706.5\" font-family=\"monospace\" font-size=\"10.00\">MseLossBackwardBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190800&#45;&gt;1420016190704 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>1420016190800&#45;&gt;1420016190704</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M119.57,-2699.59C123.75,-2692.55 129.6,-2682.67 134.74,-2674\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"137.63,-2675.99 139.72,-2665.6 131.61,-2672.42 137.63,-2675.99\"/>\n",
       "</g>\n",
       "<!-- 1420016191808 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>1420016191808</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"82,-2664 6,-2664 6,-2644 82,-2644 82,-2664\"/>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-2650.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190800&#45;&gt;1420016191808 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>1420016190800&#45;&gt;1420016191808</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M101.81,-2699.59C91.67,-2691.77 77.01,-2680.46 64.99,-2671.19\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"67.31,-2668.56 57.25,-2665.22 63.03,-2674.1 67.31,-2668.56\"/>\n",
       "</g>\n",
       "<!-- 1420016190896&#45;&gt;1420016190800 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>1420016190896&#45;&gt;1420016190800</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M138.77,-2755.59C134.9,-2748.63 129.49,-2738.89 124.72,-2730.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"127.82,-2728.66 119.9,-2721.62 121.7,-2732.06 127.82,-2728.66\"/>\n",
       "</g>\n",
       "<!-- 1420016190992 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>1420016190992</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"188,-2832 112,-2832 112,-2812 188,-2812 188,-2832\"/>\n",
       "<text text-anchor=\"middle\" x=\"150\" y=\"-2818.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190992&#45;&gt;1420016190896 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>1420016190992&#45;&gt;1420016190896</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M148.95,-2811.59C148.22,-2805.01 147.22,-2795.96 146.3,-2787.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"149.78,-2787.35 145.2,-2777.8 142.83,-2788.13 149.78,-2787.35\"/>\n",
       "</g>\n",
       "<!-- 1420016191088 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>1420016191088</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"224,-2894 124,-2894 124,-2874 224,-2874 224,-2894\"/>\n",
       "<text text-anchor=\"middle\" x=\"174\" y=\"-2880.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 1420016191088&#45;&gt;1420016190992 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>1420016191088&#45;&gt;1420016190992</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M170.25,-2873.62C166.95,-2865.39 162.05,-2853.13 157.89,-2842.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"161.17,-2841.5 154.21,-2833.52 154.67,-2844.1 161.17,-2841.5\"/>\n",
       "</g>\n",
       "<!-- 1420016191328 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>1420016191328</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"152,-2384 64,-2384 64,-2364 152,-2364 152,-2384\"/>\n",
       "<text text-anchor=\"middle\" x=\"108\" y=\"-2370.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191088&#45;&gt;1420016191328 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>1420016191088&#45;&gt;1420016191328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M180.88,-2873.62C194.15,-2854.56 222,-2809.5 222,-2767 222,-2767 222,-2767 222,-2485 222,-2439.39 174.16,-2406.71 140.46,-2389.35\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"142.4,-2386.4 131.88,-2385.12 139.31,-2392.68 142.4,-2386.4\"/>\n",
       "</g>\n",
       "<!-- 1420016089936 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>1420016089936</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"206,-2968 142,-2968 142,-2936 206,-2936 206,-2968\"/>\n",
       "<text text-anchor=\"middle\" x=\"174\" y=\"-2954.5\" font-family=\"monospace\" font-size=\"10.00\">0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"174\" y=\"-2942.5\" font-family=\"monospace\" font-size=\"10.00\"> (1, 1)</text>\n",
       "</g>\n",
       "<!-- 1420016089936&#45;&gt;1420016191088 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>1420016089936&#45;&gt;1420016191088</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M174,-2935.69C174,-2926.8 174,-2915.46 174,-2905.77\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"177.5,-2905.83 174,-2895.83 170.5,-2905.83 177.5,-2905.83\"/>\n",
       "</g>\n",
       "<!-- 1420016190272 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>1420016190272</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"293,-2048 205,-2048 205,-2028 293,-2028 293,-2048\"/>\n",
       "<text text-anchor=\"middle\" x=\"249\" y=\"-2034.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190272&#45;&gt;1420016190176 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>1420016190272&#45;&gt;1420016190176</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M252.92,-2027.62C260.39,-2009.76 276.8,-1970.53 286.77,-1946.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"289.87,-1948.33 290.5,-1937.75 283.42,-1945.63 289.87,-1948.33\"/>\n",
       "</g>\n",
       "<!-- 1420016190656 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>1420016190656</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"294,-2104 200,-2104 200,-2084 294,-2084 294,-2104\"/>\n",
       "<text text-anchor=\"middle\" x=\"247\" y=\"-2090.5\" font-family=\"monospace\" font-size=\"10.00\">ViewBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190656&#45;&gt;1420016190272 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>1420016190656&#45;&gt;1420016190272</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M247.35,-2083.59C247.59,-2077.01 247.93,-2067.96 248.23,-2059.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"251.73,-2059.93 248.6,-2049.81 244.73,-2059.67 251.73,-2059.93\"/>\n",
       "</g>\n",
       "<!-- 1420016190848 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>1420016190848</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"288,-2160 200,-2160 200,-2140 288,-2140 288,-2160\"/>\n",
       "<text text-anchor=\"middle\" x=\"244\" y=\"-2146.5\" font-family=\"monospace\" font-size=\"10.00\">SumBackward1</text>\n",
       "</g>\n",
       "<!-- 1420016190848&#45;&gt;1420016190656 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>1420016190848&#45;&gt;1420016190656</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M244.52,-2139.59C244.89,-2133.01 245.39,-2123.96 245.85,-2115.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"249.34,-2115.99 246.4,-2105.81 242.35,-2115.6 249.34,-2115.99\"/>\n",
       "</g>\n",
       "<!-- 1420016191040 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>1420016191040</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"281,-2216 121,-2216 121,-2196 281,-2196 281,-2216\"/>\n",
       "<text text-anchor=\"middle\" x=\"201\" y=\"-2202.5\" font-family=\"monospace\" font-size=\"10.00\">MseLossBackwardBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191040&#45;&gt;1420016190848 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>1420016191040&#45;&gt;1420016190848</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M208.49,-2195.59C214.29,-2188.32 222.49,-2178.02 229.55,-2169.15\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"232.16,-2171.49 235.65,-2161.48 226.69,-2167.12 232.16,-2171.49\"/>\n",
       "</g>\n",
       "<!-- 1420016192288 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>1420016192288</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"181,-2160 105,-2160 105,-2140 181,-2140 181,-2160\"/>\n",
       "<text text-anchor=\"middle\" x=\"143\" y=\"-2146.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191040&#45;&gt;1420016192288 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>1420016191040&#45;&gt;1420016192288</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M190.9,-2195.59C182.75,-2188.01 171.07,-2177.13 161.28,-2168.02\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"163.8,-2165.58 154.09,-2161.33 159.03,-2170.71 163.8,-2165.58\"/>\n",
       "</g>\n",
       "<!-- 1420016191136&#45;&gt;1420016191040 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>1420016191136&#45;&gt;1420016191040</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M201,-2251.59C201,-2245.01 201,-2235.96 201,-2227.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"204.5,-2227.81 201,-2217.81 197.5,-2227.81 204.5,-2227.81\"/>\n",
       "</g>\n",
       "<!-- 1420016191232 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>1420016191232</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"232,-2328 156,-2328 156,-2308 232,-2308 232,-2328\"/>\n",
       "<text text-anchor=\"middle\" x=\"194\" y=\"-2314.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191232&#45;&gt;1420016191136 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>1420016191232&#45;&gt;1420016191136</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M195.22,-2307.59C196.08,-2300.93 197.27,-2291.75 198.35,-2283.44\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"201.79,-2284.17 199.6,-2273.8 194.84,-2283.27 201.79,-2284.17\"/>\n",
       "</g>\n",
       "<!-- 1420016191328&#45;&gt;1420016191232 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>1420016191328&#45;&gt;1420016191232</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M122.98,-2363.59C135.73,-2355.59 154.29,-2343.93 169.24,-2334.54\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"171.01,-2337.57 177.62,-2329.28 167.29,-2331.64 171.01,-2337.57\"/>\n",
       "</g>\n",
       "<!-- 1420016191904 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>1420016191904</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"152,-1880 64,-1880 64,-1860 152,-1860 152,-1880\"/>\n",
       "<text text-anchor=\"middle\" x=\"108\" y=\"-1866.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191328&#45;&gt;1420016191904 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>1420016191328&#45;&gt;1420016191904</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M102.69,-2363.63C92.67,-2345.05 72,-2301.98 72,-2263 72,-2263 72,-2263 72,-1981 72,-1947.96 86.85,-1911.98 97.48,-1890.47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"100.52,-1892.21 101.99,-1881.72 94.3,-1889 100.52,-1892.21\"/>\n",
       "</g>\n",
       "<!-- 1420016191424 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>1420016191424</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"127,-2440 39,-2440 39,-2420 127,-2420 127,-2440\"/>\n",
       "<text text-anchor=\"middle\" x=\"83\" y=\"-2426.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191424&#45;&gt;1420016191328 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>1420016191424&#45;&gt;1420016191328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M87.35,-2419.59C90.55,-2412.7 94.99,-2403.1 98.94,-2394.57\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"102.03,-2396.22 103.06,-2385.67 95.68,-2393.28 102.03,-2396.22\"/>\n",
       "</g>\n",
       "<!-- 1420016191520 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>1420016191520</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"105,-2496 29,-2496 29,-2476 105,-2476 105,-2496\"/>\n",
       "<text text-anchor=\"middle\" x=\"67\" y=\"-2482.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191520&#45;&gt;1420016191424 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>1420016191520&#45;&gt;1420016191424</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M69.79,-2475.59C71.78,-2468.86 74.55,-2459.53 77.03,-2451.15\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"80.33,-2452.33 79.81,-2441.75 73.62,-2450.34 80.33,-2452.33\"/>\n",
       "</g>\n",
       "<!-- 1420016191616 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>1420016191616</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"83,-2552 7,-2552 7,-2532 83,-2532 83,-2552\"/>\n",
       "<text text-anchor=\"middle\" x=\"45\" y=\"-2538.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191616&#45;&gt;1420016191520 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>1420016191616&#45;&gt;1420016191520</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.83,-2531.59C51.61,-2524.78 55.46,-2515.32 58.91,-2506.86\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"62.11,-2508.28 62.64,-2497.7 55.63,-2505.64 62.11,-2508.28\"/>\n",
       "</g>\n",
       "<!-- 1420016191712 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>1420016191712</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"82,-2608 0,-2608 0,-2588 82,-2588 82,-2608\"/>\n",
       "<text text-anchor=\"middle\" x=\"41\" y=\"-2594.5\" font-family=\"monospace\" font-size=\"10.00\">MmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191712&#45;&gt;1420016191616 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>1420016191712&#45;&gt;1420016191616</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M41.7,-2587.59C42.18,-2581.01 42.85,-2571.96 43.46,-2563.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"46.95,-2564.04 44.2,-2553.81 39.97,-2563.52 46.95,-2564.04\"/>\n",
       "</g>\n",
       "<!-- 1420016191808&#45;&gt;1420016191712 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>1420016191808&#45;&gt;1420016191712</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M43.48,-2643.59C43.11,-2637.01 42.61,-2627.96 42.15,-2619.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"45.65,-2619.6 41.6,-2609.81 38.66,-2619.99 45.65,-2619.6\"/>\n",
       "</g>\n",
       "<!-- 1420016190128 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>1420016190128</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"284,-1544 196,-1544 196,-1524 284,-1524 284,-1544\"/>\n",
       "<text text-anchor=\"middle\" x=\"240\" y=\"-1530.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190128&#45;&gt;1420016190032 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>1420016190128&#45;&gt;1420016190032</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M243.92,-1523.62C251.39,-1505.76 267.8,-1466.53 277.77,-1442.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"280.87,-1444.33 281.5,-1433.75 274.42,-1441.63 280.87,-1444.33\"/>\n",
       "</g>\n",
       "<!-- 1420016190752 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>1420016190752</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"285,-1600 191,-1600 191,-1580 285,-1580 285,-1600\"/>\n",
       "<text text-anchor=\"middle\" x=\"238\" y=\"-1586.5\" font-family=\"monospace\" font-size=\"10.00\">ViewBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190752&#45;&gt;1420016190128 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>1420016190752&#45;&gt;1420016190128</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M238.35,-1579.59C238.59,-1573.01 238.93,-1563.96 239.23,-1555.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"242.73,-1555.93 239.6,-1545.81 235.73,-1555.67 242.73,-1555.93\"/>\n",
       "</g>\n",
       "<!-- 1420016191184 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>1420016191184</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"281,-1656 193,-1656 193,-1636 281,-1636 281,-1656\"/>\n",
       "<text text-anchor=\"middle\" x=\"237\" y=\"-1642.5\" font-family=\"monospace\" font-size=\"10.00\">SumBackward1</text>\n",
       "</g>\n",
       "<!-- 1420016191184&#45;&gt;1420016190752 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>1420016191184&#45;&gt;1420016190752</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M237.17,-1635.59C237.3,-1629.01 237.46,-1619.96 237.62,-1611.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"241.11,-1611.87 237.8,-1601.81 234.12,-1611.75 241.11,-1611.87\"/>\n",
       "</g>\n",
       "<!-- 1420016191280 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>1420016191280</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"268,-1712 108,-1712 108,-1692 268,-1692 268,-1712\"/>\n",
       "<text text-anchor=\"middle\" x=\"188\" y=\"-1698.5\" font-family=\"monospace\" font-size=\"10.00\">MseLossBackwardBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191280&#45;&gt;1420016191184 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>1420016191280&#45;&gt;1420016191184</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.54,-1691.59C203.28,-1684.16 212.89,-1673.58 221.04,-1664.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.42,-1667.18 227.55,-1657.42 218.23,-1662.47 223.42,-1667.18\"/>\n",
       "</g>\n",
       "<!-- 1420016192768 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>1420016192768</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"172,-1656 96,-1656 96,-1636 172,-1636 172,-1656\"/>\n",
       "<text text-anchor=\"middle\" x=\"134\" y=\"-1642.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191280&#45;&gt;1420016192768 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>1420016191280&#45;&gt;1420016192768</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M178.59,-1691.59C171.08,-1684.08 160.35,-1673.35 151.3,-1664.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"153.91,-1661.96 144.37,-1657.37 148.96,-1666.91 153.91,-1661.96\"/>\n",
       "</g>\n",
       "<!-- 1420016191472&#45;&gt;1420016191280 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>1420016191472&#45;&gt;1420016191280</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M188,-1747.59C188,-1741.01 188,-1731.96 188,-1723.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"191.5,-1723.81 188,-1713.81 184.5,-1723.81 191.5,-1723.81\"/>\n",
       "</g>\n",
       "<!-- 1420016191664 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>1420016191664</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"197,-1824 121,-1824 121,-1804 197,-1804 197,-1824\"/>\n",
       "<text text-anchor=\"middle\" x=\"159\" y=\"-1810.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191664&#45;&gt;1420016191472 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>1420016191664&#45;&gt;1420016191472</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M164.05,-1803.59C167.79,-1796.63 173.02,-1786.89 177.64,-1778.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"180.64,-1780.1 182.29,-1769.63 174.48,-1776.79 180.64,-1780.1\"/>\n",
       "</g>\n",
       "<!-- 1420016191904&#45;&gt;1420016191664 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>1420016191904&#45;&gt;1420016191664</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M116.88,-1859.59C123.9,-1852.16 133.9,-1841.58 142.39,-1832.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"144.86,-1835.07 149.18,-1825.4 139.77,-1830.27 144.86,-1835.07\"/>\n",
       "</g>\n",
       "<!-- 1420016192384 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>1420016192384</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"143,-1376 55,-1376 55,-1356 143,-1356 143,-1376\"/>\n",
       "<text text-anchor=\"middle\" x=\"99\" y=\"-1362.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191904&#45;&gt;1420016192384 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>1420016191904&#45;&gt;1420016192384</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M101.17,-1859.53C88.58,-1841.12 63,-1798.82 63,-1759 63,-1759 63,-1759 63,-1477 63,-1443.96 77.85,-1407.98 88.48,-1386.47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"91.52,-1388.21 92.99,-1377.72 85.3,-1385 91.52,-1388.21\"/>\n",
       "</g>\n",
       "<!-- 1420016190368 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>1420016190368</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"188,-1936 100,-1936 100,-1916 188,-1916 188,-1936\"/>\n",
       "<text text-anchor=\"middle\" x=\"144\" y=\"-1922.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190368&#45;&gt;1420016191904 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>1420016190368&#45;&gt;1420016191904</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M137.73,-1915.59C132.98,-1908.47 126.3,-1898.45 120.48,-1889.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"123.5,-1887.94 115.04,-1881.56 117.67,-1891.82 123.5,-1887.94\"/>\n",
       "</g>\n",
       "<!-- 1420016192000 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>1420016192000</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"181,-1992 105,-1992 105,-1972 181,-1972 181,-1992\"/>\n",
       "<text text-anchor=\"middle\" x=\"143\" y=\"-1978.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192000&#45;&gt;1420016190368 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>1420016192000&#45;&gt;1420016190368</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M143.17,-1971.59C143.3,-1965.01 143.46,-1955.96 143.62,-1947.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"147.11,-1947.87 143.8,-1937.81 140.12,-1947.75 147.11,-1947.87\"/>\n",
       "</g>\n",
       "<!-- 1420016192096 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>1420016192096</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"180,-2048 104,-2048 104,-2028 180,-2028 180,-2048\"/>\n",
       "<text text-anchor=\"middle\" x=\"142\" y=\"-2034.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192096&#45;&gt;1420016192000 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>1420016192096&#45;&gt;1420016192000</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M142.17,-2027.59C142.3,-2021.01 142.46,-2011.96 142.62,-2003.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"146.11,-2003.87 142.8,-1993.81 139.12,-2003.75 146.11,-2003.87\"/>\n",
       "</g>\n",
       "<!-- 1420016192192 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>1420016192192</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"182,-2104 100,-2104 100,-2084 182,-2084 182,-2104\"/>\n",
       "<text text-anchor=\"middle\" x=\"141\" y=\"-2090.5\" font-family=\"monospace\" font-size=\"10.00\">MmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192192&#45;&gt;1420016192096 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>1420016192192&#45;&gt;1420016192096</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M141.17,-2083.59C141.3,-2077.01 141.46,-2067.96 141.62,-2059.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"145.11,-2059.87 141.8,-2049.81 138.12,-2059.75 145.11,-2059.87\"/>\n",
       "</g>\n",
       "<!-- 1420016192288&#45;&gt;1420016192192 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>1420016192288&#45;&gt;1420016192192</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M142.65,-2139.59C142.41,-2133.01 142.07,-2123.96 141.77,-2115.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"145.27,-2115.67 141.4,-2105.81 138.27,-2115.93 145.27,-2115.67\"/>\n",
       "</g>\n",
       "<!-- 1420016189984 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>1420016189984</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"275,-1040 187,-1040 187,-1020 275,-1020 275,-1040\"/>\n",
       "<text text-anchor=\"middle\" x=\"231\" y=\"-1026.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016189984&#45;&gt;1420016189696 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>1420016189984&#45;&gt;1420016189696</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M234.92,-1019.62C242.39,-1001.76 258.8,-962.53 268.77,-938.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"271.87,-940.33 272.5,-929.75 265.42,-937.63 271.87,-940.33\"/>\n",
       "</g>\n",
       "<!-- 1420016190944 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>1420016190944</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"276,-1096 182,-1096 182,-1076 276,-1076 276,-1096\"/>\n",
       "<text text-anchor=\"middle\" x=\"229\" y=\"-1082.5\" font-family=\"monospace\" font-size=\"10.00\">ViewBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190944&#45;&gt;1420016189984 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>1420016190944&#45;&gt;1420016189984</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M229.35,-1075.59C229.59,-1069.01 229.93,-1059.96 230.23,-1051.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"233.73,-1051.93 230.6,-1041.81 226.73,-1051.67 233.73,-1051.93\"/>\n",
       "</g>\n",
       "<!-- 1420016191376 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>1420016191376</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"272,-1152 184,-1152 184,-1132 272,-1132 272,-1152\"/>\n",
       "<text text-anchor=\"middle\" x=\"228\" y=\"-1138.5\" font-family=\"monospace\" font-size=\"10.00\">SumBackward1</text>\n",
       "</g>\n",
       "<!-- 1420016191376&#45;&gt;1420016190944 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>1420016191376&#45;&gt;1420016190944</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M228.17,-1131.59C228.3,-1125.01 228.46,-1115.96 228.62,-1107.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"232.11,-1107.87 228.8,-1097.81 225.12,-1107.75 232.11,-1107.87\"/>\n",
       "</g>\n",
       "<!-- 1420016191760 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>1420016191760</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"259,-1208 99,-1208 99,-1188 259,-1188 259,-1208\"/>\n",
       "<text text-anchor=\"middle\" x=\"179\" y=\"-1194.5\" font-family=\"monospace\" font-size=\"10.00\">MseLossBackwardBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191760&#45;&gt;1420016191376 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>1420016191760&#45;&gt;1420016191376</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.54,-1187.59C194.28,-1180.16 203.89,-1169.58 212.04,-1160.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"214.42,-1163.18 218.55,-1153.42 209.23,-1158.47 214.42,-1163.18\"/>\n",
       "</g>\n",
       "<!-- 1420016193248 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>1420016193248</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"163,-1152 87,-1152 87,-1132 163,-1132 163,-1152\"/>\n",
       "<text text-anchor=\"middle\" x=\"125\" y=\"-1138.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016191760&#45;&gt;1420016193248 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>1420016191760&#45;&gt;1420016193248</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.59,-1187.59C162.08,-1180.08 151.35,-1169.35 142.3,-1160.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"144.91,-1157.96 135.37,-1153.37 139.96,-1162.91 144.91,-1157.96\"/>\n",
       "</g>\n",
       "<!-- 1420016191952&#45;&gt;1420016191760 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>1420016191952&#45;&gt;1420016191760</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M179,-1243.59C179,-1237.01 179,-1227.96 179,-1219.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"182.5,-1219.81 179,-1209.81 175.5,-1219.81 182.5,-1219.81\"/>\n",
       "</g>\n",
       "<!-- 1420016192144 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>1420016192144</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"188,-1320 112,-1320 112,-1300 188,-1300 188,-1320\"/>\n",
       "<text text-anchor=\"middle\" x=\"150\" y=\"-1306.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192144&#45;&gt;1420016191952 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>1420016192144&#45;&gt;1420016191952</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M155.05,-1299.59C158.79,-1292.63 164.02,-1282.89 168.64,-1274.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"171.64,-1276.1 173.29,-1265.63 165.48,-1272.79 171.64,-1276.1\"/>\n",
       "</g>\n",
       "<!-- 1420016192384&#45;&gt;1420016192144 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>1420016192384&#45;&gt;1420016192144</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M107.88,-1355.59C114.9,-1348.16 124.9,-1337.58 133.39,-1328.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"135.86,-1331.07 140.18,-1321.4 130.77,-1326.27 135.86,-1331.07\"/>\n",
       "</g>\n",
       "<!-- 1420016192864 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>1420016192864</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"125,-872 37,-872 37,-852 125,-852 125,-872\"/>\n",
       "<text text-anchor=\"middle\" x=\"81\" y=\"-858.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192384&#45;&gt;1420016192864 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>1420016192384&#45;&gt;1420016192864</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M92.17,-1355.53C79.58,-1337.12 54,-1294.82 54,-1255 54,-1255 54,-1255 54,-973 54,-940.65 65.15,-904.44 73.12,-882.7\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"76.28,-884.23 76.59,-873.64 69.75,-881.73 76.28,-884.23\"/>\n",
       "</g>\n",
       "<!-- 1420016190224 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>1420016190224</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"179,-1432 91,-1432 91,-1412 179,-1412 179,-1432\"/>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-1418.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190224&#45;&gt;1420016192384 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>1420016190224&#45;&gt;1420016192384</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M128.73,-1411.59C123.98,-1404.47 117.3,-1394.45 111.48,-1385.72\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"114.5,-1383.94 106.04,-1377.56 108.67,-1387.82 114.5,-1383.94\"/>\n",
       "</g>\n",
       "<!-- 1420016192480 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>1420016192480</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"172,-1488 96,-1488 96,-1468 172,-1468 172,-1488\"/>\n",
       "<text text-anchor=\"middle\" x=\"134\" y=\"-1474.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192480&#45;&gt;1420016190224 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>1420016192480&#45;&gt;1420016190224</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M134.17,-1467.59C134.3,-1461.01 134.46,-1451.96 134.62,-1443.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"138.11,-1443.87 134.8,-1433.81 131.12,-1443.75 138.11,-1443.87\"/>\n",
       "</g>\n",
       "<!-- 1420016192576 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>1420016192576</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"171,-1544 95,-1544 95,-1524 171,-1524 171,-1544\"/>\n",
       "<text text-anchor=\"middle\" x=\"133\" y=\"-1530.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192576&#45;&gt;1420016192480 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>1420016192576&#45;&gt;1420016192480</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M133.17,-1523.59C133.3,-1517.01 133.46,-1507.96 133.62,-1499.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"137.11,-1499.87 133.8,-1489.81 130.12,-1499.75 137.11,-1499.87\"/>\n",
       "</g>\n",
       "<!-- 1420016192672 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>1420016192672</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"173,-1600 91,-1600 91,-1580 173,-1580 173,-1600\"/>\n",
       "<text text-anchor=\"middle\" x=\"132\" y=\"-1586.5\" font-family=\"monospace\" font-size=\"10.00\">MmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192672&#45;&gt;1420016192576 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>1420016192672&#45;&gt;1420016192576</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M132.17,-1579.59C132.3,-1573.01 132.46,-1563.96 132.62,-1555.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"136.11,-1555.87 132.8,-1545.81 129.12,-1555.75 136.11,-1555.87\"/>\n",
       "</g>\n",
       "<!-- 1420016192768&#45;&gt;1420016192672 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>1420016192768&#45;&gt;1420016192672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M133.65,-1635.59C133.41,-1629.01 133.07,-1619.96 132.77,-1611.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"136.27,-1611.67 132.4,-1601.81 129.27,-1611.93 136.27,-1611.67\"/>\n",
       "</g>\n",
       "<!-- 1420016189600 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>1420016189600</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"264,-536 176,-536 176,-516 264,-516 264,-536\"/>\n",
       "<text text-anchor=\"middle\" x=\"220\" y=\"-522.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016189600&#45;&gt;1420013972128 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>1420016189600&#45;&gt;1420013972128</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M220,-515.59C220,-509.01 220,-499.96 220,-491.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.5,-491.81 220,-481.81 216.5,-491.81 223.5,-491.81\"/>\n",
       "</g>\n",
       "<!-- 1420016190608 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>1420016190608</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"267,-592 173,-592 173,-572 267,-572 267,-592\"/>\n",
       "<text text-anchor=\"middle\" x=\"220\" y=\"-578.5\" font-family=\"monospace\" font-size=\"10.00\">ViewBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190608&#45;&gt;1420016189600 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>1420016190608&#45;&gt;1420016189600</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M220,-571.59C220,-565.01 220,-555.96 220,-547.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.5,-547.81 220,-537.81 216.5,-547.81 223.5,-547.81\"/>\n",
       "</g>\n",
       "<!-- 1420016191856 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>1420016191856</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"263,-648 175,-648 175,-628 263,-628 263,-648\"/>\n",
       "<text text-anchor=\"middle\" x=\"219\" y=\"-634.5\" font-family=\"monospace\" font-size=\"10.00\">SumBackward1</text>\n",
       "</g>\n",
       "<!-- 1420016191856&#45;&gt;1420016190608 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>1420016191856&#45;&gt;1420016190608</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M219.17,-627.59C219.3,-621.01 219.46,-611.96 219.62,-603.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.11,-603.87 219.8,-593.81 216.12,-603.75 223.11,-603.87\"/>\n",
       "</g>\n",
       "<!-- 1420016192240 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>1420016192240</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"250,-704 90,-704 90,-684 250,-684 250,-704\"/>\n",
       "<text text-anchor=\"middle\" x=\"170\" y=\"-690.5\" font-family=\"monospace\" font-size=\"10.00\">MseLossBackwardBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192240&#45;&gt;1420016191856 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>1420016192240&#45;&gt;1420016191856</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M178.54,-683.59C185.28,-676.16 194.89,-665.58 203.04,-656.58\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"205.42,-659.18 209.55,-649.42 200.23,-654.47 205.42,-659.18\"/>\n",
       "</g>\n",
       "<!-- 1420016193344 -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>1420016193344</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"154,-648 78,-648 78,-628 154,-628 154,-648\"/>\n",
       "<text text-anchor=\"middle\" x=\"116\" y=\"-634.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192240&#45;&gt;1420016193344 -->\n",
       "<g id=\"edge88\" class=\"edge\">\n",
       "<title>1420016192240&#45;&gt;1420016193344</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M160.59,-683.59C153.08,-676.08 142.35,-665.35 133.3,-656.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"135.91,-653.96 126.37,-649.37 130.96,-658.91 135.91,-653.96\"/>\n",
       "</g>\n",
       "<!-- 1420016192432&#45;&gt;1420016192240 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>1420016192432&#45;&gt;1420016192240</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M170,-739.59C170,-733.01 170,-723.96 170,-715.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"173.5,-715.81 170,-705.81 166.5,-715.81 173.5,-715.81\"/>\n",
       "</g>\n",
       "<!-- 1420016192624 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>1420016192624</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"179,-816 103,-816 103,-796 179,-796 179,-816\"/>\n",
       "<text text-anchor=\"middle\" x=\"141\" y=\"-802.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192624&#45;&gt;1420016192432 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>1420016192624&#45;&gt;1420016192432</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M146.05,-795.59C149.79,-788.63 155.02,-778.89 159.64,-770.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"162.64,-772.1 164.29,-761.63 156.48,-768.79 162.64,-772.1\"/>\n",
       "</g>\n",
       "<!-- 1420016192864&#45;&gt;1420016192624 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>1420016192864&#45;&gt;1420016192624</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M91.45,-851.59C99.97,-843.93 112.21,-832.91 122.39,-823.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"124.45,-826.6 129.54,-817.31 119.77,-821.4 124.45,-826.6\"/>\n",
       "</g>\n",
       "<!-- 1420016191568 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>1420016191568</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"161,-368 73,-368 73,-348 161,-348 161,-368\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-354.5\" font-family=\"monospace\" font-size=\"10.00\">SubBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192864&#45;&gt;1420016191568 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>1420016192864&#45;&gt;1420016191568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M75.69,-851.63C65.67,-833.05 45,-789.98 45,-751 45,-751 45,-751 45,-469 45,-438.9 47.62,-429.25 64,-404 71.37,-392.64 82.45,-382.6 92.58,-374.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"94.53,-377.81 100.62,-369.15 90.45,-372.12 94.53,-377.81\"/>\n",
       "</g>\n",
       "<!-- 1420016190080 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>1420016190080</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"170,-928 82,-928 82,-908 170,-908 170,-928\"/>\n",
       "<text text-anchor=\"middle\" x=\"126\" y=\"-914.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016190080&#45;&gt;1420016192864 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>1420016190080&#45;&gt;1420016192864</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M118.16,-907.59C112.03,-900.24 103.33,-889.8 95.89,-880.87\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"98.81,-878.9 89.72,-873.46 93.43,-883.38 98.81,-878.9\"/>\n",
       "</g>\n",
       "<!-- 1420016192960 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>1420016192960</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"163,-984 87,-984 87,-964 163,-964 163,-984\"/>\n",
       "<text text-anchor=\"middle\" x=\"125\" y=\"-970.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192960&#45;&gt;1420016190080 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>1420016192960&#45;&gt;1420016190080</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M125.17,-963.59C125.3,-957.01 125.46,-947.96 125.62,-939.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"129.11,-939.87 125.8,-929.81 122.12,-939.75 129.11,-939.87\"/>\n",
       "</g>\n",
       "<!-- 1420016193056 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>1420016193056</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"162,-1040 86,-1040 86,-1020 162,-1020 162,-1040\"/>\n",
       "<text text-anchor=\"middle\" x=\"124\" y=\"-1026.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016193056&#45;&gt;1420016192960 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>1420016193056&#45;&gt;1420016192960</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M124.17,-1019.59C124.3,-1013.01 124.46,-1003.96 124.62,-995.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"128.11,-995.87 124.8,-985.81 121.12,-995.75 128.11,-995.87\"/>\n",
       "</g>\n",
       "<!-- 1420016193152 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>1420016193152</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"164,-1096 82,-1096 82,-1076 164,-1076 164,-1096\"/>\n",
       "<text text-anchor=\"middle\" x=\"123\" y=\"-1082.5\" font-family=\"monospace\" font-size=\"10.00\">MmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016193152&#45;&gt;1420016193056 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>1420016193152&#45;&gt;1420016193056</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M123.17,-1075.59C123.3,-1069.01 123.46,-1059.96 123.62,-1051.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"127.11,-1051.87 123.8,-1041.81 120.12,-1051.75 127.11,-1051.87\"/>\n",
       "</g>\n",
       "<!-- 1420016193248&#45;&gt;1420016193152 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>1420016193248&#45;&gt;1420016193152</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M124.65,-1131.59C124.41,-1125.01 124.07,-1115.96 123.77,-1107.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"127.27,-1107.67 123.4,-1097.81 120.27,-1107.93 127.27,-1107.67\"/>\n",
       "</g>\n",
       "<!-- 1420013966800 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>1420013966800</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"173,-312 97,-312 97,-292 173,-292 173,-312\"/>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-298.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420013966800&#45;&gt;1420013803488 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>1420013966800&#45;&gt;1420013803488</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M140.75,-291.59C145.05,-284.55 151.09,-274.67 156.39,-266\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"159.3,-267.95 161.53,-257.59 153.33,-264.3 159.3,-267.95\"/>\n",
       "</g>\n",
       "<!-- 1420016191568&#45;&gt;1420013966800 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>1420016191568&#45;&gt;1420013966800</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M120.14,-347.59C122.38,-340.86 125.49,-331.53 128.28,-323.15\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"131.58,-324.33 131.42,-313.73 124.94,-322.11 131.58,-324.33\"/>\n",
       "</g>\n",
       "<!-- 1420016192336 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>1420016192336</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"161,-424 73,-424 73,-404 161,-404 161,-424\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-410.5\" font-family=\"monospace\" font-size=\"10.00\">MulBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192336&#45;&gt;1420016191568 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>1420016192336&#45;&gt;1420016191568</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M117,-403.59C117,-397.01 117,-387.96 117,-379.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"120.5,-379.81 117,-369.81 113.5,-379.81 120.5,-379.81\"/>\n",
       "</g>\n",
       "<!-- 1420016192720 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>1420016192720</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"154,-480 78,-480 78,-460 154,-460 154,-480\"/>\n",
       "<text text-anchor=\"middle\" x=\"116\" y=\"-466.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192720&#45;&gt;1420016192336 -->\n",
       "<g id=\"edge84\" class=\"edge\">\n",
       "<title>1420016192720&#45;&gt;1420016192336</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M116.17,-459.59C116.3,-453.01 116.46,-443.96 116.62,-435.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"120.11,-435.87 116.8,-425.81 113.12,-435.75 120.11,-435.87\"/>\n",
       "</g>\n",
       "<!-- 1420016192912 -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>1420016192912</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"153,-536 77,-536 77,-516 153,-516 153,-536\"/>\n",
       "<text text-anchor=\"middle\" x=\"115\" y=\"-522.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016192912&#45;&gt;1420016192720 -->\n",
       "<g id=\"edge85\" class=\"edge\">\n",
       "<title>1420016192912&#45;&gt;1420016192720</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M115.17,-515.59C115.3,-509.01 115.46,-499.96 115.62,-491.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"119.11,-491.87 115.8,-481.81 112.12,-491.75 119.11,-491.87\"/>\n",
       "</g>\n",
       "<!-- 1420016193104 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>1420016193104</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"155,-592 73,-592 73,-572 155,-572 155,-592\"/>\n",
       "<text text-anchor=\"middle\" x=\"114\" y=\"-578.5\" font-family=\"monospace\" font-size=\"10.00\">MmBackward0</text>\n",
       "</g>\n",
       "<!-- 1420016193104&#45;&gt;1420016192912 -->\n",
       "<g id=\"edge86\" class=\"edge\">\n",
       "<title>1420016193104&#45;&gt;1420016192912</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M114.17,-571.59C114.3,-565.01 114.46,-555.96 114.62,-547.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"118.11,-547.87 114.8,-537.81 111.12,-547.75 118.11,-547.87\"/>\n",
       "</g>\n",
       "<!-- 1420016193344&#45;&gt;1420016193104 -->\n",
       "<g id=\"edge87\" class=\"edge\">\n",
       "<title>1420016193344&#45;&gt;1420016193104</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M115.65,-627.59C115.41,-621.01 115.07,-611.96 114.77,-603.73\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"118.27,-603.67 114.4,-593.81 111.27,-603.93 118.27,-603.67\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x14a9f8cb3a0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torchviz import make_dot\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(1, 1),\n",
    "    # torch.nn.ReLU(),\n",
    "    # torch.nn.Linear(40, 1)\n",
    ")\n",
    "\n",
    "maml = MAML(\n",
    "    model=model,\n",
    "    meta_lr=1e-3, \n",
    "    inner_lr=0.01, \n",
    "    inner_step=5, \n",
    "    ts_inner_step=1, \n",
    "    iterations=400, \n",
    "    path=data_root + \"/model.pth\"\n",
    ")\n",
    "\n",
    "train_loader = MetaLoader(num_tasks=1, samples_per_task=3)\n",
    "maml.model.train()\n",
    "num_tasks = len(train_loader)\n",
    "all_supp_loss, all_query_loss = 0, 0\n",
    "\n",
    "for (x_support, y_support), (x_query, y_query) in train_loader:           \n",
    "    # we are calling the private method of the class _inner_loop, \n",
    "    # which is something that you should avoid in general.\n",
    "    # I'm doing it here only because it's the quickest way to create the torchviz plot\n",
    "\n",
    "    # inner loop for each task on support loss\n",
    "    res = maml._inner_loop(x_support, y_support, x_query, y_query)\n",
    "    all_supp_loss += res[0]\n",
    "    all_query_loss += res[1]   \n",
    "\n",
    "make_dot(all_query_loss.sum(), params=dict(maml.model.named_parameters()))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercises\n",
    "\n",
    "if you studied the implementation above, understanding, and now you feel adventurous and want to test your knowledge, you can try to:\n",
    "- implement first order MAML by truncating the gradient. You can use `torchviz` to debug your code. Plot the computational graph and check that the gradient is truncated.\n",
    "- you can implement some of the MAML improvements in Antoniou, Antreas, Harrison Edwards, and Amos Storkey. \"How to train your MAML.\" arXiv preprint arXiv:1810.09502 (2018).\n",
    "    - some are quick and easy to implement, others may require more work\n",
    " \n",
    "Keep in mind that these exercises are much more difficult than the previous notebooks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}