DCL5 - Regularization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Regularization Methods\n",
    "\n",
    "Most regularization methods have two components:\n",
    "- a function that computes the regularization loss\n",
    "- a function that updates the internal state (teacher model, weight importance, ...) after each experience\n",
    "\n",
    "We are going to see this with two examples: Synaptic Intelligence and Learning without Forgetting.\n",
    "\n",
    "First, let's install Avalanche. You can skip this step if you have installed it already."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install avalanche-lib==0.3.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Setup Benchmark and Model\n",
    "We use Permuted MNIST, a task-agnostic domain-incremental benchmark. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from avalanche.benchmarks import PermutedMNIST\n",
    "from avalanche.benchmarks.utils.data_loader import GroupBalancedDataLoader\n",
    "from avalanche.models import SimpleMLP\n",
    "from torch.optim import SGD\n",
    "from torch.nn import CrossEntropyLoss\n",
    "\n",
    "benchmark = PermutedMNIST(5, return_task_id=False)\n",
    "# model\n",
    "model = SimpleMLP()\n",
    "optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9)\n",
    "criterion = CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Synaptic Intelligence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from fnmatch import fnmatch\n",
    "from typing import Sequence, Any, Set, List, Tuple, Dict, Union, TYPE_CHECKING\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import Tensor\n",
    "from torch.nn import Module\n",
    "from torch.nn.modules.batchnorm import _NormBase\n",
    "\n",
    "from avalanche.training.plugins.ewc import EwcDataType, ParamDict\n",
    "from avalanche.training.plugins.strategy_plugin import SupervisedPlugin\n",
    "from avalanche.training.utils import get_layers_and_params, ParamData\n",
    "from avalanche.training.templates import SupervisedTemplate\n",
    "\n",
    "SynDataType = Dict[str, Dict[str, Union[ParamData, Tensor]]]\n",
    "\n",
    "\n",
    "class SynapticIntelligencePlugin(SupervisedPlugin):\n",
    "    \"\"\"Synaptic Intelligence plugin.\n",
    "\n",
    "    This is the Synaptic Intelligence PyTorch implementation of the\n",
    "    algorithm described in the paper\n",
    "    \"Continuous Learning in Single-Incremental-Task Scenarios\"\n",
    "    (https://arxiv.org/abs/1806.08568)\n",
    "\n",
    "    The original implementation has been proposed in the paper\n",
    "    \"Continual Learning Through Synaptic Intelligence\"\n",
    "    (https://arxiv.org/abs/1703.04200).\n",
    "\n",
    "    This plugin can be attached to existing strategies to achieve a\n",
    "    regularization effect.\n",
    "\n",
    "    This plugin will require the strategy `loss` field to be set before the\n",
    "    `before_backward` callback is invoked. The loss Tensor will be updated to\n",
    "    achieve the S.I. regularization effect.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        si_lambda: Union[float, Sequence[float]],\n",
    "        eps: float = 0.0000001,\n",
    "        excluded_parameters: Sequence[\"str\"] = None,\n",
    "        device: Any = \"as_strategy\",\n",
    "    ):\n",
    "        \"\"\"Creates an instance of the Synaptic Intelligence plugin.\n",
    "\n",
    "        :param si_lambda: Synaptic Intelligence lambda term.\n",
    "            If list, one lambda for each experience. If the list has less\n",
    "            elements than the number of experiences, last lambda will be\n",
    "            used for the remaining experiences.\n",
    "        :param eps: Synaptic Intelligence damping parameter.\n",
    "        :param device: The device to use to run the S.I. experiences.\n",
    "            Defaults to \"as_strategy\", which means that the `device` field of\n",
    "            the strategy will be used. Using a different device may lead to a\n",
    "            performance drop due to the required data transfer.\n",
    "        \"\"\"\n",
    "\n",
    "        super().__init__()\n",
    "\n",
    "        if excluded_parameters is None:\n",
    "            excluded_parameters = []\n",
    "        self.si_lambda = (\n",
    "            si_lambda if isinstance(si_lambda, (list, tuple)) else [si_lambda]\n",
    "        )\n",
    "        self.eps: float = eps\n",
    "        self.excluded_parameters: Set[str] = set(excluded_parameters)\n",
    "        self.ewc_data: EwcDataType = (dict(), dict())\n",
    "        \"\"\"\n",
    "        The first dictionary contains the params at loss minimum while the \n",
    "        second one contains the parameter importance.\n",
    "        \"\"\"\n",
    "\n",
    "        self.syn_data: SynDataType = {\n",
    "            \"old_theta\": dict(),\n",
    "            \"new_theta\": dict(),\n",
    "            \"grad\": dict(),\n",
    "            \"trajectory\": dict(),\n",
    "            \"cum_trajectory\": dict(),\n",
    "        }\n",
    "\n",
    "        self._device = device\n",
    "\n",
    "    def before_training_exp(self, strategy: \"SupervisedTemplate\", **kwargs):\n",
    "        super().before_training_exp(strategy, **kwargs)\n",
    "        SynapticIntelligencePlugin.create_syn_data(\n",
    "            strategy.model,\n",
    "            self.ewc_data,\n",
    "            self.syn_data,\n",
    "            self.excluded_parameters,\n",
    "        )\n",
    "\n",
    "        SynapticIntelligencePlugin.init_batch(\n",
    "            strategy.model,\n",
    "            self.ewc_data,\n",
    "            self.syn_data,\n",
    "            self.excluded_parameters,\n",
    "        )\n",
    "\n",
    "    def before_backward(self, strategy: \"SupervisedTemplate\", **kwargs):\n",
    "        super().before_backward(strategy, **kwargs)\n",
    "\n",
    "        exp_id = strategy.clock.train_exp_counter\n",
    "        try:\n",
    "            si_lamb = self.si_lambda[exp_id]\n",
    "        except IndexError:  # less than one lambda per experience, take last\n",
    "            si_lamb = self.si_lambda[-1]\n",
    "\n",
    "        syn_loss = SynapticIntelligencePlugin.compute_ewc_loss(\n",
    "            strategy.model,\n",
    "            self.ewc_data,\n",
    "            self.excluded_parameters,\n",
    "            lambd=si_lamb,\n",
    "            device=self.device(strategy),\n",
    "        )\n",
    "\n",
    "        if syn_loss is not None:\n",
    "            strategy.loss += syn_loss.to(strategy.device)\n",
    "\n",
    "    def before_training_iteration(\n",
    "        self, strategy: \"SupervisedTemplate\", **kwargs\n",
    "    ):\n",
    "        super().before_training_iteration(strategy, **kwargs)\n",
    "        SynapticIntelligencePlugin.pre_update(\n",
    "            strategy.model, self.syn_data, self.excluded_parameters\n",
    "        )\n",
    "\n",
    "    def after_training_iteration(\n",
    "        self, strategy: \"SupervisedTemplate\", **kwargs\n",
    "    ):\n",
    "        super().after_training_iteration(strategy, **kwargs)\n",
    "        SynapticIntelligencePlugin.post_update(\n",
    "            strategy.model, self.syn_data, self.excluded_parameters\n",
    "        )\n",
    "\n",
    "    def after_training_exp(self, strategy: \"SupervisedTemplate\", **kwargs):\n",
    "        super().after_training_exp(strategy, **kwargs)\n",
    "        SynapticIntelligencePlugin.update_ewc_data(\n",
    "            strategy.model,\n",
    "            self.ewc_data,\n",
    "            self.syn_data,\n",
    "            0.001,\n",
    "            self.excluded_parameters,\n",
    "            1,\n",
    "            eps=self.eps,\n",
    "        )\n",
    "\n",
    "    def device(self, strategy: \"SupervisedTemplate\"):\n",
    "        if self._device == \"as_strategy\":\n",
    "            return strategy.device\n",
    "\n",
    "        return self._device\n",
    "\n",
    "    @staticmethod\n",
    "    @torch.no_grad()\n",
    "    def create_syn_data(\n",
    "        model: Module,\n",
    "        ewc_data: EwcDataType,\n",
    "        syn_data: SynDataType,\n",
    "        excluded_parameters: Set[str],\n",
    "    ):\n",
    "        params = SynapticIntelligencePlugin.allowed_parameters(\n",
    "            model, excluded_parameters\n",
    "        )\n",
    "\n",
    "        for param_name, param in params:\n",
    "            if param_name not in ewc_data[0]:\n",
    "                # new parameter\n",
    "                ewc_data[0][param_name] = ParamData(\n",
    "                    param_name, param.flatten().shape)\n",
    "                ewc_data[1][param_name] = ParamData(\n",
    "                    f\"imp_{param_name}\", param.flatten().shape)\n",
    "                syn_data[\"old_theta\"][param_name] = ParamData(\n",
    "                    f\"old_theta_{param_name}\", param.flatten().shape)\n",
    "                syn_data[\"new_theta\"][param_name] = ParamData(\n",
    "                    f\"new_theta_{param_name}\", param.flatten().shape)\n",
    "                syn_data[\"grad\"][param_name] = ParamData(\n",
    "                    f\"grad{param_name}\", param.flatten().shape)\n",
    "                syn_data[\"trajectory\"][param_name] = ParamData(\n",
    "                    f\"trajectory_{param_name}\", param.flatten().shape)\n",
    "                syn_data[\"cum_trajectory\"][param_name] = ParamData(\n",
    "                    f\"cum_trajectory_{param_name}\", param.flatten().shape)\n",
    "            elif ewc_data[0][param_name].shape != param.shape:\n",
    "                # parameter expansion\n",
    "                ewc_data[0][param_name].expand(param.flatten().shape)\n",
    "                ewc_data[1][param_name].expand(param.flatten().shape)\n",
    "                syn_data[\"old_theta\"][param_name].expand(param.flatten().shape)\n",
    "                syn_data[\"new_theta\"][param_name].expand(param.flatten().shape)\n",
    "                syn_data[\"grad\"][param_name].expand(param.flatten().shape)\n",
    "                syn_data[\"trajectory\"][param_name]\\\n",
    "                    .expand(param.flatten().shape)\n",
    "                syn_data[\"cum_trajectory\"][param_name]\\\n",
    "                    .expand(param.flatten().shape)\n",
    "\n",
    "    @staticmethod\n",
    "    @torch.no_grad()\n",
    "    def extract_weights(\n",
    "        model: Module, target: ParamDict, excluded_parameters: Set[str]\n",
    "    ):\n",
    "        params = SynapticIntelligencePlugin.allowed_parameters(\n",
    "            model, excluded_parameters\n",
    "        )\n",
    "\n",
    "        for name, param in params:\n",
    "            target[name].data = param.detach().cpu().flatten()\n",
    "\n",
    "    @staticmethod\n",
    "    @torch.no_grad()\n",
    "    def extract_grad(model, target: ParamDict, excluded_parameters: Set[str]):\n",
    "        params = SynapticIntelligencePlugin.allowed_parameters(\n",
    "            model, excluded_parameters\n",
    "        )\n",
    "\n",
    "        # Store the gradients into target\n",
    "        for name, param in params:\n",
    "            target[name].data = param.grad.detach().cpu().flatten()\n",
    "\n",
    "    @staticmethod\n",
    "    @torch.no_grad()\n",
    "    def init_batch(\n",
    "        model,\n",
    "        ewc_data: EwcDataType,\n",
    "        syn_data: SynDataType,\n",
    "        excluded_parameters: Set[str],\n",
    "    ):\n",
    "        # Keep initial weights\n",
    "        SynapticIntelligencePlugin.extract_weights(\n",
    "            model, ewc_data[0], excluded_parameters\n",
    "        )\n",
    "        for param_name, param_trajectory in syn_data[\"trajectory\"].items():\n",
    "            param_trajectory.data.fill_(0.0)\n",
    "\n",
    "    @staticmethod\n",
    "    @torch.no_grad()\n",
    "    def pre_update(model, syn_data: SynDataType, excluded_parameters: Set[str]):\n",
    "        SynapticIntelligencePlugin.extract_weights(\n",
    "            model, syn_data[\"old_theta\"], excluded_parameters\n",
    "        )\n",
    "\n",
    "    @staticmethod\n",
    "    @torch.no_grad()\n",
    "    def post_update(\n",
    "        model, syn_data: SynDataType, excluded_parameters: Set[str]\n",
    "    ):\n",
    "        SynapticIntelligencePlugin.extract_weights(\n",
    "            model, syn_data[\"new_theta\"], excluded_parameters\n",
    "        )\n",
    "        SynapticIntelligencePlugin.extract_grad(\n",
    "            model, syn_data[\"grad\"], excluded_parameters\n",
    "        )\n",
    "\n",
    "        for param_name in syn_data[\"trajectory\"]:\n",
    "            syn_data[\"trajectory\"][param_name].data += syn_data[\"grad\"][\n",
    "                param_name\n",
    "            ].data * (\n",
    "                syn_data[\"new_theta\"][param_name].data\n",
    "                - syn_data[\"old_theta\"][param_name].data\n",
    "            )\n",
    "\n",
    "    @staticmethod\n",
    "    def compute_ewc_loss(\n",
    "        model,\n",
    "        ewc_data: EwcDataType,\n",
    "        excluded_parameters: Set[str],\n",
    "        device,\n",
    "        lambd=0.0,\n",
    "    ):\n",
    "        params = SynapticIntelligencePlugin.allowed_parameters(\n",
    "            model, excluded_parameters\n",
    "        )\n",
    "\n",
    "        loss = None\n",
    "        for name, param in params:\n",
    "            weights = param.to(device).flatten()  # Flat, not detached\n",
    "            ewc_data0 = ewc_data[0][name].data.to(device)  # Flat, detached\n",
    "            ewc_data1 = ewc_data[1][name].data.to(device)  # Flat, detached\n",
    "            syn_loss: Tensor = torch.dot(\n",
    "                ewc_data1, (weights - ewc_data0) ** 2\n",
    "            ) * (lambd / 2)\n",
    "\n",
    "            if loss is None:\n",
    "                loss = syn_loss\n",
    "            else:\n",
    "                loss += syn_loss\n",
    "\n",
    "        return loss\n",
    "\n",
    "    @staticmethod\n",
    "    @torch.no_grad()\n",
    "    def update_ewc_data(\n",
    "        net,\n",
    "        ewc_data: EwcDataType,\n",
    "        syn_data: SynDataType,\n",
    "        clip_to: float,\n",
    "        excluded_parameters: Set[str],\n",
    "        c=0.0015,\n",
    "        eps: float = 0.0000001,\n",
    "    ):\n",
    "        SynapticIntelligencePlugin.extract_weights(\n",
    "            net, syn_data[\"new_theta\"], excluded_parameters\n",
    "        )\n",
    "\n",
    "        for param_name in syn_data[\"cum_trajectory\"]:\n",
    "            syn_data[\"cum_trajectory\"][param_name].data += (\n",
    "                c\n",
    "                * syn_data[\"trajectory\"][param_name].data\n",
    "                / (\n",
    "                    np.square(\n",
    "                        syn_data[\"new_theta\"][param_name].data\n",
    "                        - ewc_data[0][param_name].data\n",
    "                    )\n",
    "                    + eps\n",
    "                )\n",
    "            )\n",
    "\n",
    "        for param_name in syn_data[\"cum_trajectory\"]:\n",
    "            ewc_data[1][param_name].data = torch.empty_like(\n",
    "                syn_data[\"cum_trajectory\"][param_name].data\n",
    "            ).copy_(-syn_data[\"cum_trajectory\"][param_name].data)\n",
    "\n",
    "        # change sign here because the Ewc regularization\n",
    "        # in Caffe (theta - thetaold) is inverted w.r.t. syn equation [4]\n",
    "        # (thetaold - theta)\n",
    "        for param_name in ewc_data[1]:\n",
    "            ewc_data[1][param_name].data = torch.clamp(\n",
    "                ewc_data[1][param_name].data, max=clip_to\n",
    "            )\n",
    "            ewc_data[0][param_name].data = \\\n",
    "                syn_data[\"new_theta\"][param_name].data.clone()\n",
    "\n",
    "    @staticmethod\n",
    "    def explode_excluded_parameters(excluded: Set[str]) -> Set[str]:\n",
    "        \"\"\"\n",
    "        Explodes a list of excluded parameters by adding a generic final \".*\"\n",
    "        wildcard at its end.\n",
    "\n",
    "        :param excluded: The original set of excluded parameters.\n",
    "\n",
    "        :return: The set of excluded parameters in which \".*\" patterns have been\n",
    "            added.\n",
    "        \"\"\"\n",
    "        result = set()\n",
    "        for x in excluded:\n",
    "            result.add(x)\n",
    "            if not x.endswith(\"*\"):\n",
    "                result.add(x + \".*\")\n",
    "        return result\n",
    "\n",
    "    @staticmethod\n",
    "    def not_excluded_parameters(\n",
    "        model: Module, excluded_parameters: Set[str]\n",
    "    ) -> Sequence[Tuple[str, Tensor]]:\n",
    "        # Add wildcards \".*\" to all excluded parameter names\n",
    "        result: List[Tuple[str, Tensor]] = []\n",
    "        excluded_parameters = (\n",
    "            SynapticIntelligencePlugin.explode_excluded_parameters(\n",
    "                excluded_parameters\n",
    "            )\n",
    "        )\n",
    "        layers_params = get_layers_and_params(model)\n",
    "\n",
    "        for lp in layers_params:\n",
    "            if isinstance(lp.layer, _NormBase):\n",
    "                # Exclude batch norm parameters\n",
    "                excluded_parameters.add(lp.parameter_name)\n",
    "\n",
    "        for name, param in model.named_parameters():\n",
    "            accepted = True\n",
    "            for exclusion_pattern in excluded_parameters:\n",
    "                if fnmatch(name, exclusion_pattern):\n",
    "                    accepted = False\n",
    "                    break\n",
    "\n",
    "            if accepted:\n",
    "                result.append((name, param))\n",
    "\n",
    "        return result\n",
    "\n",
    "    @staticmethod\n",
    "    def allowed_parameters(\n",
    "        model: Module, excluded_parameters: Set[str]\n",
    "    ) -> List[Tuple[str, Tensor]]:\n",
    "\n",
    "        allow_list = SynapticIntelligencePlugin.not_excluded_parameters(\n",
    "            model, excluded_parameters\n",
    "        )\n",
    "\n",
    "        result = []\n",
    "        for name, param in allow_list:\n",
    "            if param.requires_grad:\n",
    "                result.append((name, param))\n",
    "\n",
    "        return result\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting experiment...\n",
      "Start of experience:  0\n",
      "Current Classes:  [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
      "-- >> Start of training phase << --\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 469/469 [00:21<00:00, 21.39it/s]\n",
      "Epoch 0 ended.\n",
      "\tLoss_Epoch/train_phase/train_stream/Task000 = 0.8542\n",
      "\tTop1_Acc_Epoch/train_phase/train_stream/Task000 = 0.7657\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 469/469 [00:22<00:00, 21.16it/s]\n",
      "Epoch 1 ended.\n",
      "\tLoss_Epoch/train_phase/train_stream/Task000 = 0.4157\n",
      "\tTop1_Acc_Epoch/train_phase/train_stream/Task000 = 0.8808\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 469/469 [00:23<00:00, 20.05it/s]\n",
      "Epoch 2 ended.\n",
      "\tLoss_Epoch/train_phase/train_stream/Task000 = 0.3462\n",
      "\tTop1_Acc_Epoch/train_phase/train_stream/Task000 = 0.8985\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 469/469 [00:25<00:00, 18.62it/s]\n",
      "Epoch 3 ended.\n",
      "\tLoss_Epoch/train_phase/train_stream/Task000 = 0.3102\n",
      "\tTop1_Acc_Epoch/train_phase/train_stream/Task000 = 0.9095\n",
      "-- >> End of training phase << --\n",
      "Training completed\n",
      "Computing accuracy on the whole test set\n",
      "-- >> Start of eval phase << --\n",
      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:02<00:00, 26.64it/s]\n",
      "> Eval on experience 0 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp000 = 0.2454\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp000 = 0.9301\n",
      "-- Starting eval on experience 1 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:02<00:00, 26.50it/s]\n",
      "> Eval on experience 1 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp001 = 2.3530\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp001 = 0.1091\n",
      "-- Starting eval on experience 2 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:02<00:00, 29.14it/s]\n",
      "> Eval on experience 2 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp002 = 2.5626\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp002 = 0.1030\n",
      "-- Starting eval on experience 3 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:02<00:00, 28.08it/s]\n",
      "> Eval on experience 3 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp003 = 2.4715\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp003 = 0.0849\n",
      "-- Starting eval on experience 4 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:02<00:00, 30.44it/s]\n",
      "> Eval on experience 4 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp004 = 2.5565\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp004 = 0.0907\n",
      "-- >> End of eval phase << --\n",
      "\tLoss_Stream/eval_phase/test_stream/Task000 = 2.0378\n",
      "\tTop1_Acc_Stream/eval_phase/test_stream/Task000 = 0.2636\n",
      "Start of experience:  1\n",
      "Current Classes:  [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
      "-- >> Start of training phase << --\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 469/469 [00:23<00:00, 20.16it/s]\n",
      "Epoch 0 ended.\n",
      "\tLoss_Epoch/train_phase/train_stream/Task000 = 0.6907\n",
      "\tTop1_Acc_Epoch/train_phase/train_stream/Task000 = 0.7926\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 469/469 [00:21<00:00, 21.63it/s]\n",
      "Epoch 1 ended.\n",
      "\tLoss_Epoch/train_phase/train_stream/Task000 = 0.3723\n",
      "\tTop1_Acc_Epoch/train_phase/train_stream/Task000 = 0.8897\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 469/469 [00:21<00:00, 21.81it/s]\n",
      "Epoch 2 ended.\n",
      "\tLoss_Epoch/train_phase/train_stream/Task000 = 0.3111\n",
      "\tTop1_Acc_Epoch/train_phase/train_stream/Task000 = 0.9091\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 469/469 [00:23<00:00, 19.92it/s]\n",
      "Epoch 3 ended.\n",
      "\tLoss_Epoch/train_phase/train_stream/Task000 = 0.2770\n",
      "\tTop1_Acc_Epoch/train_phase/train_stream/Task000 = 0.9191\n",
      "-- >> End of training phase << --\n",
      "Training completed\n",
      "Computing accuracy on the whole test set\n",
      "-- >> Start of eval phase << --\n",
      "-- Starting eval on experience 0 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:03<00:00, 25.88it/s]\n",
      "> Eval on experience 0 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp000 = 0.2766\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp000 = 0.9184\n",
      "-- Starting eval on experience 1 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:03<00:00, 25.77it/s]\n",
      "> Eval on experience 1 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp001 = 0.2168\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp001 = 0.9387\n",
      "-- Starting eval on experience 2 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:03<00:00, 24.54it/s]\n",
      "> Eval on experience 2 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp002 = 2.7838\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp002 = 0.0981\n",
      "-- Starting eval on experience 3 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:02<00:00, 26.80it/s]\n",
      "> Eval on experience 3 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp003 = 2.9593\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp003 = 0.0752\n",
      "-- Starting eval on experience 4 (Task 0) from test stream --\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 79/79 [00:02<00:00, 26.59it/s]\n",
      "> Eval on experience 4 (Task 0) from test stream ended.\n",
      "\tLoss_Exp/eval_phase/test_stream/Task000/Exp004 = 3.0098\n",
      "\tTop1_Acc_Exp/eval_phase/test_stream/Task000/Exp004 = 0.0721\n",
      "-- >> End of eval phase << --\n",
      "\tLoss_Stream/eval_phase/test_stream/Task000 = 1.8492\n",
      "\tTop1_Acc_Stream/eval_phase/test_stream/Task000 = 0.4205\n",
      "Start of experience:  2\n",
      "Current Classes:  [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
      "-- >> Start of training phase << --\n",
      "  6%|█████▏                                                                           | 30/469 [00:01<00:25, 17.37it/s]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[16], line 21\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mStart of experience: \u001b[39m\u001b[38;5;124m\"\u001b[39m, experience\u001b[38;5;241m.\u001b[39mcurrent_experience)\n\u001b[0;32m     19\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCurrent Classes: \u001b[39m\u001b[38;5;124m\"\u001b[39m, experience\u001b[38;5;241m.\u001b[39mclasses_in_this_experience)\n\u001b[1;32m---> 21\u001b[0m \u001b[43mcl_strategy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexperience\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     22\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTraining completed\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m     24\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mComputing accuracy on the whole test set\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\training\\templates\\base_sgd.py:146\u001b[0m, in \u001b[0;36mBaseSGDTemplate.train\u001b[1;34m(self, experiences, eval_streams, **kwargs)\u001b[0m\n\u001b[0;32m    139\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtrain\u001b[39m(\u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    140\u001b[0m           experiences: Union[CLExperience,\n\u001b[0;32m    141\u001b[0m                              ExpSequence],\n\u001b[0;32m    142\u001b[0m           eval_streams: Optional[Sequence[Union[CLExperience,\n\u001b[0;32m    143\u001b[0m                                                 ExpSequence]]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m    144\u001b[0m           \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m--> 146\u001b[0m     \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mtrain(experiences, eval_streams, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    147\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mevaluator\u001b[38;5;241m.\u001b[39mget_last_metrics()\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\training\\templates\\base.py:116\u001b[0m, in \u001b[0;36mBaseTemplate.train\u001b[1;34m(self, experiences, eval_streams, **kwargs)\u001b[0m\n\u001b[0;32m    114\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperience \u001b[38;5;129;01min\u001b[39;00m experiences:\n\u001b[0;32m    115\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_before_training_exp(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m--> 116\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_train_exp(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperience, eval_streams, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    117\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_after_training_exp(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    118\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_after_training(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\training\\templates\\base_sgd.py:264\u001b[0m, in \u001b[0;36mBaseSGDTemplate._train_exp\u001b[1;34m(self, experience, eval_streams, **kwargs)\u001b[0m\n\u001b[0;32m    261\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stop_training \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m    262\u001b[0m     \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m--> 264\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining_epoch(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    265\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_after_training_epoch(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\training\\templates\\update_type\\sgd_update.py:9\u001b[0m, in \u001b[0;36mSGDUpdate.training_epoch\u001b[1;34m(self, **kwargs)\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtraining_epoch\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m      4\u001b[0m     \u001b[38;5;124;03m\"\"\"Training epoch.\u001b[39;00m\n\u001b[0;32m      5\u001b[0m \n\u001b[0;32m      6\u001b[0m \u001b[38;5;124;03m    :param kwargs:\u001b[39;00m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;124;03m    :return:\u001b[39;00m\n\u001b[0;32m      8\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m----> 9\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmbatch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataloader:\n\u001b[0;32m     10\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stop_training:\n\u001b[0;32m     11\u001b[0m             \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\benchmarks\\utils\\data_loader.py:104\u001b[0m, in \u001b[0;36mTaskBalancedDataLoader.__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    103\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__iter__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m--> 104\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m el \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dl\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__iter__\u001b[39m():\n\u001b[0;32m    105\u001b[0m         \u001b[38;5;28;01myield\u001b[39;00m el\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\benchmarks\\utils\\data_loader.py:216\u001b[0m, in \u001b[0;36mGroupBalancedDataLoader.__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    212\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m tid, (t_loader, t_loader_sampler) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(\n\u001b[0;32m    213\u001b[0m     \u001b[38;5;28mzip\u001b[39m(iter_dataloaders, samplers)\n\u001b[0;32m    214\u001b[0m ):\n\u001b[0;32m    215\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 216\u001b[0m         batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mt_loader\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    217\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[0;32m    218\u001b[0m         \u001b[38;5;66;03m# StopIteration is thrown if dataset ends.\u001b[39;00m\n\u001b[0;32m    219\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moversample_small_groups:\n\u001b[0;32m    220\u001b[0m             \u001b[38;5;66;03m# reinitialize data loader\u001b[39;00m\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\torch\\utils\\data\\dataloader.py:634\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    631\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    632\u001b[0m     \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[0;32m    633\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset()  \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[1;32m--> 634\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    635\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m    636\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[0;32m    637\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[0;32m    638\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called:\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\torch\\utils\\data\\dataloader.py:678\u001b[0m, in \u001b[0;36m_SingleProcessDataLoaderIter._next_data\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    676\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_next_data\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    677\u001b[0m     index \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_next_index()  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[1;32m--> 678\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset_fetcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[0;32m    679\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory:\n\u001b[0;32m    680\u001b[0m         data \u001b[38;5;241m=\u001b[39m _utils\u001b[38;5;241m.\u001b[39mpin_memory\u001b[38;5;241m.\u001b[39mpin_memory(data, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory_device)\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\torch\\utils\\data\\_utils\\fetch.py:51\u001b[0m, in \u001b[0;36m_MapDatasetFetcher.fetch\u001b[1;34m(self, possibly_batched_index)\u001b[0m\n\u001b[0;32m     49\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39m__getitems__(possibly_batched_index)\n\u001b[0;32m     50\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 51\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[idx] \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[0;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m     53\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\torch\\utils\\data\\_utils\\fetch.py:51\u001b[0m, in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m     49\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39m__getitems__(possibly_batched_index)\n\u001b[0;32m     50\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 51\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdataset\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[0;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m     53\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\benchmarks\\utils\\data.py:278\u001b[0m, in \u001b[0;36mAvalancheDataset.__getitem__\u001b[1;34m(self, idx)\u001b[0m\n\u001b[0;32m    277\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__getitem__\u001b[39m(\u001b[38;5;28mself\u001b[39m, idx) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[T_co, Sequence[T_co]]:\n\u001b[1;32m--> 278\u001b[0m     elem \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem_recursive_call\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    279\u001b[0m \u001b[43m        \u001b[49m\u001b[43midx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_transform_groups\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcurrent_group\u001b[49m\n\u001b[0;32m    280\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    281\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m da \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data_attributes\u001b[38;5;241m.\u001b[39mvalues():\n\u001b[0;32m    282\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m da\u001b[38;5;241m.\u001b[39muse_in_getitem:\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\benchmarks\\utils\\data.py:265\u001b[0m, in \u001b[0;36mAvalancheDataset._getitem_recursive_call\u001b[1;34m(self, idx, group_name)\u001b[0m\n\u001b[0;32m    263\u001b[0m dd \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_datasets[dataset_idx]\n\u001b[0;32m    264\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(dd, AvalancheDataset):\n\u001b[1;32m--> 265\u001b[0m     element \u001b[38;5;241m=\u001b[39m \u001b[43mdd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem_recursive_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43midx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroup_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup_name\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    266\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    267\u001b[0m     element \u001b[38;5;241m=\u001b[39m dd[idx]\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\benchmarks\\utils\\data.py:265\u001b[0m, in \u001b[0;36mAvalancheDataset._getitem_recursive_call\u001b[1;34m(self, idx, group_name)\u001b[0m\n\u001b[0;32m    263\u001b[0m dd \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_datasets[dataset_idx]\n\u001b[0;32m    264\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(dd, AvalancheDataset):\n\u001b[1;32m--> 265\u001b[0m     element \u001b[38;5;241m=\u001b[39m \u001b[43mdd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem_recursive_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43midx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroup_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup_name\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    266\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    267\u001b[0m     element \u001b[38;5;241m=\u001b[39m dd[idx]\n",
      "    \u001b[1;31m[... skipping similar frames: AvalancheDataset._getitem_recursive_call at line 265 (4 times)]\u001b[0m\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\benchmarks\\utils\\data.py:265\u001b[0m, in \u001b[0;36mAvalancheDataset._getitem_recursive_call\u001b[1;34m(self, idx, group_name)\u001b[0m\n\u001b[0;32m    263\u001b[0m dd \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_datasets[dataset_idx]\n\u001b[0;32m    264\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(dd, AvalancheDataset):\n\u001b[1;32m--> 265\u001b[0m     element \u001b[38;5;241m=\u001b[39m \u001b[43mdd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem_recursive_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43midx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroup_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup_name\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    266\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    267\u001b[0m     element \u001b[38;5;241m=\u001b[39m dd[idx]\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\benchmarks\\utils\\data.py:267\u001b[0m, in \u001b[0;36mAvalancheDataset._getitem_recursive_call\u001b[1;34m(self, idx, group_name)\u001b[0m\n\u001b[0;32m    265\u001b[0m     element \u001b[38;5;241m=\u001b[39m dd\u001b[38;5;241m.\u001b[39m_getitem_recursive_call(idx, group_name\u001b[38;5;241m=\u001b[39mgroup_name)\n\u001b[0;32m    266\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 267\u001b[0m     element \u001b[38;5;241m=\u001b[39m \u001b[43mdd\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m\n\u001b[0;32m    269\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_frozen_transform_groups \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    270\u001b[0m     element \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_frozen_transform_groups(\n\u001b[0;32m    271\u001b[0m         element, group_name\u001b[38;5;241m=\u001b[39mgroup_name\n\u001b[0;32m    272\u001b[0m     )\n",
      "File \u001b[1;32m~\\Anaconda3\\envs\\avl031\\lib\\site-packages\\avalanche\\benchmarks\\datasets\\external_datasets\\mnist.py:15\u001b[0m, in \u001b[0;36mTensorMNIST.__getitem__\u001b[1;34m(self, index)\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__getitem__\u001b[39m(\u001b[38;5;28mself\u001b[39m, index: \u001b[38;5;28mint\u001b[39m):\n\u001b[0;32m      8\u001b[0m     \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m      9\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m     10\u001b[0m \u001b[38;5;124;03m        index (int): Index\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;124;03m        tuple: (image, target) where target is index of the target class.\u001b[39;00m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m---> 15\u001b[0m     img \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m[\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39munsqueeze(\u001b[38;5;241m0\u001b[39m) \u001b[38;5;241m/\u001b[39m \u001b[38;5;241m255.\u001b[39m\n\u001b[0;32m     16\u001b[0m     target \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mint\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtargets[index])\n\u001b[0;32m     18\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtransform \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from avalanche.training import Naive\n",
    "\n",
    "device='cpu'\n",
    "\n",
    "cl_strategy = Naive(\n",
    "    model, optimizer, criterion,\n",
    "    train_mb_size=128,\n",
    "    train_epochs=4,\n",
    "    eval_mb_size=128,\n",
    "    device=device,\n",
    "    plugins=[SynapticIntelligencePlugin(si_lambda=0.0001)]\n",
    ")\n",
    "\n",
    "# TRAINING LOOP\n",
    "print(\"Starting experiment...\")\n",
    "results = []\n",
    "for experience in benchmark.train_stream:\n",
    "    print(\"Start of experience: \", experience.current_experience)\n",
    "    print(\"Current Classes: \", experience.classes_in_this_experience)\n",
    "\n",
    "    cl_strategy.train(experience)\n",
    "    print(\"Training completed\")\n",
    "\n",
    "    print(\"Computing accuracy on the whole test set\")\n",
    "    results.append(cl_strategy.eval(benchmark.test_stream))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning without Forgetting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "from collections import defaultdict\n",
    "from typing import List\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from avalanche.models import MultiTaskModule, avalanche_forward\n",
    "from avalanche.training.regularization import RegularizationMethod\n",
    "\n",
    "\n",
    "def cross_entropy_with_oh_targets(outputs, targets, eps=1e-5):\n",
    "    \"\"\" Calculates cross-entropy with temperature scaling, \n",
    "    targets can also be soft targets but they must sum to 1 \"\"\"\n",
    "    outputs = torch.nn.functional.softmax(outputs, dim=1)\n",
    "    ce = -(targets * outputs.log()).sum(1)\n",
    "    ce = ce.mean()\n",
    "    return ce\n",
    "\n",
    "\n",
    "class LearningWithoutForgetting(RegularizationMethod):\n",
    "    \"\"\"Learning Without Forgetting.\n",
    "    The method applies knowledge distilllation to mitigate forgetting.\n",
    "    The teacher is the model checkpoint after the last experience.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, alpha=1, temperature=2):\n",
    "        \"\"\"\n",
    "        :param alpha: distillation hyperparameter. It can be either a float\n",
    "                number or a list containing alpha for each experience.\n",
    "        :param temperature: softmax temperature for distillation\n",
    "        \"\"\"\n",
    "        self.alpha = alpha\n",
    "        self.temperature = temperature\n",
    "        self.prev_model = None\n",
    "        self.expcount = 0\n",
    "        # count number of experiences (used to increase alpha)\n",
    "        self.prev_classes_by_task = defaultdict(set)\n",
    "        \"\"\" In Avalanche, targets of different experiences are not ordered. \n",
    "        As a result, some units may be allocated even though their \n",
    "        corresponding class has never been seen by the model.\n",
    "        Knowledge distillation uses only units corresponding\n",
    "        to old classes. \n",
    "        \"\"\"\n",
    "\n",
    "    def _distillation_loss(self, out, prev_out, active_units):\n",
    "        \"\"\"Compute distillation loss between output of the current model and\n",
    "        and output of the previous (saved) model.\n",
    "        \"\"\"\n",
    "        # we compute the loss only on the previously active units.\n",
    "        au = list(active_units)\n",
    "\n",
    "        # some people use the crossentropy instead of the KL\n",
    "        # They are equivalent. We compute \n",
    "        # kl_div(log_p_curr, p_prev) = p_prev * (log (p_prev / p_curr)) = \n",
    "        #   p_prev * log(p_prev) - p_prev * log(p_curr).\n",
    "        # Now, the first term is constant (we don't optimize the teacher), \n",
    "        # so optimizing the crossentropy and kl-div are equivalent.\n",
    "        log_p = torch.log_softmax(out[:, au] / self.temperature, dim=1)\n",
    "        q = torch.softmax(prev_out[:, au] / self.temperature, dim=1)\n",
    "        res = torch.nn.functional.kl_div(log_p, q, reduction=\"batchmean\")\n",
    "        return res\n",
    "\n",
    "    def _lwf_penalty(self, out, x, curr_model):\n",
    "        \"\"\"\n",
    "        Compute weighted distillation loss.\n",
    "        \"\"\"\n",
    "        if self.prev_model is None:\n",
    "            return 0\n",
    "        else:\n",
    "            if isinstance(self.prev_model, MultiTaskModule):\n",
    "                # output from previous output heads.\n",
    "                with torch.no_grad():\n",
    "                    y_prev = avalanche_forward(self.prev_model, x, None)\n",
    "                y_prev = {k: v for k, v in y_prev.items()}\n",
    "                # in a multitask scenario we need to compute the output\n",
    "                # from all the heads, so we need to call forward again.\n",
    "                # TODO: can we avoid this?\n",
    "                y_curr = avalanche_forward(curr_model, x, None)\n",
    "                y_curr = {k: v for k, v in y_curr.items()}\n",
    "            else:  # no task labels. Single task LwF\n",
    "                with torch.no_grad():\n",
    "                    y_prev = {0: self.prev_model(x)}\n",
    "                y_curr = {0: out}\n",
    "\n",
    "            dist_loss = 0\n",
    "            for task_id in y_prev.keys():\n",
    "                # compute kd only for previous heads and only for seen units.\n",
    "                if task_id in self.prev_classes_by_task:\n",
    "                    yp = y_prev[task_id]\n",
    "                    yc = y_curr[task_id]\n",
    "                    au = self.prev_classes_by_task[task_id]\n",
    "                    dist_loss += self._distillation_loss(yc, yp, au)\n",
    "            return dist_loss\n",
    "\n",
    "    def __call__(self, mb_x, mb_pred, model):\n",
    "        \"\"\"\n",
    "        Add distillation loss\n",
    "        \"\"\"\n",
    "        alpha = (\n",
    "            self.alpha[self.expcount]\n",
    "            if isinstance(self.alpha, (list, tuple))\n",
    "            else self.alpha\n",
    "        )\n",
    "        return alpha * self._lwf_penalty(mb_pred, mb_x, model)\n",
    "\n",
    "    def update(self, experience, model):\n",
    "        \"\"\"Save a copy of the model after each experience and\n",
    "        update self.prev_classes to include the newly learned classes.\n",
    "        :param experience: current experience\n",
    "        :param model: current model\n",
    "        \"\"\"\n",
    "\n",
    "        self.expcount += 1\n",
    "        self.prev_model = copy.deepcopy(model)\n",
    "        task_ids = experience.dataset.targets_task_labels.uniques\n",
    "\n",
    "        for task_id in task_ids:\n",
    "            task_data = experience.dataset.task_set[task_id]\n",
    "            pc = set(task_data.targets.uniques)\n",
    "\n",
    "            if task_id not in self.prev_classes_by_task:\n",
    "                self.prev_classes_by_task[task_id] = pc\n",
    "            else:\n",
    "                self.prev_classes_by_task[task_id] = self.prev_classes_by_task[\n",
    "                    task_id\n",
    "                ].union(pc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n",
    "\n",
    "we use a custom loop here instead of Avalanche `Naive`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experience (0)\n",
      "Train Epoch: 0 \tLoss: 0.528850\n",
      "Train Epoch: 1 \tLoss: 0.409638\n",
      "Experience (1)\n",
      "Train Epoch: 0 \tLoss: 0.966667\n",
      "Train Epoch: 1 \tLoss: 0.897288\n",
      "Experience (2)\n",
      "Train Epoch: 0 \tLoss: 0.986454\n",
      "Train Epoch: 1 \tLoss: 0.922763\n",
      "Experience (3)\n",
      "Train Epoch: 0 \tLoss: 0.933775\n",
      "Train Epoch: 1 \tLoss: 0.869438\n",
      "Experience (4)\n",
      "Train Epoch: 0 \tLoss: 0.972159\n",
      "Train Epoch: 1 \tLoss: 0.892220\n"
     ]
    }
   ],
   "source": [
    "from avalanche.models.utils import avalanche_model_adaptation\n",
    "from avalanche.models.dynamic_optimizers import reset_optimizer\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "model = SimpleMLP()\n",
    "optimizer = SGD(model.parameters(), lr=0.001, momentum=0.9)\n",
    "criterion = CrossEntropyLoss()\n",
    "lwf = LearningWithoutForgetting(alpha=1, temperature=2)\n",
    "\n",
    "device='cpu'\n",
    "num_epochs = 2\n",
    "\n",
    "for exp in benchmark.train_stream:\n",
    "    print(f\"Experience ({exp.current_experience})\")\n",
    "    model.train()\n",
    "    avalanche_model_adaptation(model, exp)\n",
    "    reset_optimizer(optimizer, model)   \n",
    "    dataset = exp.dataset\n",
    "    dataset = dataset.train()\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        dl = DataLoader(dataset, batch_size=128)\n",
    "        for x, y, t in dl:\n",
    "          x, y, t = x.to(device), y.to(device), t.to(device)\n",
    "\n",
    "          optimizer.zero_grad()\n",
    "          output = model(x)\n",
    "          loss = F.cross_entropy(output, y)\n",
    "          # ADD LWF loss\n",
    "          loss += lwf(x, output, model)\n",
    "          loss.backward()\n",
    "          optimizer.step()\n",
    "        print('Train Epoch: {} \\tLoss: {:.6f}'.format(epoch, loss.item()))\n",
    "    \n",
    "    # UPDATE LWF TEACHER\n",
    "    lwf.update(exp, model)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercises\n",
    "- test LwF in a task-aware scenario\n",
    "- test \"Stable SGD\": use a high learning rate on the first experience and decrease over time lr and batch size for the following experiences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}