Project3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "import gym\n",
    "import numpy as np\n",
    "import time\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Categorical\n",
    "from scipy.special import softmax\n",
    "from multiprocessing_env import SubprocVecEnv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Actor Critic Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.normal_(m.weight, mean=0., std=0.1)\n",
    "        nn.init.constant_(m.bias, 0.1)\n",
    "        \n",
    "\n",
    "class AC_Network(nn.Module):\n",
    "    def __init__(self, imput_dims, output_dims, hidden_layer_1, hidden_layer_2):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.critic = nn.Sequential(nn.Linear(imput_dims, hidden_layer_1),\n",
    "                                    nn.ReLU(),\n",
    "                                    nn.Linear(hidden_layer_1, hidden_layer_2),\n",
    "                                    nn.ReLU(),\n",
    "                                    nn.Linear(hidden_layer_2, 1))\n",
    "        \n",
    "        self.actor = nn.Sequential(nn.Linear(imput_dims, hidden_layer_1),\n",
    "                                    nn.ReLU(),\n",
    "                                    nn.Linear(hidden_layer_1, hidden_layer_2),\n",
    "                                    nn.ReLU(),\n",
    "                                    nn.Linear(hidden_layer_2, output_dims))        \n",
    "        self.apply(init_weights)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        state_value = self.critic(x)\n",
    "        action_vector = self.actor(x)\n",
    "        action_probs = softmax(action_vector.cpu().detach().numpy())\n",
    "        np.clip(action_probs,1e-10,1,out=action_probs)\n",
    "        action_distribution  = Categorical(torch.FloatTensor(action_probs).to(device))\n",
    "        return action_distribution, state_value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computing Generalized Advantage Estimation (for smoother training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_adv_est(next_value, rewards, masks, values, gamma=0.99, tau=0.95):\n",
    "    values = values + [next_value]\n",
    "    gae = 0\n",
    "    returns = []\n",
    "    for step in reversed(range(len(rewards))):\n",
    "        delta = rewards[step] + gamma * values[step + 1] * masks[step] - values[step]\n",
    "        gae = delta + gamma * tau * masks[step] * gae\n",
    "        returns.insert(0, gae + values[step])\n",
    "    return returns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utility Funtions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(rewards, avg_reward):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.plot(rewards,label = 'Reward Per Test Session')\n",
    "    plt.plot(avg_reward,label = \"Average Reward over all Tests\")\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_env(vis=False):\n",
    "    state = main_env.reset()\n",
    "    done = False\n",
    "    average_reward_current_policy = 0\n",
    "    while (not done):\n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        action_distribution, _ = model(state.view(1,-1))\n",
    "        next_state, reward, done, _ = main_env.step(action_distribution.sample().cpu().numpy()[0])\n",
    "        state = next_state\n",
    "        if vis: \n",
    "            main_env.render()\n",
    "        average_reward_current_policy += reward\n",
    "    return average_reward_current_policy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Proximal Policy Optimization Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ppo_step(mini_batch_size, states, actions, log_probs, returns, advantage):\n",
    "    batch_size = states.size(0)\n",
    "    for _ in range(batch_size // mini_batch_size):\n",
    "        rand_ids = np.random.randint(0, batch_size, mini_batch_size)\n",
    "        yield states[rand_ids, :], actions[rand_ids, :], log_probs[rand_ids, :], returns[rand_ids, :], advantage[rand_ids, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ppo_parameter_update(ppo_epochs, mini_batch_size, states, actions, log_probs, returns, advantages, clip_param=0.2):\n",
    "    for _ in range(ppo_epochs):\n",
    "        for state, action, prev_log_probs, return_, advantage in ppo_step(mini_batch_size, states, actions, log_probs, returns, advantages):\n",
    "            action_distribution, state_value = model(state)\n",
    "            entropy = action_distribution.entropy().mean()\n",
    "            new_log_probs = action_distribution.log_prob(action)\n",
    "\n",
    "           \n",
    "            #Ratio of current policy over old policy in logarithmic space\n",
    "            log_ratio = (new_log_probs - prev_log_probs).exp()\n",
    "            \n",
    "            #surrogate losses (for actor)\n",
    "            surrogate_loss_1 = log_ratio * advantage\n",
    "            surrogate_loss_2 = torch.clamp(log_ratio, 1.0 - clip_param, 1.0 + clip_param) * advantage\n",
    "            \n",
    "            #following are the actor and critic network losses which will be combined in the next step.\n",
    "            actor_loss  = - torch.min(surrogate_loss_1, surrogate_loss_2).mean()\n",
    "            critic_mse_loss = (return_ - state_value).pow(2).mean()\n",
    "\n",
    "            #Combined loss function to update both actor and critic network\n",
    "            combined_loss = 0.5 * critic_mse_loss + actor_loss - 0.001 * entropy\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            combined_loss.backward()\n",
    "            optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Function."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## After every 500 steps of training, we test the policy on the actual environment and record rewards."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def start_learning(envs, max_steps, curr_step, test_reards, avg_reward, moving_average_over_tests):\n",
    "    state = envs.reset()\n",
    "    while curr_step < max_steps:\n",
    "        log_probs = []\n",
    "        values = []\n",
    "        states = []\n",
    "        actions = []\n",
    "        rewards = []\n",
    "        masks = []\n",
    "        entropy = 0\n",
    "        for step in range(steps):\n",
    "            state = torch.FloatTensor(state).to(device)\n",
    "            state = state.view(num_envs,-1)\n",
    "            action_distribution, value = model(state)\n",
    "            action = action_distribution.sample()\n",
    "            next_state, reward, done, _ = envs.step(action.cpu().numpy())\n",
    "\n",
    "            log_prob = action_distribution.log_prob(action)\n",
    "            entropy += action_distribution.entropy().mean()\n",
    "\n",
    "            log_probs.append(log_prob)\n",
    "            values.append(value)\n",
    "            rewards.append(torch.FloatTensor(reward).unsqueeze(1).to(device))\n",
    "            masks.append(torch.FloatTensor(1 - done).unsqueeze(1).to(device))\n",
    "\n",
    "            states.append(state)\n",
    "            actions.append(action)\n",
    "\n",
    "            state = next_state\n",
    "            curr_step += 1\n",
    "\n",
    "            if curr_step % 500 == 0:\n",
    "                test_reward = np.mean([test_env() for _ in range(8)])\n",
    "                test_rewards.append(test_reward)\n",
    "                moving_average_over_tests += ((1/len(test_rewards)) * (test_reward - moving_average_over_tests))\n",
    "                avg_reward.append(moving_average_over_tests)\n",
    "                plot(test_rewards, avg_reward)\n",
    "\n",
    "        next_state = torch.FloatTensor(next_state).to(device)\n",
    "        _, next_value = model(next_state.view(num_envs,-1))\n",
    "        returns = gen_adv_est(next_value, rewards, masks, values)\n",
    "\n",
    "        returns   = torch.cat(returns).detach()\n",
    "        log_probs = torch.cat(log_probs).detach()\n",
    "        values    = torch.cat(values).detach()\n",
    "        states    = torch.cat(states)\n",
    "        actions   = torch.cat(actions)\n",
    "        advantage = returns - values\n",
    "\n",
    "        ppo_parameter_update(ppo_epochs, mini_batch_size, states, actions.unsqueeze(1), log_probs.unsqueeze(1), returns, advantage)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Starting Training ... "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initializing Environment Vectors for Atari Space Invaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_envs = 18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name = \"SpaceInvaders-v0\"\n",
    "def make_env():\n",
    "    def _envs():\n",
    "        env = gym.make(env_name)\n",
    "        return env\n",
    "\n",
    "    return _envs\n",
    "\n",
    "envs = [make_env() for i in range(num_envs)]\n",
    "envs = SubprocVecEnv(envs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_env = gym.make(env_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dims  = envs.observation_space.shape[0]*envs.observation_space.shape[1]*envs.observation_space.shape[2]\n",
    "output_dims = 6\n",
    "hidden_layer_1  = 1024\n",
    "hidden_layer_2  = 256\n",
    "ac_network_alpha = 1e-3\n",
    "steps = 16\n",
    "mini_batch_size = 6\n",
    "ppo_epochs = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AC_Network(input_dims, output_dims, hidden_layer_1, hidden_layer_2).to(device)\n",
    "optimizer = optim.Adam(model.parameters(), lr=ac_network_alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_steps = 8000\n",
    "curr_step  = 0\n",
    "test_rewards = []\n",
    "avg_reward = []\n",
    "moving_average_over_tests = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEvCAYAAABcwZodAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydeZibZbn/P0+WSTL7lrazdm9pp3Tf2EtFZBOURdCDIKgIB88RRVw5x5+c41EBxeMRRRQVFZVVRGQRaaGAtKUtbWlpofvszWyZyUy2SfL8/niTaaazJTNZ3mSez3XlmsybN+8802a+uXM/3/u+hZQShUKhUGQehnQvQKFQKBTjQwm4QqFQZChKwBUKhSJDUQKuUCgUGYoScIVCochQlIArFApFhmJK9wIAysvL5YwZM9K9DIVCodAl27dvb5dS2k8+rgsBnzFjBtu2bUv3MhQKhUKXCCGODXdcpVAUCoUiQ1ECrlAoFBmKEnCFQqHIUJSAKxQKRYaiBFyhUCgyFCXgCoVCkaGMKeBCCKsQYqsQYpcQYq8Q4tvh478RQhwRQuwM35aGjwshxI+FEAeFELuFEMuT/UsoFArFZCQWH7gPWC+l7BVCmIHXhRDPhx+7Q0r5xEnnXwjMDd/WAD8Lf1UoFApFAhkzApcaveFvzeHbaFMgLgN+G37eZqBYCFEx8aUqFAqFIpqYcuBCCKMQYifgAF6SUm4JP/SdcJrkPiGEJXysCmiIenpj+Niko77DzaG23rFPVCgUinEQk4BLKYNSyqVANbBaCLEI+DpwCrAKKAW+Gj5dDHeJkw8IIW4SQmwTQmxra2sb1+L1zjeffocvP74r3ctQKBRZSlwuFCmlE3gFuEBK2RJOk/iAXwOrw6c1AjVRT6sGmoe51oNSypVSypV2+5AeLVnB4bY+WpzedC9DoVBkKbG4UOxCiOLwfRtwHrA/ktcWQgjgI8Ce8FOeAa4Lu1HWAt1SypakrF7H+AMhmrs9tPf6CIXU4GiFQpF4YnGhVAAPCyGMaIL/mJTyWSHEBiGEHS1lshO4OXz+c8BFwEHADdyQ+GXrn2anBykhICVOTz+leTnpXtIQPP4gZqPAZFTlAApFJjKmgEspdwPLhjm+foTzJXDrxJeW2dR3ugfut7l8uhTwC/93E5ctreKLH5yX7qUoFIpxoEKvJHGygOsNb3+Qox1u3jzcke6lKBSKcaIEPEk0RAt4r/42Mtt7tTeVd5t7VI5eochQlIAnifpON1MLNWu8HiNwR3hNvb4Ax6LebBQKReagBDxJ1He6WVBRiNVs0KWAR69pb3N3GleiUCjGixLwJCClpL7DzfTSXOwFFt0L+J6mnjSuRKFQjBddDDXONro9/bh8AWpKc7HnW2jr1Z+AO1w+hID5UwtUBK5QZCgqAk8CEQdKrc4j8LK8HBZXF7G3uQfN/alQKDIJJeBJoKHTA0BtmZ4F3Et5voVFVUV09vlp7dGfU0ahH155z8GT2xvTvQzFSSgBTwKRCLymJBd7vpUudz/9wVCaVzWYNpePKYVW6ioLAZUHV4zOg5sO8+2/7iWoLKe6Qgl4EqjvdFOWl0OexYS9QLMSdvT607yqwThcPuz5FhZUFCKEcqIoRqfJ6aHHG2B/q3qj1xNKwJNAQ6ebmtJcgAEB11MaJRSStPf6mFJoITfHxKzyPBWBK0YkFJIDXTW3HO5M82oU0SgBTwL1nW5qTxZwHVVjOj399Acl9nxtbYuqilQErhiR9l4f/nAKcLNqvaArlIAnmEAwRJPTM1TAdRSBR9YyJVwpuqiyiJZuLx06tDsq0k+TU9uUtxdY2Hq0U7Ve0BFKwBNMS7eXYEgOCHh5vtaFUE8C7nBpnwYiEXhkI3Nvs0qjKIYSEfCPLK3E6e7nfYcrzStSRFACnmAGHChhAbeYjBTZzLoS8BMRuBWAusoiQAm4YniaujQB/+iyakDlwfWEEvAEM1DEU5Y7cMxeoK9qzEgjq0h6pyjXTHWJjT0qD64YhmanhwKriQUVBVQWWdlyROXB9YIS8ATT0OnGbBRMC0e3oKUq9BaB5+YYybec6KSwqLKId1UErhiGJqeHqmIbQgjWzCpj65FOVbmrE5SAJ5j6TjfVJbkYDWLgmN6qMR0u30D0HaGuspAj7X24vP1pWpVCrzQ5vVQV2wBYO6uU9l4/h9p607wqBSgBTzgNnW6qS2yDjulNwNtc3oENzAiLqrQ8+L4WtUGlGExTl5uq8Gt6zcwyADarPLguUAKeYKI94BHsBRb6/EH6fIE0rWowDpdvwEIY4URJvcqDK07g8vbT4w0MRODTy3KZWmhRfnCdoAQ8gfR4++ly9w8V8HC0266Tjcy2cBl9NFMKrdgLLGojUzGI5nAFZmVYwIUQrJlZxhaVB9cFSsATSENUG9lo9FTM4+0P4vIGBiyE0SyqLFQbmYpBNDm113RVVFpwzaxS2lw+jrT3pWtZijBKwBNIw0ke8Ah6EvDIGk6OwEHzgx9w9OLtD6Z6WQqd0hSOwCMpFDiRB99yROXB040S8AQynAccovuhpF/AB6owC4cK+KKqQoIhyXutaiNTodHU5SHHaBj0hj/bnkd5voUtKg+edpSAJ5CGTg/FuWYKreZBx0tyczAaREZE4IDKgysGaHJ6qCi2YoiyxWp+8FKVB9cBYwq4EMIqhNgqhNglhNgrhPh2+PgjQoj3hBB7hBC/EkKYw8fXCSG6hRA7w7f/TPYvoReGc6AAGA2CsrwcXQi446RGVtFUl9gotJpUSb1igGanh8oi25Dja2eW0tLtHZg+pUgPsUTgPmC9lHIJsBS4QAixFngEOAU4FbABn4l6zmtSyqXh212JXrReaeh0U1MyVMABynVSjdnm8mEQUJY3VMCFENRVFrFXWQkVYZq6PIM2MCOsmRX2g+ugrF5KyScf2sIvNh1O91JSzpgCLjUiZVfm8E1KKZ8LPyaBrUB1Etepe4IhSWOXZ8gGZgS99ENx9Pgoy7cMqhSNZlFVIftaXbobAadIPf3BEMdd3gELYTRzp+RTmpejCz/4AUcvrx1o52/vtKR7KSknphy4EMIohNgJOICXpJRboh4zA58EXoh6ymnhlMvzQoi6Ea55kxBimxBiW1tb2wR+BX1wvMeLPxgaNoUC+qnGbOsd6gGPZlFVEf5ASJVKK2jt9iIlVA8j4EIIVs8o1UVnwhf2tALaWMDJ5qCKScCllEEp5VK0KHu1EGJR1MM/BTZJKV8Lf78DmB5Oufwf8PQI13xQSrlSSrnSbreP/zfQCfUjeMAj2AsstPf60t4M3+HyDpv/jqCGHCsiNIbbyA6XQgHND97k9NDY5U7lsobw/J5WckwG+oNy0u3fxOVCkVI6gVeACwCEEN8C7MCXos7piaRcpJTPAWYhRHmiFqxXxhTwfAv9QUm3J73NooarwoxmZnk+NrNRjVhT0Bwe5DBcCgWi/OBpjMKPdfSxr6WHG06fAcDb9V1pW0s6iMWFYhdCFIfv24DzgP1CiM8AHwI+LqUMRZ0/TQghwvdXh39G+hNlSaah043RIKgoHlrhCPrwgmvDjP2jRuBGg2BBRQF7VQQ+6YlM4qkoGv41fcq0Aops5rT2B39xr5Y+uXbtdKpLbOyYZAJuGvsUKoCHhRBGNDF+TEr5rBAiABwD3gzr9VNhx8mVwC3hxz3ANXISmEXrO91UFlsxG4d/T4yuxpw3tSCVSxug0+0nGJKjRuCg5cGf2tFEKCQH+X8Vk4tmp4fyfAtWs3HYxw0GweqZpWmtyHxhTyt1lYXUlOayvLaEt46mPyefSsYUcCnlbmDZMMeHfa6U8ifATya+tMyiYQQPeAQ9lNOfPEptJOoqC/ntm8c41ulmZnleKpam0CFNzuEthNGsmVnKS+8ep7Xby7QRIvVkcbzHy456J18+fx4Ay2uLeWZXMy3dHiqG8a5nI6oSM0HUd3pG9ICDPgT85FFqI3FiRqbKg09mmro8VI2QEoywdlakL0rq0yh/D6dPLlg0DYBltSUAvF3vTPla0oUS8ATg9gdo7/WN6AEHKLCYsJgMac2BD0TgYwj4vKkFmI1COVEmMVLKgVFqo7GgopACqyktfvDn97Qy257HnCkFA2uxmAzsODZ58uBKwBNApJx4tBSKECLtXvCBRlZjCHiOycC8qQUqAp/EdPT58QVCYwq40SBYlQY/eGefny1HOgeib9Bet6dWFfF2g4rAFXEwloUwQroFvM3lI99iIjdn7L3rRZVF7G3uUc2KJiljWQijWTOzlMPtfTh6vMle1gD/2HecYEhyQV3FoOPLp5fwTlM3/sDkqCRWAp4AYhbwNPdDGW6Y8UjUVRXS2eenpTt1f5QK/dA0RhFPNGtmpb4/+It7WqkqtrGoqnDQ8WU1xfgDoUnz6VEJeAJo6HRTYDFRnGse9bx090Npi0fABzYyVR58MhLxgI+VQgFtklNejjFlG5m9vgCvHWjngkXTCFuYB1g+fXJtZCoBTwD1nW5qSnOHvJhOxl5gobPPn7ZGUfEI+IKKAoRQQ44nK01OD3k5RopsowclACajgZUpzINv3O/AHwwNyn9HmFpopap48hT0KAFPACP1AT+ZiHh29PqTvaRhaXP5xnSgRMjNMTHbnq8i8ElKpI3sWEFJhDWzSjng6KUjBZ8wX9jTSnm+heVh2+DJLK0tVhG4IjaklFoRT1kMAp6fPi+42x+g1xeIOQIHraBnsuQSFYNp7vbEtIEZIdIXZWuS8+De/iAb33Nwft3UEVsiL68tocnpSemmarpQAj5B2lw+fIEQNTFs9pzoh5L6F9Zoo9RGYlFlES3d3pREVQp9oRXxxC7gi6uLsJmNSfeDv3agHbc/yAV1Q9MnEZbVFgNMijSKEvAJUj/CJPrhSGc1piPGMvpo6sI7/CqNMrlw+wN0ufvjisDNRgMrppck3Ynywp5WCq2mgQrQ4airLCTHaJgUaRQl4BMkVgshaGPVID0CPp4IvK5CDTmejEQ84NUxfKqMZs3MUva3uujqS84eT38wxD/2Hee8hVPJMY0sXRaTkbqqQhWBK8amvtONELH5Za1mI4VWU3oi8HA+cLRWsidTlGumptSmIvBJxsAghzgicIC1s8N58CR1BNxyuJNuT/+o6ZMIy2tL2N3YnfWjAZWAT5D6TjcVhVYspuFbbp5Murzgbb0+jAZBaW5OXM+rq1BDjicbzU7tzT6eFApoeXCLyZA0O+Hze1qwmY2cPW/sCV7La0vwBULsa8nu4EMJ+ARpCHvAYyVd5fSOHh/l+Tlx9/deVFXI0Q43Lm96JwkpUkeTUxtOMjWO/RLQUhfLa0uSUtATDEle3Hucc0+xj9ifPJqBjcwsb2ylBHyCxOoBj2AvsKYnB94bexFPNJGKzHdVGmXS0NTlYVqhdUSb3mismVXKuy09CR8d+HZ9F+29Pj4UQ/oEtE8P0wqtWd/YSgn4BPD2Bzne44tPwNPUD8XR42NKQfwN95UTZfLR7PTGtKczHGtmliElbEtwHvyFPa3kGA2sP2VKzM9ZVluc9RuZSsAnQGSzJ94USp8/SJ8vkKxlDUtb7+jDjEdiSoEVe4FFOVEmEU1OD9Vx5r8jLKstJsdoSKgfXErJC3tbOXNuOQXWsUv7IyyvLaGh05PWBnLJRgn4BGiIwwMeIZLGaE/hRmYwJOno9cXlQIlmUWWhSqFMEgLBEK093rg3MCNYzUaW1hQn1A++t7mHxi5PTO6TaJZP1/Lg2TypXgn4BIjHAx4hHcU8HX0+QnLsQQ4jsaiqiAOOXrz9wQSvTKE3jrt8BENy3CkUgLWzStnT1J2wje8X9rRiEHDewqlxPa+usgizUbAjiwt6lIBPgPpONzazkfL82K156eiH4uiJbZTaSNRVFhIMSfa3uhK5LIUOifQBH28EDlp/8JCEbQlygLywt5U1M8sozYvPAms1G1lYWaQicMXwRBwosXZsg+h+KKkT8MjPGm8EroYcTx6a4+gDPhLLa0swG0VC/OAHHS4OOnqHbR0bC8tqitnd2E0gSwt6lIBPgHg94ACleTkYRGoj8LaBCDx+FwpoJdVFNrMacjwJiGeQw0jYcowsri5OiB/8xb3HATi/Lr70SYTl00vw9Aez9tOjEvBxIqWM2wMO2hDYshRbCScagQshqKss5F0VgWc9TU4PpXk52HJiqyweiTUzS3mnsRu3f2Juqxf2tLK0ppiKonG6YmqyeyNTCfg46ejz4/YHqS2N/4WVai94m8tHgdUUUwXbSNRVFrKv1ZX1vSUmO/G2kR2JNbPKCIQk2yeQB2/scvNOUzcXjjN9AtqnR3uBJWs7E44p4EIIqxBiqxBilxBirxDi2+HjM4UQW4QQB4QQjwohcsLHLeHvD4Yfn5HcXyE9xNNG9mRS3Q/F4fKOO/qOsKiqCH8gxKG23gStSqFHmpweKovHl2qLZsX0EowGMSE/+At7WgFirr4cDiEEy7O4oCeWCNwHrJdSLgGWAhcIIdYC3wfuk1LOBbqAT4fP/zTQJaWcA9wXPi/raBiHhTBCqvuhxDNKbSTqKrWKTJUHz16klDQ7PVQVx/+aPpl8i4lTq4omtJH54t5WTplWwIzyvAmtZVltCUc73Fk5mGRMAZcakbDLHL5JYD3wRPj4w8BHwvcvC39P+PEPiHhsGhlCRMCrS8Yn4O29PkIhmehlDYvD5cM+zg3MCDPL87GZjWrIcRbjdPfj9gcn5AGPZs2sUnY1OvH4468fcLi8bDvWNW73STSR2Zk7s7AvSkw5cCGEUQixE3AALwGHAKeUMrJD0QhUhe9XAQ0A4ce7gSHjM4QQNwkhtgkhtrW1tU3st0gD9Z1uphRYxrXZY8+30B+UCW/4MxKJiMCNBsFCVZGZ1ZxwoEw8hQKwdmYZ/UE5rg3El949jpQkRMBPrSrCZBBZmUaJScCllEEp5VKgGlgNLBjutPDX4aLtIaGmlPJBKeVKKeVKu33s/r56YzwOlAip9IL3+gK4/cEJ58DhxJDjVH1yUKSWEwI+8RQKwMoZJRgEbB5HWf0Le1qZUZbL/KkFE16HLcfIgorCrNzIjMuFIqV0Aq8Aa4FiIYQp/FA10By+3wjUAIQfLwKSOygvDTR0eiYu4CnIg0d+xkQjcNCGHPf5gxwLp48U2cWJKszEROAFVjN1lUVsiXMjs9vdz5uHOrhgUUVcRXKjsay2mF0NToJZFnzE4kKxCyGKw/dtwHnAPmAjcGX4tOuBv4TvPxP+nvDjG6SUWfWv5g+EaO72jMuBAqkV8MgotURE4AsHNjJVHjwbaXZ6sJoNcZesj8aamaW83eCMq4/OP/YdJxCSCUmfRFheW0KfP8h7WVbQE0sEXgFsFELsBt4CXpJSPgt8FfiSEOIgWo77ofD5DwFl4eNfAr6W+GWnlyanBynH50CBFEfgvROrwoxm3tQCzEaheoNnKU1OzQOeSM/Bmlll+AMhdsWxgfjC3lYqiqwsripK2DoiG5lvN2RXHtw01glSyt3AsmGOH0bLh5983AtclZDV6ZSJeMABCiwmLCZDSnLgkUZWiYjAc0wG5k0tUD1RspRmp2dCTayGY/WMUoSAzYc7WTNriJdhCH2+AJveb+Pjq2vjHv83GjWlNsrycthxzMm/rJmesOumG1WJOQ7G00Y2GiFEyrzgbb0+TAZBsS32RvijsaiyiL3NPWRZVkxBeJBDgiyEEYpyzSyYVhhzX5RX32/DFwhNqHhnOIQQLKstyboIXAn4OGjsdJNjMkxoYzBVAu7o0WZhJiqaWVRVSGefn5Zub0Kup9AH3v4g7b1+KsfZc2Q01swqZUd9F/7A2G0YXtjTSmleDqtmlCR8Hctqiznc1ofT7U/4tdOFEvBxUN/ppqbENiFRTFU/lPEOMx6JheHWsmojM7sYaCOb4AgctDmZ3v4QuxtHz4P7AkE27Hdw/sKpmIyJl6aBPHgW2QmVgI+DiXjAI6SqH4qjx5sQC2GEBRUFGIQacpxtJKKN7EisnlkKMOaYtTcOttPrC/ChBLpPollcXYRBZFdnQiXgcSKlpL4jMQLe2edPene/9gRH4Lk5JmbZ89VGZpYRicATvYkJWg/8+VMLxmxs9cKeVgosJk6fPfZm53jIs5g4ZVphVo1YUwIeJ92efly+wLgdKBEiotrRm7x8XCAYoqPPP+E+KCezqLJQReBZRlOXB4OAaUWJfa1EWDOrlO3HukYMWALBEC+9e5z1C6ZgMU2sF/loLJ9ezM4sKuhRAh4nE3WgREjFbMyOPj9yAsOMR6KusoiWbm9WdnebrDQ6PUwttGJOQu4ZtDy42x8cce9k69FOutz9cU+ej5dlNSX0+gIcdGRHW2Ql4HEyIOBliYnA23qT5+aY6DDjkair0ioyVRSePTQ7EzPIYSTWzNLy4JtHaC/74p5WrGYD58xPbl+k5dMjG5nZkQdXAh4nA0U842gjG00qqjEjbw4Jj8Arwk4UlQfPGpqcnqQ4UCKU51uYMyV/WD94KCR5ce9xzplnJzdnzNrCCTGjLJeSXHPWdCZUAh4nDZ0eyvJyyLPE8EIL+KB/+Ai7PAUplGRF4EW5ZmpKbSoCzxKCIUlrtzcpG5jRrJlZyrajXUMmxO9sdNLa401o75ORiBT0ZMtGphLwOIl5Er2U8Psr4MF1mpCfhNVspNBqSm4EHr525M0ikSyqLGKv8oJnBW0uH/1BmdQUCmh9UXp9Ad5tGfzG/+KeVkwGwfpTxjd5Pl6W1xZz0NGbsn78yUQJeJzE7AE/+jocfQ3a9sEbPx72lGR7wR0uH0U284SGGY9EXWUhRzvc9Hgz/49gstPk1NKCyRbwtRE/eFQeXErJC3tbOX1OOUUJavcwFsuyaEKPEvA4CARDNDlj7AP+2g8gbwrMvxg23QMdh4ackuxy+jZXYj3g0dSFO8XtU2mUjKfJqaX5kpkDB5hSaGVmed6gPPj+VhfHOtxJd59Es6SmGJElBT1KwOOgpdtLMCTHFvCm7XB4I5z+ebj4XjDmwHNf1tIqUdgLrMnNgbsSW4UZzcCQYyXgGc+JQQ7JFXDQ8uBbj3QO+LBf2NOKEPDBhalJn4A2cHn+1IKsyIMrAY+DmNvIvvZDsBbByhuhsBLW3wmHNsDePw86Ldn9UBLdByWaKQVWphRYJm1F5l93NfOh+zZlRQqpyemmyGYmP5aN+QmydlYZPd4A+8J58Bf3trJqRmnSXqcjsay2hLfruzJ+PKAS8DiIyQPu2Af7n4U1N4MlPM9v1WegYgm88HXwnohY7QUW+vxB+nyBES42fqSUOHomPsx4NOoqC9nbNPki8NcOtPGlx3by3nEXB45n/oSXZqc36fnvCBE/+JYjnRxp72N/qyul6ZMIy2qLcXkDHG7P7IIeJeBxUN/pxmwUTCscpdz4tR+COU8T8AhGE1xyH/Qeh43fGTgciTrak7CR6fIF8AVCSY1sFlUVcbCtN65xWZnOO43d3Py77ZTlaf+ujeH0QybT1JX4QQ4jUVFko7Y0ly2HO3hhTytA0ppXjUakM+GOY5mdRlECHgf1nW6qim0YR2oj23kE9jwBK2+A3NLBj1Wt0CLxrQ9C89tAcot5TnjAk9PbArSS+mBIsj/L5gyOxLGOPm74zVaKc3P4001rgewQ8OYkDHIYjTUzS9l6tJPn97SwuLooZdF/NLPK8yiymTN+wIMS8DhoHMsD/sb/gsEEp//b8I+vvxNyy+HZL0IomNR+KJFrJjMCj2xkToY8eJvLx3W/2kowJHn4xtXMKM+jPD+Hxi53upc2ISLN2RI1iT4W1swqw+nuZ3djd8In78SKwSBYWlOsIvDJxKge8J5m2PkILLsWCkZ4UdqK4YLvahH4tl9F9UNJgoD3xlCF6XOB6ziExtfStrrERpHNzJ4sz4P3+gLc8JutHO/x8tCnVjFnSj4AVSW5GR+BRxwoVcUTaw0RD2tmnvh0morqy5FYXlvC+w5XRm9EJ3/bOUvo8fbT5e4fWcDfvB9CQTjjC6NfaNEV8Pbv4OW7KJ1/CQaRrBTKGH1QWvfAby4Cb7dmcyyshMJqKKqGoirta/T31qETwoUQ2kZmFkfg/kCIW36/nX0tLn5x3YqB3Clob2DvZriNMpmTeEaipjSXqmIbuTlGZtvzU/ZzT2ZZbTFSwu6Gbs6cW562dUwEJeAx0jBaG9m+Dtj2Kzj1KiiZMfqFhICLfwg/PQ3jS3dSlv/x5KRQen3kGA3DV7c5G+CRK7XN1nXfAFczdDdBdyMce0P7NCFP2pi0FEJh1RCBv6ggyK+P9dPvW4HZkvpcZjIJhSR3PLGL1w60c/eVi4eUeleX2Hhp73FCIZnQCeqppGlgkEPqUigA9129FKs5vQmApbVaQc+O+i4l4NlOw2ge8C0PQL8bzvxibBcrmw1nfQle+S7n5y+h1TU0up0obeFhxkKcJCzuTq1Hi98NNz4PU+uGPjkY0Bwz3Y3Q06h9jQh8TyM07wC3Vk13LXCtCfjubVrlqbUQTNYTN/NI921gsoAp/NVsG3qe2Qa2Usgrh5x87c0vhfzPc/v4y85m7vjQfD62smbI49UlufiDIdp6fUwdzZmkY5qdHnJMBsrzUuvDXj2zdOyTkkyh1cwce35GV2QqAY+RET3g3h7Y+nNY8GGYckrsFzzjNtj9GP/W8zNudS1K4Eo1hi3i6ffAHz8OXUfg2qeGF2/QbI9FVdqNNcOf43dDTzNNx97nR09t5FN1ZuryXeDrhYBXu/V7tTeMgFf72QEfBMJf+z1AHEUURgvk2SGvTPuaW64Je1551Pfhx3PLIScvfsEPBcHXA94ennxzH7ve2Mv/1BXw8dJe2Lpx4DF8PeBzcV5viJ9xDo1d7owV8Eanh8oia8Z+gpgoy2tLePHdVqSUQ4OdDEAJeIzUd2rVaoXWk1IS2x7S8shnfim+C5qtcPEPmPa7j3Ch84/A+oStFTQb4aA3m1AQnvwMNGyBq34NM8+a2A/IyYXyOUwrnc2zTxvIy6+h7tIR3hCGQ0oI+k8I/YDoRwl9v0d7A3C3Q1+blqrqa9O+b3sP+tq184bDZDsh8BFxtxWDv29AgAfEOPLVf6Ko4wrgCgtwKHyLYDBp6SRrIVNdDv5meZH39plh+ifG8Y+Yfpq6ktsHXNLD3jwAACAASURBVO8sn17Mo9saONLex6w05uPHy5gCLoSoAX4LTANCwINSyv8VQjwKzA+fVgw4pZRLhRAzgH3Ae+HHNkspbyYJyFCItsb3mVIbR+Q7Tuo7h2li1e/RNi9nr4eq5fFfdPa5vFt2Pte1P0Wo7SsY7HMTs1i0CHzljPCGm5Tw3B1ahegF34e6jybs5xgNgoWVhfFv5gkRTqFYht0gjQkpNUF2tw8W9742Tdzd4WN9bVqFrNeppWXCAoylEAqmgqVo4PtDPYJfvtXBFPsUbr1wGTm5JSfOjaSHwpGar/U9mn96FWs33wKm9+HcO7VPLxlEs9PDuiRPwdEzkc6EO+qd2SngQAC4XUq5QwhRAGwXQrwkpbw6coIQ4gdAtBXhkJRyaYLXOoTNv/gCp7T8Ge/nX8daPiOpP6uh083CisLBB3f8ThOHs7487uvuWvgVqje9ju2vX8JwwzMJyfP6AyE6+/wnUiiv3at9UjjjC7A28e+liyoLeWJ7Y0I386SUNIYrBEcsnBICLPnabazN4xjY1eDk47/YzHR7Ho9+bi05J3/aOgnbtPl81vQ/3Ff8KGtevw/qt8CVD2mOngzAFwjicPlSVoWpR+bY8ymwmNhR38WVK6rTvZy4GXMbWErZIqXcEb7vQouuqyKPCy1x9DHgj8la5EhYV1+HUQbw/P4TI06+SQTBkKSpyzN4AzPg1wp3atbC9NPHfe388iruDlyNuX4T7HkyAauFjr6oIp63fw8b/hsWXw0f+H8Juf7J1FUW0ecPcrSjb0LXkVKyt7mb77+wn7Pv2chZd2/kvB++yp+21uMLJLdc/0h7Hzf+5i1K83J4+IZVQ1NlI2AvLeYneZ+Hy38BLbvggbPg4MtJXWuiaO0Ot5GdxAJuMAiW1hbzdoZ2JozLxxNOjywDtkQdPgs4LqU8EHVsphDibSHEq0KIYZOtQoibhBDbhBDb2tra4ly2xtKlK/m+9TZKnHvh+a+M6xqxcLzHiz8YGpxCeedxzZFx1u0TiprtBRb+EPwArrLFWrMrz8RfSJEy+oW9m+GZf9dSPJf+BAzJsW1NdMjxgeMufvjS+3zgB69y8Y9f58FNh5lVns/XLzyFfIuJrz31DmffvZFfbDpMbxIafzlcXq771RYk8NsbVzMljg3J6pJcrRhm8cfgple0XPvvr4AN39H2HXTMiSKeySvgoKVR3mvtScprK9nEnLATQuQDTwK3SSmj/1I/zuDouwWolVJ2CCFWAE8LIepOeg5SygeBBwFWrlw5rp6OQgiqTruS+/+xl1t3PAzVq2D5J8dzqVGpP9kDHgrC6z+EaafC3A9O6Nr2AgshDGw/9T9Z9+rHYMN/wcU/mNA121w+loiDLH7zuzBtEXzst2DKmdA1R2PulALMRsGe5m4+vCS29MHR9j6e3d3Ms7tb2N/qQgg4bVYZnzlrFhcsmkZpnrbem86exRsHO/jZqwf5znP7+L8NB7j+9Bl86vQZlCVgVJzL28+nfvUW7S4/f7xpbdx50OoSGy/tC3vB7fPgsxu03u+b7ob6N+GKh7Q8uw5pTEMRT0Jxd2p99w9tgMOvalZeaxFYi8Nfwzdb9PfFQx5fXmUjJGF3o5PTZ2eWHzwmARdCmNHE+xEp5VNRx03A5cCKyDEppQ/whe9vF0IcAuYB2xK47gGuXF7NGX+/mosKW5n5t9s1Ua1MbPp9iIDvewY6DsJVv5lwzjqSpz5onM261Z/TPOVLPgHVK8Z45sh4Wt/nVzn3IHPt8InHT7S1TRI5JgPzpxWMuZHZ2OXmb7tbeHZ3C++E52munF7Cty+t48JTpw3beEsIwZlzyzlzbjk7G5w88MohfrLxIL947TBXr6zhs2fPorpkfGXgvkCQm3+/nfePu/jl9StZWlMc9zWqS2z4AyHae31a5J6TCx/5qZZW+9uX4ednwRW/hJlnj2uNyaTZ6UEImFaUIRbIgB8a39IE+9DL0LwTkJoQzzxH+/Tj7Q7fnNDTpN33OCE4crHcOmC/xUzw0UIoLAsLfGTD2gAGo+Y+EsbwfWPU/chxw0nnmIY+d9Y6mLowof8ksbhQBPAQsE9K+cOTHj4P2C+lbIw63w50SimDQohZwFzgcALXPIgphVbWnTKNzx67mZfy/gPx2CfhpleHdgOcAA2dbowGQUWxVXM+bPoBlM2FBZdO+NoFFhMWk0Grxlz/DXj3aXj2NvjsxvE5GnodnLX1cwSB0LVPpSz6q6so4u/D+GkdPV7+9k4Lf93VPDABZUl1EXdevICLTq2IawNtaU0xD3xyBQcdvTy46RB/2FrP77fUc9mSSj53zmzmT4v9jSoUktz+2C7eONjBD65awrr5U2L/ZaOIvHk0dHkGp16WXQuVy+Gx6+C3l2kVr2fdnrQ01nho6vJgz7dgMSV+ZmpCkBI6D2uCffBlbcasv1cTw+pVsO7rJxxghjF+h37vYHGPvu9x8pfX9jAtx8c5U8zacXcHBPshFNA+cctg+H4o6n4w6rHwMRkEOUJvoUv/L/UCDpwBfBJ4RwixM3zsG1LK54BrGLp5eTZwlxAiAASBm6WUnSSRa1bV8Ol3j7Nl3Y9Y+8q/aH7nf3l87P/UGKnvdFNZbMVsNMD7f4fj78BlP03I9YUQJ2ZjWgu1ZlePfwre+gWsvSW+i/l64ZGryPV3cKPhP/n9lMTZEsdiUVUhj25roKXbi8Vk4Pk9rTy7u5ktRzqREhZUFHLHh+bz4cWVow/EiIE5U/K5+8ol3HbePB56/Qh/3FrPU283cd6CKdyybjYrpo/+5i2l5L/+9i7P7m7haxeewhUTcB9E2rA2drlZMb1k8INTF2p58Wdvg43/raVULn9Q86brgOZuHXrAPU44sulElO2s144XT9f2GWav1z7NxGs9NYcrfEcIaN46vosN+x1sv+q8iRf0SDm8sJsS/289poBLKV8Hhv2NpJSfGubYk2jplpRxzjw7UwstPHioiLUX3q39wbz6fTj3Gwm5fn2nm5qSXO0/5rV7oahGezEliEHT6Rd+BOacp22CLbwsdktasF+L9lrf4edT78LhSXx152gsrNT+oG749VscbOslGJLMtufxhQ/M5ZLFlQMd/BJJZbGN/7hkIZ8/dw6/ffMYv/nnEa742ZusnlHKLetms26+fdg/xp9vOsyv3zjKjWfM5HNnz5rQGqoGBHyEgiJLvuZQmX46PP81zaVy5a9g+mkT+rmJoKnLMzCcOm0EA1prhkMbtFvjNk3scgo0oT49vAlfNjupy1heW8IT2xs51uFmRnnexC4mRPjTc/JrAjKr6mAETEYDV62o4aevHKTlI1dTsXSbJuCVy2H+BRO+fkOnm/MWTNUaPTVsgYvuBWNsNrNYsOdbBvLsCAEX3QM/PU1zpXzs4bEvICU8829axHLpT3j5zVlMKUjtf+3CikJKcs14+oPcfM4sLllcySnTClJSnlySl8MXzpvLZ8+eyZ+2NvCL1w5zw2/e4pRpBdyybjYXn1qByailLp7c3sj3nt/Ph5dUcufFCya8vtwcE2V5OaO3lRVCm49atQIeux5+czF84D81cUpTSiUUkjQ7vcnrxx0psvL1hFMV4a++nhMpjOadcORV7T4CKpdpPYJmr9dSJAn8GxuLZbXa/sfbDV0TF/CT6Pb0873n9/Ov62aPPU83TrJCwAE+trKGn2w8yBPbm/i3i++F1t3w55u0j7Cl44+y+nwB2nv92j/8a9/SGjYtuzZh6wYtAt9+LKqhTuksOPvLmn/7wEtjO11e/jbs+iOc+01Y/kna/r6BmQl+EY6FLcfIW988D6NBpK2nRG6OiRvPnMm1a6fzzK5mHnj1EF/4007u/ft73HT2bOz5Fr7y5G7OmFPGvVctTljRUXWJLbbBDhVL4HOvam+2//gWHPsnfPSBhO7XxEp7nw9/MBR7CsXZAC07hxHj7hO3Qd/3DO1oeTKFVdo+0uz12gZfGv4dIsybWkC+xcSOY04+uixxBT2vH2jnjid24XD5WF5brAR8JGrLcjljThmPbmvg1nPnYLj6d/Dzc+DR6+DTf9fcAeMgElmdKg5pH/HO+7ZWjp1A7AUWOt1++oMhLc8OWnS2+zH42+1w65aRf+aWB+H1+2DFDXD2HUgpaXMld5jxSESi3HSTYzJw5YpqLl9WxUv7jvPTVw7xH0/vAbQpQg9cuyKhG3dVJbbYx8pZi+Cqh7XRei9+E35+tuZmql6ZsPXEQsQDXlk0ymtZSu0T5+afwr6/Dt2cyynQ9m2sRVqrgfxpUD5/8DFr0YnvrcWDj0W1JUg3RoNgSU0ROxLUmdDtD/C95/fz2zePMcuex5O3nD4ul9NYZI2AA1y9qpZ//+Pb/PNQB2fOnaHZtx65Shth9tEHxvViiaQ2Fh95SHvhrbwxwavWBFxK6Ozzn+hqZ7JofcMfvgQ23Qsf+I+hT9z7tFbANP9izTsuBD3ufvzB5A4zzhQMBsGH6qZx/sKpbD7cyd/fbeWWdbMpiLHKMlaqS3J5eZ8j9o52QsCaz2mi/din4FcXwAfv0jatUyRozc5wFeZwEXiwH979i9bnp3mH9ro//d+0/ZncUk2ELYUZ1/dlLJbVlPCzVw/h9gfIzRn/77b9WBe3P7aTox1ubjxjJl+5YD5Wc3KcPln1P3D+wqkU55r501v1WoP2uR+EdV+DV74LNau0ocJxUt/pZo5opPjYi3D2V7TIIcFEz8Yc1JZ05lmw+BqtZH/xx8A+/8RjR9+Ap26CmtVa/42wI8bhGmMSzyRECMFps8s4bXZZUq5fXWLDF9D6gsc1RLpqBdy8CZ7+V3jx69onvBXXaymFnOSmwJqcWmAyyMbp7oTtv4Gtv9CGfJTO1vZ7lnxc24zNcpZPLyYYkuxu7GbtrPhfK75AkB/94wA/f/UQFUU2/vjZtUl7zUXIKgG3mo18dFkVj2yup7PPr1Xznf0VaNquOQCmLdGEPA4aOt18IeevSHMeIl5bX4yMOp3+/P+G95/XUinX/1WL0I6/q/X1LpkOH//ToPRKKoYZKwZTHeVEiUvAAWwlcM0ftGh3091w8CUttTBrHcy/COZdkBQvf1OXhwKLSZvY1H4ANv9M20fpd2tFMZfcB3PP15VvPdksrdFsoG/XO+MW8L3N3dz+2C72t7q4emUNd16yIOGf9IYj6/53rl5Vgz8Y4s9vN2kHDAb46M81O95j10FvfH1XPMcPcpF4A7HyhqRtsowq4Pl2Le9+9DXY/ag2Fef3V2g5/WufHLImhysyzDhDquuygEgxz7gHHAsBp38e7jgE1z2j7Wc43oW//jv8YD788jx47YdaD3Q5rq4TQ2jq8nBJ/ntaivEnK7U5rXWXw81vwPXPaO6tSSTeAKV5Ocwsz4srDx4IhvjJhgN85P436Ojz89D1K/n+lYtTIt6QZRE4wCnTCllSU8yjb9Vz4xkztJxkbilc/Tt46Hx44gb45NMx5+/OdPyBEEaMp30+aWsuj6RQRppOv/x6beL9i9/UyoX9vXDD81BcO+RUFYGnnkgzqJicKKNhNMOsc7TbBd+F43vhvedg/980p9HL39bSGqdcpO171KyOv5is3wvvPM43G+5lZvAoNJXDOV+DVZ+G/PFVo2YTy2qK2XSgPab9jENtvdz+2C52Nji5eHEF/33ZIkryktdzaDiy8i32mlU1vH+8l7cbojr7VSzRPhYefQ023BXTdULdzXzI/w/esV8MhRVJWq2W+im0mkYebmwwaGv3dGk9WK55RGtSNQwOl1YJWWjNuvdm3ZJnMVE6lhc8XoTQ/o/P+YpmPfziu1o+umQGbH4Afn0B3DtXy5/ve1YbcTcavQ7Y+D9wXx0883n6gyGeqvkGfHEvnPt1Jd5hlk0vob3XN+r/ZSgk+fUbR7j4x69xtKOP//v4Mu7/xPKUizdkYQQO8OEllfzXs+/y6NYGltdGlTcv/YTWDOeN/4WqlbBw9F4mnk0/xkKIY6d8lnHM24mLgXL6kZh2qlbBZysZtTFSm2uEYcaKpKJ5wRMo4CdTVAWrP6vdvD1w8B9adL7vWe3TmckKs87VovN5F5wQ5NZ34M2fwp4ntBF28y7As+JznP9rN1+dvUArL1cMsDxc0LOjvmtYz3Zjl5s7Ht/Nm4c7OHe+ne9fsTiu9sOJJisFPN9i4pLFFfx1dzP/8eGF5Fuifs0Lvqc13n/6X2HKAigfoV+IuxPrrof5S+h0SqvnJX3NYwo4QN1HxryOI00e8MlOdTxe8IliLYRFl2u3YL9WIbz/OU3Q338eEFp6xZijfeI058Ly62DNLVA+h/pWF7BJf31QdMD8qQXk5hh5u97JZUsH5tYgpeTxbY3c9ey7SCn53uWncvWqmrQHSlmZQgHNE+72B3l2V/PgB0yWE/2xH71WawA1HFsewBhw87PApUNnYSYBe4F15Bx4HEQicEVqiQx2kAnaZIwZo1lzrFx0N9z2DnzuNa1LX8CrbXif920tTXLxD6B8DqC1kQU1yGE4TEYDi6sHF/Q4XF4+8/A2vvLkbuoqC3nhtrO5ZnVt2sUbsljAl9cWM3dKPn96q2Hog0XVWjqi/X2trPnkPzpvD2x5gIOl6zhIdUoiFXt+DBF4DMTtRVYkhKpizQve3utP3yKEgIrFsO6r8LlN8IWdcOZtQ5xKjUrAR2VZbQnvNvfg7Q/yt90tfOi+Tbx2sJ07L17AHz+7NuHl8BMhawVcCMHVq2rY2eBkf+swgwZmrdMaCu19SvPARrPtV+Dt5q9F11BRaE1Jv2R7gYVeXwC3f/xjnXyBIE53v4rA00B0W1m90+z0YDYKlWobgeW1JQRCkmt/uYVb/7CDmtJcnvv3M/nMWbMS1j8nUWStgANcvrwas1Hw6HBROMAZt8Epl8Df79QaCwH0e7Siilnn8k/P9JS920ZEt901/gguEv2pP8zUM2EveApp6vIwrciqOzHSC5HOhDsbnHzpg/N48pbTmTMluVOtxktWC3hpXg7n103jz283DT/VXAht/FXJDG2IgqtVm+Le54Czv6z1AU+xgLf1esd9DUePKqNPF2P2BdcRTU6PSp+MQnm+hfs/sZy/fP4M/v0Dc080mNMh+l1ZgrhmVQ1Odz9/33t8+BOsRXD178Hn0no1v/G/ULMGb+Vajvf4UrKBCYP7oYyXNlWFmTbyLSZKcs0Zk0KpKtZPHlePXLy4grrKNA+7iIGsF/AzZpdTVWwbOY0C2uirS/8PGjZDdwOcdfvARk/KBHy0cvoYcagqzLRSXZKr+wi8PxjieI+XqmL1Jp8NZL2AGwzaZubrB9tp6BwlOjr1Sm3w7IJLYe75A21kU5VCKc3LwSAmHoELAWX5qa8IU8Qx2CGNtHZ7CckR2sgqMo6sF3CAK1dUYxDw2LZRonDQ7FdX/w6EoL5D+0NMVQRuNAjK8i0T8oI7XD5Kc3N0nbPLZiLVmCn3gsdBU/iTZaXKgWcFk+IvvbLYxjnz7Dy+rZFAMDT2E4CGLg82s5HyFEazE/WCqyKe9FJdkpt+L/gYRCbxqE3M7GBSCDholZmtPV42HYitnWx9p5va0tyUVluVx1JOPwptLq8S8DSSCV7wZhWBZxWTRsA/sGAK5fk5/GnrGGmUMA0ptBBGUBF4ZhPxgkfSFHqkyemhPD8naSO+FKll0gi42WjgihXVvLzfMTB2bCSklAMReCqxF2g58PHkUKWUqow+zWSCF1x5wLOLSSPgAFevrCEYkjy5vWnU8zr6/Lj9QWpKU/tCtxdY6A9Kuj39cT/X6e6nPyhVBJ5G8i0minXuBW9yelT6JIsYU8CFEDVCiI1CiH1CiL1CiC+Ej/8/IUSTEGJn+HZR1HO+LoQ4KIR4TwjxoWT+AvEwy57P6pmlPPpW/ahRbsRCmI4IHMZnJTwxSk0JeDpJel/wCSClDBfxKAHPFmKJwAPA7VLKBcBa4FYhxMLwY/dJKZeGb88BhB+7BqgDLgB+KoTQTcLtmlU1HO1ws+VI54jnNKRLwCdQjalGqemD6mL9FvN09vnx9oeUBzyLGFPApZQtUsod4fsuYB9QNcpTLgP+JKX0SSmPAAeB1YlYbCK4cFEFBRbTqJWZEQ94ZFMqVZzohzKeCFzL66sIPL1Einn06AVXHvDsI64cuBBiBrAM2BI+9HkhxG4hxK+EEJHZZVVAtDo2MozgCyFuEkJsE0Jsa2uLb1L8RLDlGLlsWSXPvdNCt3v4XHNDl5spBRZsOan94DCRFIqKwPVBdYkNb3+Ijj79ecHVIIfsI2YBF0LkA08Ct0kpe4CfAbOBpUAL8IPIqcM8fUg4IqV8UEq5Ukq50m63x73wiXDNqlp8gRB/2TX8ZmY6HCgAhVYTOSbDuHPgNrNx8Pg4RcrRc1vZRlXEk3XEJOBCCDOaeD8ipXwKQEp5XEoZlFKGgF9wIk3SCNREPb0aOGmuWXpZVFVEXWUhf9zaMOxH3YZOT1oEXAgxbi+4GmasD6pL9VvM0+T0kJtjpDjXnO6lKBJELC4UATwE7JNS/jDqeEXUaR8F9oTvPwNcI4SwCCFmAnOBrYlbcmK4ZlUN+1p62NM0eFqPPxCiuduTtrFJES94vDhcXpX/1gGR6FaPEXjEgaLe5LOHWCLwM4BPAutPsgzeLYR4RwixGzgX+CKAlHIv8BjwLvACcKuUcphpCunl0qVVWEwG/vRW/aDjTU4PUqauC+HJxDSdfhhUFaY+KLCaKc41D/Qc0RPKA559jJkwlVK+zvB57edGec53gO9MYF1Jp8hm5uJTK3hmZzPfvHgBuTnaP0W6POAR7AUWdhzrGvvEk3C4fJw5pzwJK1LES1WxPtvKNju9LK4uTvcyFAlkUlVinszVq2pw+QI8907rwLG0C3i+hU63n/4YuyYCePuDuLwBFYHrBD0W87j9ATr7/GoDM8uY1AK+emYpM8vzeDQqjdLQ6SbHZEhbPtleYEFKregiVtQoNX0RmcyjJy+4shBmJ5NawIXQpvW8dbSLg45eQCviqSmxpW1i93i84GqUmr6oLrHh6Q/G9SacbJqcWqGXqsLMLia1gANcvrwKk0EMTOtp6EqPBzzCeAS8zaWm0esJPXrBI5uqahMzu5j0Aj6lwMoHFkzhye2N+AMh6jvSLODj6IfSphpZ6YpqHbaVbXZ6MBoEU9VrJKuY9AIOWmVmR5+fJ3c04vIF0mYhhPH1Q2lz+TAIKMtXf5x6oEqHk3manB6mFVoxqXmpWYX63wTOnmdnWqGVH798AEifBxzAajZSYDXFnQMvzbNgTFPeXjGYQquZIptZVxF4U5dqI5uNKAFHmwj/sZXVtHRrueR0plAg/mKeNpdPpU90RqQroV7QiniUSynbUAIe5qqVNUQqjNMZgUP8szEdqgpTd1SX2HQzGzMQDNHa41UOlCxECXiYmtJczpxTzpQCS9o7+sXbD0WV0euPqmL9eMEdLh/BkKSqOL2BiSLxqN6jUdxz5RLax9FIKtHEk0IJhSTtvSqFojeqS2y4/UG63P2U5uWkdS0nBjmoFEq2oSLwKKYVWVlUVZTuZWAvsNDrC+D2B8Y8t8vtJxBSw4z1RrWOnCgRD3i1SqFkHUrAdUjEC97uGruSz6HK6HWJnop51Ci17EUJuA454QX3jnmuGqWmT/TkBW9yeijJNQ903FRkD0rAdUg85fQOVYWpS4psZgqtJl1E4M1Oj3KgZClKwHVIPAKuInD9EulKmG6aujxUFikBz0aUgOuQsjwLBhFrBO4lL8dInhpmrDv0UMwjpaRJReBZixJwHWI0CErzYvOCKw+4ftFDX/BuTz9uf1CV0WcpSsB1SqxecIfLpxwoOiXiBXe6+9O2hkgKRwl4dqIEXKfEKuDtKgLXLXpoKzswiUelULISJeA6JdZ+KKoPin7Rg5VQecCzGyXgOiXSD2W0/KnbH6DXp4YZ6xU9FPM0dXmwmAyUpbmcX5EclIDrFHuBhf6gpNszcv5UTeLRN0U2MwVWU1oj8OZurQ+4EKpXfDaiBFynxOIFVx5w/ZNuL3hTl7IQZjNKwHVKLLMxVR8U/aN5wdMo4E6vcqBkMWMKuBCiRgixUQixTwixVwjxhfDxe4QQ+4UQu4UQfxZCFIePzxBCeIQQO8O3B5L9S2QjsczGVBG4/okU86TDC+7tD9Le61MbmFlMLBF4ALhdSrkAWAvcKoRYCLwELJJSLgbeB74e9ZxDUsql4dvNCV/1JCCWFIrD5Q0X/agNKr1SXZJLX5q84AMWQiXgWcuYAi6lbJFS7gjfdwH7gCop5d+llJGG1ZuB6uQtc/JRaDWRYzKMmQMvy8tRw4x1TDq94M1OrZulisCzl7hy4EKIGcAyYMtJD90IPB/1/UwhxNtCiFeFEGeNcK2bhBDbhBDb2tra4lnGpEAIMaYX3OHyMaVQpU/0TETAm5ypd6JEfqYa5JC9xCzgQoh84EngNillT9Txb6KlWR4JH2oBaqWUy4AvAX8QQhSefD0p5YNSypVSypV2u30iv0PWMtZszDaXb2CzU6FP0ukFb3J6EUKbNKXITmIScCGEGU28H5FSPhV1/HrgEuBfZHiXRkrpk1J2hO9vBw4B8xK98MnAWOX0qg+K/imymSmwpKcveFOXh6kFVsxGZTbLVmJxoQjgIWCflPKHUccvAL4KXCqldEcdtwshjOH7s4C5wOFEL3wyMJqAB0OSjl5VRp8JVKWprWyT06084FlOLE2kzwA+CbwjhNgZPvYN4MeABXgpXOW1Oew4ORu4SwgRAILAzVLKzoSvfBJgz7fQ6fbTHwwNiaI6+nyEJCoHngFoxTypF/Bmp5clNcUp/7mK1DGmgEspXweGszk8N8L5T6KlWxQTxF5gQUro7PMztXBwqmTAA65y4LqnusTG5sMdSClTVtIeCklauj1cdGpFSn6eIj2o5JiOGc0LPtAHRUXguqe6xEavLzBqX5tE09broz8oVQoly1ECrmNGE3DHQASuNjH1y2kYagAAHrJJREFUTjqcKCcGOajXRzajBFzHjNYPRZXRZw7VaegL3jRQhZmbsp+pSD1KwHXMaP1Q2lw+CiwmbDnGVC9LESc1aYjAmwcGOagIPJtRAq5jrGYjBVbTiBG4ir4zg0KbKeVe8EOOXopzzRRYzSn7mYrUowRc54zkBXe4vErAMwQhRNgLnhoBl1Ky6UAbp88uS8nPU6QPJeA6Z6R+KCoCzyxS6QXf29zD8R4f586fkpKfp0gfSsB1zkj9UFQZfWZRXWKjqcuTkr7gG/c7AFinBDzrUQKuc4ZLofT5Arj9QRWBZxDVJTZcvgA9nsDYJ0+QDe85WFJdpF4fkwAl4DrHXmCh1xfA7T/xh+9Qw4wzjoiVsCHJaZSOXh87G5yce4qKvicDSsB1TsQL3u7yDxxTHvDMI1XFPK++34aUsF4J+KRACbjOGc4L7nBpk1ZUGX3mkKping37HZTnW1hUWZTUn6PQB0rAdc5w5fSqkVXmUWQzk59kL3ggGGLT+22cO9+OQY3ZmxQoAdc5w0fgPkwGQUmuGmacKQghwhPqkyfg24910eMNqPTJJEIJuM4py7NgEEMj8PJ8i4qyMozqJA922PCeA7NRcObc8qT9DIW+UAKuc4wGQWneYCuhGmacmVSX5A40mUoGG/c7WDWjVJXPTyKUgGcAJ3vB1TDjzKS6xIbLm5y+4I1dbt4/3qvSJ5MMJeAZwMnVmG0ur4rAM5Cq4uQ5USLVl8r/PblQAp4B2PMttIcj8EAwREefX0XgGUgyveAb9juoLc1lVnlewq+t0C9KwDOASApFSklHnx8pwV6o+qBkGie84IkVcI8/yD8PdbD+lCkpm7mp0AdKwDMAe4EFfzBEjyegPOAZTHGumbwcY8JTKG8ebscXCKn0ySRECXgGcMIL7lVVmBmM5gXPTXgEvmG/A5vZyJqZpQm9rkL/KAHPACLRtsPlUxF4hpPoYh4pJRv3t3HGnHKsZjVeb7KhBDwDiC6nd/SoRlaZTKKLeQ44emlyepR9cJIypoALIWqEEBuFEPuEEHuFEF8IHy8VQrwkhDgQ/loSPi6EED8WQhwUQuwWQixP9i+R7UQLeFuvj0KrSUVbGUp1SW5CveAbBuyD9oRcT5FZxBKBB4DbpZQLgLXArUKIhcDXgJellHOBl8PfA1wIzA3fbgJ+lvBVTzIKrSZyTAbaerUIfIpyoGQsESdKU4LSKBv2O1hQUUhFkS0h11NkFmMKuJSyRUq5I3zfBewDqoDLgIfDpz0MfCR8/zLgt1JjM1AshKhI+MonEUKIgdmYbb2qCjOTOeEFn3gapdvdz/ZjXaxX0fekJa4cuBBiBrAM2AJMlVK2gCbyQCQJVwU0RD2tMXxMMQEiXnA1zDizSaQXfNOBNoIhqfLfk5iYBVwIkQ88CdwmpewZ7dRhjg2Z5CqEuEkIsU0Isa2trS3WZUxaIgLucHnVKLUMpjjXTG6OMSECvnG/g5JcM0trShKwMkUmEpOACyHMaOL9iJTyqfDh45HUSPirI3y8EaiJeno10HzyNaWUD0opV0opV9rt6iPgWNgLLBzrcOPtD6kIPIM50Rd8YimUYEjyyvttnDPPjlG1FZ60xOJCEcBDwD4p5Q+jHnoGuD58/3rgL1HHrwu7UdYC3ZFUi2L82PMtePqDgCriyXQSUcyzq9FJZ59fVV9OckwxnHMG8EngHSHEzvCxbwDfAx4TQnwaqAeuCj/2HHARcBBwAzckdMWTlOio256vXCiZTHWJjW1HOyd0jY37HRgEnDNPfXqdzIwp4FLK1xk+rw3wgWHOl8CtE1yX4iSiBVxF4JlNdYmNnrAXvMg2vuELG/Y7WDG9hGI1Vm9SoyoxM4TBEbgS8EwmYiUcrxf8eI+Xvc09Kn2iUAKeKURE22wUFOeqkVmZzAkr4fg2MiPDG5R9UKEEPEOIROD2fIvq+ZzhDETg45yPuWG/g8oiK/OnFiRyWYoMRAl4hmA1GymwmtQghyygZAJecF8gyOsH2zlXDW9QoAQ8o6gqtlFVrAQ805mIF3zrkU7c/qBKnyiA2GyECp3wk08sJzdHdSHMBqqKx9cXfMN+BxaTgdNnlydhVYpMQwl4BjFnSn66l6BIENUlueyod8b9vI37HZw2uwybeiNXoFIoCkVaqC6x0e3pp8cbe1/ww229HO1wq/SJYgAl4ApFGhiPF3xgeMN8JeAKDSXgCkUaGE9b2Y3vOZg7JZ+a0txkLUuRYSgBVyjSQLzFPL2+AFuPdKrqS8UglIArFGmgNC8Hmzl2L/jrB9roD0qVPlEMQgm4QpEGIl7wWHPgG/Y7KLCaWDlDDW9QnEAJuEKRJqpLbDQ6x06hhEKSje+1cfZcO2aj+pNVnEC3PvD+/n4aGxvxer3pXopCkRRuWmzB4zexb98+rFYr1dXVmM1DG5Xtbe6hzeVT+W/FEHQr4I2NjRQUFDBjxgzV80GRlbS5vLR0e5lbUYCzq4vGxkZmzpw55LwN+x0IAevmq+ENisHo9vOY1+ulrKxMibcia4mkQ/pDUFZWNuKnzQ3vOVhcXUy56gOvOAndCjigxFuR1eSYwgIeCI34Wm/v9bG70cl65T5RDIOuBTzdGI1Gli5dyqJFi/jwhz+M0xl/74pEcPToURYtWjTscZvNxtKlS1m4cCE333wzoVBoXD/jox/9KEuXLmXOnDkUFRWxdOlSli5dyj//+c+4rrNhwwY2b9487GMtLS1cdNFFLFmyhIULF3LppZeOa63Dcf/99/PII48k7HqpIBKB+4Mj/5+98l4bUqrhDYrh0W0OXA/YbDZ27tTmOF9//fXcf//9fPOb30z6zw0GgxiNsTUrmj17Njt37iQQCLB+/XqefvppLr/88jGfJ6VESonBoInIn//8ZwBeeeUV7r33Xp599tlxrX3Dhg2Ul5ezdu3aIY/deeedXHzxxdx6qzYydffu3eP6GcMRuWYmYTIIDELgD4ws4Bv3O7AXWKirLEzhyhSZgorAY+S0006jqalp4Pt77rmHVatWsXjxYr71rW8BcPfdd/PjH/8YgC9+8YusX78egJdffplrr70WgFtuuYWVK1dSV1c38DyAGTNmcNddd3HmmWfy+OOPs337dpYsWcJpp53G/fffP+b6TCYTp59+OgcPHhxxfUePHmXBggX867/+K8uXL6ehoSGm3/2tt97inHPOYcWKFVx44YUcP34cgPvuu4+FCxeyZMkSrr32Wg4dOsQvf/lL7rnnnmGj95aWFqqrqwe+X7x48cD9733ve6xevZrFixdz1113AeByubjwwgtZsmQJixYt4oknngDgjjvuYOHChSxevJivfvWrgPbm8KMf/QiAHTt2/P/27j065nPf4/j7i0SEiEZotWjiqNRtZEJCKhO3EFJ1qeVQtxat3a469OY07DpV3T2rW9P2HNrNtkv0tBq6KbUtNIRUojYS7BCJhApiKxEkcYtcnvPHZGZPbpKImpl6XmtZMr+Z/OYzk5nvPPPMM98fvXv3xmAwMGbMGPLy8gAICQkhMjKSoKAg/Pz86vzu4l4TEVwaNqComhF4UUkpuzNyGODXigYN9HSiVplTjMDf+1sqx/6Zf0/32eXR5rz7TNdaXbakpIS4uDimT58OQGxsLJmZmezfvx+lFCNGjGD37t2Ehoby8ccfM2vWLJKSkigsLKSoqIjExERMJhMAH3zwAV5eXpSUlDBo0CBSUlKshczNzY3ExETAXNyWLFlCv379mDNnTo0Zb9y4QVxcHAsXLqw2X/v27Tl+/DjR0dH86U9/qtVtLywsZPbs2WzatAlvb29Wr17N/PnzWb58OYsWLeL06dO4urpy9epVWrRowYsvvoi3tzevvfZapX3NnDmTCRMmEBAQQFhYGFOnTqVNmzZs2bKFM2fOsG/fPpRSRERE8NNPP3H27Fl8fHzYunUrAHl5eVy4cIEtW7aQmpqKiFQ5rTVp0iSWL19OSEgI8+bN4/333ycqKgowv/PYv38/mzZtYuHChWzbtq1W98OvxbVRg2pH4ElZVygoLNbTJ1q19Aj8Dm7evIm/vz8tW7bk8uXLDB48GDAX8NjYWIxGIwEBAaSnp5OZmUnPnj1JTk6moKCAxo0bExwcTFJSEgkJCdYC/u233xIQEIDRaCQ1NZVjx45Zr2/cuHGAuVBdvXqVfv36ATB58uRqM548eRJ/f3/69u3L008/zbBhw6rNB/D4449XOb1RnbS0NFJTUwkLC8Pf358PP/zQOnLv2rUrkyZNYvXq1VWuX64oIiKCkydPMn36dI4dO4bRaCQ3N5fY2Fi2bt1qzXvixAkyMjIwGAxs27aNyMhI9uzZg6enJ15eXjRo0ICXXnqJDRs20LRp03LXkZuby61btwgJCQHMU1+7d++2nm+ZXurZsydZWVm1vh9+La4NpdoR+K7jF3FpKIQ8oZcPalVzihF4bUfK95plDjwvL4/hw4fz+eefM2vWLJRSzJ07l9/97neVfsfHx4fo6GieeuopDAYDu3bt4uTJk3Tu3JlTp04RFRXFgQMHeOihh3jhhRfKLR2zFCOlVK1X4FjmwG1Vly8rK6tSwauJUgqDwUBCQkKl83744Qd+/PFHvv/+e/7whz9w9OjRGvfXsmVLJk6cyMSJExk6dCiJiYkopXjnnXes73BsJSUlsWXLFubMmcPw4cOZN28eSUlJbN++nTVr1rB06VJiY2PL5b2Txo3NS/EaNmxIcXFxjXl/bS6NGlBcqigprZx7Z/pFgny9aNbYKZ6mmh3oEXgteHp6snjxYqKioigqKiI8PJyVK1dy7do1AM6dO8fFi+ZezaGhoURFRREaGorJZGLZsmX4+/sjIuTn59O0aVM8PT25cOGCdWqgohYtWuDp6WmdTqnr6oo75aurLl26cO7cOfbv3w/A7du3SU1NpaSkhOzsbAYOHMhHH31ETk4ON27cwMPDg4KCgir3FRcXx82b5t4f+fn5nDp1ivbt2xMeHs6KFSu4fv06YP4S16VLlzh37hzNmjVj8uTJvPHGGxw8eJCCggLy8/MZPnw4n376KYcOHSp3Hd7e3jRp0sQ6v/3VV19Z38k4IlfLWvAKo/Czl29w4uI13bxKu6MaX9pFZCUwHLiolOpWtm0t4Fd2kRbAVaWUv4j4AGnA8bLz/q6Uevleh7YHo9FIjx49WLNmDZMnTyYtLY3g4GAAmjVrxtdff03r1q0xmUx88MEHBAcH07RpU9zc3KzTJz169MBoNNK1a1c6dOhA3759q72+6Ohopk2bhru7O+Hh4XXKOmTIkCrz1XZli63GjRuzbt06Zs2aRUFBAcXFxbz55pt07NiRCRMmUFBQQGlpKW+//TYeHh6MHDmSsWPH8t133/H555/z1FNPWfd14MABZs6ciYuLC6WlpbzyyisYjUaMRiPp6enWqR0PDw+++eYbjh07RmRkJA0aNMDV1ZVly5aRl5fHs88+S2FhIaWlpXzyySeVMn/11Ve88sor3Lx5k44dOxIdHV3n232/WNaCV5wHtxy8Qc9/a3ciNb3lFJFQ4Brwf5YCXuH8j4E8pdTCsgK+uarL3UmvXr1UUlJSuW1paWl07ty5LrvRNKdTVFJK2vl8Hm3RhJyzP1sf88+v3M/p3OvEzxlg54SaIxCRZKVUr4rba5xCUUrtBi5Xs1MB/h2IqXdCTXsAWdaCF9mMwG/cLmbvz7m6eZVWo/rOgZuAC0qpTJttviJySER+FBFTPfevab9plrXgtt/G/OlELreLS/X0iVaj+n68/RzlR9/ngfZKqVwR6QlsFJGuSqlKi7hFZAYwA6B9+/b1jKFpzqviWvCdxy/i7tqQIF8vO6bSnMFdj8BFpBHwLLDWsk0pVaiUyi37ORk4CXSq6veVUsuVUr2UUr1atdLrXLUHl+1acKUU8ekXCenoTeNGdf/QWXuw1GcKJQxIV0plWzaISCsRaVj2cwfgCeDn+kXUtN82y1rwUqU4fqGAf+bd0tMnWq3UWMBFJAbYC/iJSLaIWL5tMZ7KH16GAiki8g9gHfCyUqrKD0A1TTOzrAUvKVXW5YP6A0ytNmqzCuU5pVQbpZSLUqqtUmpF2fYXlFLLKlx2vVKqq1Kqh1IqQCn1t18r+P2yYcMGRIT09HR7R6lRfHw8np6eGI1GnnzySd566y27ZVm1ahUzZ8602/VXp1mzZkDVLXqPHDlibaPr5eWFr68v/v7+hIWF1fl6Pvnkk1ofDtC2gO9Kv0jXR5vzcHO3Ol+n9uDR38SsQUxMDCEhIaxZs+ae7K+kpOSe7Kc6JpOJQ4cOcejQITZv3syePXt+1esD87zt3fYhvxfu1Vfiu3fvzuHDhzl8+DAjRozgo48+4vDhw+zYsaPO+6pLAXdp9K++4Mmnr+jpE63WdAG/g2vXrrFnzx5WrFhRroCPGzeOLVu2WE+/8MILrF+/npKSEubMmWNt4/rnP/8ZMI+MBwwYwIQJE+jevTsAo0aNomfPnnTt2pXly5db97VixQo6depE//79eemll6yj2JycHMaMGUNgYCCBgYE1FmbLgR4sLXCvX7/OtGnTCAwMxGg08v333wPmBlOWvtxGo9HaynX+/Pl88cUXXLt2jUGDBhEQEED37t2tv1dVa9ro6Gg6depEv379qs13+fJlRo0ahcFgoE+fPqSkpFBaWoqPj0+5zoIdO3bkwoUL1d7uBQsWMGPGDIYMGcKUKVMq/d2qylxftW15++mnn3Lx4kVMJhNhYWEUFxczefJkunfvTrdu3awthy0sa8FvFJZQqqC//vq8VkvO0SVnayT8cuTe7vOR7jDswzteZOPGjQwdOpROnTrh5eXFwYMHCQgIYPz48axdu5aIiAhu375NXFwcS5cuZcWKFXh6enLgwAEKCwvp27cvQ4YMAWD//v0cPXrUetDalStX4uXlxc2bNwkMDGTMmDEUFhby/vvvc/DgQTw8PBg4cCA9evQAYPbs2bz++uuEhIRw5swZwsPDSUtLqzb7lStXyMzMJDQ0FDC3sR04cCArV67k6tWrBAUFERYWRmhoKAkJCfj4+NCoUSNrgUxMTGTSpEm4ubmxYcMGmjdvzqVLl+jTp4/1SDq2rWnPnz/Pu+++S3JyMp6engwYMACj0Vgp17vvvovRaGTjxo3s3LmTKVOmcPjwYUaOHMmGDRuYOnUq+/btw8fHh4cffpgJEyZUe7uTk5NJTEykSZMm5a6jusz1OURfXVreenp68vHHH5OQkECLFi3Yt28fly5d4sgR82O4Ygtcy1rw4lKFV1NX/Nu1uOuc2oPFOQq4ncTExFj7Wo8fP56YmBgCAgIYNmwYs2bNorCwkG3bthEaGkqTJk2IjY0lJSXFeuCBvLw8MjMzcXV1JSgoqNwRxxcvXmw9Cs7Zs2fJzMzkl19+oV+/fnh5mdf/jh07loyMDAB27NhRrvVsfn4+BQUFeHh4lMuckJCAwWDg+PHjREZG8sgjjwDmFribNm2y9sW+desWZ86cwWQysXjxYnx9fXn66afZvn07N27cICsrCz8/P4qKipg3bx67d++mQYMGnDt3znpAB9vWtPv27aN///5YloSOGzfOmt1WYmIi69evB2DgwIHk5uaSl5fHuHHjWLhwIVOnTmXNmjXW1rrV3W6AESNGVCreYJ7SqSqz5b64G7Ytb8E8ys/IyKB3795ERkYSGRnJM888U2V/m44dO3L8+HFmz55NRESE9UXdlqUnSr9OrWioD96g1ZJzFPAaRsq/htzcXHbu3MnRo0cREUpKShARFi1ahJubG/379+eHH35g7dq1PPfcc4C5cCxZsqRS86n4+PhybVzj4+PZsWMHe/fuxd3dnf79+3Pr1q07tkItLS1l7969VRYsWyaTic2bN5ORkUFISIj1WJdKKdavX4+fn1+5y9++fZukpCQ6dOjA4MGDuXTpEn/5y1/o2bMnYO6EmJOTQ3JyMi4uLvj4+Fjndiu2pq3NCLeq2ygiBAcHc+LECXJycti4cSPvvPNOjbe7uta4d8p8t+ra8tZWy5YtSUlJYevWrSxevJj169eXmzYDcGlovu/06hOtLvQceDXWrVvHlClTOH36NFlZWZw9exZfX19ri9fx48cTHR1NQkKCtWCHh4ezdOlSioqKAMjIyLC2SLWVl5fHQw89hLu7O+np6daDAAcFBfHjjz9y5coViouLrSNVMHcY/Oyzz6ynK/YAr6hTp07MnTuXP/7xj9ZsS5YssRZQSxtWV1dX2rVrx7fffkufPn0wmUxERUVZOyjm5eXRunVrXFxc2LVrF6dPn67y+nr37k18fDy5ubkUFRXx17/+tcrLhYaGWtvjxsfH4+3tTfPmzRERRo8ezRtvvEHnzp1p2bLlXd3uumSui7q0vAXKtdXNyclBKcXYsWN57733rJex1cSlIQ0E+umDN2h14BwjcDuIiYkhMjKy3LYxY8bwzTffYDKZrB+ejRgxAldXVwBefPFFsrKyCAgIQClFq1at2LhxY6V9Dx06lGXLlmEwGPDz87NOQzz22GPMmzeP3r178+ijj9KlSxc8PT0B85TLq6++isFgoLi4mNDQUJYtW1Zp37ZefvlloqKiOHXqFPPnz+e1117DYDCglMLHx8d64GKTyURcXBzu7u6YTCays7OtBXzixIk888wz9OrVC39/f5588skqr6tNmzYsWLCA4OBg2rRpQ0BAQJUrbhYsWMDUqVMxGAy4u7vz5ZdfWs8bN24cgYGBrFq1yrrtbm53bTPXRURERK1b3gLMmDGDsLAw2rVrx6JFi5g+fbr1QB2WF1VbXk1deaS5G57uNR/ZSNMsamwnez/odrL/cu3aNZo1a0ZxcTGjR49m2rRpjB492t6xtPvgQX3MazW763ay2v21YMEC/P396datG76+vowaNcrekTRNc1B6CsXBWFaJaJqm1USPwDVN05yUQxdwR5if17T7QT/WtbvhsAXczc2N3Nxc/cDWfvOUUuTm5uLmphtYaXXjsHPgbdu2JTs7m5ycHHtH0bRfnZubG23btrV3DM3JOGwBd3FxKffVc03TNK08h51C0TRN0+5MF3BN0zQnpQu4pmmak3KIr9KLSA5wtx2HvIFL9zDOvabz1Y8j53PkbKDz1Zcj5XtcKVWp05lDFPD6EJGkqnoEOAqdr34cOZ8jZwOdr74cPR/oKRRN0zSnpQu4pmmak/otFPDlNV/ErnS++nHkfI6cDXS++nL0fM4/B65pmvag+i2MwDVN0x5ITl3ARWSoiBwXkRMiElnzb9w/ItJORHaJSJqIpIrIbHtnqkhEGorIIRHZbO8sFYlICxFZJyLpZfdhsL0z2RKR18v+rkdFJEZE7NqJSkRWishFETlqs81LRLaLSGbZ/w85WL6Pyv6+KSKyQURaOFI+m/PeEhElIt72yHYnTlvARaQh8DkwDOgCPCciXeybqpxi4E2lVGegD/Cqg+UDmA2k2TtENf4X2KaUehLogQPlFJHHgFlAL6VUN6AhMN6+qVgFDK2wLRKIU0o9AcSVnbaXVVTOtx3oppQyABnA3PsdysYqKudDRNoBg4Ez9ztQbThtAQeCgBNKqZ+VUreBNcBIO2eyUkqdV0odLPu5AHMBesy+qf5FRNoCTwNf2DtLRSLSHAgFVgAopW4rpa7aN1UljYAmItIIcAf+ac8wSqndwOUKm0cClqNGfwnY7fh8VeVTSsUqpYrLTv4dsFs7xmruP4BPgf8EHPLDQmcu4I8BZ21OZ+NABdKWiPgARmCffZOU8z+YH5il9g5ShQ5ADhBdNsXzhYg0tXcoC6XUOSAK86jsPJCnlIq1b6oqPayUOg/mAQXQ2s557mQasNXeIWyJyAjgnFLqH/bOUh1nLuBSxTaHe5UUkWbAeuA1pVS+vfMAiMhw4KJSKtneWarRCAgAliqljMB17Pv2v5yyueSRgC/wKNBURCbZN5XzEpHfY55yXG3vLBYi4g78Hvgve2e5E2cu4NlAO5vTbbHz29iKRMQFc/FerZT6zt55bPQFRohIFuapp4Ei8rV9I5WTDWQrpSzvWNZhLuiOIgw4pZTKUUoVAd8BT9k5U1UuiEgbgLL/L9o5TyUi8jwwHJioHGtN879hfoH+R9nzpC1wUEQesWuqCpy5gB8AnhARXxFxxfwh0iY7Z7ISEcE8h5umlPrE3nlsKaXmKqXaKqV8MN9vO5VSDjOCVEr9ApwVEb+yTYOAY3aMVNEZoI+IuJf9nQfhQB+y2tgEPF/28/PA93bMUomIDAXeBkYopW7YO48tpdQRpVRrpZRP2fMkGwgoe2w6DKct4GUffswEfsD85PlWKZVq31Tl9AUmYx7dHi77F2HvUE7kP4DVIpIC+AP/bec8VmXvDNYBB4EjmJ9Hdv3WnojEAHsBPxHJFpHpwIfAYBHJxLyS4kMHy/cZ4AFsL3t+LHOwfA5PfxNT0zTNSTntCFzTNO1Bpwu4pmmak9IFXNM0zUnpAq5pmuakdAHXNE1zUrqAa5qmOSldwDVN05yULuCapmlO6v8B5t0h4DtH85AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "start_learning(envs, max_steps, curr_step, test_rewards, avg_reward, moving_average_over_tests)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initializing Environment Vectors for Atari BreakOut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_envs = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name = \"Breakout-v0\"\n",
    "def make_env():\n",
    "    def _envs():\n",
    "        env = gym.make(env_name)\n",
    "        return env\n",
    "\n",
    "    return _envs\n",
    "\n",
    "envs = [make_env() for i in range(num_envs)]\n",
    "envs = SubprocVecEnv(envs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_env = gym.make(env_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dims  = envs.observation_space.shape[0]*envs.observation_space.shape[1]*envs.observation_space.shape[2]\n",
    "output_dims = 4\n",
    "hidden_layer_1  = 1024\n",
    "hidden_layer_2  = 256\n",
    "ac_network_alpha = 1e-3\n",
    "steps = 20\n",
    "mini_batch_size = 6\n",
    "ppo_epochs = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model= AC_Network(input_dims, output_dims, hidden_layer_1, hidden_layer_2).to(device)\n",
    "optimizer = optim.Adam(model.parameters(), lr=ac_network_alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_steps = 8000\n",
    "curr_step  = 0\n",
    "test_rewards = []\n",
    "avg_reward = []\n",
    "moving_average_over_tests = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_learning(envs, max_steps, curr_step, test_rewards, avg_reward, moving_average_over_tests)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We ran into a few issues while training for Breakout. We successfully generated a graph of rewards vs testing sessions but we were not satisfied with the results hence we thought of hyperparameter tuning, unfortunately we could not complete the training for 15000 steps as it was taking a long time to converge. We had to abort the training. Kindly consider the graph that is put in the report for Atari Breakout."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}