My_Segregated_Dendrites.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m registry at `~/.julia/registries/General`\n",
      "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Warning: Some registries failed to update:\n",
      "│     — `~/.julia/registries/General` — registry dirty\n",
      "└ @ Pkg.Types /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.3/Pkg/src/Types.jl:1199\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `/scratch/users/sozcelik19/.julia/environments/v1.3/Project.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `/scratch/users/sozcelik19/.julia/environments/v1.3/Manifest.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n",
      "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `/scratch/users/sozcelik19/.julia/environments/v1.3/Project.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `/scratch/users/sozcelik19/.julia/environments/v1.3/Manifest.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n",
      "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `/scratch/users/sozcelik19/.julia/environments/v1.3/Project.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `/scratch/users/sozcelik19/.julia/environments/v1.3/Manifest.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n"
     ]
    }
   ],
   "source": [
    "using MLDatasets\n",
    "using Random\n",
    "import Pkg\n",
    "Pkg.add(\"Flux\")\n",
    "using Flux: onehotbatch\n",
    "Pkg.add(\"PoissonRandom\")\n",
    "using PoissonRandom\n",
    "Pkg.add(\"Distributions\")\n",
    "using Distributions\n",
    "using Statistics\n",
    "using Printf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_full_test  = 10000 # number of examples to use for full tests  (every epoch)\n",
    "n_quick_test = 100   # number of examples to use for quick tests (every 1000 examples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Global Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# global variables\n",
    "l_f_phase      = 2  # length of forward phase (time steps)\n",
    "l_t_phase      = 2  # length of target phase (time steps)\n",
    "l_f_phase_test = 2  # length of forward phase for tests (time steps)\n",
    "mem            = 1  # spike memory (time steps) - used to limit PSP integration of past spikes (for performance)\n",
    "dt          = 1.0    # time step (ms)\n",
    "lambda_max  = 0.2*dt # maximum spike rate (spikes per time step)\n",
    "# integration_time = 1 # time steps of integration of neuronal variables used for plasticity\n",
    "# we don't use integration time, this makes index start from 0 where julia indexes start from 1 - directly use 1 inst 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "499.9999999999999"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# kernel parameters\n",
    "tau_L = 10.0 # leak time constant\n",
    "\n",
    "# conductance parameters\n",
    "g_B = 0.6                                   # basal conductance\n",
    "g_L = 1.0/tau_L                             # leak conductance\n",
    "g_D = g_B                                   # dendritic conductance in output layer\n",
    "\n",
    "E_E = 8                                     # excitation reversal potential\n",
    "E_I = -8                                    # inhibition reversal potential\n",
    "\n",
    "# steady state constants\n",
    "k_B = g_B/(g_L + g_B)\n",
    "k_D = g_D/(g_L + g_D)\n",
    "k_I = 1.0/(g_L + g_D)\n",
    "\n",
    "# weight update constants\n",
    "P_hidden = 20.0/lambda_max      # hidden layer error signal scaling factor\n",
    "P_final  = 20.0/(lambda_max^2)  # final layer error signal scaling factor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sigma (generic function with 1 method)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sigmoid function\n",
    "function sigma(x) \n",
    "    return 1 / (1 + exp(-x))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "deriv_sigma (generic function with 1 method)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# derivative sigmoid function\n",
    "function deriv_sigma(x)\n",
    "    return exp(-x) / (1 + exp(-x))^2\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mutable struct HiddenLayer\n",
    "    m # The layer number, eg. m = 1 for the first hidden layer.\n",
    "    PSP_B_hist\n",
    "    PSP_A_hist\n",
    "#   integration_counter # we don't use. since integration_counter % integration_time = 0 - not index num. use 1 instead\n",
    "    B\n",
    "    A\n",
    "    A_hist\n",
    "    C\n",
    "    C_hist\n",
    "    lambda_C\n",
    "    lambda_C_hist\n",
    "    S_hist\n",
    "    average_A_f\n",
    "    alpha_f\n",
    "    average_C_f\n",
    "    average_lambda_C_f\n",
    "    average_PSP_B_f\n",
    "    average_A_t\n",
    "    alpha_t\n",
    "    E\n",
    "    average_C_t\n",
    "    average_lambda_C_t\n",
    "    delta_W\n",
    "    delta_b\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "mutable struct FinalLayer\n",
    "    m # The layer number, ie. m = M where M is the total number of layers.\n",
    "    lambda_C # something about soma\n",
    "    PSP_B_hist\n",
    "    B\n",
    "    C\n",
    "    C_hist\n",
    "    lambda_C_hist\n",
    "    S_hist\n",
    "    average_C_f\n",
    "    average_lambda_C_f\n",
    "    average_PSP_B_f\n",
    "    k_D2\n",
    "    k_E\n",
    "    k_I\n",
    "    E\n",
    "    average_C_t\n",
    "    average_lambda_C_t\n",
    "    delta_W\n",
    "    delta_b\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "mutable struct Network\n",
    "    n # Tuple - Number of units in each layer of the network, eg. (500, 10) here.\n",
    "    x_hist # tensor - Input spike history eg. (784, 2) here \n",
    "    hiddenLayer::HiddenLayer\n",
    "    finalLayer::FinalLayer\n",
    "    W\n",
    "    b\n",
    "    Y\n",
    "    f_etas\n",
    "    loss\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization Sub-Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "initializeWbY (generic function with 1 method)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function initializeWbY(ln, n, n_in)\n",
    "    W = Array{Any}(undef, ln)\n",
    "    b = Array{Any}(undef, ln)\n",
    "    Y = Array{Any}(undef, ln-1)\n",
    "    \n",
    "    # weight optimization parameters\n",
    "    V_avg = 3   # desired average of dendritic potential\n",
    "    V_sd  = 3   # desired standard deviation of dendritic potential\n",
    "    b_avg = 0.8  # desired average of bias\n",
    "    b_sd  = 0.001 # desired standard deviation of bias\n",
    "    nu    = lambda_max*0.25  # slope of linear region of activation function\n",
    "    V_sm  = V_sd^2 + V_avg^2 # second moment of dendritic potential\n",
    "    \n",
    "    for m in 2:-1:1\n",
    "        # get number of units in the layer below\n",
    "        if m != 1\n",
    "            N = n[m-1] #500 W2\n",
    "        else\n",
    "            N = n_in  #784 W1\n",
    "        end\n",
    "            \n",
    "        # generate feedforward weights & biases\n",
    "        # calculate weight variables needed to get desired average & standard deviations of somatic potentials\n",
    "        W_avg = (V_avg - b_avg)/(nu*N*V_avg) \n",
    "        W_sm  = (V_sm + (nu^2)*(N - N^2)*(W_avg^2)*(V_avg^2) - 2*N*nu*b_avg*V_avg*W_avg - (b_avg^2))/(N*(nu^2)*V_sm)\n",
    "        W_sd  = sqrt(W_sm - W_avg^2)\n",
    " \n",
    "        W[m] = W_avg .+ 3.465*W_sd*rand(Uniform(-1,1), n[m], N) # (500,784) , (10,500)\n",
    "        b[m] = b_avg .+ 3.465*b_sd*rand(Uniform(-1,1), n[m], 1) # (500,1) , (10,1)\n",
    "        \n",
    "        if m != 1\n",
    "            Y[m-1] = W_avg .+ 3.465*W_sd*rand(Uniform(-1,1), N, n[end]) # (500,10)\n",
    "        end\n",
    "    end\n",
    "        \n",
    "    return W, b, Y\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initializations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HiddenLayer"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# f_input_size - The size of feedforward input, eg. 784 for MNIST input. (784, 1) here.\n",
    "# b_input_size - The size of feedback input. This is the same as the number of units in the next layer. (10,1) here.\n",
    "function HiddenLayer(m, n, f_input_size, b_input_size, size_W) # m=1\n",
    "    PSP_B_hist = zeros(f_input_size, 1)                # (784,1) m=1 in hidden here\n",
    "    PSP_A_hist = zeros(b_input_size, 1)                # (10,1) m=1 in hidden here \n",
    "    B = zeros(n[m], 1)                                 # (500,1) m=1 in hidden here\n",
    "    A = zeros(n[m], 1)\n",
    "    A_hist = zeros(n[m], 1)\n",
    "    C = zeros(n[m], 1)\n",
    "    C_hist = zeros(n[m], 1)\n",
    "    lambda_C = zeros(n[m], 1)\n",
    "    lambda_C_hist = zeros(n[m], 1)\n",
    "    S_hist = zeros(n[m], mem)\n",
    "    average_A_f = zeros(n[m], 1)\n",
    "    alpha_f = zeros(n[m], 1)\n",
    "    average_C_f = zeros(n[m], 1)\n",
    "    average_lambda_C_f = zeros(n[m], 1)\n",
    "    average_PSP_B_f = zeros(f_input_size, 1)           # (784,1) m=1 in hidden here\n",
    "    average_A_t = zeros(n[m], 1)\n",
    "    alpha_t = zeros(n[m], 1)\n",
    "    E = zeros(n[m], 1)\n",
    "    average_C_t = zeros(n[m], 1)\n",
    "    average_lambda_C_t = zeros(n[m], 1)\n",
    "    delta_W = zeros(size_W)\n",
    "    delta_b = zeros(n[m], 1)\n",
    "    return HiddenLayer(m, PSP_B_hist, PSP_A_hist, B, A, A_hist, C, C_hist, lambda_C, lambda_C_hist, S_hist, average_A_f, alpha_f, average_C_f, average_lambda_C_f, average_PSP_B_f, average_A_t, alpha_t, E, average_C_t, average_lambda_C_t, delta_W, delta_b)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FinalLayer"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# f_input_size - The size of feedforward input. This is the same as the number of units in the previous layer. (500,1)\n",
    "function FinalLayer(m, n, f_input_size, size_W) # m=2\n",
    "    lambda_C = zeros(n[m], 1)                         # (10,1) m=2 in final here\n",
    "    PSP_B_hist = zeros(f_input_size, 1)               # (500,1) in size \n",
    "    B = zeros(n[m], 1)                                \n",
    "    C = zeros(n[m], 1)\n",
    "    C_hist = zeros(n[m], 1)\n",
    "    lambda_C_hist = zeros(n[m], 1)\n",
    "    S_hist = zeros(n[m], mem)\n",
    "    average_C_f = zeros(n[m], 1)\n",
    "    average_lambda_C_f = zeros(n[m], 1)\n",
    "    average_PSP_B_f = zeros(f_input_size, 1)\n",
    "    k_D2 = zeros(n[m], 1)\n",
    "    k_E = zeros(n[m], 1)\n",
    "    k_I = zeros(n[m], 1)\n",
    "    E = zeros(n[m], 1)\n",
    "    average_C_t = zeros(n[m], 1)\n",
    "    average_lambda_C_t = zeros(n[m], 1)\n",
    "    delta_W = zeros(size_W)\n",
    "    delta_b = zeros(n[m], 1)\n",
    "    return FinalLayer(m, lambda_C, PSP_B_hist, B, C, C_hist, lambda_C_hist, S_hist, average_C_f, average_lambda_C_f, average_PSP_B_f, k_D2, k_E, k_I, E, average_C_t, average_lambda_C_t, delta_W, delta_b)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Network"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function Network(n, n_in)\n",
    "    x_hist = zeros(n_in, mem)\n",
    "    W, b, Y = initializeWbY(length(n), n, n_in)\n",
    "    hiddenLayer = HiddenLayer(1, n, n_in, n[end], size(W[1]))\n",
    "    finalLayer = FinalLayer(2, n, n[end-1], size(W[2]))\n",
    "    f_etas = (0.21, 0.21)       #Learning rates for each layer's feedforward weights\n",
    "    loss = 0\n",
    "    return Network(n, x_hist, hiddenLayer, finalLayer, W, b, Y, f_etas, loss)\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Callable Objects Sub-Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "update_B (generic function with 1 method)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update basal potentials.\n",
    "# f_input Feedforward input. (784, 1) here\n",
    "function update_B(hidden::HiddenLayer, net::Network, f_input)\n",
    "    hidden.PSP_B_hist[:, 1] = f_input\n",
    "    hidden.B = net.W[hidden.m]*f_input .+ net.b[hidden.m] # (500,784) * (784,1) .+ (500,1) m=1 in hidden here\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "update_A (generic function with 1 method)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update apical potentials.\n",
    "# b_input Feedback input. (10,1) here\n",
    "function update_A(hidden::HiddenLayer, net::Network, b_input)\n",
    "    hidden.PSP_A_hist[:, 1] = b_input\n",
    "    hidden.A = net.Y[hidden.m]*b_input # (500,10) * (10,1) m=1 in hidden here\n",
    "    hidden.A_hist[:, 1] = hidden.A[:, 1]\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "update_C (generic function with 1 method)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update somatic potentials & calculate firing rates.\n",
    "function update_C(hidden::HiddenLayer, net::Network)\n",
    "    hidden.C = k_B*hidden.B\n",
    "    hidden.C_hist[:, 1] = hidden.C[:, 1]\n",
    "    hidden.lambda_C = lambda_max*sigma.(hidden.C)\n",
    "    hidden.lambda_C_hist[:, 1] = hidden.lambda_C[:, 1]\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "spike (generic function with 1 method)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate Poisson spikes based on the firing rates of the neurons.\n",
    "function spike(hidden::HiddenLayer)\n",
    "    hidden.S_hist = hcat(hidden.S_hist[:, 2:end], pois_rand.(hidden.lambda_C))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "update_B (generic function with 2 methods)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update basal potentials.\n",
    "# f_input Feedforward input. (500, 1) here\n",
    "function update_B(final::FinalLayer, net::Network, f_input)\n",
    "    final.PSP_B_hist[:, 1] = f_input\n",
    "    final.B = net.W[final.m]*f_input .+ net.b[final.m] # (10,500) * (500, 1) .+ (10,1) m=2 in final here\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "update_C_forward (generic function with 1 method)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update somatic potentials & calculate firing rates.\n",
    "function update_C_forward(final::FinalLayer, net::Network)\n",
    "    final.C = k_D.*final.B\n",
    "    final.C_hist[:, 1] = final.C[:, 1]\n",
    "    final.lambda_C = lambda_max*sigma.(final.C)\n",
    "    final.lambda_C_hist[:, 1] = final.lambda_C[:, 1]\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "spike (generic function with 2 methods)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate Poisson spikes based on the firing rates of the neurons.\n",
    "function spike(final::FinalLayer)\n",
    "    final.S_hist = hcat(final.S_hist[:, 2:end], pois_rand.(final.lambda_C))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "update_I (generic function with 1 method)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update injected perisomatic currents.\n",
    "# b_input : Target input, eg. if the target label is 8,\n",
    "# b_input = np.array([0, 0, 0, 0, 0, 0, 0, 0, 1, 0]).\n",
    "function update_I(final::FinalLayer, b_input)\n",
    "    g_E = b_input\n",
    "    g_I = -g_E .+ 1\n",
    "    final.k_D2 = g_D./((g_L + g_D) .+ g_E + g_I)\n",
    "    final.k_E  = g_E./((g_L + g_D) .+ g_E + g_I)\n",
    "    final.k_I  = g_I./((g_L + g_D) .+ g_E + g_I)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "update_C_target (generic function with 1 method)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update somatic potentials & calculate firing rates.\n",
    "function update_C_target(final::FinalLayer, net::Network)\n",
    "    final.C = final.k_D2.*final.B + final.k_E*E_E + final.k_I*E_I\n",
    "    final.C_hist[:, 1] = final.C[:, 1]\n",
    "    final.lambda_C = lambda_max*sigma.(final.C)\n",
    "    final.lambda_C_hist[:, 1] = final.lambda_C[:, 1]\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Callable Objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hidden layer out_f / out_t\n",
    "# Perform a forward phase pass. / Perform a target phase pass.\n",
    "# f_input Feedforward input. (784, 1) here\n",
    "# b_input Feedback input. (10,1) here\n",
    "\n",
    "function (hidden::HiddenLayer)(net::Network, f_input, b_input)\n",
    "    update_B(hidden, net, f_input)\n",
    "    update_A(hidden, net, b_input)\n",
    "    update_C(hidden, net)\n",
    "    spike(hidden)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hidden layer plateau_f / plateau_t\n",
    "# Calculate forward phase apical plateau potentials.\n",
    "# plateau_indices - Indices of neurons that are undergoing apical plateau potentials. [1 2 .. 500] array\n",
    "\n",
    "function (hidden::HiddenLayer)(plateau_indices, t_f)\n",
    "    if t_f == \"f\"\n",
    "        # calculate average apical potentials for neurons undergoing plateau potentials\n",
    "        hidden.average_A_f[plateau_indices] = mean(hidden.A_hist[plateau_indices], dims=2) # axis=-1\n",
    "\n",
    "        # calculate apical calcium spike nonlinearity\n",
    "        hidden.alpha_f[plateau_indices] = sigma.(hidden.average_A_f[plateau_indices])\n",
    "    end\n",
    "    if t_f == \"t\"\n",
    "        # calculate average apical potentials for neurons undergoing plateau potentials\n",
    "        hidden.average_A_t[plateau_indices] = mean(hidden.A_hist[plateau_indices], dims=2) # axis=-1\n",
    "\n",
    "        # calculate apical calcium spike nonlinearity\n",
    "        hidden.alpha_t[plateau_indices] = sigma.(hidden.average_A_t[plateau_indices])\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hidden layer calc_averages\n",
    "# Calculate averages of dynamic variables. This is done at the end of each forward & target phase.\n",
    "# phase (string) : Current phase of the network, \"forward\" or \"target\".\n",
    "\n",
    "function (hidden::HiddenLayer)(phase::String)\n",
    "    if phase == \"forward\"\n",
    "        hidden.average_C_f        = mean(hidden.C_hist, dims=2) # axis=-1 hepsinde\n",
    "        hidden.average_lambda_C_f = mean(hidden.lambda_C_hist, dims=2)\n",
    "        hidden.average_PSP_B_f    = mean(hidden.PSP_B_hist, dims=2)\n",
    "    elseif phase == \"target\"\n",
    "        hidden.average_C_t        = mean(hidden.C_hist, dims=2)\n",
    "        hidden.average_lambda_C_t = mean(hidden.lambda_C_hist, dims=2)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hidden layer update_W\n",
    "# Update feedforward weights.\n",
    "\n",
    "function (hidden::HiddenLayer)(net::Network)\n",
    "    hidden.E = (hidden.alpha_t - hidden.alpha_f).*((-k_B)*lambda_max*deriv_sigma.(hidden.average_C_f))\n",
    "    hidden.delta_W = hidden.E * hidden.average_PSP_B_f'   #((500,1)*(1,784) = (500,784)\n",
    "    net.W[hidden.m] += -net.f_etas[hidden.m]*P_hidden*hidden.delta_W\n",
    "    hidden.delta_b = hidden.E\n",
    "    net.b[hidden.m] += -net.f_etas[hidden.m]*P_hidden*hidden.delta_b\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final layer out_f\n",
    "# Perform a forward phase pass.\n",
    "# f_input Feedforward input. (500, 1) here\n",
    "\n",
    "function (final::FinalLayer)(net::Network, f_input)\n",
    "    update_B(final, net, f_input)\n",
    "    update_C_forward(final, net)\n",
    "    spike(final)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final layer calc_averages\n",
    "# Calculate averages of dynamic variables. This is done at the end of each forward & target phase.\n",
    "# phase (string) : Current phase of the network, \"forward\" or \"target\".\n",
    "\n",
    "function (final::FinalLayer)(phase::String)\n",
    "    if phase == \"forward\"\n",
    "        final.average_C_f        = mean(final.C_hist, dims=2) # axis=-1 hepsinde\n",
    "        final.average_lambda_C_f = mean(final.lambda_C_hist, dims=2)\n",
    "        final.average_PSP_B_f    = mean(final.PSP_B_hist, dims=2)\n",
    "    elseif phase == \"target\"\n",
    "        final.average_C_t        = mean(final.C_hist, dims=2)\n",
    "        final.average_lambda_C_t = mean(final.lambda_C_hist, dims=2)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final layer out_t\n",
    "# Perform a target phase pass.\n",
    "# f_input (ndarray) : Feedforward input.\n",
    "# b_input (ndarray) : Target input.\n",
    "\n",
    "function (final::FinalLayer)(net::Network, f_input, b_input)\n",
    "    update_B(final, net, f_input)\n",
    "    update_I(final, b_input)\n",
    "    update_C_target(final, net)\n",
    "    spike(final)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final layer update_W\n",
    "# Update feedforward weights.\n",
    "\n",
    "function (final::FinalLayer)(net::Network)\n",
    "    final.E = (final.average_lambda_C_t - lambda_max*sigma.(final.average_C_f)).*((-k_D)*lambda_max*deriv_sigma.(final.average_C_f))\n",
    "    final.delta_W = final.E * final.average_PSP_B_f'   #(10,1)*(1,500) = (10,500)\n",
    "    net.W[final.m] += -net.f_etas[final.m]*P_final*final.delta_W\n",
    "    final.delta_b = final.E\n",
    "    net.b[final.m] += -net.f_etas[final.m]*P_final*final.delta_b\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# net out_f\n",
    "# Perform a forward phase pass through the network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# net out_t\n",
    "# Perform a target phase pass through the network. This is the same as a forward phase pass,           \n",
    "# but with a target introduced at the top layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# net f_phase\n",
    "# Perform a forward phase.\n",
    "# x : Input array of size (X, 1) where X is the size of the input, eg. (784, 1).\n",
    "function (net::Network)(x)\n",
    "    for time in 1:l_f_phase\n",
    "        # update input spike history\n",
    "        net.x_hist = hcat(net.x_hist[:, 2:end], pois_rand.(x))\n",
    "        # do a forward pass <--> net out_f -- line 530\n",
    "        net.hiddenLayer(net, x, net.finalLayer.lambda_C) # hidden layer out_f\n",
    "        net.finalLayer(net, net.hiddenLayer.lambda_C) # final layer out_f\n",
    "    end\n",
    "    m = 1\n",
    "    plateau_indices = [1:1:net.n[m];] # 500 = n[1]\n",
    "    # calculate plateau potentials for hidden layer neurons\n",
    "    net.hiddenLayer(plateau_indices, \"f\") #hidden layer plateau_f\n",
    "    # calculate averages\n",
    "    net.finalLayer(\"forward\") #final layer calc_averages\n",
    "    net.hiddenLayer(\"forward\") #hidden layer calc_averages\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# net t_phase\n",
    "# Perform a target phase.\n",
    "# x : Input array of size (X, 1) where X is the size of the input, eg. (784, 1).\n",
    "# t : Target array of size (T, 1) where T is the size of the target, eg. (10, 1).\n",
    "\n",
    "function (net::Network)(x, t)\n",
    "    for time in 1:l_t_phase\n",
    "        # update input history\n",
    "        net.x_hist = hcat(net.x_hist[:, 2:end], pois_rand.(x))\n",
    "        # do a target pass <--> net out_t\n",
    "        net.hiddenLayer(net, x, net.finalLayer.lambda_C) # hidden layer out_t\n",
    "        net.finalLayer(net, net.hiddenLayer.lambda_C, t) # final layer out_t\n",
    "    end\n",
    "    m = 1\n",
    "    plateau_indices = [1:1:net.n[m];] # 500 = n[1]\n",
    "    # calculate plateau potentials for hidden layer neurons\n",
    "    net.hiddenLayer(plateau_indices, \"t\") #hidden layer plateau_t\n",
    "    # calculate averages & update weights \n",
    "    net.finalLayer(\"target\") #final layer calc_averages\n",
    "    net.finalLayer(net)      #final layer update_W\n",
    "    net.hiddenLayer(\"target\") #hidden layer calc_averages\n",
    "    net.hiddenLayer(net)      #hidden layer update_W\n",
    "    net.loss = mean((net.finalLayer.average_lambda_C_t - lambda_max*sigma.(net.finalLayer.average_C_f)).^2)\n",
    "    # reset averages\n",
    "    net.finalLayer.average_C_f .*= 0\n",
    "    net.finalLayer.average_C_t .*= 0\n",
    "    net.finalLayer.average_PSP_B_f .*= 0\n",
    "    net.hiddenLayer.average_C_f .*= 0\n",
    "    net.hiddenLayer.average_C_t .*= 0\n",
    "    net.hiddenLayer.average_PSP_B_f .*= 0\n",
    "    net.finalLayer.average_lambda_C_f .*= 0\n",
    "    net.finalLayer.average_lambda_C_t .*= 0\n",
    "    net.hiddenLayer.average_A_f .*= 0\n",
    "    net.hiddenLayer.average_A_t .*= 0\n",
    "    net.hiddenLayer.average_lambda_C_f .*= 0\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Sub-Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "clear_vars (generic function with 1 method)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function clear_vars(net::Network)\n",
    "    net.hiddenLayer.A .*= 0\n",
    "    net.hiddenLayer.B .*= 0\n",
    "    net.hiddenLayer.C .*= 0\n",
    "    net.hiddenLayer.lambda_C .*= 0\n",
    "\n",
    "    net.hiddenLayer.S_hist .*= 0\n",
    "    net.hiddenLayer.A_hist .*= 0\n",
    "    net.hiddenLayer.PSP_A_hist .*= 0\n",
    "    net.hiddenLayer.PSP_B_hist .*= 0\n",
    "    net.hiddenLayer.C_hist .*= 0\n",
    "    net.hiddenLayer.lambda_C_hist .*= 0\n",
    "\n",
    "    net.hiddenLayer.E .*= 0\n",
    "    net.hiddenLayer.delta_W .*= 0\n",
    "    net.hiddenLayer.delta_b .*= 0\n",
    "\n",
    "    net.hiddenLayer.average_C_f .*= 0\n",
    "    net.hiddenLayer.average_C_t .*= 0\n",
    "    net.hiddenLayer.average_A_f .*= 0\n",
    "    net.hiddenLayer.average_A_t .*= 0\n",
    "    net.hiddenLayer.average_lambda_C_f .*= 0\n",
    "    net.hiddenLayer.average_PSP_B_f .*= 0\n",
    "\n",
    "    net.hiddenLayer.alpha_f .*= 0\n",
    "    net.hiddenLayer.alpha_t .*= 0\n",
    "    \n",
    "    net.finalLayer.B .*= 0\n",
    "    net.finalLayer.C .*= 0\n",
    "    net.finalLayer.lambda_C .*= 0\n",
    "\n",
    "    net.finalLayer.S_hist .*= 0\n",
    "    net.finalLayer.PSP_B_hist .*= 0\n",
    "    net.finalLayer.C_hist .*= 0\n",
    "    net.finalLayer.lambda_C_hist .*= 0\n",
    "\n",
    "    net.finalLayer.E .*= 0\n",
    "    net.finalLayer.delta_W .*= 0\n",
    "    net.finalLayer.delta_b .*= 0\n",
    "\n",
    "    net.finalLayer.average_C_f .*= 0\n",
    "    net.finalLayer.average_C_t .*= 0\n",
    "    net.finalLayer.average_lambda_C_f .*= 0\n",
    "    net.finalLayer.average_lambda_C_t .*= 0\n",
    "    net.finalLayer.average_PSP_B_f .*= 0\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_weights (generic function with 1 method)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# net test_weights\n",
    "# Test the network's current weights on the test set. The network's layers are copied\n",
    "#      and restored to their previous state after testing.\n",
    "# n_test : The number of test examples to use.\n",
    "\n",
    "function test_weights(net::Network, n_test, x_test, t_test, l_f_phase, l_f_phase_test)\n",
    "    # save old length of forward phase\n",
    "    old_l_f_phase = l_f_phase\n",
    "\n",
    "    # set new length of forward phase\n",
    "    l_f_phase = l_f_phase_test\n",
    "\n",
    "    old_x_hist = net.x_hist\n",
    "\n",
    "    # initialize count of correct classifications\n",
    "    num_correct = 0\n",
    "\n",
    "    # shuffle testing data\n",
    "    order = randperm(size(x_test)[2]) #randperm(10000)\n",
    "    x_test = x_test[:,order] \n",
    "    t_test = t_test[:,order]\n",
    "    \n",
    "    for i in 1:n_test\n",
    "        # clear all layer variables\n",
    "        clear_vars(net)\n",
    "\n",
    "        # clear input spike history\n",
    "        net.x_hist .*= 0\n",
    "\n",
    "        # get testing example data\n",
    "        x = lambda_max*x_test[:, i]\n",
    "        t = t_test[:, i]\n",
    "\n",
    "        # do a forward phase & get the unit with maximum average somatic potential\n",
    "        net(x) # net f_phase\n",
    "        sel_num = argmax(net.finalLayer.average_C_f)[1]\n",
    "\n",
    "        # get the target number from testing example data\n",
    "        target_num = argmax(t)\n",
    "\n",
    "        # increment correct classification counter if they match\n",
    "        if sel_num == target_num\n",
    "            num_correct += 1\n",
    "        end\n",
    "    end\n",
    "    \n",
    "    # calculate percent error\n",
    "    err_rate = (1.0 - num_correct/n_test)*100.0\n",
    "\n",
    "    if old_x_hist != nothing\n",
    "        net.x_hist = old_x_hist\n",
    "    end\n",
    "\n",
    "    l_f_phase = old_l_f_phase\n",
    "\n",
    "    clear_vars(net)\n",
    "    \n",
    "    return err_rate\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60000"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "n_training_examples = 60000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "load_MNIST (generic function with 1 method)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function load_MNIST()\n",
    "    x_train = MNIST.convert2features(MNIST.traintensor(Float64))\n",
    "    t_train = MNIST.trainlabels()\n",
    "    t_train = onehotbatch(t_train, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])    \n",
    "    x_test = MNIST.convert2features(MNIST.testtensor(Float64))\n",
    "    t_test = MNIST.testlabels()\n",
    "    t_test = onehotbatch(t_test, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) \n",
    "    return x_train, t_train, x_test, t_test\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train (generic function with 3 methods)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# net train\n",
    "function train(net::Network, n_epochs = 10, n_training_examples = 60000)\n",
    "    \n",
    "    x_train, t_train, x_test, t_test = load_MNIST()\n",
    "    \n",
    "    current_epoch = 1\n",
    "    \n",
    "    # generate phase lengths for all training examples [2 2 2 ..]\n",
    "    l_f_phases = zeros(n_epochs*n_training_examples) .+ l_f_phase # [2 2 2 2 ..] in length 10*60000\n",
    "    l_t_phases = zeros(n_epochs*n_training_examples) .+ l_t_phase\n",
    "    \n",
    "    #record_training_labels:\n",
    "    training_labels = zeros(n_epochs*n_training_examples)\n",
    "    \n",
    "    # don't do an initial weight test\n",
    "    println(\"Start of epoch $(current_epoch)\")\n",
    "    \n",
    "    # start time used for timing how long each 1000 examples take\n",
    "    start_time = nothing\n",
    "    \n",
    "    #record_training_error\n",
    "    num_correct = 0\n",
    "    \n",
    "    for k in 1:n_epochs\n",
    "        # shuffle the training data\n",
    "        order = randperm(size(x_train)[2]) #randperm(60000)\n",
    "        x_train = x_train[:,order] \n",
    "        t_train = t_train[:,order]\n",
    "                  \n",
    "        for i in 1:n_training_examples   # n become i here\n",
    "            # set start time\n",
    "            if start_time == nothing\n",
    "                start_time = time()\n",
    "            end\n",
    "            \n",
    "            # get training example data\n",
    "            x = lambda_max*x_train[:,i]\n",
    "            t = t_train[:,i]\n",
    "            \n",
    "            # do forward phase\n",
    "            net(x) # net f_phase\n",
    "            \n",
    "            sel_num = argmax(net.finalLayer.average_C_f)[1] #axis=-1\n",
    "\n",
    "            # get the target number from testing example data\n",
    "            target_num = argmax(t)\n",
    "\n",
    "            # increment correct classification counter if they match\n",
    "            if sel_num == target_num\n",
    "                num_correct += 1\n",
    "            end\n",
    "            \n",
    "            # do target phase\n",
    "            net(x, t) # net t_phase\n",
    "            \n",
    "            #record_training_labels\n",
    "            training_labels[(k-1)*n_training_examples + i] = argmax(t)\n",
    "            \n",
    "            if i % 1000 == 0\n",
    "                if i != n_training_examples\n",
    "                    # we're partway through an epoch; do a quick weight test\n",
    "                    test_err = test_weights(net, n_quick_test, x_test, t_test, l_f_phase, l_f_phase_test)\n",
    "                    @printf(\"Epoch %d, example %d/%d. QE: %g \", current_epoch, i, n_training_examples, test_err)\n",
    "                else\n",
    "                    # we've reached the end of an epoch; do a full weight test\n",
    "                    test_err = test_weights(net, n_full_test, x_test, t_test, l_f_phase, l_f_phase_test)\n",
    "                    @printf(\"FE: %g \", test_err)\n",
    "                    \n",
    "                    # calculate percent training error for this epoch\n",
    "                    err_rate = (1.0 - num_correct/n_training_examples)*100.0\n",
    "                    @printf(\"TE: %g \", err_rate)\n",
    "\n",
    "                    num_correct = 0\n",
    "                end\n",
    "                # get end time & reset start time\n",
    "                end_time = time()\n",
    "                time_elapsed = end_time - start_time\n",
    "                @printf(\"T: %g\\n\", time_elapsed)\n",
    "                start_time = nothing\n",
    "            end\n",
    "        end\n",
    "        # update latest epoch counter\n",
    "        current_epoch += 1\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start of epoch 1\n",
      "Epoch 1, example 1000/60000. QE: 37 T: 8.64713\n",
      "Epoch 1, example 2000/60000. QE: 21 T: 3.11679\n",
      "Epoch 1, example 3000/60000. QE: 17 T: 3.09488\n",
      "Epoch 1, example 4000/60000. QE: 17 T: 3.07839\n",
      "Epoch 1, example 5000/60000. QE: 4 T: 3.10862\n",
      "Epoch 1, example 6000/60000. QE: 9 T: 3.13514\n",
      "Epoch 1, example 7000/60000. QE: 6 T: 3.19693\n",
      "Epoch 1, example 8000/60000. QE: 12 T: 3.20736\n",
      "Epoch 1, example 9000/60000. QE: 10 T: 3.20168\n",
      "Epoch 1, example 10000/60000. QE: 12 T: 3.1367\n",
      "Epoch 1, example 11000/60000. QE: 10 T: 3.17196\n",
      "Epoch 1, example 12000/60000. QE: 8 T: 3.14541\n",
      "Epoch 1, example 13000/60000. QE: 7 T: 3.12594\n",
      "Epoch 1, example 14000/60000. QE: 12 T: 3.15272\n",
      "Epoch 1, example 15000/60000. QE: 11 T: 3.18173\n",
      "Epoch 1, example 16000/60000. QE: 7 T: 3.14362\n",
      "Epoch 1, example 17000/60000. QE: 8 T: 3.10902\n",
      "Epoch 1, example 18000/60000. QE: 15 T: 3.13285\n",
      "Epoch 1, example 19000/60000. QE: 10 T: 3.14669\n",
      "Epoch 1, example 20000/60000. QE: 12 T: 3.12404\n",
      "Epoch 1, example 21000/60000. QE: 7 T: 3.11251\n",
      "Epoch 1, example 22000/60000. QE: 11 T: 3.13967\n",
      "Epoch 1, example 23000/60000. QE: 9 T: 3.09915\n",
      "Epoch 1, example 24000/60000. QE: 7 T: 3.12367\n",
      "Epoch 1, example 25000/60000. QE: 10 T: 3.11589\n",
      "Epoch 1, example 26000/60000. QE: 8 T: 3.1232\n",
      "Epoch 1, example 27000/60000. QE: 4 T: 3.1041\n",
      "Epoch 1, example 28000/60000. QE: 12 T: 3.13492\n",
      "Epoch 1, example 29000/60000. QE: 8 T: 3.09224\n",
      "Epoch 1, example 30000/60000. QE: 10 T: 3.12\n",
      "Epoch 1, example 31000/60000. QE: 6 T: 3.09861\n",
      "Epoch 1, example 32000/60000. QE: 6 T: 3.14914\n",
      "Epoch 1, example 33000/60000. QE: 10 T: 3.11386\n",
      "Epoch 1, example 34000/60000. QE: 8 T: 3.11914\n",
      "Epoch 1, example 35000/60000. QE: 9 T: 3.11641\n",
      "Epoch 1, example 36000/60000. QE: 8 T: 3.14684\n",
      "Epoch 1, example 37000/60000. QE: 7 T: 3.09919\n",
      "Epoch 1, example 38000/60000. QE: 8 T: 3.14196\n",
      "Epoch 1, example 39000/60000. QE: 7 T: 3.09419\n",
      "Epoch 1, example 40000/60000. QE: 16 T: 3.1366\n",
      "Epoch 1, example 41000/60000. QE: 5 T: 3.11332\n",
      "Epoch 1, example 42000/60000. QE: 6 T: 3.10977\n",
      "Epoch 1, example 43000/60000. QE: 8 T: 3.1046\n",
      "Epoch 1, example 44000/60000. QE: 6 T: 3.12971\n",
      "Epoch 1, example 45000/60000. QE: 6 T: 3.09447\n",
      "Epoch 1, example 46000/60000. QE: 5 T: 3.13105\n",
      "Epoch 1, example 47000/60000. QE: 10 T: 3.10392\n",
      "Epoch 1, example 48000/60000. QE: 6 T: 3.1398\n",
      "Epoch 1, example 49000/60000. QE: 10 T: 3.1064\n",
      "Epoch 1, example 50000/60000. QE: 8 T: 3.12126\n",
      "Epoch 1, example 51000/60000. QE: 4 T: 3.16714\n",
      "Epoch 1, example 52000/60000. QE: 8 T: 3.15119\n",
      "Epoch 1, example 53000/60000. QE: 10 T: 3.11615\n",
      "Epoch 1, example 54000/60000. QE: 9 T: 3.14356\n",
      "Epoch 1, example 55000/60000. QE: 9 T: 3.12167\n",
      "Epoch 1, example 56000/60000. QE: 5 T: 3.13245\n",
      "Epoch 1, example 57000/60000. QE: 9 T: 3.14036\n",
      "Epoch 1, example 58000/60000. QE: 6 T: 3.12675\n",
      "Epoch 1, example 59000/60000. QE: 5 T: 3.11192\n",
      "FE: 6.09 TE: 10.8 T: 10.4607\n",
      "Epoch 2, example 1000/60000. QE: 8 T: 3.84668\n",
      "Epoch 2, example 2000/60000. QE: 5 T: 2.89035\n",
      "Epoch 2, example 3000/60000. QE: 6 T: 2.88439\n",
      "Epoch 2, example 4000/60000. QE: 9 T: 2.89657\n",
      "Epoch 2, example 5000/60000. QE: 5 T: 2.8835\n",
      "Epoch 2, example 6000/60000. QE: 5 T: 2.88419\n",
      "Epoch 2, example 7000/60000. QE: 7 T: 2.87828\n",
      "Epoch 2, example 8000/60000. QE: 3 T: 2.89836\n",
      "Epoch 2, example 9000/60000. QE: 4 T: 2.88222\n",
      "Epoch 2, example 10000/60000. QE: 5 T: 2.89338\n",
      "Epoch 2, example 11000/60000. QE: 13 T: 2.89256\n",
      "Epoch 2, example 12000/60000. QE: 4 T: 2.86681\n",
      "Epoch 2, example 13000/60000. QE: 2 T: 2.86939\n",
      "Epoch 2, example 14000/60000. QE: 8 T: 2.86494\n",
      "Epoch 2, example 15000/60000. QE: 7 T: 2.8867\n",
      "Epoch 2, example 16000/60000. QE: 3 T: 2.85797\n",
      "Epoch 2, example 17000/60000. QE: 5 T: 2.88233\n",
      "Epoch 2, example 18000/60000. QE: 7 T: 2.88134\n",
      "Epoch 2, example 19000/60000. QE: 2 T: 2.865\n",
      "Epoch 2, example 20000/60000. QE: 6 T: 2.87481\n",
      "Epoch 2, example 21000/60000. QE: 2 T: 2.87809\n",
      "Epoch 2, example 22000/60000. QE: 6 T: 2.88502\n",
      "Epoch 2, example 23000/60000. QE: 5 T: 2.88089\n",
      "Epoch 2, example 24000/60000. QE: 6 T: 2.87505\n",
      "Epoch 2, example 25000/60000. QE: 5 T: 2.90045\n",
      "Epoch 2, example 26000/60000. QE: 5 T: 2.85739\n",
      "Epoch 2, example 27000/60000. QE: 3 T: 2.87817\n",
      "Epoch 2, example 28000/60000. QE: 3 T: 2.86334\n",
      "Epoch 2, example 29000/60000. QE: 3 T: 2.96992\n",
      "Epoch 2, example 30000/60000. QE: 7 T: 2.86755\n",
      "Epoch 2, example 31000/60000. QE: 3 T: 2.85156\n",
      "Epoch 2, example 32000/60000. QE: 3 T: 2.86881\n",
      "Epoch 2, example 33000/60000. QE: 5 T: 2.84471\n",
      "Epoch 2, example 34000/60000. QE: 4 T: 2.85463\n",
      "Epoch 2, example 35000/60000. QE: 9 T: 2.84359\n",
      "Epoch 2, example 36000/60000. QE: 13 T: 2.857\n",
      "Epoch 2, example 37000/60000. QE: 10 T: 2.85159\n",
      "Epoch 2, example 38000/60000. QE: 7 T: 2.86185\n",
      "Epoch 2, example 39000/60000. QE: 6 T: 2.87057\n",
      "Epoch 2, example 40000/60000. QE: 4 T: 2.84503\n",
      "Epoch 2, example 41000/60000. QE: 5 T: 2.85734\n",
      "Epoch 2, example 42000/60000. QE: 5 T: 2.83355\n",
      "Epoch 2, example 43000/60000. QE: 1 T: 2.86761\n",
      "Epoch 2, example 44000/60000. QE: 5 T: 2.84629\n",
      "Epoch 2, example 45000/60000. QE: 6 T: 2.85799\n",
      "Epoch 2, example 46000/60000. QE: 3 T: 2.86243\n",
      "Epoch 2, example 47000/60000. QE: 5 T: 2.84209\n",
      "Epoch 2, example 48000/60000. QE: 8 T: 2.8555\n",
      "Epoch 2, example 49000/60000. QE: 3 T: 2.84334\n",
      "Epoch 2, example 50000/60000. QE: 4 T: 2.86409\n",
      "Epoch 2, example 51000/60000. QE: 5 T: 2.83855\n",
      "Epoch 2, example 52000/60000. QE: 3 T: 2.84721\n",
      "Epoch 2, example 53000/60000. QE: 6 T: 2.8647\n",
      "Epoch 2, example 54000/60000. QE: 5 T: 2.83402\n",
      "Epoch 2, example 55000/60000. QE: 7 T: 2.85323\n",
      "Epoch 2, example 56000/60000. QE: 5 T: 2.84366\n",
      "Epoch 2, example 57000/60000. QE: 6 T: 2.86262\n",
      "Epoch 2, example 58000/60000. QE: 5 T: 2.83606\n",
      "Epoch 2, example 59000/60000. QE: 5 T: 2.86036\n",
      "FE: 4.4 TE: 5.38333 T: 9.98746\n",
      "Epoch 3, example 1000/60000. QE: 4 T: 2.91032\n",
      "Epoch 3, example 2000/60000. QE: 0 T: 2.85594\n",
      "Epoch 3, example 3000/60000. QE: 7 T: 2.90802\n",
      "Epoch 3, example 4000/60000. QE: 12 T: 2.87831\n",
      "Epoch 3, example 5000/60000. QE: 5 T: 2.89208\n",
      "Epoch 3, example 6000/60000. QE: 3 T: 2.85793\n",
      "Epoch 3, example 7000/60000. QE: 3 T: 2.85634\n",
      "Epoch 3, example 8000/60000. QE: 6 T: 2.87635\n",
      "Epoch 3, example 9000/60000. QE: 6 T: 2.87414\n",
      "Epoch 3, example 10000/60000. QE: 5 T: 2.89106\n",
      "Epoch 3, example 11000/60000. QE: 3 T: 2.91117\n",
      "Epoch 3, example 12000/60000. QE: 9 T: 2.88485\n",
      "Epoch 3, example 13000/60000. QE: 5 T: 2.87702\n",
      "Epoch 3, example 14000/60000. QE: 5 T: 2.85025\n",
      "Epoch 3, example 15000/60000. QE: 3 T: 2.88946\n",
      "Epoch 3, example 16000/60000. QE: 4 T: 2.87122\n",
      "Epoch 3, example 17000/60000. QE: 6 T: 2.88612\n",
      "Epoch 3, example 18000/60000. QE: 4 T: 2.85853\n",
      "Epoch 3, example 19000/60000. QE: 6 T: 2.88833\n",
      "Epoch 3, example 20000/60000. QE: 2 T: 2.86956\n",
      "Epoch 3, example 21000/60000. QE: 5 T: 2.8669\n",
      "Epoch 3, example 22000/60000. QE: 8 T: 2.87075\n",
      "Epoch 3, example 23000/60000. QE: 2 T: 2.86066\n",
      "Epoch 3, example 24000/60000. QE: 4 T: 2.87504\n",
      "Epoch 3, example 25000/60000. QE: 4 T: 2.88134\n",
      "Epoch 3, example 26000/60000. QE: 2 T: 2.86965\n",
      "Epoch 3, example 27000/60000. QE: 1 T: 2.87288\n",
      "Epoch 3, example 28000/60000. QE: 6 T: 2.84529\n",
      "Epoch 3, example 29000/60000. QE: 1 T: 2.87848\n",
      "Epoch 3, example 30000/60000. QE: 4 T: 2.85258\n",
      "Epoch 3, example 31000/60000. QE: 6 T: 2.87076\n",
      "Epoch 3, example 32000/60000. QE: 5 T: 2.85571\n",
      "Epoch 3, example 33000/60000. QE: 5 T: 2.88815\n",
      "Epoch 3, example 34000/60000. QE: 2 T: 2.85972\n",
      "Epoch 3, example 35000/60000. QE: 3 T: 2.85687\n",
      "Epoch 3, example 36000/60000. QE: 6 T: 2.90016\n",
      "Epoch 3, example 37000/60000. QE: 4 T: 2.86952\n",
      "Epoch 3, example 38000/60000. QE: 9 T: 2.85265\n",
      "Epoch 3, example 39000/60000. QE: 2 T: 2.87551\n",
      "Epoch 3, example 40000/60000. QE: 7 T: 2.8768\n",
      "Epoch 3, example 41000/60000. QE: 6 T: 2.87393\n",
      "Epoch 3, example 42000/60000. QE: 4 T: 2.84702\n",
      "Epoch 3, example 43000/60000. QE: 2 T: 2.89702\n",
      "Epoch 3, example 44000/60000. QE: 7 T: 2.85366\n",
      "Epoch 3, example 45000/60000. QE: 7 T: 2.87616\n",
      "Epoch 3, example 46000/60000. QE: 6 T: 2.87123\n",
      "Epoch 3, example 47000/60000. QE: 7 T: 2.86649\n",
      "Epoch 3, example 48000/60000. QE: 5 T: 2.87638\n",
      "Epoch 3, example 49000/60000. QE: 2 T: 2.85792\n",
      "Epoch 3, example 50000/60000. QE: 1 T: 2.88313\n",
      "Epoch 3, example 51000/60000. QE: 5 T: 2.925\n",
      "Epoch 3, example 52000/60000. QE: 6 T: 2.86558\n",
      "Epoch 3, example 53000/60000. QE: 5 T: 2.87787\n",
      "Epoch 3, example 54000/60000. QE: 3 T: 2.85515\n",
      "Epoch 3, example 55000/60000. QE: 3 T: 2.87686\n",
      "Epoch 3, example 56000/60000. QE: 6 T: 2.85367\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3, example 57000/60000. QE: 2 T: 2.89539\n",
      "Epoch 3, example 58000/60000. QE: 5 T: 2.84311\n",
      "Epoch 3, example 59000/60000. QE: 2 T: 2.87064\n",
      "FE: 4.28 TE: 4.045 T: 10.137\n",
      "Epoch 4, example 1000/60000. QE: 2 T: 3.51085\n",
      "Epoch 4, example 2000/60000. QE: 2 T: 2.84022\n",
      "Epoch 4, example 3000/60000. QE: 4 T: 2.85047\n",
      "Epoch 4, example 4000/60000. QE: 5 T: 2.83283\n",
      "Epoch 4, example 5000/60000. QE: 7 T: 2.85973\n",
      "Epoch 4, example 6000/60000. QE: 2 T: 2.83821\n",
      "Epoch 4, example 7000/60000. QE: 4 T: 2.8304\n",
      "Epoch 4, example 8000/60000. QE: 6 T: 2.85411\n",
      "Epoch 4, example 9000/60000. QE: 5 T: 2.84236\n",
      "Epoch 4, example 10000/60000. QE: 10 T: 2.83913\n",
      "Epoch 4, example 11000/60000. QE: 4 T: 2.83997\n",
      "Epoch 4, example 12000/60000. QE: 4 T: 2.86126\n",
      "Epoch 4, example 13000/60000. QE: 0 T: 2.84969\n",
      "Epoch 4, example 14000/60000. QE: 2 T: 2.82656\n",
      "Epoch 4, example 15000/60000. QE: 0 T: 2.86197\n",
      "Epoch 4, example 16000/60000. QE: 5 T: 2.8285\n",
      "Epoch 4, example 17000/60000. QE: 4 T: 2.84074\n",
      "Epoch 4, example 18000/60000. QE: 6 T: 2.83238\n",
      "Epoch 4, example 19000/60000. QE: 5 T: 2.85866\n",
      "Epoch 4, example 20000/60000. QE: 5 T: 2.84071\n",
      "Epoch 4, example 21000/60000. QE: 4 T: 2.8319\n",
      "Epoch 4, example 22000/60000. QE: 2 T: 2.84864\n",
      "Epoch 4, example 23000/60000. QE: 1 T: 2.83552\n",
      "Epoch 4, example 24000/60000. QE: 6 T: 2.83537\n",
      "Epoch 4, example 25000/60000. QE: 6 T: 2.83845\n",
      "Epoch 4, example 26000/60000. QE: 2 T: 2.86338\n",
      "Epoch 4, example 27000/60000. QE: 4 T: 2.84117\n",
      "Epoch 4, example 28000/60000. QE: 4 T: 2.83933\n",
      "Epoch 4, example 29000/60000. QE: 5 T: 2.85794\n",
      "Epoch 4, example 30000/60000. QE: 7 T: 2.91116\n",
      "Epoch 4, example 31000/60000. QE: 8 T: 2.83656\n",
      "Epoch 4, example 32000/60000. QE: 9 T: 2.82457\n",
      "Epoch 4, example 33000/60000. QE: 2 T: 2.85397\n",
      "Epoch 4, example 34000/60000. QE: 5 T: 2.83232\n",
      "Epoch 4, example 35000/60000. QE: 4 T: 2.83806\n",
      "Epoch 4, example 36000/60000. QE: 5 T: 2.84623\n",
      "Epoch 4, example 37000/60000. QE: 8 T: 2.8472\n",
      "Epoch 4, example 38000/60000. QE: 3 T: 2.84445\n",
      "Epoch 4, example 39000/60000. QE: 0 T: 2.83265\n",
      "Epoch 4, example 40000/60000. QE: 2 T: 2.85743\n",
      "Epoch 4, example 41000/60000. QE: 2 T: 2.83872\n",
      "Epoch 4, example 42000/60000. QE: 5 T: 2.83827\n",
      "Epoch 4, example 43000/60000. QE: 4 T: 2.85293\n",
      "Epoch 4, example 44000/60000. QE: 7 T: 2.85312\n",
      "Epoch 4, example 45000/60000. QE: 3 T: 2.8535\n",
      "Epoch 4, example 46000/60000. QE: 6 T: 2.83066\n",
      "Epoch 4, example 47000/60000. QE: 1 T: 2.85793\n",
      "Epoch 4, example 48000/60000. QE: 4 T: 2.81896\n",
      "Epoch 4, example 49000/60000. QE: 6 T: 2.84132\n",
      "Epoch 4, example 50000/60000. QE: 4 T: 2.8557\n",
      "Epoch 4, example 51000/60000. QE: 4 T: 2.85175\n",
      "Epoch 4, example 52000/60000. QE: 2 T: 2.84321\n",
      "Epoch 4, example 53000/60000. QE: 0 T: 2.84736\n",
      "Epoch 4, example 54000/60000. QE: 3 T: 2.85669\n",
      "Epoch 4, example 55000/60000. QE: 3 T: 2.8315\n",
      "Epoch 4, example 56000/60000. QE: 1 T: 2.84209\n",
      "Epoch 4, example 57000/60000. QE: 8 T: 2.85868\n",
      "Epoch 4, example 58000/60000. QE: 5 T: 2.8479\n",
      "Epoch 4, example 59000/60000. QE: 2 T: 2.84595\n",
      "FE: 3.23 TE: 3.365 T: 9.9932\n",
      "Epoch 5, example 1000/60000. QE: 5 T: 2.88496\n",
      "Epoch 5, example 2000/60000. QE: 0 T: 2.86518\n",
      "Epoch 5, example 3000/60000. QE: 2 T: 2.89963\n",
      "Epoch 5, example 4000/60000. QE: 4 T: 2.88023\n",
      "Epoch 5, example 5000/60000. QE: 5 T: 2.88936\n",
      "Epoch 5, example 6000/60000. QE: 8 T: 2.88455\n",
      "Epoch 5, example 7000/60000. QE: 1 T: 2.87486\n",
      "Epoch 5, example 8000/60000. QE: 1 T: 2.86041\n",
      "Epoch 5, example 9000/60000. QE: 1 T: 2.95897\n",
      "Epoch 5, example 10000/60000. QE: 4 T: 2.87538\n",
      "Epoch 5, example 11000/60000. QE: 3 T: 2.88109\n",
      "Epoch 5, example 12000/60000. QE: 3 T: 2.88266\n",
      "Epoch 5, example 13000/60000. QE: 5 T: 2.86074\n",
      "Epoch 5, example 14000/60000. QE: 6 T: 2.85787\n",
      "Epoch 5, example 15000/60000. QE: 5 T: 2.86605\n",
      "Epoch 5, example 16000/60000. QE: 3 T: 2.87296\n",
      "Epoch 5, example 17000/60000. QE: 4 T: 2.87966\n",
      "Epoch 5, example 18000/60000. QE: 1 T: 2.84269\n",
      "Epoch 5, example 19000/60000. QE: 2 T: 2.88418\n",
      "Epoch 5, example 20000/60000. QE: 3 T: 2.85866\n",
      "Epoch 5, example 21000/60000. QE: 1 T: 2.8675\n",
      "Epoch 5, example 22000/60000. QE: 2 T: 2.84838\n",
      "Epoch 5, example 23000/60000. QE: 2 T: 2.88025\n",
      "Epoch 5, example 24000/60000. QE: 4 T: 2.85528\n",
      "Epoch 5, example 25000/60000. QE: 3 T: 2.86741\n",
      "Epoch 5, example 26000/60000. QE: 3 T: 2.88192\n",
      "Epoch 5, example 27000/60000. QE: 2 T: 2.86251\n",
      "Epoch 5, example 28000/60000. QE: 5 T: 2.87057\n",
      "Epoch 5, example 29000/60000. QE: 4 T: 2.86829\n",
      "Epoch 5, example 30000/60000. QE: 4 T: 2.86882\n",
      "Epoch 5, example 31000/60000. QE: 2 T: 2.86988\n",
      "Epoch 5, example 32000/60000. QE: 2 T: 2.85115\n",
      "Epoch 5, example 33000/60000. QE: 3 T: 2.88257\n",
      "Epoch 5, example 34000/60000. QE: 2 T: 2.84803\n",
      "Epoch 5, example 35000/60000. QE: 1 T: 2.87165\n",
      "Epoch 5, example 36000/60000. QE: 6 T: 2.86041\n",
      "Epoch 5, example 37000/60000. QE: 2 T: 2.87675\n",
      "Epoch 5, example 38000/60000. QE: 2 T: 2.88483\n",
      "Epoch 5, example 39000/60000. QE: 4 T: 2.85848\n",
      "Epoch 5, example 40000/60000. QE: 2 T: 2.88451\n",
      "Epoch 5, example 41000/60000. QE: 2 T: 2.87088\n",
      "Epoch 5, example 42000/60000. QE: 5 T: 2.86504\n",
      "Epoch 5, example 43000/60000. QE: 2 T: 2.86809\n",
      "Epoch 5, example 44000/60000. QE: 4 T: 2.87601\n",
      "Epoch 5, example 45000/60000. QE: 6 T: 2.87179\n",
      "Epoch 5, example 46000/60000. QE: 3 T: 2.84846\n",
      "Epoch 5, example 47000/60000. QE: 8 T: 2.8832\n",
      "Epoch 5, example 48000/60000. QE: 2 T: 2.85998\n",
      "Epoch 5, example 49000/60000. QE: 6 T: 2.88193\n",
      "Epoch 5, example 50000/60000. QE: 4 T: 2.86616\n",
      "Epoch 5, example 51000/60000. QE: 3 T: 2.91961\n",
      "Epoch 5, example 52000/60000. QE: 4 T: 2.89312\n",
      "Epoch 5, example 53000/60000. QE: 1 T: 2.85983\n",
      "Epoch 5, example 54000/60000. QE: 4 T: 2.88653\n",
      "Epoch 5, example 55000/60000. QE: 2 T: 2.86031\n",
      "Epoch 5, example 56000/60000. QE: 6 T: 2.86299\n",
      "Epoch 5, example 57000/60000. QE: 3 T: 2.904\n",
      "Epoch 5, example 58000/60000. QE: 3 T: 2.8532\n",
      "Epoch 5, example 59000/60000. QE: 4 T: 2.87502\n",
      "FE: 3.34 TE: 2.83 T: 10.0784\n",
      "Epoch 6, example 1000/60000. QE: 4 T: 3.49155\n",
      "Epoch 6, example 2000/60000. QE: 4 T: 2.85265\n",
      "Epoch 6, example 3000/60000. QE: 7 T: 2.83904\n",
      "Epoch 6, example 4000/60000. QE: 4 T: 2.8316\n",
      "Epoch 6, example 5000/60000. QE: 1 T: 2.85229\n",
      "Epoch 6, example 6000/60000. QE: 1 T: 2.83859\n",
      "Epoch 6, example 7000/60000. QE: 5 T: 2.84766\n",
      "Epoch 6, example 8000/60000. QE: 7 T: 2.82819\n",
      "Epoch 6, example 9000/60000. QE: 3 T: 2.86102\n",
      "Epoch 6, example 10000/60000. QE: 4 T: 2.83567\n",
      "Epoch 6, example 11000/60000. QE: 3 T: 2.82784\n",
      "Epoch 6, example 12000/60000. QE: 1 T: 2.86791\n",
      "Epoch 6, example 13000/60000. QE: 2 T: 2.8325\n",
      "Epoch 6, example 14000/60000. QE: 2 T: 2.8358\n",
      "Epoch 6, example 15000/60000. QE: 2 T: 2.81623\n",
      "Epoch 6, example 16000/60000. QE: 2 T: 2.84696\n",
      "Epoch 6, example 17000/60000. QE: 4 T: 2.82944\n",
      "Epoch 6, example 18000/60000. QE: 1 T: 2.83899\n",
      "Epoch 6, example 19000/60000. QE: 4 T: 2.84476\n",
      "Epoch 6, example 20000/60000. QE: 5 T: 2.83516\n",
      "Epoch 6, example 21000/60000. QE: 5 T: 2.84246\n",
      "Epoch 6, example 22000/60000. QE: 0 T: 2.82406\n",
      "Epoch 6, example 23000/60000. QE: 3 T: 2.86555\n",
      "Epoch 6, example 24000/60000. QE: 1 T: 2.83891\n",
      "Epoch 6, example 25000/60000. QE: 3 T: 2.84817\n",
      "Epoch 6, example 26000/60000. QE: 3 T: 2.86702\n",
      "Epoch 6, example 27000/60000. QE: 3 T: 2.82324\n",
      "Epoch 6, example 28000/60000. QE: 8 T: 2.84674\n",
      "Epoch 6, example 29000/60000. QE: 3 T: 2.82597\n",
      "Epoch 6, example 30000/60000. QE: 5 T: 2.8655\n",
      "Epoch 6, example 31000/60000. QE: 0 T: 2.90015\n",
      "Epoch 6, example 32000/60000. QE: 5 T: 2.84485\n",
      "Epoch 6, example 33000/60000. QE: 2 T: 2.85857\n",
      "Epoch 6, example 34000/60000. QE: 4 T: 2.84016\n",
      "Epoch 6, example 35000/60000. QE: 2 T: 2.83974\n",
      "Epoch 6, example 36000/60000. QE: 3 T: 2.83154\n",
      "Epoch 6, example 37000/60000. QE: 0 T: 2.86561\n",
      "Epoch 6, example 38000/60000. QE: 5 T: 2.83077\n",
      "Epoch 6, example 39000/60000. QE: 4 T: 2.84424\n",
      "Epoch 6, example 40000/60000. QE: 1 T: 2.85957\n",
      "Epoch 6, example 41000/60000. QE: 2 T: 2.83186\n",
      "Epoch 6, example 42000/60000. QE: 3 T: 2.84729\n",
      "Epoch 6, example 43000/60000. QE: 4 T: 2.82867\n",
      "Epoch 6, example 44000/60000. QE: 5 T: 2.85136\n",
      "Epoch 6, example 45000/60000. QE: 1 T: 2.8354\n",
      "Epoch 6, example 46000/60000. QE: 0 T: 2.84459\n",
      "Epoch 6, example 47000/60000. QE: 3 T: 2.86212\n",
      "Epoch 6, example 48000/60000. QE: 5 T: 2.82362\n",
      "Epoch 6, example 49000/60000. QE: 2 T: 2.84293\n",
      "Epoch 6, example 50000/60000. QE: 4 T: 2.82945\n",
      "Epoch 6, example 51000/60000. QE: 3 T: 2.8646\n",
      "Epoch 6, example 52000/60000. QE: 2 T: 2.83107\n",
      "Epoch 6, example 53000/60000. QE: 2 T: 2.83546\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6, example 54000/60000. QE: 3 T: 2.85131\n",
      "Epoch 6, example 55000/60000. QE: 2 T: 2.83495\n",
      "Epoch 6, example 56000/60000. QE: 5 T: 2.84195\n",
      "Epoch 6, example 57000/60000. QE: 1 T: 2.84142\n",
      "Epoch 6, example 58000/60000. QE: 0 T: 2.86006\n",
      "Epoch 6, example 59000/60000. QE: 3 T: 2.83313\n",
      "FE: 3.35 TE: 2.46167 T: 9.99911\n",
      "Epoch 7, example 1000/60000. QE: 6 T: 2.89397\n",
      "Epoch 7, example 2000/60000. QE: 2 T: 2.87541\n",
      "Epoch 7, example 3000/60000. QE: 2 T: 2.90679\n",
      "Epoch 7, example 4000/60000. QE: 7 T: 2.87135\n",
      "Epoch 7, example 5000/60000. QE: 3 T: 2.86676\n",
      "Epoch 7, example 6000/60000. QE: 2 T: 2.88295\n",
      "Epoch 7, example 7000/60000. QE: 5 T: 2.85433\n",
      "Epoch 7, example 8000/60000. QE: 0 T: 2.86569\n",
      "Epoch 7, example 9000/60000. QE: 2 T: 2.88689\n",
      "Epoch 7, example 10000/60000. QE: 2 T: 2.92853\n",
      "Epoch 7, example 11000/60000. QE: 3 T: 2.87075\n",
      "Epoch 7, example 12000/60000. QE: 2 T: 2.85866\n",
      "Epoch 7, example 13000/60000. QE: 5 T: 2.88533\n",
      "Epoch 7, example 14000/60000. QE: 4 T: 2.84445\n",
      "Epoch 7, example 15000/60000. QE: 3 T: 2.86913\n",
      "Epoch 7, example 16000/60000. QE: 3 T: 2.88138\n",
      "Epoch 7, example 17000/60000. QE: 7 T: 2.87595\n",
      "Epoch 7, example 18000/60000. QE: 5 T: 2.86032\n",
      "Epoch 7, example 19000/60000. QE: 2 T: 2.86128\n",
      "Epoch 7, example 20000/60000. QE: 3 T: 2.87785\n",
      "Epoch 7, example 21000/60000. QE: 3 T: 2.86761\n",
      "Epoch 7, example 22000/60000. QE: 5 T: 2.83841\n",
      "Epoch 7, example 23000/60000. QE: 7 T: 2.8668\n",
      "Epoch 7, example 24000/60000. QE: 6 T: 2.88541\n",
      "Epoch 7, example 25000/60000. QE: 5 T: 2.86346\n",
      "Epoch 7, example 26000/60000. QE: 3 T: 2.83985\n",
      "Epoch 7, example 27000/60000. QE: 2 T: 2.87948\n",
      "Epoch 7, example 28000/60000. QE: 2 T: 2.86091\n",
      "Epoch 7, example 29000/60000. QE: 1 T: 2.84853\n",
      "Epoch 7, example 30000/60000. QE: 2 T: 2.85487\n",
      "Epoch 7, example 31000/60000. QE: 1 T: 2.83824\n",
      "Epoch 7, example 32000/60000. QE: 1 T: 2.85281\n",
      "Epoch 7, example 33000/60000. QE: 3 T: 2.8543\n",
      "Epoch 7, example 34000/60000. QE: 3 T: 2.85907\n",
      "Epoch 7, example 35000/60000. QE: 1 T: 2.8497\n",
      "Epoch 7, example 36000/60000. QE: 5 T: 2.85944\n",
      "Epoch 7, example 37000/60000. QE: 6 T: 2.86894\n",
      "Epoch 7, example 38000/60000. QE: 3 T: 2.83014\n",
      "Epoch 7, example 39000/60000. QE: 4 T: 2.8522\n",
      "Epoch 7, example 40000/60000. QE: 0 T: 2.83049\n",
      "Epoch 7, example 41000/60000. QE: 2 T: 2.89433\n",
      "Epoch 7, example 42000/60000. QE: 1 T: 2.83776\n",
      "Epoch 7, example 43000/60000. QE: 2 T: 2.84076\n",
      "Epoch 7, example 44000/60000. QE: 1 T: 2.8506\n",
      "Epoch 7, example 45000/60000. QE: 4 T: 2.83377\n",
      "Epoch 7, example 46000/60000. QE: 5 T: 2.83821\n",
      "Epoch 7, example 47000/60000. QE: 4 T: 2.83729\n",
      "Epoch 7, example 48000/60000. QE: 0 T: 2.87101\n",
      "Epoch 7, example 49000/60000. QE: 5 T: 2.84084\n",
      "Epoch 7, example 50000/60000. QE: 2 T: 2.84098\n",
      "Epoch 7, example 51000/60000. QE: 4 T: 2.85791\n",
      "Epoch 7, example 52000/60000. QE: 6 T: 2.83842\n",
      "Epoch 7, example 53000/60000. QE: 3 T: 2.84111\n",
      "Epoch 7, example 54000/60000. QE: 3 T: 2.89136\n",
      "Epoch 7, example 55000/60000. QE: 1 T: 2.86482\n",
      "Epoch 7, example 56000/60000. QE: 3 T: 2.8271\n",
      "Epoch 7, example 57000/60000. QE: 2 T: 2.88088\n",
      "Epoch 7, example 58000/60000. QE: 3 T: 2.8383\n",
      "Epoch 7, example 59000/60000. QE: 5 T: 2.84097\n",
      "FE: 3.25 TE: 2.12667 T: 9.98612\n",
      "Epoch 8, example 1000/60000. QE: 4 T: 3.47226\n",
      "Epoch 8, example 2000/60000. QE: 4 T: 2.79289\n",
      "Epoch 8, example 3000/60000. QE: 2 T: 2.82461\n",
      "Epoch 8, example 4000/60000. QE: 2 T: 2.81465\n",
      "Epoch 8, example 5000/60000. QE: 5 T: 2.79691\n",
      "Epoch 8, example 6000/60000. QE: 3 T: 2.83503\n",
      "Epoch 8, example 7000/60000. QE: 2 T: 2.7952\n",
      "Epoch 8, example 8000/60000. QE: 2 T: 2.81412\n",
      "Epoch 8, example 9000/60000. QE: 5 T: 2.79631\n",
      "Epoch 8, example 10000/60000. QE: 2 T: 2.81224\n",
      "Epoch 8, example 11000/60000. QE: 1 T: 2.80133\n",
      "Epoch 8, example 12000/60000. QE: 4 T: 2.79434\n",
      "Epoch 8, example 13000/60000. QE: 4 T: 2.8184\n",
      "Epoch 8, example 14000/60000. QE: 5 T: 2.78448\n",
      "Epoch 8, example 15000/60000. QE: 1 T: 2.80346\n",
      "Epoch 8, example 16000/60000. QE: 0 T: 2.79663\n",
      "Epoch 8, example 17000/60000. QE: 2 T: 2.81276\n",
      "Epoch 8, example 18000/60000. QE: 2 T: 2.78731\n",
      "Epoch 8, example 19000/60000. QE: 3 T: 2.81133\n",
      "Epoch 8, example 20000/60000. QE: 2 T: 2.81425\n",
      "Epoch 8, example 21000/60000. QE: 2 T: 2.79674\n",
      "Epoch 8, example 22000/60000. QE: 4 T: 2.80802\n",
      "Epoch 8, example 23000/60000. QE: 5 T: 2.78969\n",
      "Epoch 8, example 24000/60000. QE: 0 T: 2.82246\n",
      "Epoch 8, example 25000/60000. QE: 2 T: 2.79809\n",
      "Epoch 8, example 26000/60000. QE: 2 T: 2.79926\n",
      "Epoch 8, example 27000/60000. QE: 5 T: 2.83242\n",
      "Epoch 8, example 28000/60000. QE: 4 T: 2.81412\n",
      "Epoch 8, example 29000/60000. QE: 2 T: 2.81068\n",
      "Epoch 8, example 30000/60000. QE: 1 T: 2.78158\n",
      "Epoch 8, example 31000/60000. QE: 4 T: 2.82772\n",
      "Epoch 8, example 32000/60000. QE: 3 T: 2.789\n",
      "Epoch 8, example 33000/60000. QE: 3 T: 2.80353\n",
      "Epoch 8, example 34000/60000. QE: 1 T: 2.80621\n",
      "Epoch 8, example 35000/60000. QE: 4 T: 2.89729\n",
      "Epoch 8, example 36000/60000. QE: 1 T: 2.80102\n",
      "Epoch 8, example 37000/60000. QE: 5 T: 2.79986\n",
      "Epoch 8, example 38000/60000. QE: 2 T: 2.8137\n",
      "Epoch 8, example 39000/60000. QE: 0 T: 2.79837\n",
      "Epoch 8, example 40000/60000. QE: 3 T: 2.80653\n",
      "Epoch 8, example 41000/60000. QE: 3 T: 2.79973\n",
      "Epoch 8, example 42000/60000. QE: 4 T: 2.81002\n",
      "Epoch 8, example 43000/60000. QE: 3 T: 2.7975\n",
      "Epoch 8, example 44000/60000. QE: 3 T: 2.79255\n",
      "Epoch 8, example 45000/60000. QE: 5 T: 2.80304\n",
      "Epoch 8, example 46000/60000. QE: 2 T: 2.79049\n",
      "Epoch 8, example 47000/60000. QE: 1 T: 2.80539\n",
      "Epoch 8, example 48000/60000. QE: 2 T: 2.79091\n",
      "Epoch 8, example 49000/60000. QE: 5 T: 2.82662\n",
      "Epoch 8, example 50000/60000. QE: 3 T: 2.78796\n",
      "Epoch 8, example 51000/60000. QE: 2 T: 2.80145\n",
      "Epoch 8, example 52000/60000. QE: 1 T: 2.81073\n",
      "Epoch 8, example 53000/60000. QE: 3 T: 2.78978\n",
      "Epoch 8, example 54000/60000. QE: 1 T: 2.79126\n",
      "Epoch 8, example 55000/60000. QE: 3 T: 2.78921\n",
      "Epoch 8, example 56000/60000. QE: 3 T: 2.81134\n",
      "Epoch 8, example 57000/60000. QE: 2 T: 2.78717\n",
      "Epoch 8, example 58000/60000. QE: 1 T: 2.79668\n",
      "Epoch 8, example 59000/60000. QE: 3 T: 2.7901\n",
      "FE: 2.53 TE: 1.92167 T: 9.85507\n",
      "Epoch 9, example 1000/60000. QE: 2 T: 2.88851\n",
      "Epoch 9, example 2000/60000. QE: 0 T: 2.81798\n",
      "Epoch 9, example 3000/60000. QE: 4 T: 2.85596\n",
      "Epoch 9, example 4000/60000. QE: 6 T: 2.84645\n",
      "Epoch 9, example 5000/60000. QE: 2 T: 2.82826\n",
      "Epoch 9, example 6000/60000. QE: 2 T: 2.82915\n",
      "Epoch 9, example 7000/60000. QE: 2 T: 2.83576\n",
      "Epoch 9, example 8000/60000. QE: 1 T: 2.85269\n",
      "Epoch 9, example 9000/60000. QE: 5 T: 2.84194\n",
      "Epoch 9, example 10000/60000. QE: 3 T: 2.82052\n",
      "Epoch 9, example 11000/60000. QE: 3 T: 2.85845\n",
      "Epoch 9, example 12000/60000. QE: 2 T: 2.90542\n",
      "Epoch 9, example 13000/60000. QE: 1 T: 2.84101\n",
      "Epoch 9, example 14000/60000. QE: 4 T: 2.82996\n",
      "Epoch 9, example 15000/60000. QE: 4 T: 2.85521\n",
      "Epoch 9, example 16000/60000. QE: 2 T: 2.83718\n",
      "Epoch 9, example 17000/60000. QE: 3 T: 2.84045\n",
      "Epoch 9, example 18000/60000. QE: 2 T: 2.83627\n",
      "Epoch 9, example 19000/60000. QE: 1 T: 2.82323\n",
      "Epoch 9, example 20000/60000. QE: 3 T: 2.84494\n",
      "Epoch 9, example 21000/60000. QE: 1 T: 2.83857\n",
      "Epoch 9, example 22000/60000. QE: 3 T: 2.85153\n",
      "Epoch 9, example 23000/60000. QE: 3 T: 2.84248\n",
      "Epoch 9, example 24000/60000. QE: 2 T: 2.82911\n",
      "Epoch 9, example 25000/60000. QE: 2 T: 2.85107\n",
      "Epoch 9, example 26000/60000. QE: 0 T: 2.82196\n",
      "Epoch 9, example 27000/60000. QE: 3 T: 2.85887\n",
      "Epoch 9, example 28000/60000. QE: 3 T: 2.83085\n",
      "Epoch 9, example 29000/60000. QE: 2 T: 2.8576\n",
      "Epoch 9, example 30000/60000. QE: 3 T: 2.81921\n",
      "Epoch 9, example 31000/60000. QE: 3 T: 2.83222\n",
      "Epoch 9, example 32000/60000. QE: 4 T: 2.83987\n",
      "Epoch 9, example 33000/60000. QE: 5 T: 2.81684\n",
      "Epoch 9, example 34000/60000. QE: 4 T: 2.81382\n",
      "Epoch 9, example 35000/60000. QE: 2 T: 2.81543\n",
      "Epoch 9, example 36000/60000. QE: 1 T: 2.84296\n",
      "Epoch 9, example 37000/60000. QE: 3 T: 2.82274\n",
      "Epoch 9, example 38000/60000. QE: 2 T: 2.82908\n",
      "Epoch 9, example 39000/60000. QE: 2 T: 2.8451\n",
      "Epoch 9, example 40000/60000. QE: 3 T: 2.82252\n",
      "Epoch 9, example 41000/60000. QE: 3 T: 2.84417\n",
      "Epoch 9, example 42000/60000. QE: 1 T: 2.80675\n",
      "Epoch 9, example 43000/60000. QE: 3 T: 2.84825\n",
      "Epoch 9, example 44000/60000. QE: 3 T: 2.80963\n",
      "Epoch 9, example 45000/60000. QE: 4 T: 2.84734\n",
      "Epoch 9, example 46000/60000. QE: 3 T: 2.83103\n",
      "Epoch 9, example 47000/60000. QE: 0 T: 2.82451\n",
      "Epoch 9, example 48000/60000. QE: 0 T: 2.83337\n",
      "Epoch 9, example 49000/60000. QE: 4 T: 2.82254\n",
      "Epoch 9, example 50000/60000. QE: 1 T: 2.85474\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9, example 51000/60000. QE: 1 T: 2.8186\n",
      "Epoch 9, example 52000/60000. QE: 5 T: 2.82501\n",
      "Epoch 9, example 53000/60000. QE: 5 T: 2.83917\n",
      "Epoch 9, example 54000/60000. QE: 3 T: 2.81859\n",
      "Epoch 9, example 55000/60000. QE: 1 T: 2.88851\n",
      "Epoch 9, example 56000/60000. QE: 5 T: 2.80281\n",
      "Epoch 9, example 57000/60000. QE: 4 T: 2.85086\n",
      "Epoch 9, example 58000/60000. QE: 1 T: 2.80719\n",
      "Epoch 9, example 59000/60000. QE: 3 T: 2.83678\n",
      "FE: 2.37 TE: 1.655 T: 9.93334\n",
      "Epoch 10, example 1000/60000. QE: 1 T: 3.46709\n",
      "Epoch 10, example 2000/60000. QE: 1 T: 2.79964\n",
      "Epoch 10, example 3000/60000. QE: 3 T: 2.80832\n",
      "Epoch 10, example 4000/60000. QE: 3 T: 2.78348\n",
      "Epoch 10, example 5000/60000. QE: 0 T: 2.80181\n",
      "Epoch 10, example 6000/60000. QE: 1 T: 2.781\n",
      "Epoch 10, example 7000/60000. QE: 2 T: 2.79542\n",
      "Epoch 10, example 8000/60000. QE: 2 T: 2.81524\n",
      "Epoch 10, example 9000/60000. QE: 3 T: 2.79356\n",
      "Epoch 10, example 10000/60000. QE: 4 T: 2.8056\n",
      "Epoch 10, example 11000/60000. QE: 2 T: 2.78824\n",
      "Epoch 10, example 12000/60000. QE: 4 T: 2.80865\n",
      "Epoch 10, example 13000/60000. QE: 3 T: 2.78829\n",
      "Epoch 10, example 14000/60000. QE: 2 T: 2.80339\n",
      "Epoch 10, example 15000/60000. QE: 2 T: 2.78535\n",
      "Epoch 10, example 16000/60000. QE: 2 T: 2.80684\n",
      "Epoch 10, example 17000/60000. QE: 1 T: 2.7961\n",
      "Epoch 10, example 18000/60000. QE: 1 T: 2.78718\n",
      "Epoch 10, example 19000/60000. QE: 4 T: 2.80042\n",
      "Epoch 10, example 20000/60000. QE: 1 T: 2.78821\n",
      "Epoch 10, example 21000/60000. QE: 5 T: 2.79674\n",
      "Epoch 10, example 22000/60000. QE: 1 T: 2.78161\n",
      "Epoch 10, example 23000/60000. QE: 0 T: 2.81663\n",
      "Epoch 10, example 24000/60000. QE: 2 T: 2.78877\n",
      "Epoch 10, example 25000/60000. QE: 3 T: 2.80336\n",
      "Epoch 10, example 26000/60000. QE: 5 T: 2.80351\n",
      "Epoch 10, example 27000/60000. QE: 3 T: 2.7848\n",
      "Epoch 10, example 28000/60000. QE: 4 T: 2.79926\n",
      "Epoch 10, example 29000/60000. QE: 3 T: 2.78977\n",
      "Epoch 10, example 30000/60000. QE: 3 T: 2.82687\n",
      "Epoch 10, example 31000/60000. QE: 2 T: 2.78815\n",
      "Epoch 10, example 32000/60000. QE: 3 T: 2.80374\n",
      "Epoch 10, example 33000/60000. QE: 2 T: 2.80798\n",
      "Epoch 10, example 34000/60000. QE: 4 T: 2.86423\n",
      "Epoch 10, example 35000/60000. QE: 6 T: 2.80201\n",
      "Epoch 10, example 36000/60000. QE: 0 T: 2.78571\n",
      "Epoch 10, example 37000/60000. QE: 1 T: 2.81154\n",
      "Epoch 10, example 38000/60000. QE: 5 T: 2.78682\n",
      "Epoch 10, example 39000/60000. QE: 2 T: 2.80117\n",
      "Epoch 10, example 40000/60000. QE: 2 T: 2.78479\n",
      "Epoch 10, example 41000/60000. QE: 1 T: 2.80954\n",
      "Epoch 10, example 42000/60000. QE: 4 T: 2.79054\n",
      "Epoch 10, example 43000/60000. QE: 1 T: 2.7843\n",
      "Epoch 10, example 44000/60000. QE: 1 T: 2.80721\n",
      "Epoch 10, example 45000/60000. QE: 1 T: 2.79484\n",
      "Epoch 10, example 46000/60000. QE: 2 T: 2.79082\n",
      "Epoch 10, example 47000/60000. QE: 3 T: 2.78862\n",
      "Epoch 10, example 48000/60000. QE: 0 T: 2.80516\n",
      "Epoch 10, example 49000/60000. QE: 2 T: 2.79679\n",
      "Epoch 10, example 50000/60000. QE: 2 T: 2.7954\n",
      "Epoch 10, example 51000/60000. QE: 3 T: 2.82104\n",
      "Epoch 10, example 52000/60000. QE: 1 T: 2.77769\n",
      "Epoch 10, example 53000/60000. QE: 2 T: 2.79718\n",
      "Epoch 10, example 54000/60000. QE: 4 T: 2.78283\n",
      "Epoch 10, example 55000/60000. QE: 2 T: 2.80959\n",
      "Epoch 10, example 56000/60000. QE: 2 T: 2.78236\n",
      "Epoch 10, example 57000/60000. QE: 5 T: 2.80413\n",
      "Epoch 10, example 58000/60000. QE: 2 T: 2.82531\n",
      "Epoch 10, example 59000/60000. QE: 4 T: 2.79232\n",
      "FE: 2.48 TE: 1.51833 T: 9.872\n"
     ]
    }
   ],
   "source": [
    "n = (500, 10) #for now building everything for having one hidden layer\n",
    "n_in  = 784\n",
    "net = Network(n, n_in)\n",
    "train(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.3.1",
   "language": "julia",
   "name": "julia-1.3"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.3.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}