cross-entropy/CEM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross-Entropy Method\n",
    "\n",
    "---\n",
    "\n",
    "In this notebook, we will train the Cross-Entropy Method with OpenAI Gym's MountainCarContinuous environment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import math\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2(a). Define the Agent class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent(nn.Module):\n",
    "    def __init__(self, env, h_size=16):\n",
    "        super(Agent, self).__init__()\n",
    "        self.env = env\n",
    "        \n",
    "        # state, hidden layer, action sizes\n",
    "        self.s_size = env.observation_space.shape[0]\n",
    "        self.h_size = h_size\n",
    "        self.a_size = env.action_space.shape[0]\n",
    "        \n",
    "        # define layers\n",
    "        self.fc1 = nn.Linear(self.s_size, self.h_size)\n",
    "        self.fc2 = nn.Linear(self.h_size, self.a_size)\n",
    "        \n",
    "    def set_weights(self, weights):\n",
    "        s_size = self.s_size\n",
    "        h_size = self.h_size\n",
    "        a_size = self.a_size\n",
    "        \n",
    "        # separate the weights for each layer\n",
    "        fc1_end = (s_size*h_size)+h_size\n",
    "        fc1_W = torch.from_numpy(weights[:s_size*h_size].reshape(s_size, h_size))\n",
    "        fc1_b = torch.from_numpy(weights[s_size*h_size:fc1_end])\n",
    "        fc2_W = torch.from_numpy(weights[fc1_end:fc1_end+(h_size*a_size)].reshape(h_size, a_size))\n",
    "        fc2_b = torch.from_numpy(weights[fc1_end+(h_size*a_size):])\n",
    "        \n",
    "        # set the weights for each layer\n",
    "        self.fc1.weight.data.copy_(fc1_W.view_as(self.fc1.weight.data))\n",
    "        self.fc1.bias.data.copy_(fc1_b.view_as(self.fc1.bias.data))\n",
    "        self.fc2.weight.data.copy_(fc2_W.view_as(self.fc2.weight.data))\n",
    "        self.fc2.bias.data.copy_(fc2_b.view_as(self.fc2.bias.data))\n",
    "    \n",
    "    def get_weights_dim(self):\n",
    "        return (self.s_size+1)*self.h_size + (self.h_size+1)*self.a_size\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.tanh(self.fc2(x))\n",
    "        \n",
    "        return x.cpu().data\n",
    "        \n",
    "    def evaluate(self, weights, gamma=1.0, max_t=5000):\n",
    "        self.set_weights(weights)\n",
    "        episode_return = 0.0\n",
    "        state = self.env.reset()\n",
    "        for t in range(max_t):\n",
    "            state = torch.from_numpy(state).float().to(device)\n",
    "            action = self.forward(state)\n",
    "            state, reward, done, _ = self.env.step(action)\n",
    "            episode_return += reward * math.pow(gamma, t)\n",
    "            if done:\n",
    "                break\n",
    "                \n",
    "        return episode_return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2(b). Instantiate the Environment and Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation space: Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "action space: Box([-1.], [1.], (1,), float32)\n",
      "  - low: [-1.]\n",
      "  - high: [1.]\n"
     ]
    }
   ],
   "source": [
    "# Instantiate the environment\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "env = gym.make('MountainCarContinuous-v0')\n",
    "env.seed(101)\n",
    "np.random.seed(101)\n",
    "\n",
    "print('observation space:', env.observation_space)\n",
    "print('action space:', env.action_space)\n",
    "print('  - low:', env.action_space.low)\n",
    "print('  - high:', env.action_space.high)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate the agent\n",
    "agent = Agent(env).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the Agent with the Cross-Entropy Method\n",
    "\n",
    "Run the code cell below to train the agent from scratch.  Alternatively, you can skip to the next code cell to load the pre-trained weights from file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "def cem(n_iterations=500, max_t=1000, gamma=1.0, print_every=10, pop_size=50, elite_frac=0.2, sigma=0.5):\n",
    "    \"\"\"PyTorch implementation of the cross-entropy method.\n",
    "        \n",
    "    Params\n",
    "    ======\n",
    "        n_iterations (int): maximum number of training iterations\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        gamma (float): discount rate\n",
    "        print_every (int): how often to print average score (over last 100 episodes)\n",
    "        pop_size (int): size of population at each iteration\n",
    "        elite_frac (float): percentage of top performers to use in update\n",
    "        sigma (float): standard deviation of additive noise\n",
    "    \"\"\"\n",
    "    n_elite=int(pop_size*elite_frac)\n",
    "\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    best_weight = sigma*np.random.randn(agent.get_weights_dim())\n",
    "\n",
    "    for i_iteration in range(1, n_iterations+1):\n",
    "        weights_pop = [best_weight + (sigma*np.random.randn(agent.get_weights_dim())) for i in range(pop_size)]\n",
    "        rewards = np.array([agent.evaluate(weights, gamma, max_t) for weights in weights_pop])\n",
    "\n",
    "        elite_idxs = rewards.argsort()[-n_elite:]\n",
    "        elite_weights = [weights_pop[i] for i in elite_idxs]\n",
    "        best_weight = np.array(elite_weights).mean(axis=0)\n",
    "\n",
    "        reward = agent.evaluate(best_weight, gamma=1.0)\n",
    "        scores_deque.append(reward)\n",
    "        scores.append(reward)\n",
    "        \n",
    "        torch.save(agent.state_dict(), 'checkpoint.pth')\n",
    "        \n",
    "        if i_iteration % print_every == 0:\n",
    "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_iteration, np.mean(scores_deque)))\n",
    "\n",
    "        if np.mean(scores_deque)>=90.0:\n",
    "            print('\\nEnvironment solved in {:d} iterations!\\tAverage Score: {:.2f}'.format(i_iteration-100, np.mean(scores_deque)))\n",
    "            break\n",
    "            \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/kenpower/opt/anaconda3/envs/DRLND/lib/python3.9/site-packages/torch/nn/functional.py:1698: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n",
      "  warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 10\tAverage Score: -1.44\n",
      "Episode 20\tAverage Score: -3.98\n",
      "Episode 30\tAverage Score: -4.18\n",
      "Episode 40\tAverage Score: 2.57\n",
      "Episode 50\tAverage Score: 18.74\n",
      "Episode 60\tAverage Score: 29.35\n",
      "Episode 70\tAverage Score: 38.69\n",
      "Episode 80\tAverage Score: 45.65\n",
      "Episode 90\tAverage Score: 47.98\n",
      "Episode 100\tAverage Score: 52.56\n",
      "Episode 110\tAverage Score: 62.09\n",
      "Episode 120\tAverage Score: 72.28\n",
      "Episode 130\tAverage Score: 82.21\n",
      "Episode 140\tAverage Score: 89.48\n",
      "\n",
      "Environment solved in 47 iterations!\tAverage Score: 90.83\n",
      "CPU times: user 14min 4s, sys: 1.34 s, total: 14min 5s\n",
      "Wall time: 14min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# get the scores using the cross-entroy method\n",
    "scores = cem()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEGCAYAAACO8lkDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABBu0lEQVR4nO2dd5xcdbn/38/MbE2yu9lsyWZTNqQ3SFkCEsDQISBNkSgICl6ugmJHQO8V71WvP1EUFdCIIIqiXIiXSA+9lySQ3knbtN30umVmvr8/Zs7s1OzM7Jmc2ZPn/XrlNTvnnJnzzMnM93Oe8n2+YoxBURRFUaLxOG2AoiiKkn+oOCiKoigJqDgoiqIoCag4KIqiKAmoOCiKoigJ+Jw2wA6qqqpMQ0OD02YoiqL0KObPn7/DGFOdbJ8rxKGhoYF58+Y5bYaiKEqPQkQ2pNqnYSVFURQlARUHRVEUJQEVB0VRFCWBnIuDiDwgIs0isiRqW6WIzBWR1eHHvlH7bhORNSKyUkTOy7V9iqIoSiJHw3P4E3B+3LZbgReNMSOAF8PPEZGxwExgXPg194qI9yjYqCiKokSRc3EwxrwG7IrbfAnwUPjvh4BLo7b/3RjTZoxZB6wBpubaRkVRFCUWp3IOtcaYrQDhx5rw9npgU9RxTeFtCYjIDSIyT0TmtbS05NRYRVGUY418S0hLkm1Je4obY2YZYxqNMY3V1UnncCiKovQ49h7u4OF3NnCwze+oHU5NgtsuInXGmK0iUgc0h7c3AYOijhsIbDnq1im20Ly/lQOtfo6r7p3Vaxc37WV18wHOHVub1XsoSk/ghWXbeWx+E9efNpT6ihI+/+B7rNp+gKcWbeXBL5xIRyDIY/Ob2LavlcPtARr69WLq0ErGDShDJNn9tD04JQ5zgGuBn4Yfn4ja/jcRuQsYAIwA3nPEQqVbGGO44c/z2bTrEG/ddiZFvlBdwZrm/fxy7mqWbtnLM187nZLC2HqDNn+Ae15aw72vrMUfDDmNzyzeyj9vnIbHk7sfwpFY1LSHpt2HmTGhzpHzu53New5TV1Yc+f/95qMfMqG+nC9MGwqAPxBERPCm+f9vjGHPoQ4qSgtSDp6H2wMUF3i6PbgaY3h+2XY+NqwfZcUFGb/+4Xc28J9PLEFEeHbpNnoVevGI8KWPD+P3r63lylnv0LTrEDsPtlPo81Ds87CvNeRRXDFlIHdecUK37D8SORcHEXkEmA5UiUgT8ANCovCoiFwPbASuADDGLBWRR4FlgB+4yRgTyLWNiv28uqqFDzftAeDZJdu4ZGI9j89v4juPLSQ85rP7UDslhSW0dgT49Yur2bDzEEu27GXDzkNcNqmeq08ezIpt+/neP5cw+4PNXDihju88tpACr4fbZ4yhuk9RwnmNMazfeYg9h9op8nkZO6CsW5+jafchPvfH9zjQ5mdCfTmDKkt5ctEW7n5hNT+8ZBynDKs64usPtPnpVejN6R1eT2XBxt386oXVvLaqhe+eP5ovTx/Gqu37mb1gM/9auIVTh1cxqLKUz/zhHVr2t/Hbz05m4qAKOgJBOgJBSgtjh6/9rR385qU1PLtkGxt3HaJxSF9uOmM4p42owuftjKD7A0Eu/PXrVPUp4s/XTaW4IP2CyHU7DvKTp5dz+4wxDK3qxT/e38Stsxdz/rj+3Hf15Jj/55Xb9vPPDzbz9todDK/pw39dMo5eRSGb9xxq5/89u4JH3tvEWaNruPOKE/jH+5t4eUUzP7xkHGPqyhhaVcqtsxdz0tBKHrhgDCcMqgBgy57D3P/6Oh54cx0nHdePT00Z2I3/hdSIG5YJbWxsNNpbKX8wxnDZvW/Rsr8Nn1eo6VPE/deeyBk/f4WhVb04b1wtP3l6Ba/fcgaDKkuZv2EXn7zvbeorShha1YvrTm3gzNG1AASDhsvue4stew4ztF8v3t+wiwKPh9IiL7+8ciJnjKqJOfcf31jHfz+5LPL8skn13HHxOMpL0rurM8bw02dWsPtQO9efehzffXwRa5oP0O4P8ukTB3LbBWOY/vNXaNnfhkfgyhMHsWVPKyu27WNkbR8mD+7LWWNqGNW/D7Ne/YjfvLyGEwaW8z+XT2B4TZ+E8+1v7eDx+U08Oq8JfzDI/1w+gSlDKrtx9eHmRz6gvKSA/7pk3BFFqXl/Ky8tb2b6qBr6lxd365ypWNtygDdW7+CzJw2mIGqAfnlFM9c99D59SwspK/bR7g/y+nfP5M7nVvKH1z+id5GP4TW9GVBRwpOLtlDdu4hdB9s5ZXgVH2zYTVGBh6dvPo2aspDdxhhu+Mt8XlrRzGkjqphQX87j85vYsreVkgIvEwdV8P2LxjBuQDlPL97KjX9dAMDZY2r43dVTaA8E8Yh0KRTX/el9XlrRzIia3sy6ppGLf/sGHhH2Hu7g15+ZxMUnDABg54E2pt/5Coc7AoyvL2dR0x5G9S/ja2cNZ8nmfTzy3kb2HO7gi6cO5TvnjYoRr2j2HGqnvCTRAwoEDZ/9wzssatrLkzefyrAsw64iMt8Y05h0n4qDYjevrGzm8w++z/9cPoH9rR385OkVnD2mhhdXNPOvr5zKqu37+eajC3nl29NpqOrFOx/tZOasd/jbF0/ilOGJd+ILNu7m8nvfwucR7rpyImPr+vD5B9+npk8Rs2+cFnPsZfe+ycE2P7deMJoPN+3lnpfX0L+smH999VQqexVyoM3Pl/4yn+mjqvnCtKEJoYrfv7qW/3lmBT6PRMJa93x2Mm+saeHxBZv5dONAHn5nIw9ffxKzFzQx+4PNDKvuxfj6clZtP8DKbfsIGigu8NDaEWT6qGo+2LiHQ+1+fnLZBK5oHBRzvn//yzyeW7qd4weWs+tgO1v3tvLNc0Zy0xnDs7r2ew93MOm/nido4FvnjOSrZ40AQoPnnc+tZN763fznJ8bSp9jH1X98l027DiMCp4+o5jefnRQJjWzb20ptWVFkULr/9Y9YtX0/bf4gbR1B2vwBPnHCAC6fnPyuNRA0PPDGOn7+/Era/EFmTOjP3TMnUeD1sHXvYWbc/Tq1ZcU89uVTeGN1C196eAG//9wU7pizlDF1ZcyYUMe3/3chALecP4qrpg7h+08sYcnmvUxtqOSJhZs5+bh+PPj5ExERZr22lp88vYL/uGgs158aCke1+4O8sHw7763bxb8WbqFf70Keuvk0rrr/XbbsOcy/nXYcP5izlNJCL4faAxT6PEwZ3JdThvXjlOH9qCgtZO6y7ew62M6N04exbMs+Pnv/u1w4oY5nlmyluMCLP2h4+uZT+db/LmLjzoM8943TqelTzPf/bzGPvLeJZ792GiNq+/Dqqha+8rcF7G/14/UIJw2t5PsXju2WZ7ttbysX3P0aEwZW8Ofrsqv4P5I4uKIrq3L0aPcHufy+N/n2uaOYHnfXbvHHN9ZRX1HCJycP5GCbn58/v4oXljdzxZSBjK8vZ23LAQAC4RuTQHgQThVTnjy4Lz/71PEMrCiJiMeImt7sONAec9z2fa18sHEP3z53JGeOruXM0bU0DunLNQ+8x4vLt3NF4yDeXLODN8L/nlu6jVsvGM3kwaEJ+s8v285Pn13BhRPquOPicfz57fWUFRdw4fF1jBtQxj/e38TD72xkxoT+nDqiilNHVPGTyyfE3G3uPtjOC8u38+66XZw/rj9nj61lx4E2bn7kA27/52IaqnpxYkPIMzjQ5uflFS18/pQG7rh4HPtbO7ht9mLufG4l9RUlXDopaRX3EXn3o50EDUyoL+cXc1fh83q4dNIAfvPSGv727kZKC71ces+b9C72IcCsz01hyZZ9/PrF1fzhtY/41rmjeGvtDq66/11+PXMSnzhhABt2HuRHTy2nb2kBZSUFFPk8HGoP8O3/XUhtWTHTkgj6719by8+eXcnZY2oYX1/Or15YTbt/AWeMrubx+U20+YPcc9Vkehf5OHtMLbVlRfzgiaVs29fK7TPGcNHxdby1dgdlxQV8+ePDEBF+85lJkfcfU9eHO/61jLvmrqLNH+SPb6zjgvH9uW5aQ+SYQp+HGRPqmDGhjmnDq/i3P8/ju48t4r11u/jejDFce0oDvYp8LNy0h/7lxew51M6ba3byi7mr+MXczs/iEXh68VZKCrzUV5Twi0+fwMRBFfz46eV857xRDK/pwy+uOJ4Lf/0Gn/7d29xy/mj+9u5GPnfyEEbUhrzFj4+sZu43Ps6GnQcZX18eCS91h/7lxdx/bSODK3t1+72SoeKgZMS+1g6WbN7Hw+9sSCoOHYEg89bv5soTB1Ho81DoK+TiEwbwzOKtfPu8UUCnCFii0JU4AHw67o7b65HI6yzmLtsOwLnj+ke2nTq8in69Cnl77U6uaBzEW2t2UFLg5Y6Lx/Ljp5bzyfve5rjqXhxqC7BtXytj6sq484rjKS308a1zR0Xep6GqFxcdP4BnlmzllvNGR7bHhyH69irkisZBMR5CVe8i7rt6Cpf89g2+/PAC/vXVadSVl/DaqhbaA0EuGB+yt09xAb+6ciLb97XyvX8u5oRBFQytiv3hH2jzc8ecpXgEavoU84VpDfTr3Zl7eWvtTooLPDxyw8n8+1/m8f+eXcH/e3YFAF+ePox/P/04fvTUcj7YuJvfXT2FEbV9OHdcf9Y2H+CBN9bxuZOH8IMnlmIMPPHhZj5xwgCeXxq6rnO+ciqDKksBONjm55J73uTmRz5g1jVT2L6vjdqyYqYM6UswaHjkvY187Lh+/OGaRkSEXoU+/ueZ5bywfDsegbs+PTESCvF5Pcw8cTB3v7iaPkU+zhlbi4hw16cnpvw+XPOxBl5Y3sxvXlqDzyOcObqGn33q+JRhtHPG1nL2mFpmf7CZkgJv5Pv0qSkDE2L2uw628+5HO9l1qJ3po2po3tfKlx9ewOrdB7h75kSKC7x88bShnDayilHhwX94TR/++sWT+NLDC7jxrwsoK/bx9bNHxrxv//Ji28N33Q1BHgkVByUjrAH5tdU7ONDmp3fcHdDSLfs43BGgsSHSLosfXjyOr501gtpwfNgrceIQ9iAyqUZKJg7PL9tOQ79SRtR0xl89HuHkYf14c+0OjDG8tXYnJw6t5MoTB3Ph8QN4atEWnvhwCxWlBXx8ZDUXTKhLSHRa/Oiy8dx4xjAaqjK/UysvKWDWNY1cds+b3PLYIv5y/UnMXbadvqUFTBnSea18Xg93z5zEBXe/zs2PfMCcr0yLGfD+8f4mHpvfRHWfIlr2t1FXUcxVJw2J7H9r7Q5ObKikd5GPh68/iZXb9/PyihZqy4q4bFI9IsLPk1S4fOOcETyzZCtX/P5tNuw8xPj6Ml5btYN9rR08t3QbY+vKIsIA0KvIx++unsKl97zJJ+97G4DSQi+vfucM1rYcYNOuw3zrnFER2//t9OOYOXUQh9sDFHg99O1VGHP+z0wdzD0vr2HGhLq0EsQej/Dbz07inY928rFhVWnllO64eCxvrd3B5ZPrKS9NfXxlr0IuiKpMq68o4cmbT+Xdj3ZFhFxEGN0/NiTU2FDJv746je//cwmfOGFAwmfsaag4KBlhDcjt/iAvr2jmE+EEnMW89aFOKVboBEIDSbQbneA5BEKPvkzFISpftq+1g7fX7uAL04Ym3D1OG1bFU4u28s5Hu1jdfIBPhu8Uexf5uPLEwVx54uC0zllWXEBZ/8zLFS1G1vbhG+eM5EdPLefF5dt5aUUzZ4+pTUhGDqgo4d8/fhw/e3Ylew51RAaZQNDw0FvraRzSl99/bgpTfvRCjEA2729l1fYDkTyANYDFD2LJGF7Th0sn1TN7wWbOHF3DTWcM55P3vcU/3tvE/I27+fpZI5O8pjezbzyFRU17qexVwA1/ns+vX1zN4Y4AvYt8nBflwUHIM+qTotyzf3ko/9DQrzTp/mRUlBZy/vj0y4sH9i3l5W9PT7s4IZqq3kVceHzX56orL+GPnz8x4/fPR1QclIyIHoyeXbItQRzeW7eLwZWlES8hGQniYHkOGZR7ekQIRtnyysoWOgKGc8fWJhw7bXg/AO6auxKAU4b1S/s8dnPNxxr4yzsb+MY/PmRfq59zktgLRDyyaAF8eUUzG3cd4pbzRyVcQ4C31+4Esv983zp3FO3+IN89fzT1FSUMKC/mrrmrMAbOHZfczpG1fRgZDq18ZupgHnlvIz6vcOnE+oQ5LF0xMVyqmUuO9L1UYsm39hlKnmMNRuUlBby8spnWjs5pKMYY5m3YHeM1JMMKH1kDXzCNnEM88Z7Dok17KPJ5mDS4b8KxgytLqa8o4f31u+lT7GPcgPK0z2M3hT4P3z1/NPta/RT5PJw+Mvk8CUsoowXwwbfWUVdezHnj+ke8o2hxeGvNTsq68fnqK0r47WcnM6iyFI9HuGBCHYc7AgyuLGV0/8Qy3HhuPmsERb5QlVauau+Vo4eKg5IRVnnneeNqOdQe4NVVnU0P17YcZNfBdk5sSBygo7HCR9bAZ71npmElf6BzYPQHDUU+T1KBEZHI3fTJx/XLSIRywQXj+3PaiCouPEJ+wxcnoB+1HODNNTu5+uQhFHg7P2cwSiDfWbfT1s9nzQg/N5wg7orqPkXccv5opo+qjsmjKD0TFQclI6zBaNrwKgp9HuZv2B3ZF8k3DD2y52AlpC1RCGaTkBaJGRgDQXPEQfGUcGjJyZCShYjw5+umcteVE1MeY10LSwCbdh8G4KTwte1M6ne+ZtfBdgZUlNhm5+TBFfz3peO54ePHpf2aa09p4E9fmKozwl2A5hyUjLAGqyKfh/5lxWzb2xrZ9976XVT2KuS4Lqp5PHGeQyBLzyE6pOIPGrye1Pc654ztz1Un7Y7MYHWargZPX5xnYHkQlgBaHzVaIINdCGQ2Nn7u5CFdH6i4EhUHJSOCkUEqLA77OsVhcdNeJg+u6HLg88aFTCwPIqOEdJw4hAbG1Mf3LvLx48smpP3+TpO6oiv0Ib1JchIBY3A4Yqa4CA0rKRnhjySPoba8mO1hcTDGsHnP4bRma1oDXySslEVC2heXkPYHTWTgdAOelHNB4vbHeA6ZheYU5Ui459ekHBU6ZzN76F9WxLa9rRhj2Hu4g0PtAer7dh3zjr/rzSYh7ZE4z8EYXKQNCQnp+Fnk8aE5CF0Dr8b6FZtw0c9JORpEBikRasuKafMH2Xu4I5Iwra/ouo48PmSSVULaEzvPIRB018DoiQ8rJRHQ+HLegLE356Ac26g4KBkRfQdr9YnZtq+VLXtC4pBOtUzK3koZDO7RXVOt93DTwJjQYiRJXiZUsRX62xiDMZnlbRTlSKg4KBkRLQ7WbNNte7MUh/iQiTezhHQmpaw9DetaJHoOnT9Zjyex4stN10BxFq1WUjIiuqSyJrwS2/Z9rWzZ20qRz0O/NJqNJSRbs/AcvJJZKWtPI5KXiStljf6I0XmX+FJXReku7vk1KUeFQDA068rrEWrKQuKwbW8bm3cfpr6iJK3JT13V8KdDyHMIhVOs9zpSKWtPI1LRFUjtOXhFolqQhLZpWEmxC0d/TiLyDRFZKiJLROQRESkWkUoRmSsiq8OPOg8/j7Bm5Po8QpHPS2WvQrbta2XznsNpz85NGPgC2ZWyhuzpHDzdlJBOFXqL8RyikvKdzQuPopGKq3FMHESkHrgZaDTGjAe8wEzgVuBFY8wI4MXwcyVPsDwH6w61tqyY5nBCekAalUoQVYYZ7zlkElZKMni6KaSSTtLe6+lMSAc1rKTYjNOOuA8oEREfUApsAS4BHgrvfwi41BnTlGREPIdwwrR/WRGbdh+ieX8b9RXp9eLvvOsPPQ9G7oozm+cQeq1ll7vEIVVeJiYhHRNWynyWuaIcCcfEwRizGfg5sBHYCuw1xjwP1BpjtoaP2QokXahYRG4QkXkiMq+lpSXZIUoO8EflHCC0SMvq5tCa0Gl7DpGBLxh+T5PRBDjoFBjLHreJQ0JeJllYSbRaSckdToaV+hLyEoYCA4BeInJ1uq83xswyxjQaYxqrq6tzZaYSR3x4o7asGKuitD7DnEN0pU2mbR86ZwgTeQ83DYwJeRmTJCHtSaxW0vYZil04GVY6G1hnjGkxxnQAs4FTgO0iUgcQfmx20EYljvg71P5RK2ul0zoj+rXWcgzBLDwHa0pEdM7BTSEVT3wpa1LPIbFayU1JecVZnBSHjcDJIlIqofrHs4DlwBzg2vAx1wJPOGSfkoR4cagt7xSH/uXphZU6PYfOsFKmg1qyhG2mApPP+CKT4Ag/JvccLK+tsxz46NmouBvHJsEZY94VkceABYAf+ACYBfQGHhWR6wkJyBVO2agk0hneiPUcqvsUUeRLb83g+IVqgsHMw0rWhLeYUlYXiYPlOUTnVELbO4+JDitZuQddZEexC0dnSBtjfgD8IG5zGyEvQslDAnGVRZY4ZLICWfwSl9kkpK07ZLeXsgbjPl/04C+SpAWJioNiE+qEKhkR3x20orSAQp+HgVmIg5VsDWaTkI5r++22hHR8uW8gSTtur3ROgtN5DordaG8lJSPiPQcR4QunNDBpcPoT2a3xK+auP8M7Xl9cY7qg2xLScXmZQDBxvYqYsJJWKyk2o+KgZESydQVumzEmo/cQkZj1GPxZhIQ6Y/LZh6bymQTPIclKd56olt0BrVZSbEbDSkpGZLPeczK8IjHLhGYqDsli8m66a45fBjRUqht7jDeqbXlnFdnRs1FxN/pVUjIimMRzyAaPJ7q3Uuax8mSL4bjJc4iU6gY6w0o+b7znkGQ1PfUcFJtQcVAywh+0J/Hp83iiBvZg1p5DwKUJ6fiJggGTmFPxJPUc3HMNFGdRcVAyImhC4Y3u1tNH3/Vmk5BOWIfaZQnpSNjMukaBRM8oesGjgHoOis2oOCgZkU3yOBkxfYGCmVfZRKp5ujFXIp/xxiXck3lG0Z6D0WolxWZUHJSMyCZ5nAyvR6KSrcGsu7IGoz0HFw2MyRPucceIRLUs79ymKHag4qBkRDZ9kJLh9UhkBbiAyfyON/7O2nWeQ9LeUXEJac+RV4pTlO6gXyUlI+xqUxG7/nHmA7snPiafxSzrfMb6KP4ocYj/eB5JnASnnoNiFyoOSkbYJQ7R6x/7g8HsE9LdmGWdz8RPFEzmOSSf5+Cea6A4i4qDkhGhxGj3vzY+T+xaBJm+pdtbdkPsRMFknpFXJGEdbjd5T4qzqDgoGREIGFtm4Xo8EpUvCCbcFXdF9CS4bNag7gnETBRMIn4ejySuw+0i70lxFhUHJSMCJjG8kQ3RHUWzSkhHeQ6WyLjNc4idKJjoOUSvIW31WHJTaE1xFhUHJSOSlVRmQ/xCNd4Mx7ToUk+3diRNnCgYuz+2HFirlRR70a+SkhHJEqPZEC0OoYl1GYaVrDUhojwHt901x04UTNGVVddzUHKEioOSEclKKrPB64ktZc00j+GJyjm4tVLH6/HEdmVNsp5DQrWSywRScQ5HxUFEKkTkMRFZISLLReRjIlIpInNFZHX4Mf1VZJSckwvPIZumeTFhJdeKA1ETBZOUskbPFXFpaE1xDqc9h7uBZ40xo4ETgOXArcCLxpgRwIvh50qe4LepTUVM07gswkrRi+G4NSEdPfgnu+4S0z5DPQfFXhwTBxEpA04H/ghgjGk3xuwBLgEeCh/2EHCpE/YpyQkae+YTeOLi6ZkmpKOX0XTrXbPXKzG9oxK6snpiE9ahbe66BopzOOk5HAe0AA+KyAcicr+I9AJqjTFbAcKPNcleLCI3iMg8EZnX0tJy9Kw+xrGrK6svLl6ecUJaEj0Ht901x0yCS9KSPCZvE3502SVQHMRJcfABk4H7jDGTgINkEEIyxswyxjQaYxqrq6tzZaMSh51dWf0xYaXMXm9pScDVOYfYUtX4a+QRibTqjsxzcNk1UJzDSXFoApqMMe+Gnz9GSCy2i0gdQPix2SH7lCT4s1i1LRmemElwmQuOlZwNBIKuDanE9FZKlpCOC82B+7wnxTkcEwdjzDZgk4iMCm86C1gGzAGuDW+7FnjCAfOUFASD9gxAvoRS1izXkDb2LV2ab3jiw0oJM6QTu7K6Le+iOIfP4fN/FfiriBQCHwFfICRYj4rI9cBG4AoH7VPi8AeDFBV0/2vj8Qj+QNQkuAwFx7qJDgaNayeA+bzxXVkTxcEKJ6nnoNiNo+JgjPkQaEyy66yjbIqSJgFjT3O36I6i2aziFgkrGRMRGbcNjNGlrMkT0onVSuo5KHbh9DwHpYeRzZKeyfB6Y9tnZL7Yj2WPez2H+HLfZF1Zg3HVSm67BopzqDgoGREI2nN3GjMJLotV3LzHQPsMX1z/qSOu5xCeDOeyS6A4iIqDkhG2eQ7xCelsV4KLbrznspExPuGcrJQ1ISHtstCa4hwqDkpGJKuayYZQmWbo72zCSiISWs/AuDes1GVXVk8oIW1cPNdDcQ4VByUj7FqOMzT7N9itVdysiXSuTUh7ukhIi9V8sHOZULddA8U5VByUjAiYzENAybCWuOzOoGZNpHOz5xBTyupNrFaC2M60Wq2k2IWKg5IRoTWk7eutFEkmZ9p5j86ErVsT0l31VvJE5V2ymWWuKEdCxUHJCLsGIa9H8Ee3vsjGcwiHlVwrDnFrXiSbBAeERVZDSoq9qDgoGRGwqfGeNbs30I2QkDfe+3CpOJjwZ0xWygqdcz1UGxQ7UXFQMsIucbBm91ornWUlDuFSTiv04rYyTk84IR3puJoirBQM2tctV1EsVByUjLBrPQdrfWQ7PAcrIR2fsO3p+MIJacszSkhIh58Gw9dRw0qKnag4KBkRyGLCWjIsz6E79fnecPM+tzadsxLSgRSeUWQiYLhaSSuVFDtRcVAyIhA0WVUWxRMfEsq2lDXg4pyDx/IcLM8oyRrSQOQYt31+xVlUHJSMsM9zCH31rAls2dz1Wi2t3SoO1poXgRTXKNpzCATdl3NRnEXFQcmIZCWV2WBN4GoPBIDEu+K03kOEgCFl2KWnY3VlTeU5xFQrZbHUqqIcCf06KWkTDBqMsWcWrvUe7f7s7/pDg2ewc/B0YUI6FHoLNaFKWAku/NwY+2auK4qFioOSNqnuYLPBeo/2cK/pbO76rbyFWxPSVtdVq0Fh/OfzRq9pETSRHISi2IGKg5I2dq42ZolBuz808mUVVrL6M7k05+CN8xxSzZAOhMt53fb5FWdxXBxExCsiH4jIk+HnlSIyV0RWhx/7Om2jEiJSb29T+wyAjkDykEm67xEIBl0rDlZC2vIcEsJKMdVK7vv8irM4Lg7A14DlUc9vBV40xowAXgw/V/IAO2ciR8JKYc8h295K0Qlptw2OnvCaFykT0vHzHNz18RWHcVQcRGQgcCFwf9TmS4CHwn8/BFx6lM1SUhC00XOw7oLbLHHIsitr9DwAt4mDteZFIFVCOuI52NfWRFEsnPYcfgXcAgSjttUaY7YChB9rkr1QRG4QkXkiMq+lpSXnhirYuhyn5SlYYaVsPIf4hLTbSlm94ZXe/ClE2fp/sNpnuO3zK87imDiIyEVAszFmfjavN8bMMsY0GmMaq6urbbZOSUbnojrd/9p448JK2XgjHqt5n40eTT4Rycv4k4uf9XE75zm46/MrzuJz8NzTgItFZAZQDJSJyMPAdhGpM8ZsFZE6oNlBG5UoUt3BZkNEHLqZkO4IuDch3XmNAjHPLTxROQf1HBS7ccxzMMbcZowZaIxpAGYCLxljrgbmANeGD7sWeMIhE5U47FyKMr5aKbvGe56I5+ARXFfn3+ldHXmGdGipVF0iVLEXp3MOyfgpcI6IrAbOCT9X8oCceA7+boiDRLWrduHAaA3+qbyrzpxDeD0H910CxUGcDCtFMMa8ArwS/nsncJaT9ijJsXMSnDXwtXWjlNVq2R1Msr6yG/B0kZfxRPVW0molxW7y0XNQ8hQ7E7/xA193FvvxB+1pBphvxM8FSZWQ1molJReoOChpY2fJaHxvpWzFIZJzcKE4eOIS0gkrwXk6PQetVlLsRsVBSZtceA4d3fAcPFHzHNzoOUTmgqQqZY2rVlJxUOwkbXEQkRIRGZVLY5T8xs6ZyAnJ1ixzDm4eGC3Ba0vhXVnX0Bj35l0U50hLHETkE8CHwLPh5xNFZE4O7VLyEKuNgx0DcXw8PfuurO4dGLtKSHeGlQjnHI6ufYq7SddzuAOYCuwBMMZ8CDTkwiAlfwlY6wrYmZDuTs5BQr2V3JqQjqyWlyIhbT211nxwo/ekOEe64uA3xuzNqSVK3uO30XOwZZ6DR/BbnoMLB0arTUnEc0iRkA6G13Nwo/ekOEe68xyWiMhnAa+IjABuBt7KnVlKPhK00XOwY4a0x+WlrPHNCeMHf6/Oc1BySLqew1eBcUAb8DdgL/D1HNmk5Cm2eg5xK8Flc9drrbEcMG71HGJDbwmT4OK7srrwGijO0aXnICJeYI4x5mzge7k3SclXIl1ZbQhfdDXwpYNHQmGlQMC4bv1o6Dr0FumtFK5WcuM1UJyjS8/BGBMADolI+VGwR8lj/AH7Slk98TX82c6QDrq3lNVKSLelEIfO9hm49hoozpFuzqEVWCwic4GD1kZjzM05sUrJS4I2znOwkqupavjTeo/IGsvuHBithHSqvIy1rEYwXK2kjoNiJ+mKw1Phf8oxjJ1dWT1xOYfsFvsJrbHs9oR0qrxMzBrSRsNKir2kJQ7GmIdEpBAYGd600hjTkTuzlHzE1q6scdVKWc2QDq+xHHRpMtYTN88h5XoORquVFPtJSxxEZDrwELAeEGCQiFxrjHktZ5YpeYedvZXiZ0hnX8oayoW48a7ZFxdWihfASLVS0LhWIBXnSDes9AvgXGPMSgARGQk8AkzJlWFK/mFnV9b41hDZjGuWwPiDQVfeNUdmSKcqZY2f5+BCgVScI915DgWWMAAYY1YBBbkxSclXIp6DDUuORU/w8nokqyU+o0s93SgO1uCfqlopMgnOoGElxXbS9Rzmicgfgb+En18FzM+NSUq+EsjFPAd/MOv3ix48y0rcNzD64tpnHLFaydjj0SmKRbqew5eBpYTaZnwNWAZ8qTsnFpFBIvKyiCwXkaUi8rXw9koRmSsiq8OPfbtzHsU+LM/BzvYZbYHs7/qjFwxy411zfEI6XkSjeyuFPIejap7ictL9OvmAu40xlxtjLgN+DXi7eW4/8C1jzBjgZOAmERkL3Aq8aIwZAbwYfq7kAbaKg3Q/JOSxwfvIZ+JnkSckpCNhJW28p9hPuuLwIlAS9bwEeKE7JzbGbDXGLAj/vR9YDtQDlxCqjCL8eGl3zqPYh63iEJW3yPb9rLfocKnnEF3RlaxCzBIDrVZSckG64lBsjDlgPQn/XWqXESLSAEwC3gVqjTFbw+fZCtSkeM0NIjJPROa1tLTYZYpyBHLhOXTn/Y6VhHS7P5h04I9Z7EerlRSbSVccDorIZOuJiDQCh+0wQER6A48DXzfG7Ev3dcaYWcaYRmNMY3V1tR2mKF3gt1EcPFHfvGzDIZ3tJdxZqRM9UTC55xB6DIWV7JmcqCgW6VYrfR34XxHZAhhgAHBld08uIgWEhOGvxpjZ4c3bRaTOGLNVROqA5u6eR7GHYGQSXPczn9Hvke2kuuiV0twsDqlyKiKCR8AfSJ6wVpTucMRfuYicKCL9jTHvA6OBfxBKJD8LrOvOiSVU2P5HYLkx5q6oXXOAa8N/Xws80Z3zKPbhj0yC6/57Rb9H1glpK+wScHdCui0QjMnRxB/T6dEdNdOUY4Cuvk6/B9rDf38MuB24B9gNzOrmuacBnwPOFJEPw/9mAD8FzhGR1cA54edKHhAMt4XOZsJaPNZdL8SGmDLBZ0NSO5+JqehKcc1FpHOWuQuvgeIcXYWVvMaYXeG/rwRmGWMeBx4XkQ+7c2JjzBuE+jQl46zuvLeSG/w2Jz19Hg/tgWDWYSqPDUntfCb6M6Ua+L0i3WpeqCip6OpX6RURS0DOAl6K2pduvkJxCXa3aLA0Idu3jLbF7eKQKi/j9XSKgxtDa4pzdDXAPwK8KiI7CFUnvQ4gIsMJrSOtHEPYLQ7WYJb9PIdjRxxSeQWhhLR9rdQVxeKI4mCM+bGIvAjUAc8bE26uE/I4vppr45T8wnZx8FjikF1YKZ3BsycT4zkcISHdYSWk3XcJFAfpMjRkjHknybZVuTFHyWdyJw7dez3Ys8ZEvhGTU0npOQgd3VgTQ1FSocVvStr4cyUO2XZldXnOwZfG5/NE5Rw0rKTYiYqDkjZBm6uVOj2H7nVl7c575DPpJNy9IpHGfJqQVuxExUFJG9s9B01IHxERwfqIqXIq0dVKbsy7KM6h4qCkjTUJzi483fQcPC5PSEOnd5QqIe3xaLWSkhtUHJS08QeNrYnf7oaV3J6Qhk7RS13KGlWtpL9mxUb066SkTTBo75oB1uCefVfWrmcQ93Ssz5hyElxUtZJbvSfFGVQclLTxB5O3js4WK2eQdVdWcb/nEBHQNKqV3Jh3UZxDxUFJm0DQ3rtTO8NKbh0Y0/IctFpJyQEqDkraBILBlInRbOhuWCn6dW4NqXRV0RXyHDQhrdiPioOSNgGTG88hW8GJfp2dopVPdCWgHkE9ByUnqDgoaROwOefQVSVOuq/vznvkO12GlWJmSB81s5RjAP06KWkTsLlayWdjzsGtCemIgKYKK4l0znNwqUAqzqDioKRNwOZ5Dt2dBBdti1vj7Va47IieQ1CrlRT7UXFQ0iZn6znY0XjPpXfN3i48h1C1kok5VlHsIG/FQUTOF5GVIrJGRG512h7FfnGI3BVnmUyOmefg8oR0Ks9BJPT/Au71nhRnyEtxEBEvcA9wATAW+IyIjHXWKiVg7O3K2u2EdNS3163x9q7amh8Lcz0UZ8jXdaCnAmuMMR8BiMjfgUuAZbk4mTGGpxdvo80foLykgIrSAipKCzmuqhcS9aNc23KAnz+3koWb9gAwuq6MX1xxAn17FWZ0vtkLmnhu6Tbmb9jDl6cP4/pTh3bL/ub9rVx+71v896XjOWNUTcznWrX9AHUVxZQVFyR97d/e3chTi7dw+4wxjBtQzu6D7by7bhfb97USNIZrPtYQGXT8gVwt9pNtzsET9bc7B8auEtJuXw1PcY58FYd6YFPU8ybgpOgDROQG4AaAwYMHd+tkCzbu4aa/LUjYftmkeu769AkA/HLuKu55ZS0lBV7OGVuLAE8u3son73uLh66byqDK0rTONXtBE998dCH1FSUUeoWH3lrPddMaYkQoU+av303T7sPc8tgi5n7jdPoUF/Dnt9fz8DsbWNtykLryYu69ajKTBveNed2+1g5++sxy9rX6ufi3bzK1oZJ5G3ZFYtgAw2t6c9qIasD+rqzdniEdXcrqUnHoKiEdW857VExSjhHyVRySfc1NzBNjZgGzABobG02S49PmqUVbKfR5mPOVabR1BNlzuINXVjbz4JvrmTKkL/tb/fz6pTVcPqme2y8cQ1XvIgA+c9JgvvjQPC7+7Rv8x0VjuWxS/REH+Y9aDvD9/1vC1KGV/O2LJzH7g83c8tgiFjbtZeKgiqztX7Z1Hx6B3QfbufXxxRxs9/P66h1MGdKX/7hoCH96ax2f/v3bNA6ppOVAGyc2VPLfl4zjoTfXs6/Vz1+/eBL/WriFdz7ayTUfa2DGhP7U9CnmrF+8yhurd0TEIWfrOdgQVnJrMjYTz0HDSoqd5Ks4NAGDop4PBLbk4kTBoOHpxVuZPrKa0f3LIttPG17FRy0H+eG/ltIRMFx8wgB+fsUJMT/SExsqmX3jKXz7fxfyzUcXMmfhFv5wTSMFSXonr99xkJv+9gGFPg93z5yIz+vh/PH9+f7/LeGJDzenJQ7rdhzk0Xmb+M65o2LsWL51H8NrejNjQh2/emE1hT4PP718AjOnhjyqT00eyA/mLGHDrkPUlRfzyHsbOdjm59VVLZw9poZpw6uYNrwq4XxThvTltdU7uC3qWuWT5xAdVvIeownp6M0aVlLsJC8T0sD7wAgRGSoihcBMYE4uTjR/42627WvlwuPrYrZ7PMIvr5xI//JiTh1exZ1XHJ/07m1YdW8e/9Ip3D5jNK+sbOGR9zbG7A8GDXfMWcrZd73K+h0H+eWnJ1JXXgJAWXEBZ46q4clFWyMVJ0di9oIm7ntlLZv3HI7ZvmzLPsbUlXHTGcP55jkjmf3lUyLCAFBeWsCvZk7inzdO4y/Xn8Q3zxnJnIVb2Hu4g6+dNTLl+U4dUcXyrfto2d8G5HAN6awX+4l6L5cOjF23z1DPQckNeek5GGP8IvIV4DnACzxgjFmai3M9tWgrRT4PZ42pTdhX2auQF775cQq9niOGizwe4d9OO46XV7TwqxdWc+mk+kgC+MOmPfzprfVcNqme22aMpqZPccxrL544gGeXbuONNTs4+bjKI55rTfMBADbvORzJcew51M6Wva2MrSujwOvh5rNGdPmZv3rmcAq8Hva1djBhYHnK404fUc2dz63kzTU7uHRSPcF8W+znWGjZ3UVbcw0rKbkiXz0HjDFPG2NGGmOGGWN+nItzBIKGpxZv5YxRNfQuSq6TRT5vWsliEeH2GWPYdbCd+15ZG9m+dU8rAP/+8eMShAHgzNGhc1/7wHuM+v6z3DZ7ccpzWOKwJcpzWLZ1HwBjB5QlfU0qW788fRjfPX/0EY8bN6CMvqUFvLa6BbDfc+huKeuxtNjPkbqyRv52qfekOEPeisPR4P31u2jZ38ZFJ9R1fXAaTBhYzmWT6nngjXXsOdQOwNa9oYG8f1miMAAUF3i596rJfOe8UYzu34f5G3YnPa4jEGTdjoNAnDhsCYnDmLr0xSFdPB5h2vAq3li9A2OM7dVKvi7i6V0hIpGYu1vvmrta7MerYSUlRxzT4jB5cF8e/MKJnDm6puuD0+SSiQNo8wcjd/nb97VS5PNQXpJ8ngHA6SOruemM4UwbXkXT7sMYk5h/2LDzEP5wXmJznOdQ06coUkFlN6ePqKZ5fxurmw+EPAc7J8F1MfClQ3dDU/lOJglpt+ZdFGc4psWh0OfhjFE1lBbal3oZ2DeUbLYG8G372uhfXpxWaKq+ooTDHQF2H+pI2GeJTZHPw+ZwqApg+db9OfEaLI4fFMpJrNq+P9w+w76vjFXU1Z2B3dPNcth8p6vQW7SwuvQSKA5xTItDLhhQERKHpt0hcdi+t5XaFCGleCxhadp9KGHfmub9AEwdWhkJK7X7g6xp3p9RviFT6qM+T0gc7HtvqxS1O8nk7rb9zne6Cr1pWEnJFSoONlNa6KOyV2FEHLbta6WuPD1xqLe8jt2HE/ataT7AgPJiRtT0YXM49LS6eT8dAZNTz6FPcaidSNPuQ7Z7Dt1NSEP3237nO13mHLRaSckRKg45YGDfEjbvCQ3g2/a1pkxGJ7yuIlSeGj+PAWBNywGG1/ahvm8o9LTnUAeLmvYCMD6HngOEPk8uPAfrvbrTUdXtOQdPVzkHrVZScoSKQw6oryhh8+5D7D7UQbs/mHZYqazER58iX8TreHllMz97dgXBoGFN8wGGV/emviL0Xpv3HObDjXsoLylgaFWvnH0WCIlW0+7Doa6stuYcQu/VnUEt0oLDpeLQVdgsJiHt0mugOENeToLr6dRXlPDyyubOMtY0w0oiQn34Lh3ggTfW8frqHQC0dgQZXtM7ktPYvOcwH27aw8RBFd1q2pcOA/uGPo8x9iZ+7UhId9XSuqfTVegtJufg0mugOIN6DjlgYN8SWjuCLN8aSiKn6zlYr23afYhg0LBw0x5E4N7wpLrhNb0jCeLV2/ezqnl/txr2ZWJTmz+0FKWdi+rYcdfv9rBSV6G3mLCS/poVG9GvUw6o7xvKHczfsAsg7YQ0hENSew6zbudB9rX6+cbZIyNzJEbU9KayVyFFPg/PLt2GMTBxcIXt9sczsG9nO3I749oeG+76PS4PK3UVeotpW66eg2IjGlbKAdbd/bz1uxGB6j7pT1Ab2LeU/a1+XlsVallx/vj+TBxUwUsrmiOLCtVXlLBkc2hm9MSBFfYan8ymypLI33b2MLKjDNW6o3avOIQetbeScrRRccgBVknq6uYDVPcpStrCu6vXPrloK72LfAyr7s3I2j6cPrI65piPdhykoV9pxqvQZYMldmBvDyM7ylDdn5AOew4p15BWz0HJDRpWygHlJQX0KQ7pbrplrBbWRLj5G3Zz/MDypIPegHDL76ORb4DOuQ5gr+dgx8BuR2gqn7EG/NSeQ/Tf7rwGijOoOOQI62473Uql+NcBTEqRT7Aqlo6WOECnXXZ6Dl2tVZDWe1gC49rFfqzHdHIOR8Mi5VhBxSFHWB5App5DZa9CSgq8AEwc1DfpMUP6hRLEk4ck358LrM+Ti/UcuvOebi9l7SohHWleKOS8pFk5ttCcQ46wKnwy9RysuQ5rmg+k9AwumNCfspJGjj8KyWgL6/PYO89BS1m7osuEtMtzLopzqDjkCCsMk8kcB4shlaW0+QMpq5yKfF7OHJ24cl0usTyHXCwT2p1Qlet7K8mRr5HHhtCcoiRDxSFHZBtWAviPi8ZysN1vt0ndIuI55CAhbUtXVpcOjt4uOtfa0bxQUZLhSM5BRO4UkRUiskhE/ikiFVH7bhORNSKyUkTOc8I+O5g+qoZvnzuSqUMrM35tQ1Uvxg1IvbazE1j9m1Itp5oNdtz1ekUQcfMyoaHH1F1ZrUd3fn7FOZxKSM8FxhtjjgdWAbcBiMhYYCYwDjgfuFdEvA7Z2C1KCr185cwRFPrckfMfXtObf954CtNHVXd9cJpE1iroRqWRx+NerwHS6Moa8RyOmknKMYIjI5cx5nljjBU3eQcYGP77EuDvxpg2Y8w6YA0w1QkblUQmDe6Lz8ae3baUsnrE1XfNXYXN3J6QV5wjH25rrwOeCf9dD2yK2tcU3paAiNwgIvNEZF5LS0uOTVRygR19kbwej6sHRk9XCWmtVlJyRM4S0iLyAtA/ya7vGWOeCB/zPcAP/NV6WZLjTbL3N8bMAmYBNDY2Jj1GyW+6WgIzHbzi7rBSV3NBtFpJyRU5EwdjzNlH2i8i1wIXAWcZY6zBvQkYFHXYQGBLbixUnGZ4TW8GV5ZGZnxng9cjrp0dDZ2ikDIhrZ6DkiOcqlY6H/gucLEx5lDUrjnATBEpEpGhwAjgPSdsVHLPiNo+vHbLGVR2o3mgR8TVnkNXCelINZOLr4HiDE7Nc/gtUATMDU/5f8cY8yVjzFIReRRYRijcdJMxJuCQjUoPwOd1d0La28U8hs6cxFEzSTlGcEQcjDHDj7Dvx8CPj6I5Sg/GIy4Xhy7KfSMJafUcFJvR+w2lR+P1iKtDKl01FrSjBYmiJEPbZyg9mgvG92dETW+nzcgZdeUlFPo8KRd1cvt6FopzqDgoPZrzx9c5bUJOOfm4Sj78z3MoLUz+U9VqJSVXaFhJUfIYEUkpDKDVSkruUHFQlB6MaLWSkiP0K6UoPRivVispOULFQVF6MFqtpOQKFQdF6cFotZKSK1QcFKUH09UyooqSLSoOitKDsTRBPQfFblQcFKUH49HFfpQcoeKgKD0YTUgruULFQVF6MLqGtJIrVBwUpQfTVWM+RckWFQdF6cFYHoOGlRS7UXFQlB6Mrueg5AoVB0XpwXi1WknJESoOitKD0WolJVc4Kg4i8m0RMSJSFbXtNhFZIyIrReQ8J+1TlHynM6zksCGK63BssR8RGQScA2yM2jYWmAmMAwYAL4jISGNMwBkrFSW/iSSkNeeg2IyTnsMvgVsAE7XtEuDvxpg2Y8w6YA0w1QnjFKUnoGElJVc4Ig4icjGw2RizMG5XPbAp6nlTeFuy97hBROaJyLyWlpYcWaoo+Y1WKym5ImdhJRF5AeifZNf3gNuBc5O9LMk2k2QbxphZwCyAxsbGpMcoittRz0HJFTkTB2PM2cm2i8gEYCiwMLzE4UBggYhMJeQpDIo6fCCwJVc2KkpPp7OU1WFDFNdx1L9SxpjFxpgaY0yDMaaBkCBMNsZsA+YAM0WkSESGAiOA9462jYrSU7CiSRpWUuzGsWqlZBhjlorIo8AywA/cpJVKipIaXexHyRWOi0PYe4h+/mPgx85Yoyg9C228p+QKjVQqSg/GowlpJUeoOChKDyYSVlLPQbEZFQdF6cFE5jnoL1mxGf1KKUoPxhP+BWvOQbEbFQdF6cFotZKSKxyvVlIUJXt8Xg+3zxjNmaNrnDZFcRkqDorSw7nh9GFOm6C4EA0rKYqiKAmoOCiKoigJqDgoiqIoCag4KIqiKAmoOCiKoigJqDgoiqIoCag4KIqiKAmoOCiKoigJiDE9f/llEWkBNmT4sipgRw7MsRu10156gp09wUZQO+3EKRuHGGOqk+1whThkg4jMM8Y0Om1HV6id9tIT7OwJNoLaaSf5aKOGlRRFUZQEVBwURVGUBI5lcZjltAFponbaS0+wsyfYCGqnneSdjcdszkFRFEVJzbHsOSiKoigpUHFQFEVREjgmxUFEzheRlSKyRkRuddoeCxEZJCIvi8hyEVkqIl8Lb68Ukbkisjr82DcPbPWKyAci8mQe21ghIo+JyIrwNf1YvtkpIt8I/18vEZFHRKQ4H2wUkQdEpFlElkRtS2mXiNwW/j2tFJHzHLbzzvD/+SIR+aeIVOSjnVH7vi0iRkSqnLYzmmNOHETEC9wDXACMBT4jImOdtSqCH/iWMWYMcDJwU9i2W4EXjTEjgBfDz53ma8DyqOf5aOPdwLPGmNHACYTszRs7RaQeuBloNMaMB7zAzDyx8U/A+XHbktoV/o7OBMaFX3Nv+HfmlJ1zgfHGmOOBVcBteWonIjIIOAfYGLXNSTsjHHPiAEwF1hhjPjLGtAN/By5x2CYAjDFbjTELwn/vJzSY1ROy76HwYQ8BlzpiYBgRGQhcCNwftTnfbCwDTgf+CGCMaTfG7CHP7CS0VG+JiPiAUmALeWCjMeY1YFfc5lR2XQL83RjTZoxZB6wh9DtzxE5jzPPGGH/46TvAwHy0M8wvgVuA6Mogx+yM5lgUh3pgU9TzpvC2vEJEGoBJwLtArTFmK4QEBHB6NflfEfpCB6O25ZuNxwEtwIPh8Nf9ItKLPLLTGLMZ+Dmhu8atwF5jzPP5ZGMcqezK59/UdcAz4b/zyk4RuRjYbIxZGLcrL+w8FsVBkmzLq3peEekNPA583Rizz2l7ohGRi4BmY8x8p23pAh8wGbjPGDMJOEh+hLoihGP2lwBDgQFALxG52lmrsiIvf1Mi8j1Codq/WpuSHOaInSJSCnwP+M9ku5NsO+p2Hovi0AQMino+kJArnxeISAEhYfirMWZ2ePN2EakL768Dmp2yD5gGXCwi6wmF5M4UkYfJLxsh9P/cZIx5N/z8MUJikU92ng2sM8a0GGM6gNnAKXlmYzSp7Mq735SIXAtcBFxlOidz5ZOdwwjdFCwM/5YGAgtEpD95YuexKA7vAyNEZKiIFBJK/Mxx2CYAREQIxciXG2Puito1B7g2/Pe1wBNH2zYLY8xtxpiBxpgGQtfuJWPM1eSRjQDGmG3AJhEZFd50FrCM/LJzI3CyiJSG/+/PIpRnyicbo0ll1xxgpogUichQYATwngP2AaFqROC7wMXGmENRu/LGTmPMYmNMjTGmIfxbagImh7+3+WGnMeaY+wfMIFTFsBb4ntP2RNl1KiH3cRHwYfjfDKAfoeqQ1eHHSqdtDds7HXgy/Hfe2QhMBOaFr+f/AX3zzU7gh8AKYAnwF6AoH2wEHiGUB+kgNHBdfyS7CIVI1gIrgQsctnMNoZi99Rv6XT7aGbd/PVDltJ3R/7R9hqIoipLAsRhWUhRFUbpAxUFRFEVJQMVBURRFSUDFQVEURUlAxUFRFEVJQMVBOaYRkYCIfBj174izqEXkSyJyjQ3nXR/dhTOD150nIneISF8Rebq7dihKKnxOG6AoDnPYGDMx3YONMb/LoS3pcBrwMqGmgm86bIviYlQcFCUJ4ZYG/wDOCG/6rDFmjYjcARwwxvxcRG4GvkSof88yY8xMEakEHiDU+O8QcIMxZpGI9CM0Eaqa0GxXiTrX1YRadxcSarR4ozEmEGfPlYRaTx9HqB9TLbBPRE4yxlyci2ugHNtoWEk51imJCytdGbVvnzFmKvBbQp1o47kVmGRC6wZ8Kbzth8AH4W23A38Ob/8B8IYJNQGcAwwGEJExwJXAtLAHEwCuij+RMeYfhHpDLTHGTCA0o3qSCoOSK9RzUI51jhRWeiTq8ZdJ9i8C/ioi/0eoPQeEWqB8EsAY85KI9BORckJhoMvD258Skd3h488CpgDvh9orUULqRnsjCLVUACg1oTU/FCUnqDgoSmpMir8tLiQ06F8M/IeIjOPI7ZaTvYcADxljbjuSISIyD6gCfCKyDKgTkQ+BrxpjXj/ip1CULNCwkqKk5sqox7ejd4iIBxhkjHmZ0MJHFUBv4DXCYSERmQ7sMKE1OaK3X0CoCSCEGth9SkRqwvsqRWRIvCHGmEbgKUL5hp8Rahg5UYVByRXqOSjHOiXhO3CLZ40xVjlrkYi8S+gm6jNxr/MCD4dDRgL80hizJ5ywflBEFhFKSFstrn8IPCIiC4BXCa8ZbIxZJiLfB54PC04HcBOwIYmtkwklrm8E7kqyX1FsQ7uyKkoSwtVKjcaYHU7boihOoGElRVEUJQH1HBRFUZQE1HNQFEVRElBxUBRFURJQcVAURVESUHFQFEVRElBxUBRFURL4/01qDj2mmVcmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Watch a Smart Agent!\n",
    "\n",
    "In the next code cell, you will load the trained weights from file to watch a smart agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "# load the weights from file\n",
    "agent.load_state_dict(torch.load('checkpoint.pth'))\n",
    "\n",
    "state = env.reset()\n",
    "while True:\n",
    "    state = torch.from_numpy(state).float().to(device)\n",
    "    with torch.no_grad():\n",
    "        action = agent(state)\n",
    "    env.render()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    if done:\n",
    "        break\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "DRLND",
   "language": "python",
   "name": "drlnd"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}