diff --git a/how-to-guides/07-param-schedulers.ipynb b/how-to-guides/07-param-schedulers.ipynb new file mode 100644 index 0000000..f188323 --- /dev/null +++ b/how-to-guides/07-param-schedulers.ipynb @@ -0,0 +1,754 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# How to use ParamScheduler with Ignite" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import ipywidgets as widgets\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import ast" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from torch.optim import SGD" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook aims at presenting the use of ignite parameter schedulers.\n", + "\n", + "See the [PyTorch-Ignite implementation](https://github.com/pytorch/ignite/blob/master/ignite/handlers/param_scheduler.py) and [documentation](https://pytorch.org/ignite/master/contrib/handlers.html#module-ignite.handlers.param_scheduler) as well as the [PyTorch documentation](https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate).\n", + "\n", + "# `PyTorch` schedulers\n", + "\n", + "We use the `LRScheduler` tool of `PyTorch-Ignite` which allows to easily integrate the schedulers of `PyTorch` and in particular to display the values." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from ignite.handlers import LRScheduler" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> Beware that not all `PyTorch` schedulers are compatible because they have more or less exotic modes." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.optim.lr_scheduler import (\n", + " StepLR,\n", + " ExponentialLR,\n", + " CosineAnnealingLR,\n", + " MultiStepLR,\n", + " LambdaLR,\n", + " MultiplicativeLR,\n", + " CyclicLR,\n", + " OneCycleLR\n", + " # ReduceLROnPlateau : not compatible\n", + " # CosineAnnealingWarmRestarts : not compatible\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def print_optimizer(cls, num_events=50, lr=1.0, *args, **kwargs):\n", + " optimizer = SGD(params=[torch.zeros(10)], lr=lr)\n", + " lr_scheduler = cls(optimizer=optimizer, *args, **kwargs)\n", + " LRScheduler.plot_values(num_events=num_events, lr_scheduler=lr_scheduler)\n", + " plt.title(\"{}\".format(cls))\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LambdaLR\n", + "\n", + "See the [documentation](https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.LambdaLR.html#torch.optim.lr_scheduler.LambdaLR)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0ada03fd78ca4fc394f48b9942de4437", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(FloatSlider(value=0.05, description='lr', max=0.1, min=0.01, step=0.01), IntSlider(value…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_lambdalr(**kwargs):\n", + " print_optimizer(LambdaLR, lr_lambda=lambda epoch: 1 / (epoch+1), **kwargs)\n", + " \n", + "lambdalr_params = {\n", + " \"lr\": (0.01, 0.1, 0.01), \n", + " \"num_events\": (5, 100, 5),\n", + "}\n", + "\n", + "_ = widgets.interact(print_lambdalr, **lambdalr_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## MultiplicativeLR\n", + "\n", + "See the [documentation](https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.MultiplicativeLR.html#torch.optim.lr_scheduler.MultiplicativeLR)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ae424390a0cf452ba245f67c168d5229", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(FloatSlider(value=0.5, description='factor', max=1.0, min=0.05, step=0.05), FloatSlider(…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_multiplicativelr(factor, **kwargs):\n", + " print_optimizer(MultiplicativeLR, lr_lambda=lambda epoch: factor, **kwargs)\n", + "\n", + "multiplicativelr_params = {\n", + " \"lr\": (0.01, 0.1, 0.01), \n", + " \"num_events\": (5, 100, 5), \n", + " \"factor\": (0.05, 1.0, 0.05),\n", + "}\n", + " \n", + "_ = widgets.interact(print_multiplicativelr, **multiplicativelr_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## CyclicLR" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "51b759a6c9ec40c6bfa0495f23cf5b2b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(FloatSlider(value=0.05, description='base_lr', max=0.1, min=0.01, step=0.01), FloatSlide…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_cycliclr(**kwargs):\n", + " print_optimizer(CyclicLR, **kwargs)\n", + "\n", + "cycliclr_params = {\n", + " \"base_lr\": (0.01, 0.1, 0.01),\n", + " \"max_lr\": (0.01, 0.05, 0.01), \n", + " \"num_events\": (5, 100, 5), \n", + " \"step_size_up\": (1, 10, 1),\n", + " \"step_size_down\": (1, 10, 1),\n", + " \"mode\": ['triangular', 'triangular2', 'exp_range'],\n", + " \"gamma\": (0.1, 2.0, 0.01),\n", + "}\n", + " \n", + "_ = widgets.interact(print_cycliclr, **cycliclr_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## OneCycleLR\n", + "\n", + "See the [documentation](https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.OneCycleLR.html#torch.optim.lr_scheduler.OneCycleLR)." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "991afe00432946d2a697b5affbb05105", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(FloatSlider(value=0.02, description='max_lr', max=0.05, min=0.01, step=0.01), IntSlider(…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_onecyclelr(**kwargs):\n", + " print_optimizer(OneCycleLR, **kwargs, num_events=kwargs[\"total_steps\"])\n", + "\n", + "onecyclelr_params = {\n", + " \"max_lr\": (0.01, 0.05, 0.01), \n", + " \"total_steps\": (1, 100, 1),\n", + " \"pct_start\": (0.1, 1.0, 0.1),\n", + " \"anneal_strategy\": ['cos', 'linear'],\n", + "} \n", + " \n", + "_ = widgets.interact(print_onecyclelr, **onecyclelr_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## StepLR\n", + "\n", + "See the [documentation](https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.StepLR.html#torch.optim.lr_scheduler.StepLR)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b56e289a5aa74cecb3136bb865281fc3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(FloatSlider(value=0.05, description='lr', max=0.1, min=0.01, step=0.01), IntSlider(value…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_steplr(**kwargs):\n", + " print_optimizer(StepLR, **kwargs)\n", + "\n", + "steplr_params = {\n", + " \"lr\": (0.01, 0.1, 0.01), \n", + " \"num_events\": (5, 100, 5),\n", + " \"step_size\": (1, 10, 1), \n", + " \"gamma\": (0.01, 1.0, 0.01),\n", + "} \n", + " \n", + "_ = widgets.interact(print_steplr, **steplr_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ExponentialLR\n", + "\n", + "See the [documentation](https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.ExponentialLR.html#torch.optim.lr_scheduler.ExponentialLR)." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5da7df3041724625a8f71ddb5db0bfbf", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(FloatSlider(value=0.05, description='lr', max=0.1, min=0.01, step=0.01), IntSlider(value…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_exponentiallr(**kwargs):\n", + " print_optimizer(ExponentialLR, **kwargs)\n", + "\n", + "exponentiallr_params = {\n", + " \"lr\": (0.01, 0.1, 0.01), \n", + " \"num_events\": (5, 100, 5),\n", + " \"gamma\": (0.01, 1.0, 0.01),\n", + "} \n", + " \n", + "_ = widgets.interact(print_exponentiallr, **exponentiallr_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## CosineAnnealingLR\n", + "\n", + "See the [documentation](https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.CosineAnnealingLR.html#torch.optim.lr_scheduler.CosineAnnealingLR)." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3502c0393e3743be98d031bdab54d3f2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(FloatSlider(value=0.05, description='lr', max=0.1, min=0.01, step=0.01), IntSlider(value…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_cosineannealinglr(**kwargs):\n", + " print_optimizer(CosineAnnealingLR, **kwargs)\n", + "\n", + "cosineannealinglr_params = {\n", + " \"lr\": (0.01, 0.1, 0.01), \n", + " \"num_events\": (5, 100, 5),\n", + " \"T_max\": (1, 20, 1),\n", + " \"eta_min\": (0.0, 1.0, 0.1),\n", + "} \n", + " \n", + "_ = widgets.interact(print_cosineannealinglr, **cosineannealinglr_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## MultiStepLR\n", + "\n", + "See the [documentation](https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.MultiStepLR.html#torch.optim.lr_scheduler.MultiStepLR)." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "adde4c098a7043708c1bc8d39cd3ff75", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(FloatSlider(value=0.05, description='lr', max=0.1, min=0.01, step=0.01), IntSlider(value…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_multisteplr(**kwargs):\n", + " kwargs[\"milestones\"] = list(ast.literal_eval(kwargs[\"milestones\"]))\n", + " print_optimizer(MultiStepLR, **kwargs)\n", + "\n", + "multisteplr_params = {\n", + " \"lr\": (0.01, 0.1, 0.01), \n", + " \"num_events\": (5, 100, 5),\n", + " \"milestones\": \"10, 30, 40\",\n", + " \"gamma\": (0.01, 1.0, 0.01),\n", + "} \n", + " \n", + "_ = widgets.interact(print_multisteplr, **multisteplr_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# `PyTorch-Ignite` Schedulers\n", + "\n", + "Each scheduler can apply on a part of the parameters associated with the optimizer. This is a strong difference with `PyTorch`. On the other hand, there is an operator to concatenate schedulers. " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from ignite.contrib.handlers import (\n", + " LinearCyclicalScheduler,\n", + " PiecewiseLinear,\n", + " CosineAnnealingScheduler\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_values(cls, **kwargs):\n", + " cls.plot_values(**kwargs)\n", + " plt.title(\"{}\".format(cls))\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LinearCyclicalScheduler\n", + "\n", + "See the [documentation](https://pytorch.org/ignite/generated/ignite.handlers.param_scheduler.LinearCyclicalScheduler.html#ignite.handlers.param_scheduler.LinearCyclicalScheduler)." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "68b547d45efa4412b0c2d110f2af5dad", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(IntSlider(value=27, description='num_events', max=50, min=5), FloatSlider(value=0.55, de…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_linearcyclicalscheduler(**kwargs):\n", + " plot_values(LinearCyclicalScheduler, param_name=\"lr\", **kwargs)\n", + " \n", + "linearcyclicalscheduler_params = {\n", + " \"num_events\": (5, 50, 1),\n", + " \"start_value\": (0.1, 1., 0.01),\n", + " \"end_value\": (0., 0.1, 0.001),\n", + " \"cycle_size\": (0, 20, 1),\n", + "}\n", + "\n", + "_ = widgets.interact(print_linearcyclicalscheduler, **linearcyclicalscheduler_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PiecewiseLinear\n", + "\n", + "See the [documentation](https://pytorch.org/ignite/generated/ignite.handlers.param_scheduler.PiecewiseLinear.html#ignite.handlers.param_scheduler.PiecewiseLinear)." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0e39d64ccf60429a9575221783bb3e05", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(IntSlider(value=52, description='num_events', min=5), Text(value='(10, 0.5), (20, 0.45),…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_piecewiselinear(**kwargs):\n", + " kwargs[\"milestones_values\"] = list(ast.literal_eval(kwargs[\"milestones_values\"]))\n", + " plot_values(PiecewiseLinear, param_name=\"lr\", **kwargs)\n", + " \n", + "piecewiselinear_params = {\n", + " \"num_events\": (5, 100, 1),\n", + " \"milestones_values\": \"(10, 0.5), (20, 0.45), (21, 0.3), (30, 0.1), (40, 0.1)\",\n", + "}\n", + "\n", + "_ = widgets.interact(print_piecewiselinear, **piecewiselinear_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## CosineAnnealingScheduler\n", + "\n", + "See the [documentation](https://pytorch.org/ignite/generated/ignite.handlers.param_scheduler.CosineAnnealingScheduler.html#ignite.handlers.param_scheduler.CosineAnnealingScheduler)." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "676d6df2adf84e3b91066ac3d0221e8b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(IntSlider(value=77, description='num_events', max=150, min=5), FloatSlider(value=1.0, de…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_cosineannealingscheduler(**kwargs):\n", + " plot_values(CosineAnnealingScheduler, param_name=\"lr\", **kwargs)\n", + " \n", + "cosineannealingscheduler_params = {\n", + " \"num_events\": (5, 150, 1),\n", + " \"start_value\": (0.1, 2.0, 0.1),\n", + " \"end_value\": (0.001, 0.01, 0.001),\n", + " \"cycle_size\": (1, 20, 1),\n", + " \"start_value_mult\": (0.0, 1.9, 0.05),\n", + " \"end_value_mult\": (0.0, 3.0, 0.05),\n", + "}\n", + "\n", + "_ = widgets.interact(print_cosineannealingscheduler, **cosineannealingscheduler_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Warmup\n", + "\n", + "See the [documentation](https://pytorch.org/ignite/generated/ignite.handlers.param_scheduler.create_lr_scheduler_with_warmup.html#ignite.handlers.param_scheduler.create_lr_scheduler_with_warmup)." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "from ignite.contrib.handlers import create_lr_scheduler_with_warmup" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c109dfdc40f742bcb5984c3e9c7ffe28", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(IntSlider(value=50, description='num_events', max=150, min=5), FloatSlider(value=0.25, d…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def print_warmup_optimizer(num_events=50, **kwargs):\n", + " optimizer = SGD(params=[torch.zeros(10)], lr=kwargs[\"warmup_end_value\"])\n", + " lr_scheduler = ExponentialLR(optimizer=optimizer, gamma=kwargs[\"gamma\"])\n", + " lr_values = [None] * num_events\n", + " del kwargs[\"gamma\"]\n", + " create_lr_scheduler_with_warmup(\n", + " lr_scheduler=lr_scheduler,\n", + " **kwargs,\n", + " output_simulated_values=lr_values,\n", + " )\n", + " lr_values = np.array(lr_values)\n", + " plt.plot(lr_values[:, 0], lr_values[:, 1], label=\"learning rate\")\n", + " plt.title(\"create_lr_scheduler_with_warmup + ExponentialLR\")\n", + " plt.show()\n", + " \n", + "warmup_params = {\n", + " \"num_events\": (5, 150, 1),\n", + " \"warmup_start_value\": (0.01, 0.5, 0.01),\n", + " \"warmup_end_value\": (0.5, 2.0, 0.1),\n", + " \"warmup_duration\": (5, 20, 1),\n", + " \"gamma\": (0.5, 1.0, 0.01),\n", + "}\n", + "\n", + "_ = widgets.interact(print_warmup_optimizer, **warmup_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example of an advanced case using multiparameters\n", + "\n", + "In the following case, we consider an optimizer with 3 groups of parameters. We associate a `PyTorch` scheduler on each group with warmup." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABF8UlEQVR4nO3dd3hUVfrA8e+ZSe+9kEoghF5DbwoWOiiKoOu69vqz7a7rFl3XXXdX1+66KvaOikqxgAhKkZrQOyGkEEgnhRTSzu+PO4EAacAkQ2bez/PcJzN37sx9hwlvzpz7nnOU1hohhBAdn8nWAQghhLAOSehCCGEnJKELIYSdkIQuhBB2QhK6EELYCSdbnTgoKEjHxsba6vRCCNEhJScn52utgxt7zGYJPTY2lqSkJFudXgghOiSlVHpTj0mXixBC2AlJ6EIIYSckoQshhJ2QhC6EEHZCEroQQtiJViV0pdQEpdQ+pVSKUurRRh5/QSm11bLtV0oVWT1SIYQQzWqxbFEpZQZeBS4HDgOblFKLtNa764/RWj/U4Pj/Awa0QaxCCCGa0ZoW+hAgRWudqrWuAuYB05s5fg7wqTWCuxhsSC0gOb3Q1mEIIUSLWpPQI4DMBvcPW/adRSkVA3QGVjTx+B1KqSSlVFJeXt65xmoTjy3cyY1vbyQlt9TWoQghRLOsfVF0NjBfa13b2INa67la60StdWJwcKMjVy8qdXWatIJyyqtqufPDZI6fqLF1SEII0aTWJPQsIKrB/UjLvsbMxo66W7JLKqmqqWNav04cyi/jD/O3Iys8CSEuVq1J6JuAeKVUZ6WUC0bSXnTmQUqp7oA/sM66IdpOWkEZALMSo3hkQne+3XGUt9ccsnFUQgjRuBarXLTWNUqp+4ClgBl4R2u9Syn1JJCkta5P7rOBedqOmrDpBeUAxAR6MLJrIFsyjvGv7/fSq5Mvw7sE2jg6IYQ4Xav60LXW32mtu2mtu2itn7Lse7xBMkdr/YTW+qwa9Y4sraAMZ7Oik587SimevbYfsYEe3PfJZo4UVdg6PCGEOI2MFG1GRkE5UQEemE0KAG83Z+b+OpETNXXc9VEyldWNXvsVQgibkITejLSCcmIDPU/b1yXYi+dn9WP74WIeW7BTLpIKIS4aktCboLUmvaCMmECPsx67olcY94+P54vkw3ywrsm55oUQol1JQm9C3vETlFfVEhNwdkIHeHB8PJf1COXJb3az7mBBO0cnhBBnk4TehJMVLkGejT5uMileuK4fnYM8uefjZDILy9szPCGEOIsk9Cak5Rs16Gf2oTfk7ebM3BsHUVOnuePDZMpkJKkQwoYkoTcho7Acs0kR4efe7HFxwV68MmcA+7JL+O3n26irk4ukQgjbkITehLSCciL83HFxavmf6JKEEP40qQdLdmXz4vID7RCdEEKcrcWRoo6qqQqXptw6qjP7skt5efkBuoV6MaVvpzaMTgghziYt9EZorTmUf24JXSnFP67qTWKMP7/9fBvbMovaLkAhhGiEJPRGFJVXU1pZ0+wF0ca4Opl5/cZBBHu7ctsHSTI9gBCiXUlCb0T9LIsx55jQAYK8XHn7psFUVNVy2/tJUvkihGg3ktAbUV+DHnsOXS4NJYR588r1A9ibXcKDn22lVipfhBDtQBJ6I9ILylEKopoYJdoalyaE8PiUnizbncO/vttjxeiEEKJxUuXSiPSCMsJ83HBzNl/Q6/xmZGfSCsp5a80hYgI9uHF4rHUCFEKIRkhCb0TaOZYsNuexKT3JLCznr4t2EenvwaXdQ6zyukIIcSbpcmlEekE5MQHnfkG0MWaT4uU5A+gR7sN9n2xmZ1axVV5XCCHOJAn9DCWV1RSUVRHbxKRc58PT1Yl3fjMYX3dnbnlvE1lSziiEaAOS0M+QcYEVLk0J9XHjvVuGUFFdy2/e2UhxRbVVX18IIVqV0JVSE5RS+5RSKUqpRtcNVUrNUkrtVkrtUkp9Yt0w28+phaGt10Kv1y3UmzduHERaQRl3fpjEiRpZwk4IYT0tJnSllBl4FZgI9ATmKKV6nnFMPPBHYKTWuhfwoPVDbR+nBhVZt4Veb0SXIP5zTT/Wpxbyuy+2y+yMQgiraU2VyxAgRWudCqCUmgdMB3Y3OOZ24FWt9TEArXWutQNtL+kFZQR7u+Lp2nYFQDMGRJBdUsm/v99LiLcrj03p2fKThBCiBa3JWhFAZoP7h4GhZxzTDUAp9QtgBp7QWi8584WUUncAdwBER0efT7xtLq2gvMll56zpzjFxZBdX8vaaQ4T5uHH7mLg2P6cQwr5Z66KoExAPXALMAd5USvmdeZDWeq7WOlFrnRgcHGylU1uXMW2u9fvPz6SU4vEpPZncJ5ynvtvDgi1ZbX5OIYR9a00LPQuIanA/0rKvocPABq11NXBIKbUfI8FvskqU7aSiqpackhNWr3BpismkeG5WPwrLqvjdF9vw9XDm0gQZeCSEOD+taaFvAuKVUp2VUi7AbGDRGccswGido5QKwuiCSbVemO0jo7D5haHbgpuzmbm/HkRCmDd3f5RMcvqxdju3EMK+tJjQtdY1wH3AUmAP8LnWepdS6kml1DTLYUuBAqXUbuAn4Pda64K2Crqt1Fe4tFcLvZ63mzPv3TyEMB83bnlvE/tzStv1/EII+9CqPnSt9Xda625a6y5a66cs+x7XWi+y3NZa64e11j211n201vPaMui2kl5fsmilYf/nItjblQ9vHYqrk4lfvbWBTMu3BSGEaC0ZKdpAWkE5/h7O+Ho42+T8UQEefHjrUE7U1HHDWxvILam0SRxCiI5JEnoD7VXh0pyEMG/eu3kw+cdPcOPbGykqr7JpPEKIjkMSegPpBeXt3n/emAHR/rz560QO5Zdx07ubOC7L2AkhWkESusWJmlqOFFUQbeMWer2RXYP47/UD2JlVzK3vbaKiSuZ9EUI0TxK6xeFjFdTp9q9wac4VvcJ4flY/NqYVctdHyTKZlxCiWZLQLU5WuFwkLfR60/tH8K+r+rByfx73f7qF6to6W4ckhLhISUK3SMtvm3nQrWH2kGj+OrUnS3fl8NBnW6mRpC6EaISsKWqRXlCGt6sTAZ4utg6lUTeP7ExVTR3/+n4vLmYTz17bD5NJ2TosIcRFRBK6RXphOTFBHih18SbJO8d2oaqmjueW7cfJrPj31X0lqQshTpKEbpFeUE7PTj62DqNF/zc+nuraOl5ekYJJKf55VR9J6kIIQBI6ADW1dWQWljOxd5itQ2mVhy7vRq3WvPrTQUwmxT+m95akLoSQhA5wpKiSmjpN7EVW4dIUpRS/uyKBOg2v/XwQBfxdkroQDk8SOm2/jmhbUErxyJUJ1GnNGytTqdPw1AxJ6kI4MknonKpBj23HedCtQSnFoxO6Y1aK//18EK219KkL4cAkoWNcEHVzNhHi7WrrUM6ZUorfX5mAk0nx8ooUaus0/57ZF7MkdSEcjiR0jGlzYwM9L+qSxeYopXj4igRMJsWLPx6gqraO567th5NZxo0J4UgkoWN0ucQFd6zulsY8eFk3XJxMPLNkH1U1dbw0ewAuTpLUhXAUDv+/va5OG4OKOkiFS0vuuaQrj03pyfc7s7n7o2Qqq2VCLyEchcMn9OySSqpq6jpUhUtLbh3VmX/M6M3yvbnc+v4mymQ+dSEcgsMn9FMLQ9tHC73er4bF8Pysfqw7WMCNb2+guKLa1iEJIdpYqxK6UmqCUmqfUipFKfVoI4//RimVp5Taatlus36obSOjwJhl0Z5a6PWuHhjJ/24YyI6sYubMXU/+8RO2DkkI0YZaTOhKKTPwKjAR6AnMUUr1bOTQz7TW/S3bW1aOs82kFZTjYjYR7utu61DaxITe4bz560RS848z6/V1ZBVV2DokIUQbaU0LfQiQorVO1VpXAfOA6W0bVvtJLygjMsDdruu2L0kI4aNbh5J3/ATXvraWg3nHbR2SEKINtCahRwCZDe4ftuw700yl1Hal1HylVFRjL6SUukMplaSUSsrLyzuPcK2vvgbd3iXGBvDZHcOpqq3j2tfXsf1wka1DEkJYmbUuii4GYrXWfYFlwPuNHaS1nqu1TtRaJwYHB1vp1OdPa016QZld9p83pmcnH764awQeLmbmzF3PmgP5tg5JCGFFrUnoWUDDFnekZd9JWusCrXX9Fbe3gEHWCa9t5R0/QXlVrUO00Ot1DvLky7tHEBXgwc3vbWTxtiO2DkkIYSWtSeibgHilVGellAswG1jU8AClVHiDu9OAPdYLse3Yc4VLc0J93PjszuH0j/Lj/nlbePeXQ7YOSQhhBS0mdK11DXAfsBQjUX+utd6llHpSKTXNctj9SqldSqltwP3Ab9oqYGtKK6hfGNpxWuj1fN2d+fDWoVzeI5S/Ld7Nv77fQ12dtnVYQogL0Kq5XLTW3wHfnbHv8Qa3/wj80bqhtb30gjLMJkWEv32WLLbEzdnMa78axF8X7eSNlanklpzg6Zl9Zf4XITooh56cK62gnAg/d5wdeFZCs0nx9+m9CfNx49kf9pNbWslrvxqEj5uzrUMTQpwjx81k4FAVLs1RSnHfuHieu7YfG1ILufa1dRyRAUhCdDgOndDT8sscsv+8KTMHRfL+LUM4UlTBVf/7hZ1ZxbYOSQhxDhw2oReVV1FSWSMt9DOM7BrEF3cPx6QUs95Yx/I9ObYOSQjRSg6b0B25wqUl3cN8WHDvSOKCPbn9gyTek7JGIToEh03o9QtDSwu9caE+bnx+53DGdQ/licW7+evCndTU1tk6LCFEMxw2oafll6MURAVIQm+Kh4sTb9w4iNtGdeb9denc8n4SJZUyr7oQFyuHTejpBWWE+7jh5my2dSgXNbNJ8ZcpPfnX1X1Ym5LPzP+tPTnCVghxcXHYhJ5WUGY364i2hzlDovng1iHklp5g+qtrWHewwNYhCSHO4LAJPaOwnNgg6W45FyO6BLHw3pEEerly49sb+Gh9uq1DEkI04JAJvbSymvzjVdJCPw+xQZ58dc8IRscH8ZcFO/nz1zuoqpGLpUJcDBwyoafXz7IoF0TPi4+bM2/dNJi7xnbh4w0Z/OqtDeSVynqlQtiaYyd0aaGfN7NJ8ejE7rw0uz/bs4qY9t817DgsI0uFsCWHTOhpUoNuNdP7RzD/rhGYlGLm62v5Iimz5ScJIdqEQyb09IIygr1d8XR16MkmraZ3hC+L7hvJ4Fh/fj9/O48t2Cn96kLYgIMm9HJipXVuVYFerrx/8xDuHBPHh+vTmT13HUeLZcZGIdqTwyZ06T+3PieziT9O6sGr1w9kX3YpU15ew9oUWYhaiPbicAm9oqqW7JJKqXBpQ5P7hrPwvpH4e7rwq7c38OpPKbK8nRDtwOESekahpcIlSFrobalriDcL7x3J5L6d+M/Sfdz2QRLHyqpsHZYQdq1VCV0pNUEptU8plaKUerSZ42YqpbRSKtF6IVpXfYWL9KG3PU9XJ16e3Z+/T+/FmgP5THllDVsyjtk6LCHsVosJXSllBl4FJgI9gTlKqZ6NHOcNPABssHaQ1pRxclCRtNDbg1KKG4fHMv/u4SgF176+jrdWp6K1dMEIYW2taaEPAVK01qla6ypgHjC9keP+DjwNVFoxPqtLKyjD38MZX4/WLYL8/q73eSH5BU7UykjIC9E30o9v/28047qH8I9v93Db+9IFI4S1tSahRwANR4sctuw7SSk1EIjSWn/b3Asppe5QSiUppZLy8vLOOVhrOJcKl9q6Wt7Y9gbv7HyHG769gUPFsnLPhfD1cOaNGwfxxNSerD6Qz6SXV7PxUKGtwxLCblzwRVGllAl4HvhtS8dqredqrRO11onBwcEXeurzYkyb27r+8/3H9lNaXcrM+JnklOdw3TfXsSBlgXQXXAClFL8Z2Zkv7x6Bq5OJ2XPX8eKP+6mVKhghLlhrEnoWENXgfqRlXz1voDfws1IqDRgGLLoYL4xW1dRxpKii1S305JxkAO7qdxfzp86nd1BvHvvlMf6w6g+UVpW2Zah2r0+kL9/cP5oZ/SN48ccDzHlzPVlFMhBJiAvRmoS+CYhXSnVWSrkAs4FF9Q9qrYu11kFa61itdSywHpimtU5qk4gvQOaxcup062dZTMpJIsIrgjDPMEI9Q3nz8je5f8D9/JD+A9cuvpatuVvbNmA75+XqxPPX9ee5a/uxK6uYiS+u4tvtR20dlhAdVosJXWtdA9wHLAX2AJ9rrXcppZ5USk1r6wCtqb7CJbYVNeh1uo7knGQSQ0990TCbzNze93Y+mPgBADctuYlXt75KdZ2ss3khZg6K5LsHRhMX7MW9n2zm919s4/iJGluHJUSH06o+dK31d1rrblrrLlrrpyz7HtdaL2rk2EsuxtY5nFsNempRKkUnikgMO7vnqG9wX+ZPnc+UuCm8vu11bvr+JtJLZPWeCxET6MkXdw3n/8Z15cvNh5n00mqS0+WCqRDnwqFGiqYXlOPt6kSAp0uLxyblGH+TBoUOavRxLxcvnhr1FM+OfZb0knSuXXwtn+/7XC6YXgBns4nfXpHAZ3cOp05rrn19Hc/9sI/qWpm5UYjWcKiEnlZQRnSgB0qpFo9Nykki1COUSK/IZo+7MvZKvpz2Jf2D+/P39X/n3uX3kldum5JMezE4NoDvHxjNVQMieWVFClf97xcO5MhFaCFa4lAJ3Zg2t+X+c601SdlJJIYltir5h3mG8frlr/PokEfZmL2RqxZdxZJDS6wRssPydnPmuVn9eP1XAzlSVMnkV9bw1upUKW8UohkOk9BrauvILCxvVQ16ekk6BZUFTXa3NMakTNzQ4wY+n/o50d7R/H7V7/ndyt9xrFLmLrkQE3qHs/TBMYyJD+Yf3+5h9tx1pOWX2TosIS5KDpPQjxZXUlOnW9VCr+8/b1jh0lpxvnF8MPED7h9wP8szlnPVwqtYnr78nF9HnBLs7cqbvx7Es9f2Y292KRNfWs37a9NkSl4hzuAwCf1c1hFNykki0C2QWJ/Y8zqXk8mJ2/vezrzJ8wj2CObBnx/kkVWPSGv9AiiluGZQJD88NIbBnQP466JdzHlzPekF0loXop4DJfTW1aBrrUnOSWZQ6KBW9Z83JyEggU8mf8K9/e9lWfoyZiycwdK0pVIJcwHCfd15/+bBPDOzL7uPlnDli6t4e80h6VsXAgdK6On5Zbg5mwjxdm32uKzjWWSXZTdaf34+nE3O3NXvLuZNnkeYZxi/W/k7Hvr5IamEuQBKKWYNjmLZQ2MZ0SWIv3+zm2teXyuVMMLhOUxCTysoJybAs8VW94X0nzcnISCBjyd9zEODHmJN1hqmL5zOVwe+ktb6BQjzdePtmxJ54bp+pOWXMfnlNby8/ABVNVK3LhyTwyT09FbOspiUnYSvqy9d/LpYPQYnkxO39L6F+VPn082/G39d+1du/eFWGWV6AZRSXDUgkmUPj+XK3mE8v2w/U15ZTXK6XK8QjschEnpdnSa9sLxVc7gk5yQzKGQQJtV2/zSxvrG8c+U7/HX4X9lbsJerF17N3O1zqa6VOWHOV5CXK6/MGcBbv06ktLKGa15fy2MLdlJSKf+mwnE4RELPKa2kqqauxRZ6dlk2h48ftlr/eXNMysQ13a5h4YyFjI0ayytbXuHaxdeyOWdzm5/bnl3WM5RlD4/lpuGxfLQhncueW8m3249K15ZwCA6R0NPyLRUuLdSgtzR/S1sI9gjm+Uue57/j/kt5TTk3LbmJx395XEocL4CXqxNPTOvFgntGEuztyr2fbObm9zadnG1TCHvlEAm9vlY5uoV50JNzkvF29ibBP6E9wjrN2KixLJi+gJt73czig4uZtmAaXx34ijotF/jOV78oPxbeO5LHpvRk46FCLn9hJa8sP8CJmlpbhyZEm3CIhJ5WUI6zWdHJz73Z45KykxgQOgCzydxOkZ3Ow9mDhxMf5vOpnxPnG8df1/6VX3//a/YU7LFJPPbAyWzi1lGdWf7bsYzvEcJzy/Yz8cXVrD4gZaPC/jhEQk8vKCMqwAOzqemSxfyKfNJK0tq1u6Up8f7xvDvhXf4+8u9klmYy+9vZPLX+KYpPFNs6tA4r3Ned/90wiPdvGUKd1tz49kbu/ihZlr0TdsUhEnpaK2ZZrF8/1Nr15+fLpEzM6DqDRTMWcV3CdXy+/3Omfj2VL/d/SW2ddBmcr7Hdglny4Bh+d0U3ftqXy/jnfuaV5QeorJZ/U9Hx2X1C11qT0Yoa9KTsJNyd3OkR2KOdImsdX1df/jT0T3w25TM6+3bmiXVPcP1318t6phfAzdnMfePiWf7bS7g0weiGufyFlSzdlS3VMKJDs/uEnn+8irKq2lZVuAwIGYCzybmdIjs33QO6896E9/j36H+TX57Pjd/fyKOrHyW7LNvWoXVYEX7uvParQXxy21Dcnc3c+WEyN769kX3ZMoWA6JhaldCVUhOUUvuUUilKqUcbefwupdQOpdRWpdQapVRP64d6fk5WuDTTQj9WeYyUopSLov+8OUopJsdNZvFVi7m9z+0sS1vGtAXTeG3ba1TUSF/w+RrRNYjv7h/NE1N7siOrmIkvreIvC3ZQWFZl69CEOCctJnSllBl4FZgI9ATmNJKwP9Fa99Fa9weeAZ63dqDn6+Qsi8200DfnGoN5Lpb+85Z4OHtw/8D7WThjIaMiRvG/rf9j6tdTWXxwsZQ5nicns4nfjOzMz7+7hBuHxfDpxkzG/ucn3lyVKmWOosNoTQt9CJCitU7VWlcB84DpDQ/QWpc0uOsJXDQdkekFZZhNiohmShaTspNwNbvSO6h3O0Z24SK9I3n+kud5b8J7BLoH8qc1f+L6b69nU/YmW4fWYfl7uvC36b1Z8sBoEmP8eeq7PVz+/Cq+2yGjTcXFrzUJPQLIbHD/sGXfaZRS9yqlDmK00O9v7IWUUncopZKUUkl5ee1TB5xWUE6EnzsuTk2/1eScZPoG98XF7NIuMVnboNBBfDr5U/456p/kV+Rzy9JbuH/F/aQWp9o6tA4rPtSbd28ewge3DMHd2cw9H29m5mtrSU4vtHVoQjTJahdFtdavaq27AH8A/tLEMXO11ola68Tg4GBrnbpZLVW4lFaVsrdwb4fpbmmKSZmY2mUq31z1DQ8MfICN2Ru5euHVPLnuSZl7/QKM6RbMdw+M5umZfTh8rIKZr63jzg+TOJh33NahCXGW1iT0LCCqwf1Iy76mzANmXEBMVtVSDfqW3C1odIdP6PXcnNy4rc9tfHvVt8xKmMXXB75m8teT+e+W/3K8SpLQ+TCbFNcNjubn31/Cw5d3Y82BfK54YRV//GoHOSWVtg5PiJNak9A3AfFKqc5KKRdgNrCo4QFKqfgGdycDB6wX4vkrKq+iuKK62RZ6UnYSTiYn+gb3bcfI2l6geyB/GvonFs5YyJjIMbyx/Q0mfjWR93e9z4naE7YOr0PycHHi/vHxrHzkUm4cFsP8ZOPC6b+/30txuUzTK2yvxYSuta4B7gOWAnuAz7XWu5RSTyqlplkOu08ptUsptRV4GLiprQI+F/UVLjHNtNCTc5LpE9QHNye39gqrXUX7RPPs2GeZN2UePQN78mzSs0z+ajJf7v+SmroaW4fXIQV5ufLEtF78+PBYJvQK441VBxn9zApe/SmF8ir5NxW2o2x15T4xMVEnJSW16TkWbs3igXlbWfbQGOJDvc96vLy6nBGfjuCW3rdw/8BGr+PanQ1HN/Dy5pfZnr+dGJ8Y7ul3DxM6T2jTBT3s3Z6jJTy7dB/L9+YS5OXCPZd05fqh0bg522aSN2HflFLJWutG+4jt+n9xWn45SkFUE9Pmbs3dSq2utZv+89YYGj6UjyZ9xCvjXsHF7MIfVv+BmYtmsix9mdSwn6ce4T68/ZvBfHn3cOJDvHnym91c+uzPfLQ+XdY3Fe3KrhN6emEZ4T5uTbaUknKSMCsz/UP6t29gNqaU4pKoS5g/dT7/GfMfanUtD//8MLMWz2J5xnKptz5Pg2IC+PSOYXxy21DCfd34y4KdXPrsz8zbmEF1rSR20fbsO6EXlLfYf94zsCcezi0vHm2PTMrEhM4T+Hra1/xz1D+pqKngwZ8eZNY3s1iRsUIS+3ka0TWIL+8ewfu3DCHI25VHv9pxMrFLi120JTtP6E3XoFfWVLIjf4dDdbc0xWwyM7XLVBbOWMhTo56ivLqcB356gGsXXytdMedJKcXYbsEsuGcE7/wmkUBPl5OJ/eMN6TKdgGgTdpvQSyuryT9e1WQLfXvedqrrqttlQeiOwsnkxLQu004m9hO1J3j454eZuWgm36Z+K1Ux50EpxbjuoSy4dyTv3jyYYG9X/vz1TsY+8zPv/nJI5mEXVmW3CT395KRcjbfQk3OSUSiH6z9vjfrEvmD6Ap4e/TQAj65+lGkLpvHl/i+prpWa63OllOLShBC+vmcEH906lOhAD/62eDejnl7B/35OobRS/k3FhbPbhJ5R2HwNelJOEt0DuuPj4tOeYXUoZpOZSXGT+HLal7x4yYt4u3jzxLonmPDVBD7Y9QHl1eW2DrHDUUoxKj6Iz+8czmd3DKNnJ1+eWbKPEf9ewbNL95F/XAZ9ifNntwk9zTIPemN96FW1VWzL23bRz39+sTApE+NjxjNv8jzeuOwNYnxi+E/Sf7jiyyv475b/UlgpE1adj6FxgXxwyxAW3TeSkV2CePXnFEb+ewWPLdhJRoH8sRTnzsnWAbSV9Pxygr1d8XQ9+y3uzN/JidoT0n9+jpRSjIgYwYiIEWzL28bbO97mje1v8N6u95jRdQY39bqJKO+oll9InKZvpB+v3ziIg3nHmbsylXmbMvh4QzqT+oRz55gu9In0tXWIooOw24SeVlBGTBMDiuoXhB4YMrA9Q7Ir/YL78fK4l0ktTuX9Xe/z1YGv+GL/F4yPHs9NvW6iX3A/W4fY4XQJ9uLpa/ry0OXdeHftIT5Zn8E3248yLC6A20fHcWlCCCaTsnWY4iJmt0P/h/1zOSO7BvHcrLMTy53L7iS3PJevp3/dZud3NHnleXyy9xM+2/cZpVWl9A/uz409b2Rc9DicTHbbbmhTJZXVfLohg/fWpnG0uJIuwZ7cOiqOqwdGyLQCDszhhv5XVteSXVLZaIVLdV01W3K3SP25lQV7BPPAwAf48ZofeXTIo+RX5PPblb9lytdT+GDXB5RWycLL58rHzZk7x3Zh1SOX8tLs/ri7mPnT1zsY/q/lPLt0H7kyda84g10m9JMVLkFnV7jsLdhLRU0Fg8Lkgmhb8HD24IYeN/DNVd/w4iUvEuoRyn+S/sNlX1zGPzf8k7TiNFuH2OE4m01M7x/B4vtGMe+OYSTGBhgXUJ9ewYPztrA1s8jWIYqLhF1+F07Lt1S4NNKHnpRjdPNIC71tmU1mxseMZ3zMeHYX7ObjPR8zf/98Pt37KSMjRnJ99+sZFTFKZnk8B0ophsUFMiwukLT8Mj5Yl87nSZks2HqE/lF+3DQihkl9wnF1ku4YR2WXfehvrkrlqe/2sO3xK/D1cD7tsXuX30tGSQaLr1rcJucWTcuvyOeL/V/wxb4vyKvII8o7iusSrmNG1xn4ukolx/k4fqKG+UmZfLAundT8MoK8XJgzJJo5Q6Lp1MzC6KLjaq4P3S4T+p+/3sG3O46y9fErTttfW1fL6HmjuSL2Cp4Y8USbnFu0rLq2mh8zfuTTvZ+yJXcLbmY3JnaeyHUJ19ErqJetw+uQ6uo0a1LyeX9tGiv25aKAy3uG8qthMYzsEiTVMXakuYRul10uTc2yuP/YfkqrS6X+3Maczc5M7DyRiZ0nsq9wH5/u/ZTvDn3H1ylf0zOwJ7O6zWJi54kOOwvm+TCZFGO6BTOmWzCZheV8sjGDzzZlsnRXDrGBHswZEs01gyIJ9HK1daiiDdllC33U0ysYFOPPS7MHnLb/w90f8symZ1h2zTLCPMPa5Nzi/JRWlfJN6jd8vu9zUopS8HT2ZErcFK7pdg3dA7rbOrwOqbK6lqW7svl4fQYb0wpxMZuY0DuMOUOiGRYXgFLSau+IHKqFXlVTx5GiCq4eGHnWY0nZSUR6RUoyvwh5u3gzp/scZifMZmveVr7Y9wVfH/iaz/Z9Rq/AXszsNpNJnSfh6dz0/PbidG7OZqb3j2B6/wj255TyyYYMvtp8mEXbjtA5yJPrBkcxc2Akwd7SarcXrWqhK6UmAC8BZuAtrfW/z3j8YeA2oAbIA27RWqc395pt1UJPzTvOuOdW8vysfqcl9Tpdx9jPxjI2ciz/GPUPq59XWF/xiWK+Sf2G+fvnk1KUgruTO1fGXslVXa9iQMgAaWGeh8rqWr7bcZRPN2awKe0YTibF+B4hzEqMYmy3YJzMUnV0sbugFrpSygy8ClwOHAY2KaUWaa13NzhsC5CotS5XSt0NPANcd+Ghn7v6aXPP7EM/WHSQohNF0n/egfi6+nJDjxu4vvv17MjfwZcHvmTJoSUsSFlArE8s07tOZ2rcVEI9Q20daofh5mzm6oGRXD0wkpTc43yRlMmXmw+zdFcOId6uXD0wkmsGRdI1xMvWoYrz0JoulyFAitY6FUApNQ+YDpxM6Frrnxocvx74lTWDPBdNzbJ43vXnRRlQXQnB3awSnzh3Sin6Bvelb3Bf/jD4D/yQ/gNfH/ialza/xCtbXmF4p+FM7zKdS6Muxc3JzdbhdhhdQ7z446Qe/O7KBFbszeWLpEzeXJ3K6ysPMiDaj5kDI5nat9NZpb/i4tWahB4BZDa4fxgY2szxtwLfN/aAUuoO4A6A6OjoVoZ4btILyvFydSLQ0+W0/ck5yYR6hBLhFdH6F6upgncnQXEmhPeHfnOgzzXgGWTdoEWreTh7MKPrDGZ0nUFGSQaLDi5i0cFFPLLqEbycvbgy9kqmxE1hYOhAGbTUSs5mE1f2CuPKXmHkllaycMsRvkjO5C8LdvLk4t1c1jOEqwdEMjYhGGfpkrmotdiHrpS6Bpigtb7Ncv9GYKjW+r5Gjv0VcB8wVmvd7Ez9bdWH/pt3N5JXeoJv7x99cp/Wmks/v5RhnYbx79H/bubZZ0h+DxY/AEPugMwNcHQbKDN0vQz6zoKESeAipXW2VqfrSMpOYuHBhSxLX0ZFTQWdPDsxOW4yU+KmEOcXZ+sQOxytNTuzSvjSchG1sKyKAE8XpvQN56oBEfSP8pNrGDZyoVUuWUDDSa4jLfvOPMllwJ9pRTJvSxkF5fQIP30VorSSNAoqC86tu6W2GlY/B50GwsRnQCnI2Q3bP4MdX8CXS8HFC7pPgT7XQtwlYLa7oqEOwaRMDAkfwpDwIfx56J9ZkbmCbw5+w9s73+bNHW/SI6AHk+MmMyF2gvS3t5JSij6RvvSJ9OXPk3uwcl8eX2/N4rNNxqjUmEAPpvfrxPQBEXQJlv72i0VrWuhOwH5gPEYi3wRcr7Xe1eCYAcB8jJb8gdacuC1a6DW1dfR4fAm3j47jkQmnape/2P8FT657ksUzFhPrG9u6F9vyMSy8B+Z8BgkTTn+srg7S18D2z2H3IjhRDB5B0GsG9J4JUcPAJF9NbS2/Ip8lh5bwbeq37CzYiUIxKHQQEztP5PKYy/F387d1iB1OSWU1S3Zms3BrFmsPFqA19I7wYWrfTkzp14kImW6gzV3w0H+l1CTgRYyyxXe01k8ppZ4EkrTWi5RSPwJ9gKOWp2Rorac195ptkdAzC8sZ/cxPPDOzL7MGn/pS8ejqR1l/ZD0/zfqpdV8Ta2vg1cFGC/zOVUbrvCk1J+DAD7DzS9i3BGoqwCcCel1lbBGDmn++aBdpxWl8n/Y936V+R1pJGmZlZlj4MCZ0nsC46HGytux5yC2pZPH2oyzadoRtlhkfB8X4M6VvOJP6hBPqIxeo24LDzOWy+kAeN769kXl3DGNYXCBg9AVePv9y+of059mxz7buhbZ/Dl/dDtd9BD2mtj6AE6VGUt/5JaT8CHXV4BcNPadDz6sgYqAkdxvTWrO3cC9L0pawNG0pWcezcDI5MaLTCK6MvZJLoi6R5H4e0gvKWLztCN9sP8re7FKUgsGxAUzuE87E3mGESHK3GodJ6B+uT+exBTtZ/8fxhPkav0CZpZlM+moSfxr6J+Z0n9Pyi9TVwv+Gg8kMd/1y/l0nFUWw7zvY+RWk/mwkd99o6DkNekyDyMHSLWNjWmt25u9kadpSfkj/gaNlR3EyOTE8fDiXx1zOuOhxMgvkeUjJLeWb7Uf5dvtRDuQeN5J7TAATeocxoXeYzAJ5gRwmof/jm918tCGd3X+bcHJ2uQUpC3jsl8f4atpXxPvHt/wiO7+C+TfDNe8Y/eHWUHEM9n4HuxdC6k9QWwXe4cYF1R5TIWakXFC1Ma012/O382P6jyxLX0bW8SzMyszgsMFcFn0Z46LHEewRbOswO5wDOaV8u+MoS3ZmszfbWLWqf5SfpUwylDi5oHrOHCah3/5BEhkF5Sx9aMzJfX9Z8xdWHl7JyutWtlyXXFcHr48yWtP3rDda6dZWWQz7lxrJPWW50efu7g/dJkL3ydBlnJRC2pjWmt0Fu/kx40d+TP+RtJI0APoG92Vc1DjGR49v/cV1cVJq3nG+35nN0l3ZbD9cDEC3UC+u6BnGFb1C6RPhK6WQreAwCf2KF1YSG+jJ3F+feq8TvpxA94DuvHjpiy2/wJ7F8Nmv4Kq50K8dZi6oKoeDy43z7l8KlUXg5A5dLoWEidBtAniFtH0coklaaw4WHWR5xnJWZK5gd4ExQDrWJ5ZLoy9lXNQ4+gT1wdwWf/ztWFZRBUt3ZvPD7mw2HiqkTkO4rxuX9Qjlsp6hDIsLkJWXmuAQCb2uTtPj8SX8engMf57cE4Dssmwun385jwx+hBt73tj8C2gNb4yBquNw76b27wKprYb0tbD3W6PvvTgTUEaVTMIEowUf2ksuqtrY0eNH+SnzJ37K/Imk7CRqdA0BbgGMjhjNJVGXMLzTcJkR8hwVllWxYm8uP+zKZvWBfCqqa/FydWJ0fBDje4RyaUKwzOPegENMn5tTWsmJmrrTJuU6p/lbDvwA2dth+qu26c82O0PcWGOb+DTk7DT63fd/Dyv+YWy+URB/BXS7EjqPAWe5uNTewr3Cub7H9Vzf43pKqkpYc3gNPx/+mRWZK1h4cCFOJicGhw5mTOQYxkSOIdqnbaa4sCcBni5cM8iYFKyyupa1B/NZtjuXFXtz+H5nNkrBgCg/xnUP4dLuIfQM95GumSbYTQt93cEC5ry5no9uHcqoeGOulSfWPsEPaT+wevbq5r8Saw1vjYeyPPi/zUZyvZiUZhtdMgd+gIM/QXUZOLlB7GiIv9zYAmR4uy1V11WzNXcrqw6vYuXhlRwqPgRAjE8MoyNGMypiFIlhibiapaXZWvXTDyzfm8NPe3PZZul3D/Vx5ZJuIVzaPZiRXYPwdrvI/r+2MYfocpm3MYNHv9rB6kcuJSrAuKg49eupxPjE8N/x/23+ySnL4aOrYcoLkHiL1WJqEzUnIG0NHFhmJPjCg8b+gDhjjpmul0HsKHCRr/22lFmSyaqsVazOWs2mo5uoqqvCzezG4LDBjIwYychOI4nxiZGW5jnILa3k5315rNyXx6oDeZRW1uBkUgyM8Wdst2DGdgumZ7iP3a+f6hAJ/ekle3lrdSp7/z4Rs0mRX5HPpZ9fysODHubm3jc3/USt4Z0JRp/1/VvAqYO1oAoOGn+QUpbBodVG1YzZBaKHQZfxxgXW0D5S825DFTUVbMrexJqsNfyS9QsZpRkARHhFMKLTCEZ0GsGQ8CEyoOkcVNfWsTn9GCv35/Hzvjx2Hy0BIMjLhdHxwYyOD2JU1yC7HNDkEAn9no+T2ZtdyorfXgLAkrQl/H7l7/lk0if0Ce7T9BMPrYL3p8KkZ2HI7VaLxyaqKyFjnVE5k7ICci3T7XgEGZOH1W9+Uc28iGhrmSWZ/HLkF3458gubsjdRVl2GSZnoHdSbYeHDGB4+nH7B/XC+2Lr+LmK5pZWs3p/PqgN5rDmQT0FZFQAJod6MsiT3IZ0D8HTt+JcNHSKhT3ppNaE+rrx78xAA/rH+Hyw6uIi1c9biZGrmQ3xvCuQfgAe2gbOd/TUvOWqMUk39yfh5PMfYHxAHnS0XYGPHgGegLaN0aNV11ezI28HaI2tZf3Q9O/N3UqtrcXdyZ2DIQIaGD2Vo+FAS/BOkNLKV6uo0u4+WsCYln9UH8tiUdoyqmjqcTIoB0X6M6BLEyK5B9I/yw8Wp431ztfuErrWm91+Xcm1iFE9M6wXAVQuvIsQjhDcuf6PpJ6avg3cnwJX/guH3WCWWi5bWkLvHkuB/hvRfjBJNMLpkOo82LrLGjAB3PxsG6thKq0rZmL2RDUc3sOHoBlKLUwHwcfFhcNjgk1tXv66ygEcrVVbXkpR2jNUpeaxNKWDnkWK0BndnM4mx/gzvEsjwuED6RPh2iDVV7b5sMf94FWVVtcRalp07VnmMlKIUJnWe1PwTVz0DnsEw6DdtH6StKQWhPY1t+D1G3fuRLXBopdH3nvQOrP8foCC8ryW5j4SY4cZIVtEuvF28GR89nvHR4wHILc9lY/ZGNh7dyMbsjSzPWA6Av6s/iWGJDAodRGJoIvH+8ZLgm+DmbDa6XSzVb8Xl1axLLWB9agFrD+bzzJJ9AHi6mBncOYChnQMZGhdAnwjfDrdCk10k9PT6dUSDjMqOzTmbAZpfEPpwEhxcAZf9zTGH2pudIWqIsY35vdH/npUEab9A2mrY+Cas+y+gjAFNMSOMLXoEeMsiEe0lxCOEKXFTmBI3BYCs41lsyt7EpuxNJGUnsSx9GWC04AeGDiQxNJGBIQPpHtgdZ5P0wTfG18P55ERhAHmlJ9h4qJB1qfmsO1jAz/vyAPBwMTMoxp/BsQEM6RxA/yg/3Jwv7m4vO0no5QDEWgYVJeUk4Wp2pXdg76aftPIZcA+Awbe1R4gXP2c3o9wxdhTwB0uCTza6ZtLWwJaPYONc49iAOIgebtmGQWBXGcHaTiK8IojoGsGMrjMAOHL8CEk5SSTnJJOck8zPmT8D4O7kTt/gvgwKGUT/kP70De4rI1ibEOztyuS+4UzuGw4YCX5TWiEbUgvYcKiQF37cj9bgYjbRJ9KXxFh/BscEMCjGH/8z1i62NbvoQ3/+h328+vNB9jw5ARcnE7MWz8LbxZu3r3y78Scc2QJzL4FxfzFap6JltdVwdLuR4DPWG9U0FYXGYx6BEDXU0uIfCp0GyChWG8ktz2Vz7mY25xjb/mP70WhMykSCfwL9Q/rTP7g//UP6E+4ZLnXwrVBcXk1SeiEb0wpJSjvG9sNFVNcaebNLsCeJluQ+MMafLsGebf5vavcXRe//dAtbMo+x+pFxlFSVMOrTUdzd727u7n9340+Yd4PRrfDgDnCT+a7Pi9aQv99I7pkbjJ/1g5xMzhDWx0jwkYONzS9aWvE2UFpVyva87WzO3czW3K3syN9BRU0FACHuIfQL6Ue/YGPrEdhDRrK2QmV1Ldsyi0hKP0ayZSuuqAbAz8OZ/lF+DIz2Z2C0P/2ifK0+ktXuL4qmF5Sd7G7ZkrMFjW66/zx7J+z9BsY+Ksn8QigFwQnGNugmY19ZPmRuNBL84U2Q/D5seN14zDMEIhONycYiE43Ft91kIE1b83bxNkamRowEoKauhv3H9rM1dyvb8raxLW/byX54J5MTCf4J9AnqQ9/gvvQJ6kO0T7RcbD2Dm7OZoXGBDLWsilZXp0nNL2OzJblvzjh2sh9eKYgP8WJAlD/9o/3oF+lHt1CvNqumsYsWer+//cDUfuH8Y0Yfnk96no/2fMTaOWtxc2qkrvzzm4yRlQ/tkOqNtlZbbUwydjjJsm061YpHQVA3I8FHDDQSfFjvjjdS1w7kV+SzPW+7seVvZ1f+LsprjOtS3i7e9ArsRZ+gPvQK6kXvwN6EespF8ZYUV1SzNbOIrRlFbM08xpbMIorKjVa8u7OZv03rddq6x+fCrlvoReVVFFdUn3ZBtE9Qn8aTed4+Y2GJUQ9JMm8PZmejP73TgFOjcMsLjWsYWcmQtdlYe3XbJ8ZjJmejrLLTAAjvD536Q0gvcLq4LjzZmyD3IMZFj2Nc9DgAautqOVh8kB15O9hZsJNd+bt4Z+c71OpaAILdg+kV2IueQT2Nn4E9CXIPsuVbuOj4ujufnF8GjLEy6QXlbDtcxJaMIrqGts1KTa1K6EqpCcBLgBl4S2v97zMeHwO8CPQFZmut51s5zibVV7jEBHpSVl3G7oLd3NK7iQm2Vj0Lzh4w/L72Ck+cySMAuo43NjD64osPG0n+yGbj566vIfk94/H6JB/eH8L7GVtIT8csNW0nZpOZbv7d6ObfjZkYyzBW1lSyt3Avuwp2sSt/F7sLdrPy8Eo0xjf8EPcQegT2MLYAYwvzDJOLrhZKKWKDPIkN8mR6/4g2O0+LCV0pZQZeBS4HDgOblFKLtNa7GxyWAfwG+F1bBNmcNEsNemygB9tyt1Graxuf/7zgIOycbyRzGep+8VDKmFvGL8pYQBuMJH/sEBzZCke3wtFtxjerze9bnmMyumvC+hoXX+s3T2klthU3JzejQiak/8l95dXl7Cncw+6C3Se31VmrqdN1APi5+pEQkECPgB4kBCTQ3b87sb6xzU/FIS5Ia/5lhwApWutUAKXUPGA6cDKha63TLI/VtUGMzapvoUcFePD9jiTMynzaL91Jq58DsyuM+L/2DVCcO6WMWveAOOh9tbFPayjKMBYhObrd+Jn+C+z4/NTzvMONxB7aC0J7G1tgV1mAu414OHswKHQQg0IHndxXXl3O/mP72VO4h32F+9hTuIeP93xMdZ3Rf+xicqGrf1e6+XcjwT/h5DcBPzc/G70L+9Ka3/QIILPB/cPA0PM5mVLqDuAOgOho66zkklZQRrivG27OZpJykugV2AsP5zO+jhcegm3zYMgdskZnR6UU+McYW4+pp/aXFRjJPWenUcGUs9NYBMSSQDC7QnA3I7mH9DD65EN7GslfugOszsPZ46yWfHVdNWnFaewt3Mu+wn3sO7aPVYdXsSBlwcljQtxDiA+Ip5tfN+L94+nq15U4vzgpozxH7dp00VrPBeaCUeVijddMLygnJtCDipoKduTvaHzt0DUvgMkJRj5gjVOKi4lnoDHne5dLT+2rqTJq5HN2Qs4uyN1tTEi27dNTx7j5QnAPS5LvAcHdjc0rRBK9lTmbnIn3jyfeP56pXU79Mc6vyGdf4T4OHDvA/mP7OVB0gI+OfnSyNW9SJqK9o+ni14Wufl3p6teVLn5diPWJlamFm9CahJ4FNKyvibTsuyikF5RxWY9QduTtoKau5uz+86JM2PqJUSvtE26bIEX7cnIxSiDDzpj6obzQmHEyd7fl5x7LBdh3Tx3j7m9J7gkQlGC07oMSwDdSEr2VBbkHERQRdLJGHow6+YySDPYX7edg0UFSjqWQUpTCT5k/neybNysz0T7RdPHtQmffzsT5xRHnG0esT+zZ384dTGsS+iYgXinVGSORzwaub9OoWun4iRryj1cRE+hJUs4aTMrEgJABpx/0y4vGz5EPtnd44mLjEQCxI42tntbGPPG5eyBvr1HaWl/eWnHs1HHOnhAUb1yMDeoGQV2NnwFd7G8efRtyMjkZCdrv9DVyT9SeIK04jZSiFFKLU41kb0n09eWUAJ08O9HZrzOdfTrT2dfYYn1iCXIPcoiKmxYTuta6Ril1H7AUo2zxHa31LqXUk0CS1nqRUmow8DXgD0xVSv1Na92rTSPn1CyLsYEefHEkiQT/BLxdvE8dUHIENn8A/a+XVXpE45QC7zBja9hto7Ux8jXfkuDzD1imOlh3+oVYLFU6gfHGBdjArhDYxdh8o0AWpbAKV7MrCQEJJAQknLa/uraa9JJ0UotTSS1O5VDxIQ4VH2JzzuaTUxwAeDl7EeMTQ4xPDLE+scT6xhLtE02MdwxeLm1TE24LrepD11p/B3x3xr7HG9zehNEV067qK1zC/Z3Yvm0713a79vQDfnkZ6mqNgURCnAulwCvY2GJHnf5YVZlRBpu/30j0BSlQcMCY8qB+0RAw1nb172wk9/qqnfrNN1KSvRU4m53p6t+Vrv5dT9tfp+vILc8ltTiVtOI00krSSCtOY2vuVr4/9P3J+nmAQLdAYnxijATvE0OUdxTR3tFEeUd1uGTfoeu56mvQy0jjRO2J0+dvKc0x+kb7zYaAzjaKUNglF09jEZDwvqfvr+++KUgxEn7hQcvPVGPu/ZrKU8eanME/1vjd9I81En/9bb8YGTh1gUzKRJhnGGGeYYzoNOK0xyprKskozSCjJIP0kvST2y9Zv5xWeQMQ4BZApHfkyQQf6R1p/PSKvCi7cTp0Qk/PLyfIy5VdhVsAGBRyqh6Wda9AbRWM/q2NohMOp2H3zZmt+ro6KD1qJPnCQ0aSL0w1BlClr4Oq0tOP9wo1Ert/rFGq6Rdz6qdPhNTWXwA3J7eT9e9nKq8uJ7M082TCzyzNJLM0k+ScZL5N/fa0lr2b2c2Yn947wvjpFUGkVyQR3hF08uqEj0v7Tz7XoX8r0grKiA30ICk7iXj/+FODE8oKYNM70Psa4+uuELZmMoFvhLF1HnP6Y1pDeQEcSzO2wkNQlAbH0o1piXfOB91gzJ4yG6/jF2P00/tFG/34vpYRtz6RMv/NefJw9mi0rx6gqraKrONZHC49TGZpJlnHs07e35yzmePVx0873tvFmwivCDp5dqKTl2Xz7ES4VzgxPjFtsuBIh07o6QXlDO/ixy95W5neZfqpB9a/CtXl0joXHYNSxrQFnkHG1MJnqq025rspSjdGyx5Lt9zONOrrS48CDYd1KKOF7xtpSfARRrL3jTD2+UQa57rIugsudi5ml5OVM2fSWlNSVcLh44fJKs3iyPEjHD5+mKNlR8kozWDd0XWnXaT945A/cn0P6xcLdtiEXlldS3ZJJZ7e2VTkVZzqPy8vhA1zoed0COlu2yCFsAazs9G/3tS1oJoqKDlsSfqZUFy/HTamSdj3/en992CMoPXpZEn2Eadu+3QyRtH6RBgLqJtkLvTWUErh6+qLr6svvQLPLvDTWlN8opissiyOHj9Kgv/Z3wCsocMm9IxCo8Kl0ukAwKn5JDa8YfRHytJywlE4uZyqnmlMfZdOcSYUZ0FJlpHsS44Yt9PXGa38+ukS6pmcwMtyTcAnHLw7Wa4RhFvuhxv3XX2ktd8CpRR+bn74ufk1mvCtpcMm9LR8o8Ilp2r3yYEDVBbD+teg+5SzRwkK4agadul0GtD4MXV1UJ5vJPrSo5Zkf8S4XXoU8vZD6io4UXz2c53cwTvUkvzP+OkVakyn4BVqnF9KNdtUh03oRg16HQeKdzAxboKxc+Nc4xdOWudCnBuTyZJ4W5i8rqoMSrMtiT7bSPrHcyz7siFntzE52omSs5+rTEY3jmeIUd9/2s8Qyx+dEMsxQUZXkzgnHTahpxWU4euXR1nNcWP+lhOlsO5ViL/SWOlGCGF9Lp6nRsI2p6rcSPTHc+F4tuVn7ql9ZbnGoKzjuVB7ovHXcPe3JHdLgvewfMvwDAaPwFP7PAKNaR3kD0DHTejpBeX4BxymAEv/+aa3jbk3xj5i69CEEC4ezV/Irae10Zovy7ck+jwj2R/PM7qAyvKMx3L3GrcrCpt+LTffBgk+8FSi9wgA94BT993r9/nb3R+BjpvQC8twCksl0jOSMGcfWPsKdBnXeNmXEOLipJSRiN18WzdmpLbGSOpl+ZaEn29c8G24leUbVT/Z2437Z1b4NOTqA+5+RpJ39z+V6Btubn6W236nbl+kE7J1yIReVVNH1rEyAsL2MSrsMmOIf3k+jJHWuRB2zezUur7+hqrKTyX7ikKjtLni2KmfFQ3uF2UY9yuLTx/MdSYnN0ty9zN+uvmefvu0zafBbT/jj0gbjfTtkAn98LFycMnlhD5OYlBfWPxniB0NMcNtHZoQ4mLj4mFs5zLjal2d0RVUUQgVRVBZZEn+DW5XFp+6X3rUmH65sggqSzh9oFcjJj8Hg287v/fTjA6Z0NMLyjF7HAJgUG6acaFl5lu2DUoIYT9MJktXjN+5P7euzhgLU1Fk/FGoT/wnSoxkX1kMnQZaN16LDpnQ0wrKMHukEuIeSsSmdyF6uNFCF0IIWzOZTnWxtPep2/2MVpCWX4aTxyGGuASgSrKMunMZqSaEcHAdMqHvKzyIcjrOoCN7IGKQUd0ihBAOrkMm9PSynQAkFh6BsX+Q1rkQQtABE3pNbR0leg8BtZqY4J4Qf4WtQxJCiItCqxK6UmqCUmqfUipFKfVoI4+7KqU+szy+QSkVa/VILY4UVeDuvo8hFeWoMdI6F0KIei0mdKWUGXgVmAj0BOYopXqecditwDGtdVfgBeBpawdaL+nwPk44V9Lb5AcJk9rqNEII0eG0poU+BEjRWqdqrauAecD0M46ZDrxvuT0fGK/aaPXUA3uN0/TpfYtMvi+EEA20JiNGAJkN7h+27Gv0GK11DVAMBJ75QkqpO5RSSUqppLy8vPMKONAjkIGV7vQbftd5PV8IIexVuzZxtdZztdaJWuvE4ODg83qN26Y+zvt3bsTs1CHHRAkhRJtpTULPAhpOghBp2dfoMUopJ8AXKLBGgEIIIVqnNQl9ExCvlOqslHIBZgOLzjhmEXCT5fY1wAqtdQuz0wghhLCmFvsttNY1Sqn7gKWAGXhHa71LKfUkkKS1XgS8DXyolEoBCjGSvhBCiHbUqo5orfV3wHdn7Hu8we1K4FrrhiaEEOJcSN2fEELYCUnoQghhJyShCyGEnZCELoQQdkLZqrpQKZUHpJ/n04OAfCuG01E44vt2xPcMjvm+HfE9w7m/7xitdaMjM22W0C+EUipJa51o6zjamyO+b0d8z+CY79sR3zNY931Ll4sQQtgJSehCCGEnOmpCn2vrAGzEEd+3I75ncMz37YjvGaz4vjtkH7oQQoizddQWuhBCiDNIQhdCCDvR4RJ6SwtW2wOlVJRS6iel1G6l1C6l1AOW/QFKqWVKqQOWn/62jtXalFJmpdQWpdQ3lvudLQuPp1gWInexdYzWppTyU0rNV0rtVUrtUUoNd5DP+iHL7/dOpdSnSik3e/u8lVLvKKVylVI7G+xr9LNVhpct7327UmrguZ6vQyX0Vi5YbQ9qgN9qrXsCw4B7Le/zUWC51joeWG65b28eAPY0uP808IJlAfJjGAuS25uXgCVa6+5AP4z3b9eftVIqArgfSNRa98aYmns29vd5vwdMOGNfU5/tRCDest0BvHauJ+tQCZ3WLVjd4Wmtj2qtN1tul2L8B4/g9MW43wdm2CTANqKUigQmA29Z7itgHMbC42Cf79kXGIOxpgBa6yqtdRF2/llbOAHullXOPICj2NnnrbVehbFGRENNfbbTgQ+0YT3gp5QKP5fzdbSE3poFq+2KUioWGABsAEK11kctD2UDobaKq428CDwC1FnuBwJFloXHwT4/785AHvCupavpLaWUJ3b+WWuts4BngQyMRF4MJGP/nzc0/dlecH7raAndoSilvIAvgQe11iUNH7Ms8Wc3NadKqSlArtY62daxtDMnYCDwmtZ6AFDGGd0r9vZZA1j6jadj/EHrBHhydteE3bP2Z9vREnprFqy2C0opZ4xk/rHW+ivL7pz6r2CWn7m2iq8NjASmKaXSMLrSxmH0LftZvpKDfX7eh4HDWusNlvvzMRK8PX/WAJcBh7TWeVrrauArjN8Be/+8oenP9oLzW0dL6K1ZsLrDs/Qdvw3s0Vo/3+Chhotx3wQsbO/Y2orW+o9a60itdSzG57pCa30D8BPGwuNgZ+8ZQGudDWQqpRIsu8YDu7Hjz9oiAximlPKw/L7Xv2+7/rwtmvpsFwG/tlS7DAOKG3TNtI7WukNtwCRgP3AQ+LOt42mj9zgK42vYdmCrZZuE0ae8HDgA/AgE2DrWNnr/lwDfWG7HARuBFOALwNXW8bXB++0PJFk+7wWAvyN81sDfgL3ATuBDwNXePm/gU4xrBNUY38ZubeqzBRRGFd9BYAdGBdA5nU+G/gshhJ3oaF0uQgghmiAJXQgh7IQkdCGEsBOS0IUQwk5IQhdCCDshCV0IIeyEJHQhhLAT/w+H+Y/SOfzYbAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "t1 = torch.tensor([0.0])\n", + "t2 = torch.tensor([0.0])\n", + "t3 = torch.tensor([0.0])\n", + "\n", + "opt = torch.optim.SGD([\n", + " {\"params\": [t1], \"lr\": 0.7, \"weight_decay\": 0.0},\n", + " {\"params\": [t2], \"lr\": 0.2, \"weight_decay\": 0.0001},\n", + " {\"params\": [t3], \"lr\": 0.4, \"weight_decay\": 0.01},\n", + "])\n", + "\n", + "torch_lr_scheduler = ExponentialLR(optimizer=opt, gamma=0.98)\n", + "lr_values = [None] * 100\n", + "scheduler = create_lr_scheduler_with_warmup(torch_lr_scheduler,\n", + " warmup_start_value=0.0,\n", + " warmup_duration=10,\n", + " output_simulated_values=lr_values)\n", + "\n", + "lr_values = np.array(lr_values)\n", + "# lr_values.shape = (100, 3) <=> event index, lr for group 1, lr for group 2\n", + "# Plot simulated values\n", + "plt.plot(lr_values[:, 0], lr_values[:, 1], label=\"learning rate g1\")\n", + "plt.plot(lr_values[:, 0], lr_values[:, 2], label=\"learning rate g2\")\n", + "plt.plot(lr_values[:, 0], lr_values[:, 3], label=\"learning rate g3\")\n", + "plt.show()" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}