diff --git a/bonsai/models/unet/tests/UNet_segmentation_example.ipynb b/bonsai/models/unet/tests/UNet_segmentation_example.ipynb index 54a1fc37..60548937 100644 --- a/bonsai/models/unet/tests/UNet_segmentation_example.ipynb +++ b/bonsai/models/unet/tests/UNet_segmentation_example.ipynb @@ -629,7 +629,7 @@ " return state, loss\n", "\n", "\n", - "print(\"🚀 Starting training from checkpoint...\")\n", + "print(\"Starting training from checkpoint...\")\n", "train_loader, vis_loader = load_dataset()\n", "num_epochs = 100\n", "state = train_state\n", diff --git a/bonsai/models/unet/tests/UNet_segmentation_example.md b/bonsai/models/unet/tests/UNet_segmentation_example.md index 5663f428..a893efaa 100644 --- a/bonsai/models/unet/tests/UNet_segmentation_example.md +++ b/bonsai/models/unet/tests/UNet_segmentation_example.md @@ -256,7 +256,7 @@ def train_step(state: TrainState, other_vars: nnx.State, batch: tuple[jax.Array, return state, loss -print("🚀 Starting training from checkpoint...") +print("Starting training from checkpoint...") train_loader, vis_loader = load_dataset() num_epochs = 100 state = train_state diff --git a/bonsai/tutorials/JAX_machine_translation.ipynb b/bonsai/tutorials/JAX_machine_translation.ipynb new file mode 100644 index 00000000..cb478b90 --- /dev/null +++ b/bonsai/tutorials/JAX_machine_translation.ipynb @@ -0,0 +1,1253 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ee3e1116-f6cd-497e-b617-1d89d5d1f744", + "metadata": {}, + "source": [ + "# Machine Translation with encoder-decoder transformer model\n" + ] + }, + { + "cell_type": "markdown", + "id": "50f0bd58-dcc6-41f4-9dc4-3a08c8ef751b", + "metadata": {}, + "source": [ + "This tutorial is adapted from [Keras' documentation on English-to-Spanish translation with a sequence-to-sequence Transformer](https://keras.io/examples/nlp/neural_machine_translation_with_transformer/), which is itself an adaptation from the book [Deep Learning with Python, Second Edition by François Chollet](https://www.manning.com/books/deep-learning-with-python-second-edition)\n", + "\n", + "We step through an encoder-decoder transformer in JAX and train a model for English->Spanish translation." + ] + }, + { + "cell_type": "markdown", + "id": "0e5066d9", + "metadata": {}, + "source": [ + "### Installing Dependencies\n", + "The current versions are configured for **Google TPU v5**. To switch to the **GPU** version, use the following command:\n", + "\n", + "```python\n", + "%pip install \"jax[cuda12]==0.8.2\" \"flax==0.12.2\" numpy tiktoken tqdm grain optax matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7bf8d50f", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/lib/python3.11/pty.py:89: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n", + " pid, fd = os.forkpty()\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: numpy in /home/aries/bonsai/env/lib/python3.11/site-packages (2.4.0)\n", + "Requirement already satisfied: tiktoken in /home/aries/bonsai/env/lib/python3.11/site-packages (0.12.0)\n", + "Requirement already satisfied: flax in /home/aries/bonsai/env/lib/python3.11/site-packages (0.12.2)\n", + "Requirement already satisfied: tqdm in /home/aries/bonsai/env/lib/python3.11/site-packages (4.67.1)\n", + "Requirement already satisfied: grain in /home/aries/bonsai/env/lib/python3.11/site-packages (0.2.15)\n", + "Requirement already satisfied: optax in /home/aries/bonsai/env/lib/python3.11/site-packages (0.2.6)\n", + "Requirement already satisfied: matplotlib in /home/aries/bonsai/env/lib/python3.11/site-packages (3.10.8)\n", + "Requirement already satisfied: jax in /home/aries/bonsai/env/lib/python3.11/site-packages (0.8.2)\n", + "Requirement already satisfied: regex>=2022.1.18 in /home/aries/bonsai/env/lib/python3.11/site-packages (from tiktoken) (2025.11.3)\n", + "Requirement already satisfied: requests>=2.26.0 in /home/aries/bonsai/env/lib/python3.11/site-packages (from tiktoken) (2.32.5)\n", + "Requirement already satisfied: msgpack in /home/aries/bonsai/env/lib/python3.11/site-packages (from flax) (1.1.2)\n", + "Requirement already satisfied: orbax-checkpoint in /home/aries/bonsai/env/lib/python3.11/site-packages (from flax) (0.11.31)\n", + "Requirement already satisfied: tensorstore in /home/aries/bonsai/env/lib/python3.11/site-packages (from flax) (0.1.80)\n", + "Requirement already satisfied: rich>=11.1 in /home/aries/bonsai/env/lib/python3.11/site-packages (from flax) (14.2.0)\n", + "Requirement already satisfied: typing_extensions>=4.2 in /home/aries/bonsai/env/lib/python3.11/site-packages (from flax) (4.15.0)\n", + "Requirement already satisfied: PyYAML>=5.4.1 in /home/aries/bonsai/env/lib/python3.11/site-packages (from flax) (6.0.3)\n", + "Requirement already satisfied: treescope>=0.1.7 in /home/aries/bonsai/env/lib/python3.11/site-packages (from flax) (0.1.10)\n", + "Requirement already satisfied: absl-py in /home/aries/bonsai/env/lib/python3.11/site-packages (from grain) (2.3.1)\n", + "Requirement already satisfied: array-record>=0.8.1 in /home/aries/bonsai/env/lib/python3.11/site-packages (from grain) (0.8.3)\n", + "Requirement already satisfied: cloudpickle in /home/aries/bonsai/env/lib/python3.11/site-packages (from grain) (3.1.2)\n", + "Requirement already satisfied: etils[epath,epy] in /home/aries/bonsai/env/lib/python3.11/site-packages (from grain) (1.13.0)\n", + "Requirement already satisfied: protobuf>=5.28.3 in /home/aries/bonsai/env/lib/python3.11/site-packages (from grain) (6.33.2)\n", + "Requirement already satisfied: chex>=0.1.87 in /home/aries/bonsai/env/lib/python3.11/site-packages (from optax) (0.1.91)\n", + "Requirement already satisfied: jaxlib>=0.5.3 in /home/aries/bonsai/env/lib/python3.11/site-packages (from optax) (0.8.2)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /home/aries/bonsai/env/lib/python3.11/site-packages (from matplotlib) (1.3.3)\n", + "Requirement already satisfied: cycler>=0.10 in /home/aries/bonsai/env/lib/python3.11/site-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /home/aries/bonsai/env/lib/python3.11/site-packages (from matplotlib) (4.61.1)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /home/aries/bonsai/env/lib/python3.11/site-packages (from matplotlib) (1.4.9)\n", + "Requirement already satisfied: packaging>=20.0 in /home/aries/bonsai/env/lib/python3.11/site-packages (from matplotlib) (25.0)\n", + "Requirement already satisfied: pillow>=8 in /home/aries/bonsai/env/lib/python3.11/site-packages (from matplotlib) (12.1.0)\n", + "Requirement already satisfied: pyparsing>=3 in /home/aries/bonsai/env/lib/python3.11/site-packages (from matplotlib) (3.3.1)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /home/aries/bonsai/env/lib/python3.11/site-packages (from matplotlib) (2.9.0.post0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /home/aries/bonsai/env/lib/python3.11/site-packages (from jax) (0.5.4)\n", + "Requirement already satisfied: opt_einsum in /home/aries/bonsai/env/lib/python3.11/site-packages (from jax) (3.4.0)\n", + "Requirement already satisfied: scipy>=1.13 in /home/aries/bonsai/env/lib/python3.11/site-packages (from jax) (1.16.3)\n", + "Requirement already satisfied: toolz>=1.0.0 in /home/aries/bonsai/env/lib/python3.11/site-packages (from chex>=0.1.87->optax) (1.1.0)\n", + "Requirement already satisfied: six>=1.5 in /home/aries/bonsai/env/lib/python3.11/site-packages (from python-dateutil>=2.7->matplotlib) (1.17.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /home/aries/bonsai/env/lib/python3.11/site-packages (from requests>=2.26.0->tiktoken) (3.4.4)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/aries/bonsai/env/lib/python3.11/site-packages (from requests>=2.26.0->tiktoken) (3.11)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/aries/bonsai/env/lib/python3.11/site-packages (from requests>=2.26.0->tiktoken) (2.6.2)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/aries/bonsai/env/lib/python3.11/site-packages (from requests>=2.26.0->tiktoken) (2026.1.4)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/aries/bonsai/env/lib/python3.11/site-packages (from rich>=11.1->flax) (4.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/aries/bonsai/env/lib/python3.11/site-packages (from rich>=11.1->flax) (2.19.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /home/aries/bonsai/env/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.1->flax) (0.1.2)\n", + "Requirement already satisfied: fsspec in /home/aries/bonsai/env/lib/python3.11/site-packages (from etils[epath,epy]->grain) (2025.12.0)\n", + "Requirement already satisfied: importlib_resources in /home/aries/bonsai/env/lib/python3.11/site-packages (from etils[epath,epy]->grain) (6.5.2)\n", + "Requirement already satisfied: zipp in /home/aries/bonsai/env/lib/python3.11/site-packages (from etils[epath,epy]->grain) (3.23.0)\n", + "Requirement already satisfied: nest_asyncio in /home/aries/bonsai/env/lib/python3.11/site-packages (from orbax-checkpoint->flax) (1.6.0)\n", + "Requirement already satisfied: aiofiles in /home/aries/bonsai/env/lib/python3.11/site-packages (from orbax-checkpoint->flax) (25.1.0)\n", + "Requirement already satisfied: humanize in /home/aries/bonsai/env/lib/python3.11/site-packages (from orbax-checkpoint->flax) (4.15.0)\n", + "Requirement already satisfied: simplejson>=3.16.0 in /home/aries/bonsai/env/lib/python3.11/site-packages (from orbax-checkpoint->flax) (3.20.2)\n", + "Requirement already satisfied: psutil in /home/aries/bonsai/env/lib/python3.11/site-packages (from orbax-checkpoint->flax) (7.2.1)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install numpy tiktoken tqdm grain optax matplotlib \"jax[tpu]==0.8.2\" \"flax == 0.12.2\"" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "dd506ffa-3b91-44f1-92d1-a08ed933e78e", + "metadata": {}, + "outputs": [], + "source": [ + "import dataclasses\n", + "import pathlib\n", + "import random\n", + "import re\n", + "import string\n", + "\n", + "import grain.python as grain\n", + "import jax.numpy as jnp\n", + "import numpy as np\n", + "import optax\n", + "import tiktoken\n", + "import tqdm\n", + "from flax import nnx" + ] + }, + { + "cell_type": "markdown", + "id": "e1f324b0-140a-48fa-9fcb-d6308f098343", + "metadata": {}, + "source": [ + "## Pull down data to temp and extract into memory\n", + "\n", + "There are lots of ways to get this done, but for simplicity and clear visibility into what's happening this is downloaded to a temporary directory, extracted there, and read into a python object with processing.\n", + "\n", + "### Libraries Used:\n", + "* **tempfile**: Used to create temporary directories to store the dataset during processing.\n", + "* **zipfile**: Used to extract the contents of the downloaded zip file.\n", + "* **requests**: Used to fetch the raw dataset file from the URL.\n", + "\n", + "### Process Overview:\n", + "We extract the zip data into a folder on the local environment. During this process, we apply a critical formatting step to the Spanish target data:\n", + "\n", + "> `\"[start] \" + spa + \" [end]\"`\n", + "\n", + "### Why do we do this? (Teacher Forcing)\n", + "This is essential for the **Decoder** (the part of the AI generating the Spanish translation). It needs explicit instructions on when to begin generating text and when to stop.\n", + "\n", + "* **[start]:** Tells the model, \"Begin generating the first word now.\"\n", + "* **[end]:** Tells the model, \"The sentence is complete; stop generating.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "102943a5-8724-48e0-8d6a-f56069f03426", + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "import zipfile\n", + "\n", + "import requests\n", + "\n", + "url = \"http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip\"\n", + "\n", + "with tempfile.TemporaryDirectory() as temp_dir:\n", + " temp_path = pathlib.Path(temp_dir)\n", + " zip_file_path = temp_path / \"spa-eng.zip\"\n", + "\n", + " response = requests.get(url)\n", + " zip_file_path.write_bytes(response.content)\n", + "\n", + " with zipfile.ZipFile(zip_file_path, \"r\") as zip_ref:\n", + " zip_ref.extractall(temp_path)\n", + "\n", + " text_file = temp_path / \"spa-eng\" / \"spa.txt\"\n", + "\n", + " with open(text_file) as f:\n", + " lines = f.read().split(\"\\n\")[:-1]\n", + " text_pairs = []\n", + " for line in lines:\n", + " eng, spa = line.split(\"\\t\")\n", + " spa = \"[start] \" + spa + \" [end]\"\n", + " text_pairs.append((eng, spa))" + ] + }, + { + "cell_type": "markdown", + "id": "9524904b-fa17-493f-bcfa-335963cb7c45", + "metadata": {}, + "source": [ + "## Build train/validate/test pair sets\n", + "\n", + "\n", + "We partition the dataset into three distinct subsets: **Training**, **Validation**, and **Test**.\n", + "\n", + "* **Validation Data (15%):** Used to evaluate the model during training to tune hyperparameters and prevent overfitting.\n", + "* **Test Data (15%):** Reserved for the final evaluation to check how the model performs on completely unseen data.\n", + "* **Training Data (70%):** The remaining data used to actually teach the model." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "bee9f1b0-5f74-47dc-a7e1-a4ea3be1ef7f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "118964 total pairs\n", + "83276 training pairs\n", + "17844 validation pairs\n", + "17844 test pairs\n" + ] + } + ], + "source": [ + "random.shuffle(text_pairs)\n", + "num_val_samples = int(0.15 * len(text_pairs))\n", + "num_train_samples = len(text_pairs) - 2 * num_val_samples\n", + "train_pairs = text_pairs[:num_train_samples]\n", + "val_pairs = text_pairs[num_train_samples : num_train_samples + num_val_samples]\n", + "test_pairs = text_pairs[num_train_samples + num_val_samples :]\n", + "\n", + "print(f\"{len(text_pairs)} total pairs\")\n", + "print(f\"{len(train_pairs)} training pairs\")\n", + "print(f\"{len(val_pairs)} validation pairs\")\n", + "print(f\"{len(test_pairs)} test pairs\")" + ] + }, + { + "cell_type": "markdown", + "id": "2442289e", + "metadata": {}, + "source": [ + "# Tokenization\n", + "\n", + "This step is crucial because computers and machines do not understand English words directly; they require a **numeric representation** to process language.\n", + "\n", + "To achieve this, we use the **Tiktoken** library developed by **OpenAI**. specifically utilizing the `cl100k_base` dictionary.\n", + "\n", + "* **Tiktoken:** A fast BPE (Byte Pair Encoding) tokenizer.\n", + "* **cl100k_base:** A vocabulary containing approximately **100,000 tokens**, where every unique word or word fragment is mapped to a specific integer." + ] + }, + { + "cell_type": "markdown", + "id": "a714c4ea-9ff6-4dab-ae9c-1a884d4857e7", + "metadata": {}, + "source": [ + "We strip out punctuation to keep things simple and in line with the original tutorial - the `[` `]` are kept in so that our `[start]` and `[end]` formatting is preserved." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "07e054d3-a20c-4aed-8f8a-fb5158df8e5b", + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n", + "\n", + "strip_chars = string.punctuation + \"¿\"\n", + "strip_chars = strip_chars.replace(\"[\", \"\")\n", + "strip_chars = strip_chars.replace(\"]\", \"\")\n", + "\n", + "vocab_size = tokenizer.n_vocab\n", + "sequence_length = 20" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e2b3e5b3-8466-4c81-99da-0559c88b25ef", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_standardization(input_string):\n", + " lowercase = input_string.lower()\n", + " return re.sub(f\"[{re.escape(strip_chars)}]\", \"\", lowercase)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "5bdc0673-9723-45b5-8a42-2152295df69b", + "metadata": {}, + "outputs": [], + "source": [ + "def tokenize_and_pad(text, tokenizer, max_length):\n", + " tokens = tokenizer.encode(text)[:max_length]\n", + " padded = (\n", + " tokens + [0] * (max_length - len(tokens)) if len(tokens) < max_length else tokens\n", + " ) ##assumes list-like - (https://github.com/openai/tiktoken/blob/main/tiktoken/core.py#L81 current tiktoken out)\n", + " return padded" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "235b1221-e72d-4793-addd-7bb870bd8e75", + "metadata": {}, + "outputs": [], + "source": [ + "def format_dataset(eng, spa, tokenizer, sequence_length):\n", + " eng = custom_standardization(eng)\n", + " spa = custom_standardization(spa)\n", + " eng = tokenize_and_pad(eng, tokenizer, sequence_length)\n", + " spa = tokenize_and_pad(spa, tokenizer, sequence_length)\n", + " return {\n", + " \"encoder_inputs\": eng,\n", + " \"decoder_inputs\": spa[:-1],\n", + " \"target_output\": spa[1:],\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "ca013d07-1504-42cc-906f-2fcacc757008", + "metadata": {}, + "outputs": [], + "source": [ + "train_data = [format_dataset(eng, spa, tokenizer, sequence_length) for eng, spa in train_pairs]\n", + "val_data = [format_dataset(eng, spa, tokenizer, sequence_length) for eng, spa in val_pairs]\n", + "test_data = [format_dataset(eng, spa, tokenizer, sequence_length) for eng, spa in test_pairs]" + ] + }, + { + "cell_type": "markdown", + "id": "90bbae98-48dd-4ae4-99bb-92336d7c0a1c", + "metadata": {}, + "source": [ + "At this point we've extracted the data, applied formatting, and tokenized the phrases with padding. The data is kept in train/validate/test sets that each have dictionary entries, which look like the following:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "dcbfa780-553f-41f6-8b3e-55955db78b2a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'encoder_inputs': [29177, 499, 6604, 264, 2697, 62896, 4587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'decoder_inputs': [29563, 60, 6183, 4355, 11158, 24180, 4247, 4799, 510, 408, 60, 0, 0, 0, 0, 0, 0, 0, 0], 'target_output': [60, 6183, 4355, 11158, 24180, 4247, 4799, 510, 408, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0]}\n" + ] + } + ], + "source": [ + "## data selection example\n", + "print(train_data[135])" + ] + }, + { + "cell_type": "markdown", + "id": "24c6271b-e359-4aba-a583-f18c40eddba9", + "metadata": {}, + "source": [ + "The output should look something like\n", + "\n", + "{'encoder_inputs': [9514, 265, 3339, 264, 2466, 16930, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'decoder_inputs': [29563, 60, 1826, 7206, 71086, 37116, 653, 16109, 1493, 54189, 510, 408, 60, 0, 0, 0, 0, 0, 0], 'target_output': [60, 1826, 7206, 71086, 37116, 653, 16109, 1493, 54189, 510, 408, 60, 0, 0, 0, 0, 0, 0, 0]}" + ] + }, + { + "cell_type": "markdown", + "id": "7a906a05-bd17-4a47-afe0-4422d2ea0f50", + "metadata": {}, + "source": [ + "## Define Transformer components: Encoder, Decoder, Positional Embed\n", + "\n", + "In many ways this is very similar to the original source, with `ops` changing to `jnp` and `keras` or `layers` becoming `nnx`. Certain module-specific arguments come and go, like the rngs attached to most things in the updated version, and decode=False in the MultiHeadAttention call." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "a3f8a6fd", + "metadata": {}, + "outputs": [], + "source": [ + "@dataclasses.dataclass\n", + "class TransformerConfig:\n", + " sequence_length: int\n", + " vocab_size: int\n", + " embed_dim: int\n", + " latent_dim: int\n", + " num_heads: int\n", + " dropout_rate: float" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "121bf138-34b3-4be9-a0fc-3bbac81f469a", + "metadata": {}, + "outputs": [], + "source": [ + "class TransformerEncoder(nnx.Module):\n", + " def __init__(self, config: TransformerConfig, rngs: nnx.Rngs, **kwargs):\n", + " self.attention = nnx.MultiHeadAttention(\n", + " num_heads=config.num_heads, in_features=config.embed_dim, decode=False, rngs=rngs\n", + " )\n", + " self.dense_proj = nnx.Sequential(\n", + " nnx.Linear(config.embed_dim, config.latent_dim, rngs=rngs),\n", + " nnx.relu,\n", + " nnx.Linear(config.latent_dim, config.embed_dim, rngs=rngs),\n", + " )\n", + "\n", + " self.layernorm_1 = nnx.LayerNorm(config.embed_dim, rngs=rngs)\n", + " self.layernorm_2 = nnx.LayerNorm(config.embed_dim, rngs=rngs)\n", + "\n", + " def __call__(self, inputs, mask=None):\n", + " if mask is not None:\n", + " padding_mask = jnp.expand_dims(mask, axis=1).astype(jnp.int32)\n", + " else:\n", + " padding_mask = None\n", + "\n", + " attention_output = self.attention(\n", + " inputs_q=inputs, inputs_k=inputs, inputs_v=inputs, mask=padding_mask, decode=False\n", + " )\n", + " proj_input = self.layernorm_1(inputs + attention_output)\n", + " proj_output = self.dense_proj(proj_input)\n", + " return self.layernorm_2(proj_input + proj_output)\n", + "\n", + "\n", + "class PositionalEmbedding(nnx.Module):\n", + " def __init__(self, config: TransformerConfig, rngs: nnx.Rngs, **kwargs):\n", + " self.token_embeddings = nnx.Embed(num_embeddings=config.vocab_size, features=config.embed_dim, rngs=rngs)\n", + " self.position_embeddings = nnx.Embed(\n", + " num_embeddings=config.sequence_length, features=config.embed_dim, rngs=rngs\n", + " )\n", + "\n", + " def __call__(self, inputs, step=None):\n", + " if step is None:\n", + " length = inputs.shape[1]\n", + " positions = jnp.arange(0, length)[None, :]\n", + " else:\n", + " positions = jnp.array([step])[None, :]\n", + "\n", + " embedded_tokens = self.token_embeddings(inputs)\n", + " embedded_positions = self.position_embeddings(positions)\n", + " return embedded_tokens + embedded_positions\n", + "\n", + "\n", + "class TransformerDecoder(nnx.Module):\n", + " def __init__(self, config: TransformerConfig, rngs: nnx.Rngs, **kwargs):\n", + " self.attention_1 = nnx.MultiHeadAttention(\n", + " num_heads=config.num_heads, in_features=config.embed_dim, decode=True, rngs=rngs\n", + " )\n", + " self.attention_2 = nnx.MultiHeadAttention(\n", + " num_heads=config.num_heads, in_features=config.embed_dim, decode=False, rngs=rngs\n", + " )\n", + "\n", + " self.dense_proj = nnx.Sequential(\n", + " nnx.Linear(config.embed_dim, config.latent_dim, rngs=rngs),\n", + " nnx.relu,\n", + " nnx.Linear(config.latent_dim, config.embed_dim, rngs=rngs),\n", + " )\n", + " self.layernorm_1 = nnx.LayerNorm(config.embed_dim, rngs=rngs)\n", + " self.layernorm_2 = nnx.LayerNorm(config.embed_dim, rngs=rngs)\n", + " self.layernorm_3 = nnx.LayerNorm(config.embed_dim, rngs=rngs)\n", + "\n", + " def init_cache(self, input_shape):\n", + " self.attention_1.init_cache(input_shape=input_shape)\n", + "\n", + " def __call__(self, inputs, encoder_outputs, mask=None, decode=False):\n", + " if decode:\n", + " padding_mask = None\n", + " else:\n", + " causal_mask = self.get_causal_attention_mask(inputs.shape[1])\n", + " if mask is not None:\n", + " padding_mask = jnp.expand_dims(mask, axis=1).astype(jnp.int32)\n", + " padding_mask = jnp.minimum(padding_mask, causal_mask)\n", + " else:\n", + " padding_mask = causal_mask\n", + "\n", + " cross_mask = mask\n", + " if decode and mask is not None:\n", + " cross_mask = jnp.expand_dims(mask, axis=(1, 2)).astype(jnp.int32)\n", + "\n", + " elif not decode and mask is not None:\n", + " cross_mask = jnp.expand_dims(mask, axis=1).astype(jnp.int32)\n", + "\n", + " attention_output_1 = self.attention_1(\n", + " inputs_q=inputs, inputs_k=inputs, inputs_v=inputs, mask=padding_mask, decode=decode\n", + " )\n", + " out_1 = self.layernorm_1(inputs + attention_output_1)\n", + "\n", + " attention_output_2 = self.attention_2(\n", + " inputs_q=out_1, inputs_k=encoder_outputs, inputs_v=encoder_outputs, mask=cross_mask, decode=False\n", + " )\n", + "\n", + " out_2 = self.layernorm_2(out_1 + attention_output_2)\n", + "\n", + " proj_output = self.dense_proj(out_2)\n", + " output = self.layernorm_3(out_2 + proj_output)\n", + " return output\n", + "\n", + " def get_causal_attention_mask(self, sequence_length):\n", + " i = jnp.arange(sequence_length)[:, None]\n", + " j = jnp.arange(sequence_length)\n", + " mask = (i >= j).astype(jnp.int32)\n", + " mask = jnp.reshape(mask, (1, 1, sequence_length, sequence_length))\n", + " return mask" + ] + }, + { + "cell_type": "markdown", + "id": "d033ae31-cc43-4e61-8d7f-cdc6d55b8bf9", + "metadata": {}, + "source": [ + "Here we finally use our earlier encoder, decoder, and positional embed classes to construct the Model that we'll train and later use for inference." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c5dcfaf6-f5cd-40f4-bbf0-2754c0193327", + "metadata": {}, + "outputs": [], + "source": [ + "class TransformerModel(nnx.Module):\n", + " def __init__(self, config: TransformerConfig, rngs: nnx.Rngs):\n", + " self.config = config\n", + " self.encoder = TransformerEncoder(config, rngs=rngs)\n", + " self.positional_embedding = PositionalEmbedding(config, rngs=rngs)\n", + " self.decoder = TransformerDecoder(config, rngs=rngs)\n", + " self.dropout = nnx.Dropout(rate=config.dropout_rate, rngs=rngs)\n", + " self.dense = nnx.Linear(config.embed_dim, config.vocab_size, rngs=rngs)\n", + "\n", + " def init_cache(self, input_shape):\n", + " self.decoder.init_cache(input_shape)\n", + "\n", + " def __call__(self, encoder_inputs, decoder_inputs, mask=None, deterministic=False, decode=False):\n", + " x = self.positional_embedding(encoder_inputs)\n", + " encoder_outputs = self.encoder(x, mask=mask)\n", + "\n", + " x = self.positional_embedding(decoder_inputs)\n", + " decoder_outputs = self.decoder(x, encoder_outputs, mask=mask, decode=decode)\n", + "\n", + " decoder_outputs = self.dropout(decoder_outputs, deterministic=deterministic)\n", + " logits = self.dense(decoder_outputs)\n", + " return logits\n", + "\n", + " def decode_step(self, token_input, encoder_outputs, step_index):\n", + " x = self.positional_embedding(token_input, step=step_index)\n", + " decoder_outputs = self.decoder(x, encoder_outputs, decode=True)\n", + " logits = self.dense(decoder_outputs)\n", + " return logits" + ] + }, + { + "cell_type": "markdown", + "id": "1744cd95-afcc-4a82-9a00-18fef4f6f7df", + "metadata": {}, + "source": [ + "## Build out Data Loader and Training Definitions\n", + "It can be more computationally efficient to use pygrain for the data load stage, but this way it's abundandtly clear what's happening: data pairs go in and sets of jnp arrays come out, in step with our original dictionaries. 'Encoder_inputs', 'decoder_inputs' and 'target_output'." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "1fb8cb44-9012-4802-9286-1efc19dd2ba1", + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64 # set here for the loader and model train later on\n", + "\n", + "\n", + "class CustomPreprocessing(grain.MapTransform):\n", + " def __init__(self):\n", + " pass\n", + "\n", + " def map(self, data):\n", + " return {\n", + " \"encoder_inputs\": np.array(data[\"encoder_inputs\"]),\n", + " \"decoder_inputs\": np.array(data[\"decoder_inputs\"]),\n", + " \"target_output\": np.array(data[\"target_output\"]),\n", + " }\n", + "\n", + "\n", + "train_sampler = grain.IndexSampler(\n", + " len(train_data),\n", + " shuffle=True,\n", + " seed=12, # Seed for reproducibility\n", + " shard_options=grain.NoSharding(), # No sharding since it's a single-device setup\n", + " num_epochs=1, # Iterate over the dataset for one epoch\n", + ")\n", + "\n", + "val_sampler = grain.IndexSampler(\n", + " len(val_data),\n", + " shuffle=False,\n", + " seed=12,\n", + " shard_options=grain.NoSharding(),\n", + " num_epochs=1,\n", + ")\n", + "\n", + "train_loader = grain.DataLoader(\n", + " data_source=train_data,\n", + " sampler=train_sampler, # Sampler to determine how to access the data\n", + " worker_count=4, # Number of child processes launched to parallelize the transformations\n", + " worker_buffer_size=2, # Count of output batches to produce in advance per worker\n", + " operations=[\n", + " CustomPreprocessing(),\n", + " grain.Batch(batch_size=batch_size, drop_remainder=True),\n", + " ],\n", + ")\n", + "\n", + "val_loader = grain.DataLoader(\n", + " data_source=val_data,\n", + " sampler=val_sampler,\n", + " worker_count=4,\n", + " worker_buffer_size=2,\n", + " operations=[\n", + " CustomPreprocessing(),\n", + " grain.Batch(batch_size=batch_size),\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "40d9707d-a73c-47f5-8c12-1f336e526e61", + "metadata": {}, + "source": [ + "Optax doesn't have the identical loss function that the source tutorial uses, but this softmax cross entropy works well here - you can one_hot_encode if you don't use the `_with_integer_labels` version of the loss." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "d2f8e06f-1126-41cc-b8d8-de6bd7a5255a", + "metadata": {}, + "outputs": [], + "source": [ + "def compute_loss(logits, labels):\n", + " loss = optax.softmax_cross_entropy_with_integer_labels(logits=logits, labels=labels)\n", + " return jnp.mean(loss)" + ] + }, + { + "cell_type": "markdown", + "id": "0a1b625a-d9e7-4028-bc98-521ce1632450", + "metadata": {}, + "source": [ + "While in the original tutorial most of the model and training details happen inside keras, we make them explicit here in our step functions, which are later used in `train_one_epoch` and `eval_model`." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "279d991f-f129-48b3-9b7e-d143019c18a8", + "metadata": {}, + "outputs": [], + "source": [ + "@nnx.jit\n", + "def train_step(model, optimizer, batch):\n", + " def loss_fn(model, train_encoder_input, train_decoder_input, train_target_input):\n", + " logits = model(train_encoder_input, train_decoder_input)\n", + " loss = compute_loss(logits, train_target_input)\n", + " return loss\n", + "\n", + " grad_fn = nnx.value_and_grad(loss_fn)\n", + " loss, grads = grad_fn(\n", + " model, jnp.array(batch[\"encoder_inputs\"]), jnp.array(batch[\"decoder_inputs\"]), jnp.array(batch[\"target_output\"])\n", + " )\n", + " optimizer.update(model, grads)\n", + " return loss\n", + "\n", + "\n", + "@nnx.jit\n", + "def eval_step(model, batch, eval_metrics):\n", + " logits = model(jnp.array(batch[\"encoder_inputs\"]), jnp.array(batch[\"decoder_inputs\"]))\n", + " loss = compute_loss(logits, jnp.array(batch[\"target_output\"]))\n", + " labels = jnp.array(batch[\"target_output\"])\n", + "\n", + " eval_metrics.update(\n", + " loss=loss,\n", + " logits=logits,\n", + " labels=labels,\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "04e53ee9-6da1-431c-8b3f-f619d3fee68f", + "metadata": {}, + "source": [ + "Here, `nnx.MultiMetric` helps us keep track of general training statistics, while we make our own dictionaries to hold historical values" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "32a17edc-33d0-41bc-a516-8b8ce45c3ad7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to find host bounds for accelerator type: WARNING: could not determine TPU accelerator type, please set env var `TPU_ACCELERATOR_TYPE` manually, otherwise libtpu.so may not properly initialize.\n", + "WARNING: Logging before InitGoogle() is written to STDERR\n", + "E0000 00:00:1768499505.463938 40451 common_lib.cc:530] INVALID_ARGUMENT: Error: unexpected worker hostname 'WARNING: could not determine TPU worker hostnames or IP addresses' from env var TPU_WORKER_HOSTNAMES. Expecting a valid hostname or IP address without port number, or hostname:port:address triple. (Full TPU workers' addr string: WARNING: could not determine TPU worker hostnames or IP addresses, please set env var `TPU_WORKER_HOSTNAMES` manually, otherwise libtpu.so may not properly initialize.)\n", + "=== Source Location Trace: === \n", + "learning/45eac/tfrc/runtime/libtpu_init_utils.cc:310\n" + ] + } + ], + "source": [ + "eval_metrics = nnx.MultiMetric(\n", + " loss=nnx.metrics.Average(\"loss\"),\n", + " accuracy=nnx.metrics.Accuracy(),\n", + ")\n", + "\n", + "train_metrics_history = {\n", + " \"train_loss\": [],\n", + "}\n", + "\n", + "eval_metrics_history = {\n", + " \"test_loss\": [],\n", + " \"test_accuracy\": [],\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "1189a6a6-2cc6-4c87-9f87-b4b800a1513d", + "metadata": {}, + "outputs": [], + "source": [ + "## Hyperparameters\n", + "rng = nnx.Rngs(0)\n", + "embed_dim = 256\n", + "latent_dim = 2048\n", + "num_heads = 8\n", + "dropout_rate = 0.5\n", + "vocab_size = tokenizer.n_vocab\n", + "sequence_length = 20\n", + "learning_rate = 1.5e-3\n", + "num_epochs = 10\n", + "\n", + "config = TransformerConfig(\n", + " sequence_length=sequence_length,\n", + " vocab_size=vocab_size,\n", + " embed_dim=embed_dim,\n", + " latent_dim=latent_dim,\n", + " num_heads=num_heads,\n", + " dropout_rate=dropout_rate,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "fbeb6101-be11-4a33-9650-a3efd3656855", + "metadata": {}, + "outputs": [], + "source": [ + "bar_format = \"{desc}[{n_fmt}/{total_fmt}]{postfix} [{elapsed}<{remaining}]\"\n", + "train_total_steps = len(train_data) // batch_size\n", + "\n", + "\n", + "def train_one_epoch(epoch):\n", + " model.train() # Set model to the training mode: e.g. update batch statistics\n", + " with tqdm.tqdm(\n", + " desc=f\"[train] epoch: {epoch}/{num_epochs}, \",\n", + " total=train_total_steps,\n", + " bar_format=bar_format,\n", + " leave=True,\n", + " ) as pbar:\n", + " for batch in train_loader:\n", + " loss = train_step(model, optimizer, batch)\n", + " train_metrics_history[\"train_loss\"].append(loss.item())\n", + " pbar.set_postfix({\"loss\": loss.item()})\n", + " pbar.update(1)\n", + "\n", + "\n", + "def evaluate_model(epoch):\n", + " # Compute the metrics on the train and val sets after each training epoch.\n", + " model.eval() # Set model to evaluation model: e.g. use stored batch statistics\n", + "\n", + " eval_metrics.reset() # Reset the eval metrics\n", + " for val_batch in val_loader:\n", + " eval_step(model, val_batch, eval_metrics)\n", + "\n", + " for metric, value in eval_metrics.compute().items():\n", + " eval_metrics_history[f\"test_{metric}\"].append(value)\n", + "\n", + " print(f\"[test] epoch: {epoch + 1}/{num_epochs}\")\n", + " print(f\"- total loss: {eval_metrics_history['test_loss'][-1]:0.4f}\")\n", + " print(f\"- Accuracy: {eval_metrics_history['test_accuracy'][-1]:0.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "49a1d33a-c2e4-4d48-821b-519f5c0192c7", + "metadata": {}, + "outputs": [], + "source": [ + "model = TransformerModel(config, rngs=rng)\n", + "optimizer = nnx.Optimizer(model, optax.adamw(learning_rate), wrt=nnx.Param)" + ] + }, + { + "cell_type": "markdown", + "id": "fa7d5601-60c1-4131-a40c-c670f055ce68", + "metadata": {}, + "source": [ + "## Start the Training!\n", + "With our data loaders in place and the model, optimizer, and training/evaluation loops fully configured, it’s finally time to press go.\n", + "Training on an RTX 4050 (6 GB VRAM), the model fits comfortably within memory, and with a batch size of 64, each epoch completes in approximately 1 minute 30 seconds." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "c764510c-4d98-46ad-b877-8cfc2fa5a9ea", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 0/10, [1300/1301], loss=1.1 [02:31<00:00] \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 1/10\n", + "- total loss: 1.2200\n", + "- Accuracy: 0.7859\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 1/10, [1300/1301], loss=0.766 [01:37<00:00]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 2/10\n", + "- total loss: 0.9918\n", + "- Accuracy: 0.8195\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 2/10, [1300/1301], loss=0.63 [01:35<00:00] \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 3/10\n", + "- total loss: 0.9035\n", + "- Accuracy: 0.8343\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 3/10, [1300/1301], loss=0.568 [01:36<00:00]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 4/10\n", + "- total loss: 0.8744\n", + "- Accuracy: 0.8410\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 4/10, [1300/1301], loss=0.499 [01:35<00:00]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 5/10\n", + "- total loss: 0.8573\n", + "- Accuracy: 0.8464\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 5/10, [1300/1301], loss=0.463 [01:35<00:00]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 6/10\n", + "- total loss: 0.8596\n", + "- Accuracy: 0.8478\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 6/10, [1300/1301], loss=0.425 [01:35<00:00]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 7/10\n", + "- total loss: 0.8618\n", + "- Accuracy: 0.8502\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 7/10, [1300/1301], loss=0.43 [01:34<00:00] \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 8/10\n", + "- total loss: 0.8556\n", + "- Accuracy: 0.8514\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 8/10, [1300/1301], loss=0.442 [01:32<00:00]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 9/10\n", + "- total loss: 0.8708\n", + "- Accuracy: 0.8522\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[train] epoch: 9/10, [1300/1301], loss=0.376 [01:34<00:00]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[test] epoch: 10/10\n", + "- total loss: 0.8773\n", + "- Accuracy: 0.8537\n" + ] + } + ], + "source": [ + "for epoch in range(num_epochs):\n", + " train_one_epoch(epoch)\n", + " evaluate_model(epoch)" + ] + }, + { + "cell_type": "markdown", + "id": "f922eac4-8338-4a0d-bc6d-1f5880079bde", + "metadata": {}, + "source": [ + "We can then plot the loss over training time. That log-plot comes in handy here, or it's hard to appreciate the progress after 1000 steps or so." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "a79ecfa5-d74a-4956-9ee2-cbed86d5a82f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGdCAYAAADJ6dNTAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWc1JREFUeJzt3XlYVGX7B/DvsINsCgiCKO64IJKCe2684pJmVppZqZmmYWmWqT+3zExfbTGNbFcrTVvNt9xxVxREQRHcEHfBBVmV/fz+EMbZN2Y4M8z3c11eF3PmmXPucwq4eZb7kQiCIICIiIjITNiIHQARERGRLCYnREREZFaYnBAREZFZYXJCREREZoXJCREREZkVJidERERkVpicEBERkVlhckJERERmxU7sAPRVUVGBmzdvws3NDRKJROxwiIiISAeCICA/Px/+/v6wsdHcN2JxycnNmzcRGBgodhhERERkgGvXrqFhw4Ya21hccuLm5gbg0c25u7uLHA0RERHpIi8vD4GBgdLf45pYXHJSNZTj7u7O5ISIiMjC6DIlgxNiiYiIyKwwOSEiIiKzwuSEiIiIzIrFzTkhIvNXXl6O0tJSscMgohpka2sLOzs7o5T5YHJCREZVUFCA69evQxAEsUMhohrm4uKCBg0awMHBoVrnYXJCREZTXl6O69evw8XFBT4+PiyUSGQlBEFASUkJ7ty5g4yMDLRo0UJroTVNmJwQkdGUlpZCEAT4+PjA2dlZ7HCIqAY5OzvD3t4eV65cQUlJCZycnAw+FyfEEpHRsceEyDpVp7dE7jxGOQsRERGRkTA5ISKqRfbt2weJRIKcnJwav3bv3r0xbdq0ap9n7dq18PT0rPZ5DCWRSLB582bRrq+L999/Hx06dNDrM0FBQVixYoVJ4jE2JidEZPXGjh2LYcOGiR0GVRo5ciTOnz9v8usY8gveUMZOuN59913Exsbq9ZmEhARMnDjRaDGYEifEEhGR2SgtLYWzs7PVTqguKSnRaRmuq6srXF1d9Tq3j4+PoWHVOPacVDpw/g7e33IGW5Jvih0KEZmZ/fv3IyIiAo6OjmjQoAFmzZqFsrIy6fu///47QkJC4OzsDC8vL0RGRqKwsBDAo2GWiIgI1KlTB56enujevTuuXLmi8jrdunXDzJkz5Y7duXMH9vb2OHDgAADgp59+QqdOneDm5gY/Pz+8+OKLuH37ttrYVfUOrFixAkFBQXLHvvvuO7Ru3RpOTk4IDg7Gl19+qfGZFBYW4pVXXoGrqysaNGiATz75RKmNquERT09PrF27FgBw+fJlSCQSbNq0Cb169YKTkxPWr1+v1MtQdQ8//fQTgoKC4OHhgRdeeAH5+fnSNvn5+Rg9ejTq1KmDBg0a4LPPPtM4zLR27VosXLgQycnJkEgkkEgk0rgA4O7du3jmmWfg4uKCFi1aYMuWLXKfT0lJwcCBA+Hq6gpfX1+8/PLLuHv3rspr7du3D+PGjUNubq70Wu+//z6AR0MtixYtwiuvvAJ3d3dpz8bMmTPRsmVLuLi4oGnTppg3b55cYUPF/65VvX8ff/wxGjRoAC8vL0RHR8t9RnFYRyKR4LvvvtN4n1u2bEGLFi3g5OSEPn36YN26dTUybMjkpNKp6zlYe+Qy4tJV/89FRPoTBAEPSspE+WesInA3btzAoEGDEB4ejuTkZKxevRrff/89PvzwQwDArVu3MGrUKLz66qtIS0vDvn37MHz4cAiCgLKyMgwbNgy9evXCqVOnEBcXh4kTJ6pdzTR69Ghs3LhRLvZNmzbB398fPXv2BPCoZ2HRokVITk7G5s2bcfnyZYwdO7Za97h+/XrMnz8fixcvRlpaGj766CPMmzcP69atU/uZGTNmYP/+/fj777+xc+dO7Nu3DydOnDDo+rNmzcLUqVORlpaGqKgolW3S09OxefNm/PPPP/jnn3+wf/9+LF26VPr+9OnTcfjwYWzZsgW7du3CwYMHNcYzcuRIvPPOO2jbti1u3bqFW7duYeTIkdL3Fy5ciBEjRuDUqVMYNGgQRo8ejezsbABATk4O+vbti7CwMBw/fhzbt29HVlYWRowYofJa3bp1w4oVK+Du7i691rvvvit9/+OPP0ZoaChOnjyJefPmAQDc3Nywdu1apKam4vPPP8e3336Lzz77TONz3Lt3L9LT07F3716sW7cOa9eulUu4VNF0nxkZGXjuuecwbNgwJCcn4/XXX8ecOXM0ns9YOKyjgEUtiYznYWk52szfIcq1Uz+IgotD9X/EffnllwgMDMQXX3wBiUSC4OBg3Lx5EzNnzsT8+fNx69YtlJWVYfjw4WjcuDEAICQkBACQnZ2N3NxcPPXUU2jWrBkAoHXr1mqvNWLECEybNg2HDh2SJiMbNmzAqFGjpAnNq6++Km3ftGlTrFy5EuHh4SgoKNC7m7/KggUL8Mknn2D48OEAgCZNmiA1NRVff/01xowZo9S+oKAA33//PX7++Wf069cPALBu3To0bNjQoOtPmzZNem11KioqsHbtWri5uQEAXn75ZcTGxmLx4sXIz8/HunXrsGHDBmk8a9asgb+/v9rzOTs7w9XVFXZ2dvDz81N6f+zYsRg1ahQA4KOPPsLKlSsRHx+PAQMG4IsvvkBYWBg++ugjafsffvgBgYGBOH/+PFq2bCl3LgcHB3h4eEAikai8Vt++ffHOO+/IHZs7d67066CgILz77rvYuHEj3nvvPbX3VLduXXzxxRewtbVFcHAwBg8ejNjYWEyYMEHtZzTd59dff41WrVph+fLlAIBWrVohJSUFixcvVns+Y2HPSaWqb3wmJ0QkKy0tDV27dpXr7ejevbu0TH9oaCj69euHkJAQPP/88/j2229x//59AEC9evUwduxYREVFYciQIfj8889x69Yttdfy8fFB//79sX79egCP/nKNi4vD6NGjpW0SExMxZMgQNGrUCG5ubujVqxcA4OrVqwbdX2FhIdLT0zF+/HjpPAZXV1d8+OGHSE9PV/mZ9PR0lJSUoHPnztJj9erVQ6tWrQyKoVOnTlrbBAUFSRMTAGjQoIF0OOvSpUsoLS1FRESE9H0PDw+D4wGA9u3bS7+uU6cO3N3dpddLTk7G3r175Z5XcHAwAKh9Zpqouv9Nmzahe/fu8PPzg6urK+bOnav1v3Hbtm1ha2srfS37jNTRdJ/nzp1DeHi4XHvZZ2xK7DkhIpNxtrdF6gequ+lr4to1wdbWFrt27cKRI0ewc+dOrFq1CnPmzMGxY8fQpEkTrFmzBm+99Ra2b9+OTZs2Ye7cudi1axe6dOmi8nyjR4/GW2+9hVWrVmHDhg0ICQmR9sQUFhYiKioKUVFRWL9+PXx8fHD16lVERUWhpKRE5flsbGyUhrhk5yEUFBQAAL799lu5ZKPq3qpDIpFovHaVOnXqaD2Xvb290rkrKiqqFZ+h1ysoKMCQIUPw3//+V+lzDRo00PtaivdflZAuXLgQUVFR8PDwwMaNG1XO69E1ZmN+piaw50SBAHadEBmLRCKBi4OdKP+MVaW2devWiIuLk/sle/jwYbi5uUmHMSQSCbp3746FCxfi5MmTcHBwwF9//SVtHxYWhtmzZ+PIkSNo164dNmzYoPZ6Tz/9NIqKirB9+3Zs2LBBrtfk7NmzuHfvHpYuXYqePXsiODhY61/GPj4+yMzMlIs/KSlJ+rWvry/8/f1x6dIlNG/eXO5fkyZNVJ6zWbNmsLe3x7Fjx6TH7t+/r7T818fHR66n6MKFC3jw4IHGeA3RtGlT2NvbIyEhQXosNzdX63JkBwcHlJeX6329J554AmfOnEFQUJDSM1OXaOlzrSNHjqBx48aYM2cOOnXqhBYtWqidRG1KrVq1wvHjx+WOyT5jU2JyUonVtomsW25uLpKSkuT+Xbt2DW+88QauXbuGN998E2fPnsXff/+NBQsWYPr06bCxscGxY8fw0Ucf4fjx47h69Sr+/PNP3LlzB61bt0ZGRgZmz56NuLg4XLlyBTt37sSFCxc0zjupU6cOhg0bhnnz5iEtLU06HwAAGjVqBAcHB6xatQqXLl3Cli1bsGjRIo331bt3b9y5cwfLli1Deno6YmJisG3bNrk2CxcuxJIlS7By5UqcP38ep0+fxpo1a/Dpp5+qPKerqyvGjx+PGTNmYM+ePUhJScHYsWOVSpf37dsXX3zxBU6ePInjx49j0qRJSn+pG4ObmxvGjBmDGTNmYO/evThz5gzGjx8PGxsbjUlqUFAQMjIykJSUhLt376K4uFin60VHRyM7OxujRo1CQkIC0tPTsWPHDowbN05tAhIUFISCggLExsbi7t27GpO0Fi1a4OrVq9i4cSPS09OxcuVKuWS3prz++us4e/YsZs6cifPnz+PXX3+VTrA19RYVTE4UcM4JkXXat28fwsLC5P4tXLgQAQEB2Lp1K+Lj4xEaGopJkyZh/Pjx0gmL7u7uOHDgAAYNGoSWLVti7ty5+OSTTzBw4EC4uLjg7NmzePbZZ9GyZUtMnDgR0dHReP311zXGMnr0aCQnJ6Nnz55o1KiR9LiPjw/Wrl2L3377DW3atMHSpUvx8ccfazxX69at8eWXXyImJgahoaGIj4+XWykCAK+99hq+++47rFmzBiEhIejVqxfWrl2rtucEAJYvX46ePXtiyJAhiIyMRI8ePdCxY0e5Np988gkCAwPRs2dPvPjii3j33Xfh4uKiMV5Dffrpp+jatSueeuopREZGonv37tKl0eo8++yzGDBgAPr06QMfHx/88ssvOl3L398fhw8fRnl5Ofr374+QkBBMmzYNnp6eaveW6datGyZNmoSRI0fCx8cHy5YtU3v+oUOH4u2338aUKVPQoUMHHDlyRLqKpyY1adIEv//+O/7880+0b98eq1evlq7WcXR0NOm1JYKx1tvVkLy8PHh4eCA3Nxfu7u5GO+/qfen47/azeL5jQyx/PtRo5yWyJkVFRcjIyECTJk2qtSMpUXUVFhYiICAAn3zyCcaPHy92OLXG4sWL8dVXX+HatWsq39f0M0Cf39+cEKvAojI1IiICAJw8eRJnz55FREQEcnNz8cEHHwB4NIeHDPfll18iPDwcXl5eOHz4MJYvX44pU6aY/LpMTipVDZ9ZVj8SERFV+fjjj3Hu3Dk4ODigY8eOOHjwILy9vcUOy6JduHABH374IbKzs9GoUSO88847mD17tsmvy+SkEufDEhFZrrCwMCQmJoodRq3z2Wefaa1MawqcEKuAS4mJiIjEJUpy8swzz6Bu3bp47rnnxLi8SlxKTEREZB5ESU6mTp2KH3/8UYxLa8eOE6Jqs7BFgERkJMb63hclOendu7fcHgnmQMJZJ0TVVlXuXF0pdSKq3aqKy1W32J7eE2IPHDiA5cuXIzExEbdu3cJff/2FYcOGybWJiYnB8uXLkZmZidDQUKxatarGNguqLv69R2Q4Ozs7uLi44M6dO7C3t1dbkIqIahdBEPDgwQPcvn0bnp6e1d6XSe/kpLCwEKGhoXj11VdVbnG9adMmTJ8+HV999RU6d+6MFStWICoqCufOnUP9+vWrFawpPV5KzPSEyFASiQQNGjRARkaGKHuBEJG4PD094efnV+3z6J2cDBw4EAMHDlT7/qeffooJEyZg3LhxAICvvvoK//77L3744QfMmjVL7wCLi4vl9jvIy8vT+xxEVHMcHBzQokULDu0QWRl7e/tq95hUMWqdk5KSEiQmJsoVaLGxsUFkZCTi4uIMOueSJUuwcOFCY4WoFftNiKrPxsaG5euJyGBGHRC+e/cuysvL4evrK3fc19cXmZmZ0teRkZF4/vnnsXXrVjRs2FBj4jJ79mzk5uZK/6mr519dpt5hkYiIiHQjSoXY3bt369zW0dHR5LsfyuKUEyIiInEZtefE29sbtra2yMrKkjuelZVllAkypsR+EyIiIvNg1OSkarOl2NhY6bGKigrExsaia9euxryUybDjhIiISFx6D+sUFBTg4sWL0tcZGRlISkpCvXr10KhRI0yfPh1jxoxBp06dEBERgRUrVqCwsFC6esdccSkxERGRedA7OTl+/Dj69OkjfT19+nQAwJgxY7B27VqMHDkSd+7cwfz585GZmYkOHTpg+/btSpNkzQ2HdYiIiMyD3slJ7969tfYuTJkyBVOmTDE4KDGx34SIiEhcrC1diUuJiYiIzIPFJCcxMTFo06YNwsPDTXshdp0QERGJymKSk+joaKSmpiIhIcEk52fHCRERkXmwmOSkpgjsOiEiIhIVk5NKVR0nXElMREQkLiYnVTiuQ0REZBaYnChgzwkREZG4mJxUYr8JERGReWByooATYomIiMTF5KQSp5wQERGZByYnCjjnhIiISFwWk5yYukKspHLWCXMTIiIicVlMcsIKsURERNbBYpKTmsJhHSIiInExOanEjhMiIiLzwORECbtOiIiIxMTkpBLnnBAREZkHJicKOOeEiIhIXExOKnEpMRERkXlgclKFwzpERERmgcmJAoHjOkRERKJiclKJHSdERETmgcmJAvabEBERictikhOT763DtcRERERmwWKSE1PvrVOFU06IiIjEZTHJialV9ZswNyEiIhIXk5NKHNUhIiIyD0xOFHApMRERkbiYnFRizwkREZF5YHJCREREZoXJSSUJy7ARERGZBSYnCjjlhIiISFxMTipVzTkRuJiYiIhIVExOiIiIyKwwOalUXFYBADh88Z7IkRAREVk3JieVvj1wSewQiIiICBaUnJh64z8/DyeTnJeIiIj0YzHJiak3/nupS2MAQF0Xe5Ocn4iIiHRjMcmJqTnYPnoUgfVcRI6EiIjIujE5qVK5lLiChU6IiIhExeSkkk1loRPmJkREROJiclKpqnh9BZMTIiIiUTE5qSStEMuuEyIiIlExOanEYR0iIiLzwOSkUtWwDvfWISIiEheTk0qSyp4TzjkhIiISF5OTSpxzQkREZB6YnFR6PKxDREREYmJyUsnGhhNiiYiIzAGTk0rSnhNmJ0RERKJiclKJE2KJiIjMg8UkJzExMWjTpg3Cw8NNcn7phFjOOiEiIhKVxSQn0dHRSE1NRUJCgknOX1WEraLCJKcnIiIiHVlMcmJqEu1NiIiIqAYwOalUNaxTwQmxREREomJyUkk6rMPkhIiISFRMTiqVVS7TycorFjkSIiIi68bkpNL5zHyxQyAiIiIwOZGqqhBLRERE4mJyUkk2N2GVWCIiIvEwOalUNSEW4P46REREYmJyUkl2WIcrdoiIiMTD5KSS7LAO99chIiISD5OTSrYS9pwQERGZAyYnlSScc0JERGQWmJxUkh/WYXZCREQkFiYnlZrVd5V+zeSEiIhIPExOKjWq5yL9mhNiiYiIxMPkpJJ8nRNmJ0RERGJhclJJds7JOe6zQ0REJBqLSU5iYmLQpk0bhIeHm+T8sqt1Rn5z1CTXICIiIu0sJjmJjo5GamoqEhISxA6FiIiITMhikhMiIiKyDkxOiIiIyKwwOSEiIiKzwuSEiIiIzAqTEyIiIjIrTE7UWL7jLH6Muyx2GERERFbHTuwAzFXM3nQAwCtdg8QNhIiIyMqw54SIiIjMCpMTIiIiMitMToiIiMisMDkhIiIis8LkhIiIiMwKkxMiIiIyK0xOiIiIyKwwOSEiIiKzwuREi5KyCrFDICIisipMTrR4/afjYodARERkVZicaLH33B2cy8wXOwwiIiKrweREB1ErDogdAhERkdVgciJjcu9mYodARERk9ZicyJg5IFjsEIiIiKwekxMdnbx6X+wQiIiIrAKTEwUBns4qjydeYXJCRERUEywmOYmJiUGbNm0QHh5u0uusHBWm8niFIJj0ukRERPSIxSQn0dHRSE1NRUJCgkmvoy4JqWBuQkREVCMsJjmpKc18XFUeZ88JERFRzWByoqBeHQeVx9NvF9ZwJERERNaJyYmO/jhxXewQiIiIrAKTEz2UlnMTQCIiIlNjcqJC/za+Ko+v3pdew5EQERFZHyYnKrRv6KHy+JrDGTUcCRERkfVhcqLCiPBAlcfvPyit4UiIiIisD5MTFXxcHdGzhbfYYRAREVklJicqSCQS/DS+s8r3Rn93FPEZ2TUcERERkfVgcqKnwxfvYcTXcWKHQUREVGsxOSEiIiKzwuSEiIiIzAqTEyIiIjIrTE6IiIjIrDA5MVDQrH8x47dkscMgIiKqdZicaBDg6azx/d8SuRkgERGRsTE50cDFwVbsEIiIiKwOkxMNXBzttLbJzC2qgUiIiIisB5MTDT55PlRrm2dXH6mBSIiIiKwHkxMNmtd3xbH/66exzY2ch1h7OAP/nLpZQ1ERERHVbtrHLaycr7sTGng44ZaG4Zv3/5cKAOgQ6ImGdV1qKjQiIqJaiT0nOujdqr5O7RKv3DdxJERERLUfkxMdTOjZRKd2UzcmmTYQIiIiK8DkRAdNfVzh7+EkdhhERERWgcmJjo7M1jwxloiIiIyDyQkRERGZFSYnREREZFaYnBhZXlGp2CEQERFZNCYnRnb6eq7YIRAREVk0JidGNvq7YwhduBO5D9mDQkREZAgmJ3p4s29zndrlPizF5J8T8fPRKxi/NgFFpeUmjoyIiKj2YHKih7cjW+rc9kj6PczdnILYs7fxS/xVVFQIJoyMiIio9mByogcbG4lBn1v4v1Q0/b+t+PPEdSNHREREVPtYTHISExODNm3aIDw8XNQ4ZkS1Mviz039NNmIkREREtZPFJCfR0dFITU1FQkKCqHEEeDqLen0iIqLazmKSE3Ph5epQ7XMIgoD7hSVGiIaIiKj2YXKipx7NvTGiU0ODP5945T6mbUpC2KJd2Hkm04iRERER1Q5MTvQkkUiw7LlQgz//7Ooj+DvpJgBg+Y5zxgqLiIio1mByYqDT7/cXOwQiIqJaicmJgdyc7Kt9DokeK5MFQYAgsFYKERHVfkxORHblXqFSgbbiMvmKsmXlFRi08hAm/JhYk6ERERGJgsmJiM5nFaDX8n1497fH9U/i0u+h1dztWLH7vPRY8vVcpN3Kw+60LDHCJCIiqlFMTszAnydvSL+e/3cKAGDF7gtihUNERCQqJicWgXNNiIjIejA5qYbN0d3FDoGIiKjWYXJSDR0CPUW57t9JNzBlwwkUlZZrb0xERGRh7MQOgPQ3dWMSACAkwAOv92ombjBERERGxp4TM1FYXKb3Z7K5Pw8REdVCTE7MxJy/TuNeQTEu3inQ+TOcJktERLURk5Nq+vyFDhjZKbDa59mcdBMdP9wNVUVgWRiWiIisCZOTanq6QwD++1x7Ua7NcvZERFQbMTkxc1yRQ0RE1oardczYi98exZH0exgS6q/y/Qu3dZ+fQkREZCnYc2LGjqTfAwD8L/mm9JjsUE58RjY2JVzl8A4REdUqTE6MJCKoXo1cp+eyvdKvH5SUY+Yfp3Ho4t0auTYREVFNYHJiJCte6FAj17l+/6HSsfTK4R1BENiLQkREFo/JiZH4ezpjyfAQjIqo/rJiQwiCgJe/j8dzX8UxQSEiIovG5MSIRkU0wpLhNb+sWCKR4GFpOQ5dvIvEK/dx/f5D3Gf1WCIislBMTkxgcm9x97t5atUhhC3ahaRrOaLGQUREZAgmJyYQEuBRo9fblnJL7nXuw1IAwNrDGXLHr99/gLsFxTUWFxERkSFY58QEJDV8vaOXslUe33vuDkrLK2Bva4OcByXo8d9HK30uLx1ck+ERERHphT0nJiCp6ewEwLVs5VU8uQ9L8fHOcwCA9DuFNR0SERGRQZicmNiYro1r5DpRKw6oPL4p4VrlV/qt4CkuK8emhKu4kaOc9BAREZkSkxMT69faV+wQVEq7lYftCnNVZMXsTcfMP06j/6f7azAqIiIiJicm0cLXTfr1ky19sEykXYsBIL+oDKev5yodH/j5QUz6+QQSr9xX+bmDF+4AAApLuPEgERHVLCYnJtDMxxUbJnTG7ulPAgD6txGv96S8QsCQLw5h/3nVJe7PZ+XXcERERESacbWOiXRr5i12CHJ2pGTq3DblRi7yi8pMGA0REZF67DmpYcF+btobiejghTt4atUhXKzcr4eIiKimMTmpARKZyicvdamZ1Tu62hh/Ve71dj16WLT5bNd5LP431WjnIyIi68DkpAa4ONpKv/Z0sRclBkHNUuJkFZNlFZ3L1DwvpaC4DG9vSkJsWpb0WGl5BT6PvYBvD2bgJpcjExGRHpic1AB7WxscmtkHB9/rAwdbcR75+azHwzQbjl1Vej82LQvxGdk4fll59Y66GipVVu25gL9O3sD4dcelxypkdkYuLa8wJGQiIrJSnBBbQxrWdQFgHqtj/u+v03Kvb+Y8lCYWPm6OKj+z7shljOkWpPK9zNwipWOCfjXfiIiIpNhzUsOebOkjdghKsvKUkwtFC7acUbvLsS7V+n+Mu4zuS/fg8l2W0SciIs2YnNQwe1sbdGxcV+ww5PwYd0X6dV7ljsaqzPgtGUu2pmFl7AW54xIdNhOa//cZ3Mh5iAVbzhgeKBERWQUO6xD+OnlD+nVxmfr5IRduF+BC5RLjsd2D4O6k2+Re2dVK5RUc7yEiIs3Yc0IGKS9/nGRo6zeRXSkkxo7NRERkWZiciECQmS3a0tdVxEiMhAkHEREZEZMTkf0+uRueat9A7DD0Vi7I9pxozk60vU9ERCSLyYkIZGdduDvZw16k2ifVMXZNvM5tS8of72ysy8ogIiKybpb3W7EWeKtvCwDA8CcCAABedRzEDMcgKTfycC37AQDt80he/v5xInM+qwAf/C8VRaXlGj5BRETWjMmJCPoE18fxuZH45PlQAIClLmCp6j2RzU2y8oow+edExKXfkx67pVCk7YfDGfj+UIbS+W7lPsRHW9OkSQ8REVknLiUWibfr40qs6va9MXfpdwrxR+J1/JZ4XXpszl+nsTvtNrZp2UDwyj35Ymx/J93A1I1JAIAdZzKxf0YfAMDZzDyUlQtoF+ABQRDwsLQcLg7835aIqDZjzwlVyzu/Jcu9vn5ft03+FCfJViUmAHDl3gOUlVegrLwCA1YcxFOrDiG/qBRvbUxCm/k7cPF2AVS5W1CMvp/sw5f7Lup3E0REZFaYnJgBdfvQVM1JqY02Hb+GhMvZEARBZWG24HnbkXjl8SaEOQ9K8b/kmwCAtUeUh4QA4Is9F3HpTiGWbT+n9foPS8ox4qs4xOxlIkNEZG7YP27GnOxtxQ5Bb2czdd/Y8Pmv4tA3uD5O38hVeq+sQlBb6l5VMpdXVIriMt0n2f56/BriL2cj/nI2ovs01/lzRERkekxOzICgpusksHIn49psz9nbat9T16OkeDgrrwidP4rV67q6rBY6eukeYvZexKKn2yHIu45e5yciIsOJMqzzzz//oFWrVmjRogW+++47MUIwK+qmw77aI6gmwzA7JeXq9/mRtTstS+9zq1v+LAgCKiqHmV745igOXriLyetP6H1+IiIyXI0nJ2VlZZg+fTr27NmDkydPYvny5bh37572D9Zi6noIHO0sb1jHmK6qWVKs7nnJupmj28RcRVN+OYknl+/FwxIWjiMiEkuNJyfx8fFo27YtAgIC4OrqioEDB2Lnzp01HYZZkV1KvHJUGABg2bPtxQrHbFSoyUIUh8FUlcfvtnSPQdf899QtXL//EPvOqR9uIiIi09I7OTlw4ACGDBkCf39/SCQSbN68WalNTEwMgoKC4OTkhM6dOyM+/nGF0Js3byIg4PEqlICAANy4ccOw6GsJ2d+1Q0P9ce7DARgRHiheQGZC9rm88kO8yuOA+iGaCkutbkdEZOX0Tk4KCwsRGhqKmJgYle9v2rQJ06dPx4IFC3DixAmEhoYiKioKt2/zL1F1XghvBACICKoHQH44Z/p/WuKZsAC0qF8Ldi+uhoy7hdobKej44S7cLShW+Z4+mxFy20Iiopql92qdgQMHYuDAgWrf//TTTzFhwgSMGzcOAPDVV1/h33//xQ8//IBZs2bB399frqfkxo0biIiIUHu+4uJiFBc//gWTl5enb8hmL6ShBxLmRKKeij123ur3aB+e/p/tr+mwzNaeyiGX6b8mIb+oDH2D66tsd/9BKTp9uBvH/q8ffN2dajJEIiKqBqPOOSkpKUFiYiIiIyMfX8DGBpGRkYiLiwMAREREICUlBTdu3EBBQQG2bduGqKgotedcsmQJPDw8pP8CA2vncIePmyNsbdT/ja7LJFBrcSe/GKeu5+DPEzewKzULZ24q10mR9cr3uu+gLIYj6XeVyvkTEVkzoyYnd+/eRXl5OXx9feWO+/r6IjPz0V4rdnZ2+OSTT9CnTx906NAB77zzDry8vNSec/bs2cjNzZX+u3btmjFDthgdAj3lXnvVccAzYbW3gqw2Q784LP36fmGpxrbnsvKxO1X/5cY1IeVGLl789hh6Ld8ndihERGZDlCJsQ4cOxdChQ3Vq6+joCEdHR+0Na7l5Q9pIN9gL8HTG/hm9YWdrAx83R3xz4BIaeDgp7f5rLS7c1l6V9rUfjyNjySBIKmfPqptEq8q9whJDQ0NZeQVu5DxEYy/VRdySr+fodb6Mu4VwdbSDjxu/J4io9jJqz4m3tzdsbW2RlSX/V2pWVhb8/PyMeSmr4+5kL/16aAd/2Nk++k83e2Aw9r3bG1Mr56ZYo/NZqjcCVDRgxUGVx389rtwbp0/youjLfRfx5i8nUVEhYOJPiei1fB/+PXXL8BNWup1fhD4f70P44t3VPhcRkTkzanLi4OCAjh07Ijb2cSnxiooKxMbGomvXrsa8lFXaNrUnpv+nJd7q+zgRkUgkCPKug+7NvUWMzDKcy1Ldw/Le76cMOt+NnIf4dNd53MmXXxG0bPs5/C/5Jg5evCstz//DYdWbFeqzauicDvsWVVQISL2Zp3IzRSIiS6F3clJQUICkpCQkJSUBADIyMpCUlISrV68CAKZPn45vv/0W69atQ1paGiZPnozCwkLp6h0yXOsG7nirXws4OyhXjg2s54LDs/qKEFXtoG5/oyp/nbyutLHgi98excrYCxjxdZzKz8tWmVV7XbWbFxjmv9vPYtDKg1j0T6pRz0tEVJP0Tk6OHz+OsLAwhIU9qmQ6ffp0hIWFYf78+QCAkSNH4uOPP8b8+fPRoUMHJCUlYfv27UqTZMn4AjydxQ7BYnx78JLca8XcYs3hy3Kv396UjJg9F6WvKyoEXLn3qLx+xt1C/N9fp/WOYem2s/jhkOoeFVV06WX5+sCj+1p75LLe8RARmQu9J8T27t1b61+ZU6ZMwZQpUwwOishUfoy7jFe6BiErT34oRgCw7fTjeSHHMrKVPrvm8GW81KUxvtyXjn8U5pD8En8NS4ar33Ig5Yb8cucLWfn4an+6AXdARFT7ibIrMZnO8x0bqn3PRcVwkLWZ//cZ/Hz0itJxQRC07j6cX1yGiI9isfbIZZWVZyf/nKh2rkdxmfwOyw9LtQ/56CrxSja+3p/Ocv1EVGuIspTYEDExMYiJiUF5ufF+qNdG7w0IRlpmHl4Ib4S5m1Okx1/v1RTT/9MSp67n4vmv4kSMUHyyz6XK8Sv3q33ebSmZGPm1bs/WRsVyoPIKQWMhPnWeXf3omvXdubyYiGoHi0lOoqOjER0djby8PHh4eIgdjtnycXPEP2/2BADkPizF8h3nMHNAMCb3bgYA8GMZd5Ve+OaoUc6ja5KjKgl5WFoOV0f135LaljdfusMqs0RUO1hMckL6i+7THM93bIj6TEhEoSmZUJWcvL0pCT2ae2NURCM42HHElYisF5OTWo6JiXh2nMlU+56qYZ1dqVnYlZqFByXl0p4ucyQIAuZuTkF9NydMjbTe4n9EZDr888zKeLjYa29ERvHniRtKx5Kv5WDb6Vsa55b8d/tZJF3LMWFk1XM+qwDrj13FZ7vPix0KEdVSTE6sjGwZfKp5T8ccxuT1J7RuRDgs5rDSsWpU1JdKvJKNbktisT1Ffa/Oucx8zP7zFG7lPlT5fpERVxoREanC5MQKtW7gLnYIVm/x1jStbTYlXMWR9LsAgNLyCuQVlVX7umPXJOBmbhEm/Zwod1wQBGnSMXjlQfwSfw3RWpZWExGZCpMTKzSuW5DYIVilJTokJLJm/nEaL357DADQ/7MDSgmFIRTrrVSZ+FMigudtx/X7D1BWWS8l7Zb2vXyIiEyBE2Kt0LMdG+J2fhE6N/Wy+ponNamqtLwhMu5qXyZcUq6ceKw/dgWlZRUY272Jxs/uqhxm+vX4dY3tPtl5DnHp96SvC4rLNC5/JiIyBH+qWCFbGwmm9NV9lUUdB1sU6rCJHZmGbDKgydf75ZOfhyXlmPPXo4JzQzsEoF4dB72uq1jFtrS8Aqtk9hcCgLRbeQgPqqfxPOuPXYEgAC91aSw9lnIjF7tSszCpVzOVG1kSkXWzmGGdmJgYtGnTBuHh4WKHUqv8MLYTAjydMaLT47L38XP6wd3pcd76/Vg+czGN+la5QJwgCIhNy8LNHNWTVgGgRGYIR9Uk1lPXc/SKo0LFnlraSubnF5Vizl8pmLs5BbkPS6XHn1p1CJ/HXkDM3otIv1OAp784pHWSMBFZD4tJTqKjo5GamoqEhASxQ6lV+gb74vCsvnhTpifF3ckeifP+I31tayPBqIhGYoRHavT+eB/GrzuObkv3qG0T+sFOpWOyCcvQL5RXBOnrkpbhJtnrlaiY75J2Kw9v/XISyddz8dqPx6sdDxHVDhaTnJBpNazrjG7NvNAvuD6c7G1hb/v4fw0JAMWyHAuGtKnZAEnOlXsPxA4BADD7z9O4k6+8CWIViUyxOQGqe1lyHpSqPE5E1otzTgjAo18iGyZ0UfOe8jHZ5IXM38mrOVhzOEPlezs1VLLVxcXbBfBx02HTQTUjQDc0DE0RkXVickJaBdZzgS/L4Fu06A2qa5Ycv5yNiT9Vb4myoGIuShVjFI4jIuvD5ITU2vtubxQUlaG+mxMm9GyKT3exXHlt85wOS8mLy8pxv7AUdeuori6sZU6s6AqLy1BQXIbtKZkoLCnDG72bix0SEWnBvnlSq4l3HYQ09AAAODvYIu2DARga6o8vRz8h166pTx0xwiMTWBl7QenY4JWH0GVJrNqibO/9nozbeUUAgDM3c3E+S3W7sWsSUKqiFos2D0rKcOLqfaUemtSbefhizwUUlZbjYUk5tp2+hfwi5fkrHT/chc4fxWLBljNYtv0cMnOL9I6BiGoWkxPSmbODLVaOCsOgkAZyx2W77pc91x5O9vzfqja5eLsAAPDvqZsq37+ZW4R3fz+F/KJSDF55CP0/O4Dyyu4U2flKqbfysE3Dnj7qjPz6KIZ/eQS/Hr8md3zQyoP4eOd5fLn3IuZsPo3J608g5H3lFUpFpfIJkWL9FiIyP/wtQgZRNUkWAEZ0CsSpBVFwc+KIYW3wUKb4Xmm5+vGb9NsFyC4skWmruofEkE0DT9/IBQD8nqi6eu2Zm3lyO0Dfl4nDUPP/TsEL38RJkywiqllMTsggsj3sij++HexstE6ErMOqoBZh9f506ddrj1xW2+5GzkNsSpDv2diUcBVj1+hXl2jB3ylIv1Og12cUFZZo3iBRlwTpx7grOHopW+fqvERkXExOqPoM+OOypZ8bxnRtrL0hiUrVHBR1vtz3OJEpKa/AzD9OI+lajlwbxcQj9uxtudfr4q6g3yf7MfLrOJQp9L5cy36odEwVibpuvUp/ntC8f5Cssgr117uQlW9QTxARaWcxyQnL15sXLT//MfHJpiqP+3s8WpL8w5hwPN8p0NhhkZkYFqO6+uzX+y/hhW+0rxA6lpGNgxfvyh3LzCvC8NVHlNpm5uk3wbW4rEKulL4hdqVm4T+fHcBzXynHQ0TVZzHJCcvXW5Y3ejdHYy8XpeMrXgjD5aWDUVfPTejIsly6o76s/dFL2Tqdo1zFHJdT13Mx8POD+F/y48m5Z27mybXRNqT4Y9wVhC7ciR/jLmuNQV2nYNXk3JQbeWpa6OZBSRnuFqivsEtkrSwmOSHzomnOCQDY2EjQvqFnTYVDViTtVh7e/OWk2vc3HLuK8WsTtA65zP/7DADgwPk7+Dvphsa2ijTUndNLx0W70enD3Rq3AKiOmzkPsXTbWY0bRBKZIyYnZDI+rsolzVvUd9X4mRfCA9EvuL709ecvdMCG1zobPTYyf6/9eBw9/qt+Y0N1vth7EbFnb2PDsas6tX/lh3hM3ZiEa9ma9yvadvoW+n2yD2m38mDQRCsVqpY1n7x63yjnUzTmh3h8tT8dY9fEm+T8RKbC9Z5kENk5J+rKl0+NbIHr9x9gWFgAujXzwoOScrXDORcWD8Sp6zlo39ATuQ9LsSr2Al6IaITWDdxNET5ZiOv3Df+Lv6BY86odRXcKihFYT3kossrk9Y+2AHhj/Qk09dZceLCotByL/01Dv9b10btVfZVtch48XvKsbhLv1XsPcKegCB0b19MWPv67/Sx2pWbhrze6wc3pUTXfC5U1as5nVW8FlL7Kyitgx/23qBr4fw9VWwtfN5XHPZzt8c0rnTAopAE8XRzg7+ms9hz2tjbo2Lge7G1t4O3qiIVPt2NiQtWSrLBSSBuVObaKY4XFZZW9J+r9cDgDPx29onEp9ftbzmiN6cnle/Hs6jidllev3peOi7cL8Eu8bj1GpjLrj1MIeX8nsvScqEwki8kJVduS4SEYFdEIW6Z0FzsUIqnYs7e1FmRTV2pfm5syJfBv5xehpEx+yfENHXp8FCfyyiorr5CbKHtWzdYBKj8rcuG4jQnX8LC0XKcJx0TqMDkhg/StnBfS2MsF3q6OWDI8RO8JsMaaVEikztJtZzW+3/+zAzKvlP+HjN5wAg+0FHWLWByLoV8ckjumbam9IsXmI76OQ6cPd0tfHzh/B0/HHMYFA5MpIkvD5IQM0sDDGSfn/Qe73u4ldihEam1S2I9HXw9KyvH2piRcv/94sqyqnPpspnzScD5TfhjmzM1cvPd7Mm7lPu5RkU1gFJOZE1dz5F5vOn4NyddypPNeNDFl0l9RIWB7yq0aXf2TX1Sq1DNFtR8nxJLBqlurxFGPDQKXPdce7/1+Suf29rYSjXvBECmq+qWu+Itwx5ks7DiTpde54i8/ruVSVl6BwSsf9aycup4LX3cnDA5pYNAk1apNGMXyx4nrmFH5fXh56WCl943ds3O/sARhi3ahgYcT4mb3M+q5ybwxOSHRtKjvilERgSqXHCtyste8F8+qUWF4onFd2EiACgHwquOACT8ex8ELdzV+jqjK8Sv3sSs1C18fuKSxnb49E12XPl4OfTYzH2cz87H//B25NvoOA5lCxt1CrNh9HpN7N0Ownzvyi0qxLSUTUW384OHyaPXPoYvqv5+Ky8rxH7lhMu1u5jxE8rUcRLX1g42N8kOoSvJu5XJyrbVhckKikUgkWDK8vU5t1S1Xvrx0MPKLSqVLJ2UNaOfH5IR0pm1+SpUKNf8vrth9Hm/1bYFEhZolxi6w1mv5XjzZwgept/Kw/rXOSon7Z7vOw95WgolPNpM7fi37Ad77/RQm9mqKPiqWN49bE4/L9x5g55kspC0agLc3JWN3Whb+bHodGyd2VWovCILcEmjF+yzToeeyx3/3oEIA/vtsCEaGN1JxDa2noFqKc07IYgVULk1WlZgAQFNvzQXfiAyRrWYF0IrdF/B38g08/5X2vYMU3cwpwof/pGotBAcAV+49wE9HryDxyn1sSbqp9H5ZhYCPtp5FhcKqnXd/S0bcpXsYp2Z58+V7j679sLQc17IfYHfao6Gso5eypX8cyPZt7Dsn3/sz8uujcq+19UABj3o5AeDQRXW7PzM7sVYWk5xw4z9SpGrvHlldm3lh8TPt4Opoh3f+07KGoiJrZuickLmbU/DdoQz0XLYXe87qPr+lRMMuzYpDRfpskKhYUTZs0S7sVdhBOvl6jtzrG9WYJKuuZ5Q9J9bLYpITbvxHinT5wTW6c2OkLIzCm/1amD4gsnoxe9OrfY5X1x7Xua2mbwHF748r97T3ylRJV9i4MedBKcatlf/Zuys1C0+tOoiLt023vLmmcpOHJeVIuZGrNkmimmcxyQlZty5NvQAAbo6Pp0kJev7o+nk89+ihWqbyl2mphh4UVfKKSvHB/1KVekP0ceZmHlJu5OGtX5IMPoc2xsoV/pd8E6O/O6p2B+jhq4/gqVWH8L9Tt4xzQao2JidkEXzdnRA/px/i50Ri5oBguDra4f2hbfU6h4uj5hU/RJamsKQcvyZcQ4s52/T6XPv3d+KHwxkYtzYBN3IeIr+oVKfPlaqoPptfrNtnNREEoLzy3JfvFuKl747hyMW7ev8Bos6bv5zE4Yv3sGy76knPVdsR/JF43SjXo+rjah2yGPXdnAAAk3s3w+tPNlW59NAQ43s0wfeHMoxyLqKadDe/WO0qo2mbknQ6R/ele+Cg4yZ9/6roWbhx/yF+r+Yv9X9P38KZm7nYNb0X3tp4Eqeu5+LQxbtYNSqsWudVlPOgVOF1CY6kq5uMq15RabnW8gZUPew5IYtkSGKi7hMvdWlcvWBUmNCzidHPSaToOw1J9ZZk5ZU86miaWKtNhfBoJZAqVRVxf0+8rnUjxsv3HuBcZj5u5z0eetF0f7Iu3i7ACYUl3KooThIe/d0xvKFD1V1Zn+46j+B523Hwwh3tjclgTE7I6jXxroMFQ9ro3F5bXpT+0SDMGaz7+Yhqq65L9uD9LWfw7m/JeDrmsNb2V+49kEsgdN1ZOvLT/Rj+5RGtq6UkCn+iKG6+WFahPUlbGXsBgG67SpuLrLwixOy9qHbOjTnisA5ZDU2j1+O6N0G9Og6YujFJ63n2z+iDhnWdsfB/qWhUzwVervKfszXScBNRbbD2yGWd20ZvUN+LUVhcBhuJBM4Oj4dTrt57gCeX75W+jvx0P6L7NMOMqGCDYj2stt6KMkta1/Py98dwPqsA+8/fwa+vKxfUM0fsOSGrNLh9A6VjQ0P9sW1qT5xdNEDr5yUSCd4f2hav9miCpzsEYN5Tj3pKZMfuF+o5YZeotntYUm7wZ9su2IG2C7bLLfdVNZwUszcdZeUVOHThLgqLNe8obUx38ouVCt+Zi6p9nOIzsrW0NB9MTshqaOvPkEgkaN3AXetENx835b2AxnRtjJWjwrBvRm/psW7NvAyIkqj2aj1/u7QuSlGp/olKhSC/z869QtXDFCtjL+Cl74/h1bUJchs5GnUPI5k85MjFuwhfvBuTfk6EIAgY9c1RjF0Tr1PdlOokNQ9Kai75qmlMTog0eKKRJwBg6fAQJM6NRPz/9VOZvNjZ2mBoqD/8K0vqG8LP3cngzxJZishPD6CguAxTNAzhaNJt6R7pZF91v9J/SbgGADiWkY2/Tuq3kuiN9Yk4Xrnh4NFL9zBt40nc0zJX49uDj0r170zNws3cIsRduod95+6gQEvPzdFL9xC+eDde+1H3wntVPt99AW3m78DOM5l6f9YScM4JWSctf6jsfPtJpN3Kw9BQf+Q+LIWni4NJwujUuC6OX9G+yoCoNvliz0XsTjO8ANzHO85haKg/LilUsq0iuwlh3kPlBOFBSRnSbqmubLv1dCa2ns7E5aWD8cI3j/YLKqsQ8MWLT6iNR/bHiS69IPlFpbidX4wfKlcj7Tl7G4Ig4NT1XDSv74o6jtp/NX+2+zwAYM7mFPRv66e1vSqpN/MQ4Oks3XXanDA5IaukrbhTS183tPR1AwCTJSYA8Pvkbgia9S8AI3c5E5mxr/ZXv8y/rmS/ryQSIGbvRSzfcU6vc6jakFHdTxBNk3qr9Fy2FzkPSqWblwKPln5P3ZiEYD83bJ/2pM6xVVQIGLcmHu0beuJtPfYQS7ySjWdXx8HN0Q6nF0bp/LmawmEdIhOR/aHYq6WP9vYAZg00bJUBkbXRZQdnRRl3H+icmIz4WvPu0hl3C/HG+kSUKdSIOXU9V+7194cy8Ev8VbljVcXgZDdL/PPEDQDA2cx8lFcIaueTJF65j6FfHJK+vldYgr3n7uDzyiXOutpTuXVBfg1OGtYHkxOySl2b1uxk1Q+eVr9yZ+HQtrCzkeDTkR0wqVczbJnSXfresufaGz0WY1fdJKppV7MfoOeyvdobAki/87j2SVWZel3IrmxJvp6LkrIKpQmuW09nYscZ9btI91y2F4v+ScXsP09Ly/Pr4pkvD6PN/B14f8sZ5ClsLfDs6iNKCZAhFGu+mBsmJ2SVXuzcGJ+NDMXB9/rUyPUa1nXBhgmdsW1qT6X3xnQLwtlFA6SbG3o4Px7/HdYhQO9r9W6luZdmSKi/3uckslS/xF8zynnaLdihcshG03JlxXL5gPpVSqkyiVNV8rH2yGUs3JJqlbslW8yck5iYGMTExKC83PB18mTdfGVWw9jaSPBMWMMavX63Zt4AHlWYVfwjyk6mPkpjrzqYM6g16tYxbK6Lm5P2yW2B9ZxxLfuh1nZE9EhJeQW2nlZeGSNAQHGp9sqy649dQUFxGcrLVScaspN4Ze04k4k/TlyHm6MdTr3fX7+gNVA3x+1a9gPczHmIzjXcu6zIYnpOoqOjkZqaioSEBLFDIQvl7+mMNWPD8cdkcSsk/japK4L93LBxYhe1bSY82RTPdTRO8tQuwF3p2MB2ykXoiEh/9wpLEHdJe2XZ+X+fwbLt5xB7Vr9VSlXLkfOLy/DKD/Ea294tKMbtvCKNbRb8nYLiMuU/8qt2pu65bC9GfnMUKTeqP3RUHRaTnBAZQ5/g+ujYuF4NXe3xnyay3bIdG9fD9mlPSodxNJ5Bh2Fhdyc7nNEw2/6fNx8PJY3tFqQQGRFVx7Lt+q38UZxDoo+DF+5qfL/Th7sR8VEstp2+heKycgiCgO0p8jtJr4u7gh+PXJH7GfDV/nSEvL8Tf8jsLp18PcfgOI2ByQmRiTT1roPOTeqhfxtfuWEbY2jp6yr92tvNUa4ugqbx6Sca1330hRGzk29e7mi8kxFRtU1efwJLtp7FluSbmPSz8jwZ2VVCALB021kAwIzfVe8uLQYmJ0QmYmMjwabXu+KbVzoZ5XzPywzz/DS+Mz54ui1cHe3w6YgOcu10mTrXQGb+zTcvd0TGkkEGx9U3uL7BnyWyOjU0t3XDsauIS9cw3KSiW9acpt1azIRYInrM190Jr3QNwkudG8NGxS7Ivu6OyMorRmMvF5Wff7FzY1y4XYAezb0Nri5Zxc7WBrMHBmNJ5V9fRKReRQ2uvNF0qXwVw0uy7bMLSkwQke6YnBCZMdkfFqrmn6hKTABg48Su+OZAOib1aiZ3vKq1g50NFj8TYqQogXHdmzA5IdLB5Xv6F48zREl5BbIfqE4w1h65rPXzRSomzdYkDusQWQidiyYJQBPvOlgyvD0ae9UB8Hgn5c5N1U8GXvZcezwTpn9dFYCl94nM0a5U9QXitBG7SBt7TojMmIOdDYZ18EdBcRkaqRmiUaRq36BDM/vgQXG5xtopIzoFYkSnQPx18obeceryY6xL03o4eilbe0MiEp2aTtmau764lycibVa8EIbvxoTDzcnwvyUc7WwNLuqm6J83eygdk+jQdbKSZfOJLIfI3aFMTogsxIhOgegXXB+LNOzTYwxv9G6GjlVLjlVoF+Ah3fPn4+dDATyquDsqIhAD2/nByV71j5X6bk4qjxMRKeKwDpGFcLK3xfdjw7W2q+5igPcGBCP9TgH6fbJfbZsRnQIxNNQfTva20mNLhj9KWFrP2169AIhIdGJPI2PPCREpsZMZcP7zjW4q28gmJrKM0Ru8aFg7pWPpHw1C8gLj7S1CROaLyQlRLWOMMgqN6rlgWAd/vNSlEZ5oVBfhQeqHeRSN79Gk2td/uUtjpWO2RpyhFxLgYbRzEdVGYq/AY3JCVMtomi+iK4lEghUvhOHDYY9qoXw6ogP6tPLBhtc6a/3stMiW1b6++riUj6maoKuoqXcduQnFv0zsgjmDWhszNCIyIiYnRLXEnnd64cNh7TC2e5DRzx1YzwVrxkWgW3NvrW1lezjqV9ZXUaVqMq0uqoZ5VP0x19ZfeddlRVun9kQdh8fJiaujHSY82RShgZ5KbQe2q17FXCKqPiYnRLVEUx9XvNSlMeyNvMmgIeq62AN4NLlWnec6NsSPr0aga1Mv7J7+pNp2rz/ZVDrM42j3eJ7L3nd7I3l+f52WMTvZ26qs//JS50ZKx7xcHy+5/mNyV63nJiLj42odIjK6uNn98KCkHKe0bLv+ZEsfPNnSBwBQr44DsgtVlNuWyT0c7GywaWIXlFcIaOJdp9pxPtexIdr4u+Ol747h/gPlvUYC6+pW+I6IjEv8P7GIqNZxsrdFvToO1drl9D9tfAEAL0bI9250buql0/BSlf8b9Kj3RtVEYYlEgrb+HtIy/+raEVmbpGs5ol7fYnpOYmJiEBMTg/JycTcjIiLdNfR01rntxCebYum2s9I5H9+83BEPS8vh4mDYj6nWDdzx1xvdpEueNeUcsiNDcu3ELvZAJJJ95+6Ien2L6TmJjo5GamoqEhISxA6FiHTUwtcNK0eFYdPELgCAuYMfrZBZOlx5R+TXn2yKf97sIS1zL5FI9E5MZJcxzxvcWq4Wi2BAl4ji5meDQnSfLKtqufKacdqL6BGRBSUnRGSZhob6o3NTLwDAaz2b4vT7/fFChPJEVIlEgnYBHtWa0Nuivium9GmOqLa+6FJ5zSqDQxoAAIL93JSvLfO1phxmbDfda7j8+rryZNquCjERkWoWM6xDRLWDm5O9Sc//blQrlcdnD2qNJxrXRc8WPkrvqVvxo3g4okk9nWLo08oHzg7KFXQdjLCSqk8rH+wVucudyNSYnBCRVXCyt8XTHQJ0aGn4jFh3Jzt8NrIDereqr/Re/Jx+sKlmldu148Lxd9LNap2DyBJwWIeIrJ66lEHfVCI8qB76tfZVKrU/slOgQbsyR7b2lX6d+kEUereqD08X0/Y8EZkDJidERDJMsZT4hYhAvT8ztV8LyI4CVU0OHqbQ+9NBRZVbIkvHYR0iqjVUzfPQRd/W9XH8yn2l47pUn9WkTQN33Mx9iNYNtJfYVzSpVzNM3XhS6XhooCf2vtsbvu6OKCguQz0XBzSfs61acRKZGyYnRGTxZg0MxvHL2RhUuSJHXxN6NsWy7ecAoFqrhWRL3wOPNiUsFwS9zzkjqhWcHWzVzn6pqo5raA0YInPHYR0isniTejXDd2PCDU4s7G1tMHNAMEICPDBOZuNEbf0mQ0P9pV/3Da6PmQp7CdnYSLTG5GD3+P3B7Rtg9sBgTHyyKQDDhphmyKxW8tGw8SKRJlX/D4qFyQkREYDJvZvhf2/2gKuT7r0Rc59qjZ4tvPHVSx3xw9hweLnqnwyM6NRQ+vWkJ5vh9V7N9E6ynuv4+Byyxd9CGyoXgpM1WsXGh692172Wiy5WjgqTu0eyDANE3p2byQkRkSyZ3gptU07quznhp/Gdq/WD3FYiQfL8/tg+rSdCtCQT6nw4rJ1Bn1v8jHyl3nHdg+Bkr/xr4an2hg2XnftwAIaG+ks3dyTSFZMTIiIZjnaPJ9Vq6sFwdTTefA8PF3sE+6maNKvbuI5smX5D1avjgAVD2uLVHso9J1+8+AQ+fj5U73PKPksifTA5ISKS4eFij7mDW2P+U21QR0UC8tcb3dC5ST1srNwvyJQMmXMit4mhHp+v+pi3qyMufTQI4UF1FWLRfLJJvZrpfjE8WipNpA6nehMRKXitp/rJgGGN6mKTin1zDOWsYcWNCUqu6MTGRoLPXwjD0m1nVfakqDJrYDB2p2Xh4u0Cje3ejmyJLk3roXNTL3wee0Flm3YB7ki5kad33GQ8Ym/IzZ4TIiIdRLX11d5IT+FBdTG5t/oeB2PspCxr9sBgte8p8vd0xspRYToVefOunAi84bXOKuestPV/PJemW3Mv6UaQ6nzzcied4+Rk29qJyQkRkQ5UbRhoiOXPtUeApzO2T+uJ3yZ1g4ez+nL0xu45CajrbPBn1cXSo7k3fn4tAgBQ390J41X0tFTVZVHHW6E+TB096rcse071XJhZeiRiqiwZHqK9kYEiWyvvvWRuqluAsLqYnBARabBmXDhe69EEI8P1L0GvyvOdAnF4Vl81E2CNSwCw551eGBrqjw2vdcbAdupX3Rj6u+jn1zrL3Yu2zh5V74/voTCMZoTfi5N6NcP2aT0N/nxYI8/qB6HGR8+YLvGpLZicEBFp0KdVfcx9qk21KsfWJPkJsQKa+rhi5agwdGvuDVsbCV5XU1yrpa+bxvMONFLdC1VDVYqJkeLrd/u31HhON4XaNKMiHtVvUUwA+7TS3vvVqXFdrH+tM1r5usGumrtIq2OKuUSJcyNNcFbxWMZ3GxGRFTLFJoSK/n2rB0ZFBOKzkR00tnNzUh5+2vqWcs+EtpB1uSXFlEBbpdv9M/pgbLcg6euPnlFd92XNuAilOTGv92qKaZGPVw79Prkbujf3hkQiwcWPBmHLlO46RKwfU/x39XJ1RICn4cN25obJCRGRmTLkd5i+f+u39ffAkuHt4evupPNnwoPq4vLSwWjjr//QlKr6MIoxK9ZHUTf/oapUf706DggPqqexfZ3KTSEVJwy/1bcFmvq4qo3XxgRzLyqMnJ14uqift2SpuJSYiMhMGbJaR/Z3r61Nzf/9qS7k94e0wfX7D9FWh4TGwc4GNhKgovJcqtKDda9GoJdM5dnqjMA42Kr/sDGL7VVR91+1fUMPnLqeq/f5DrzXp3oBmSH2nBARmakuWpbcygpt6AE3RzuEBdbFgiFtEODpjHlPtTZJXJp2QxbU/Ood270J5j7VRudVIF2bab53H4V9jHTt4Jg/pI3SsX6tfdGlaT2VheSCtKw0qqK4JFzT0nPFOTJVHAzduLIyCTUomVVD7Don7DkhIjJTE3o2RV0XB3Rvrj1J+euN7iirEOBgZ4Nx3ZtgnJoN/Krz62vFyA745sAlg/fy0YedTK+PLkMrsrVUNBkV0QhtGrjj6ZjDAB49D3tbG2ycqF9hPU8Xe+Q8KJW+njkgGKv3pcu93nEmS+lzGyZ0hruTPTo2rovEK/f1uqY6VY+npZ8bbuYWGeWcYrOYnpOYmBi0adMG4eHhYodCRFQjHOxs8GLnRmjspf2vdxsbCRzsTPsjfVhYALZO7YnAei5q24yOaAwA+E8b/YrWKSZgC4e2RYCnMz54uq1OvSKB9Vzwz5s9cGRWX61ttU2w1UX35t5Kx17u0lj6tboksFuzR59b/lz7asdQper5LFdT80Xb58yRxSQn0dHRSE1NRUJCgtihEBGRGo28XJD2wQB883JHnT8jkQCtfOXnogR518HhWX3xStcguV+i9pXzQ1QVdmsX4AF/HVaseMsMCTkbuGmiqt/rqqrjypo7+PEwmz5FzjZM6Kwx8aya5Ktv0rUluode7WsSh3WIiKxIn1b18c2BS3BxMN2Owc4GnHty72bYmZqJ5zsqF7vrXtnb4FXHAYdn9UVpeYVB16jiYGeDlIVRkACw1WEmbR0HWxSWlMPFwRYPSsp1uoaq6R+eLo8r4SpeVVMYVb0txtYuwPSFAA1lMT0nRERUfV2beeHv6O44PFP78EdN8nFzxKGZfTE1Unm34vruTkicG4nDs/rCyd5WZc0Vfbk62qncdVqVXyd1Ra+WPvhVZsNHY5d3Pz73PyqHghZUTuDVdDVVoVRtCdClaT3lN1VQnAws9pAPe06IiKxMqA6b+ZkbL9fqzRPRNRFRpa2/B9a9GqF0PLJ1fexOu42+wcp75Wj75S4bT+LcSNSr46CyXSs/zZV7AfnE5Y/J3ZB0LQevdg9CfnEZ3J3sETTrX9WfkwnS39MJ47oHYc3hy1qvVxOYnBARkag07aRcXT+M7YSl287i0xEdjH7uz0Z2wO60LES2fjT5V589hnzcHLHo6bZwtLOVJl7qEhRVPn+hA3aeyYK7sx3sZJYgd2xcFx0b1wUAuOvZw9S6BvZ70hWTEyIiqrX6Bvuib7B+K4d0IcGjkv7PhDWUHnsmLAC5D0vRKaiuys94Key+/HLXILnXi55uhwclZRjTNQibk27g0p1Cucq3VdI+GABnB1s83SGg2vcha0ioP97745RRz2koJidERCSKAW39sOfsbTzzhHF/yYrFxkaCV3s8qi9z8XaB9PiKkR1wLisfvVtq3njQz8MJ61/rAgDo39YPgiBIh15WjQrDxJ8SsXBoW4MnA4cEeOD0DfUVaKszydjYmJwQEZEoVr/0BErLBZPXZzEF7RNGH4/rDG7fAMNs9U/AZOeE9G/rh3MfDlDad0gf6qr3miMmJ0REJAqJRAIHOzOuBGYkxrrD6iQm+jLlPCBdWF66SkREZOaMvPGwUQgCUF9NoTY7ETaJ1MS8oiEiIqpljF0TxVCqEqY3+zZHaKAnhpvZvB8O6xAREelJW7phJvmInApBUIrrnf6t8E7/VkptxY6fPSdERERGZo7DOgBQ381J7BB0wuSEiIhIT/oM1ZhTJ8qqUWHo1swLP4/vLHYoGnFYh4iISEftG3rg1PVcPN+xofbGZsbPwwlB3nWwYUIXsUPRiskJERGRjn6f1A1ZeUUIrOeisZ05jepseK0z1hy5jEVPtxM7FJ0xOSEiItKRg52N1sREkdiTS7s190a35t7iBqEnzjkhIiIyMheZUvDmOjnWnLHnhIiIyMga1nXBjKhWcHOyg42NOU2J1Y23q+pibTWFyQkREZEJRPdpLnYIetswoTMKisrg5yHukmMmJ0RERAQA6NbMPOamcM4JERERmRUmJ0RERGRWmJwQERGRWWFyQkRERGaFyQkRERGZFSYnREREZFaYnBAREZFZYXJCREREZoXJCREREZkVJidERERkVpicEBERkVlhckJERERmhckJERERmRWL2ZU4JiYGMTExKCsrAwDk5eWJHBERERHpqur3tiAIWttKBF1amZHr168jMDBQ7DCIiIjIANeuXUPDhg01trG45KSiogI3b96Em5sbJBKJUc+dl5eHwMBAXLt2De7u7kY9t6XiM1HGZ6Ian4syPhNlfCaqWcNzEQQB+fn58Pf3h42N5lklFjOsU8XGxkZrxlVd7u7utfZ/DkPxmSjjM1GNz0UZn4kyPhPVavtz8fDw0KkdJ8QSERGRWWFyQkRERGaFyYkMR0dHLFiwAI6OjmKHYjb4TJTxmajG56KMz0QZn4lqfC7yLG5CLBEREdVu7DkhIiIis8LkhIiIiMwKkxMiIiIyK0xOiIiIyKwwOakUExODoKAgODk5oXPnzoiPjxc7JKNYsmQJwsPD4ebmhvr162PYsGE4d+6cXJuioiJER0fDy8sLrq6uePbZZ5GVlSXX5urVqxg8eDBcXFxQv359zJgxQ7rPUZV9+/bhiSeegKOjI5o3b461a9ea+vaMZunSpZBIJJg2bZr0mDU+lxs3buCll16Cl5cXnJ2dERISguPHj0vfFwQB8+fPR4MGDeDs7IzIyEhcuHBB7hzZ2dkYPXo03N3d4enpifHjx6OgoECuzalTp9CzZ084OTkhMDAQy5Ytq5H7M0R5eTnmzZuHJk2awNnZGc2aNcOiRYvk9gep7c/lwIEDGDJkCPz9/SGRSLB582a592vy/n/77TcEBwfDyckJISEh2Lp1q9HvVxeanklpaSlmzpyJkJAQ1KlTB/7+/njllVdw8+ZNuXPUtmdiVAIJGzduFBwcHIQffvhBOHPmjDBhwgTB09NTyMrKEju0aouKihLWrFkjpKSkCElJScKgQYOERo0aCQUFBdI2kyZNEgIDA4XY2Fjh+PHjQpcuXYRu3bpJ3y8rKxPatWsnREZGCidPnhS2bt0qeHt7C7Nnz5a2uXTpkuDi4iJMnz5dSE1NFVatWiXY2toK27dvr9H7NUR8fLwQFBQktG/fXpg6dar0uLU9l+zsbKFx48bC2LFjhWPHjgmXLl0SduzYIVy8eFHaZunSpYKHh4ewefNmITk5WRg6dKjQpEkT4eHDh9I2AwYMEEJDQ4WjR48KBw8eFJo3by6MGjVK+n5ubq7g6+srjB49WkhJSRF++eUXwdnZWfj6669r9H51tXjxYsHLy0v4559/hIyMDOG3334TXF1dhc8//1zaprY/l61btwpz5swR/vzzTwGA8Ndff8m9X1P3f/jwYcHW1lZYtmyZkJqaKsydO1ewt7cXTp8+bfJnoEjTM8nJyREiIyOFTZs2CWfPnhXi4uKEiIgIoWPHjnLnqG3PxJiYnAiCEBERIURHR0tfl5eXC/7+/sKSJUtEjMo0bt++LQAQ9u/fLwjCo28ie3t74bfffpO2SUtLEwAIcXFxgiA8+ia0sbERMjMzpW1Wr14tuLu7C8XFxYIgCMJ7770ntG3bVu5aI0eOFKKiokx9S9WSn58vtGjRQti1a5fQq1cvaXJijc9l5syZQo8ePdS+X1FRIfj5+QnLly+XHsvJyREcHR2FX375RRAEQUhNTRUACAkJCdI227ZtEyQSiXDjxg1BEAThyy+/FOrWrSt9RlXXbtWqlbFvySgGDx4svPrqq3LHhg8fLowePVoQBOt7Loq/iGvy/keMGCEMHjxYLp7OnTsLr7/+ulHvUV+qEjZF8fHxAgDhypUrgiDU/mdSXVY/rFNSUoLExERERkZKj9nY2CAyMhJxcXEiRmYaubm5AIB69eoBABITE1FaWip3/8HBwWjUqJH0/uPi4hASEgJfX19pm6ioKOTl5eHMmTPSNrLnqGpj7s8wOjoagwcPVordGp/Lli1b0KlTJzz//POoX78+wsLC8O2330rfz8jIQGZmptz9eHh4oHPnznLPxNPTE506dZK2iYyMhI2NDY4dOyZt8+STT8LBwUHaJioqCufOncP9+/dNfZt669atG2JjY3H+/HkAQHJyMg4dOoSBAwcCsN7nUqUm79+Svp8U5ebmQiKRwNPTEwCfiTZWn5zcvXsX5eXlcr9gAMDX1xeZmZkiRWUaFRUVmDZtGrp374527doBADIzM+Hg4CD9hqkie/+ZmZkqn0/Ve5ra5OXl4eHDh6a4nWrbuHEjTpw4gSVLlii9Z43P5dKlS1i9ejVatGiBHTt2YPLkyXjrrbewbt06AI/vSdP3SmZmJurXry/3vp2dHerVq6fXczMns2bNwgsvvIDg4GDY29sjLCwM06ZNw+jRowFY73OpUpP3r66NOT8f4NH8tZkzZ2LUqFHSTf2s/ZloY3G7EpPhoqOjkZKSgkOHDokdiuiuXbuGqVOnYteuXXBychI7HLNQUVGBTp064aOPPgIAhIWFISUlBV999RXGjBkjcnTi+fXXX7F+/Xps2LABbdu2RVJSEqZNmwZ/f3+rfi6km9LSUowYMQKCIGD16tVih2MxrL7nxNvbG7a2tkqrMLKysuDn5ydSVMY3ZcoU/PPPP9i7dy8aNmwoPe7n54eSkhLk5OTItZe9fz8/P5XPp+o9TW3c3d3h7Oxs7NuptsTERNy+fRtPPPEE7OzsYGdnh/3792PlypWws7ODr6+v1T2XBg0aoE2bNnLHWrdujatXrwJ4fE+avlf8/Pxw+/ZtuffLysqQnZ2t13MzJzNmzJD2noSEhODll1/G22+/Le1xs9bnUqUm719dG3N9PlWJyZUrV7Br1y5prwlgvc9EV1afnDg4OKBjx46IjY2VHquoqEBsbCy6du0qYmTGIQgCpkyZgr/++gt79uxBkyZN5N7v2LEj7O3t5e7/3LlzuHr1qvT+u3btitOnT8t9I1V9o1X9MuvatavcOaramOsz7NevH06fPo2kpCTpv06dOmH06NHSr63tuXTv3l1pmfn58+fRuHFjAECTJk3g5+cndz95eXk4duyY3DPJyclBYmKitM2ePXtQUVGBzp07S9scOHAApaWl0ja7du1Cq1atULduXZPdn6EePHgAGxv5H5W2traoqKgAYL3PpUpN3r8lfT9VJSYXLlzA7t274eXlJfe+NT4TvYg9I9ccbNy4UXB0dBTWrl0rpKamChMnThQ8PT3lVmFYqsmTJwseHh7Cvn37hFu3bkn/PXjwQNpm0qRJQqNGjYQ9e/YIx48fF7p27Sp07dpV+n7Vktn+/fsLSUlJwvbt2wUfHx+VS2ZnzJghpKWlCTExMWa7ZFYd2dU6gmB9zyU+Pl6ws7MTFi9eLFy4cEFYv3694OLiIvz888/SNkuXLhU8PT2Fv//+Wzh16pTw9NNPq1wyGhYWJhw7dkw4dOiQ0KJFC7nlkTk5OYKvr6/w8ssvCykpKcLGjRsFFxcXs1gyq8qYMWOEgIAA6VLiP//8U/D29hbee+89aZva/lzy8/OFkydPCidPnhQACJ9++qlw8uRJ6cqTmrr/w4cPC3Z2dsLHH38spKWlCQsWLBBt2aymZ1JSUiIMHTpUaNiwoZCUlCT3s1d25U1teybGxOSk0qpVq4RGjRoJDg4OQkREhHD06FGxQzIKACr/rVmzRtrm4cOHwhtvvCHUrVtXcHFxEZ555hnh1q1bcue5fPmyMHDgQMHZ2Vnw9vYW3nnnHaG0tFSuzd69e4UOHToIDg4OQtOmTeWuYQkUkxNrfC7/+9//hHbt2gmOjo5CcHCw8M0338i9X1FRIcybN0/w9fUVHB0dhX79+gnnzp2Ta3Pv3j1h1KhRgqurq+Du7i6MGzdOyM/Pl2uTnJws9OjRQ3B0dBQCAgKEpUuXmvzeDJWXlydMnTpVaNSokeDk5CQ0bdpUmDNnjtwvmdr+XPbu3avy58iYMWMEQajZ+//111+Fli1bCg4ODkLbtm2Ff//912T3rYmmZ5KRkaH2Z+/evXul56htz8SYJIIgU+aQiIiISGRWP+eEiIiIzAuTEyIiIjIrTE6IiIjIrDA5ISIiIrPC5ISIiIjMCpMTIiIiMitMToiIiMisMDkhIiIis8LkhIiIiMwKkxMiIiIyK0xOiIiIyKwwOSEiIiKz8v8Xe50j5lc/lQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.plot(train_metrics_history[\"train_loss\"], label=\"Loss value during the training\")\n", + "plt.yscale(\"log\")\n", + "plt.legend()" + ] + }, + { + "cell_type": "markdown", + "id": "66250bf2-3d88-40ad-87e3-7d2b906fd860", + "metadata": {}, + "source": [ + "And eval set Loss and Accuracy - Accuracy does continue to rise, though it's hard-earned progress after about the 5th epoch. Based on the training statistics, it's fair to say the process starts overfitting after roughly that 5th epoch." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "64d54051-358b-4de8-b5b3-04bebf18018f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0MAAANECAYAAAByxfRXAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAqHNJREFUeJzs3Xl4VOXd//HPzCSZ7HtIWAIBFBCFgCARxAVFU9C4VaWgglS0Wn1aRetPEMSlStunIj4tiloR22IFK0VbKBZRiwvKZhBElH1PQoDsySSZOb8/khkYSSAJSc4s79d1zdXmzJnJd6LOmc/c9/29LYZhGAIAAACAIGM1uwAAAAAAMANhCAAAAEBQIgwBAAAACEqEIQAAAABBiTAEAAAAICgRhgAAAAAEJcIQAAAAgKBEGAIAAAAQlAhDAAAAAIISYQhowBNPPCGLxWJ2GQHpsssu02WXXWZ2GQCAILN7925ZLBbNnz/f7FLgQwhDOKX58+fLYrFo3bp1ZpcCtNizzz6rJUuWmF0GgHby4osvymKxKCsry+xSEOS2bNmiJ554Qrt37za7FDSCMAQg4BGGgOCyYMECZWRkaM2aNdq+fbvZ5SCIbdmyRU8++SRhyIcRhgAAQMDYtWuXPv/8c82aNUspKSlasGCB2SU1qry83OwSgKBHGEKr+OqrrzRq1CjFxsYqOjpaV1xxhb744guvc2pqavTkk0/q7LPPVnh4uJKSkjR8+HCtWLHCc05eXp4mTpyoLl26yG63q2PHjrruuutO+Y3K73//e1ksFu3Zs+ek+6ZMmaKwsDAdO3ZMkvTJJ5/o5ptvVteuXWW325Wenq4HH3xQlZWVp3x9p5pnbLFY9MQTT3gdO3DggH76058qNTVVdrtd5557rubNm3fK3+FWW1urp59+Wj179pTdbldGRoamTp0qh8PhdV5GRoauueYaffrppxoyZIjCw8PVo0cP/fnPf27S73G5XJo9e7bOPfdchYeHKzU1VT/72c88fytJuuaaa9SjR48GHz906FANHjzY8/Prr7+uyy+/XB06dJDdblffvn310ksvNamWhqxYsULDhw9XfHy8oqOj1bt3b02dOtXrHIfDoRkzZuiss87y/PN85JFHvP5WFotF5eXleuONN2SxWGSxWHTHHXe0uC4Avm3BggVKSEjQ1VdfrZtuuqnRMFRUVKQHH3xQGRkZstvt6tKli8aPH6/CwkLPOVVVVXriiSfUq1cvhYeHq2PHjrrxxhu1Y8cOSdLHH38si8Wijz/+2Ou5G7pm3HHHHYqOjtaOHTs0evRoxcTE6NZbb5XUvGvT1q1bdcsttyglJUURERHq3bu3HnvsMUnSRx99JIvFon/84x8nPe7NN9+UxWLR6tWrT/n327lzp26++WYlJiYqMjJSF154oZYuXep1jvt1L1q0SM8884y6dOmi8PBwXXHFFU0eiTvddTI/P18hISF68sknT3rsd999J4vFoj/+8Y+SpKNHj+rhhx9Wv379FB0drdjYWI0aNUobN25sUi0/1JTPK1LdP4ubbrpJiYmJCg8P1+DBg/Xee+957p8/f75uvvlmSdKIESM816Af/vsCc4WYXQD83zfffKOLL75YsbGxeuSRRxQaGqqXX35Zl112mf773/965mw/8cQTmjlzpiZNmqQhQ4aopKRE69at04YNG3TllVdKkn784x/rm2++0f/8z/8oIyNDBQUFWrFihfbu3auMjIwGf/8tt9yiRx55RIsWLdKvfvUrr/sWLVqkq666SgkJCZKkt99+WxUVFbr33nuVlJSkNWvW6A9/+IP279+vt99+u1X+Hvn5+brwwgtlsVh0//33KyUlRf/+97915513qqSkRA888MApHz9p0iS98cYbuummm/TQQw/pyy+/1MyZM/Xtt9+edIHbvn27brrpJt15552aMGGC5s2bpzvuuEODBg3Sueeee8rf87Of/Uzz58/XxIkT9Ytf/EK7du3SH//4R3311Vf67LPPFBoaqjFjxmj8+PFau3atLrjgAs9j9+zZoy+++EL/+7//6zn20ksv6dxzz9W1116rkJAQ/fOf/9TPf/5zuVwu3Xfffc36G37zzTe65ppr1L9/fz311FOy2+3avn27PvvsM885LpdL1157rT799FPdfffdOuecc7Rp0yY9//zz+v777z3T4v7yl794/p27++67JUk9e/ZsVj0A/MeCBQt04403KiwsTGPHjtVLL7100ntYWVmZLr74Yn377bf66U9/qvPPP1+FhYV67733tH//fiUnJ8vpdOqaa67RypUr9ZOf/ES//OUvVVpaqhUrVmjz5s0teh+pra1Vdna2hg8frt///veKjIyU1PRr09dff62LL75YoaGhuvvuu5WRkaEdO3bon//8p5555hlddtllSk9P14IFC3TDDTec9Hfp2bOnhg4d2mh9+fn5GjZsmCoqKvSLX/xCSUlJeuONN3Tttdfq73//+0nP+Zvf/EZWq1UPP/ywiouL9bvf/U633nqrvvzyy1P+HZpynUxNTdWll16qRYsWacaMGV6PX7hwoWw2mydo7Ny5U0uWLNHNN9+s7t27Kz8/Xy+//LIuvfRSbdmyRZ06dTr9P5wTNOXzyjfffKOLLrpInTt31qOPPqqoqCgtWrRI119/vd555x3dcMMNuuSSS/SLX/xC//d//6epU6fqnHPOkSTP/8JHGMApvP7664YkY+3atY2ec/311xthYWHGjh07PMcOHjxoxMTEGJdcconnWGZmpnH11Vc3+jzHjh0zJBn/+7//2+w6hw4dagwaNMjr2Jo1awxJxp///GfPsYqKipMeO3PmTMNisRh79uzxHJsxY4Zx4n8eu3btMiQZr7/++kmPl2TMmDHD8/Odd95pdOzY0SgsLPQ67yc/+YkRFxfXYA1uubm5hiRj0qRJXscffvhhQ5Lx4Ycfeo5169bNkGSsWrXKc6ygoMCw2+3GQw891OjvMAzD+OSTTwxJxoIFC7yOL1++3Ot4cXFxg8/3u9/97qS/WUOvKzs72+jRo4fXsUsvvdS49NJLT1nf888/b0gyDh8+3Og5f/nLXwyr1Wp88sknXsfnzp1rSDI+++wzz7GoqChjwoQJp/ydAPzfunXrDEnGihUrDMMwDJfLZXTp0sX45S9/6XXe448/bkgyFi9efNJzuFwuwzAMY968eYYkY9asWY2e89FHHxmSjI8++sjr/oauGRMmTDAkGY8++uhJz9fUa9Mll1xixMTEeB07sR7DMIwpU6YYdrvdKCoq8hwrKCgwQkJCvK5VDXnggQcMSV7vq6WlpUb37t2NjIwMw+l0er3uc845x3A4HJ5zX3jhBUOSsWnTplP+nqZeJ19++eUGn69v377G5Zdf7vm5qqrKU5vbrl27DLvdbjz11FNexxq7lp/odJ9XDMMwrrjiCqNfv35GVVWV55jL5TKGDRtmnH322Z5jb7/9doP/jsB3ME0OZ8TpdOo///mPrr/+eq/pVB07dtS4ceP06aefqqSkRJIUHx+vb775Rtu2bWvwuSIiIhQWFqaPP/7Ya6pWU4wZM0br16/3TF2Q6r45stvtuu6667x+h1t5ebkKCws1bNgwGYahr776qlm/syGGYeidd95RTk6ODMNQYWGh55adna3i4mJt2LCh0ccvW7ZMkjR58mSv4w899JAknTRVoW/fvrr44os9P6ekpKh3797auXPnKet8++23FRcXpyuvvNKrxkGDBik6OlofffSRJHmmGixatEiGYXgev3DhQl144YXq2rWr59iJf9vi4mIVFhbq0ksv1c6dO1VcXHzKen4oPj5ekvTuu+/K5XI1+hrOOecc9enTx+s1XH755ZLkeQ0AgseCBQuUmpqqESNGSKqbJjtmzBi99dZbcjqdnvPeeecdZWZmnjTS4X6M+5zk5GT9z//8T6PntMS999570rGmXJsOHz6sVatW6ac//anXe+8P6xk/frwcDof+/ve/e44tXLhQtbW1uu22205Z27JlyzRkyBANHz7ccyw6Olp33323du/erS1btnidP3HiRIWFhXl+dl+PTnUNas518sYbb1RISIgWLlzoefzmzZu1ZcsWjRkzxnPMbrfLaq37SOt0OnXkyBHP9OpTXXMbc7rPK0ePHtWHH36oW265RaWlpZ76jxw5ouzsbG3btk0HDhxo9u+FOQhDOCOHDx9WRUWFevfufdJ955xzjlwul/bt2ydJeuqpp1RUVKRevXqpX79++tWvfqWvv/7ac77dbtdvf/tb/fvf/1ZqaqouueQS/e53v1NeXt5p67j55ptltVo9b5iGYejtt9/2rGNy27t3r+644w4lJiYqOjpaKSkpuvTSSyWp2R/YG3L48GEVFRXplVdeUUpKitdt4sSJkqSCgoJGH79nzx5ZrVadddZZXsfT0tIUHx9/0rqoH14QJSkhIeG0YXLbtm0qLi5Whw4dTqqzrKzMq8YxY8Zo3759nnnmO3bs0Pr1670uRJL02WefaeTIkYqKilJ8fLxSUlI8a3ya+7cdM2aMLrroIk2aNEmpqan6yU9+okWLFnkFo23btumbb745qf5evXpJOvXfGUDgcTqdeuuttzRixAjt2rVL27dv1/bt25WVlaX8/HytXLnSc+6OHTt03nnnnfL5duzYod69eyskpPVWFISEhKhLly4nHW/KtckdME5Xd58+fXTBBRd4rZVasGCBLrzwwpOuLT+0Z8+eRq/n7vtP9MNrkHtK+qmuQc25TiYnJ+uKK67QokWLPI9fuHChQkJCdOONN3qOuVwuPf/88zr77LNlt9uVnJyslJQUff311y26tp/u88r27dtlGIamT59+0mtwT+njGuQ/WDOEdnPJJZdox44devfdd/Wf//xHf/rTn/T8889r7ty5mjRpkiTpgQceUE5OjpYsWaL3339f06dP18yZM/Xhhx9q4MCBjT53p06ddPHFF2vRokWaOnWqvvjiC+3du1e//e1vPec4nU5deeWVOnr0qP7f//t/6tOnj6KionTgwAHdcccdjY5ASI1/C3jiN42SPM9x2223acKECQ0+pn///o3+ntP9vh+y2WwNHj9xFKchLpdLHTp0aHRhcUpKiuf/5+TkKDIyUosWLdKwYcO0aNEiWa1Wz1xtqe5DwxVXXKE+ffpo1qxZSk9PV1hYmJYtW6bnn3/+lH/bhkRERGjVqlX66KOPtHTpUi1fvlwLFy7U5Zdfrv/85z+y2WxyuVzq16+fZs2a1eBzpKenN+t3AvBvH374oQ4dOqS33npLb7311kn3L1iwQFdddVWr/s6mXhvcThzBOPHcll6bGjN+/Hj98pe/1P79++VwOPTFF194mg20ppZcg5p7nfzJT36iiRMnKjc3VwMGDNCiRYt0xRVXKDk52XPOs88+q+nTp+unP/2pnn76aSUmJspqteqBBx5o0d/vdJ9X3M/58MMPKzs7u8HnOF3whO8gDOGMpKSkKDIyUt99991J923dulVWq9XrQ2liYqImTpyoiRMnqqysTJdccomeeOIJTxiS6ha3P/TQQ3rooYe0bds2DRgwQM8995z++te/nrKWMWPG6Oc//7m+++47LVy4UJGRkcrJyfHcv2nTJn3//fd64403NH78eM/xH3aHaYj7266ioiKv4z/8liwlJUUxMTFyOp0aOXLkaZ/3h7p16yaXy6Vt27Z5LbDMz89XUVGRunXr1uznbEjPnj31wQcf6KKLLvKantGQqKgoXXPNNXr77bc1a9YsLVy4UBdffLHXgtR//vOfcjgceu+997y+KTyTqWpWq1VXXHGFrrjiCs2aNUvPPvusHnvsMX300UcaOXKkevbsqY0bN+qKK644bXg8kyktAPzDggUL1KFDB82ZM+ek+xYvXqx//OMfmjt3riIiItSzZ09t3rz5lM/Xs2dPffnll6qpqVFoaGiD5zT12nAqTb02uaein65uqS5ATJ48WX/7299UWVnpaYhzOt26dWv0eu6+/0w19zp5/fXX62c/+5ln5sf333+vKVOmeJ3z97//XSNGjNBrr73mdbyoqMgrNDXHqT6vuP9ZhIaGnvY1cP3xfUyTwxmx2Wy66qqr9O6773q1v87Pz9ebb76p4cOHe6apHTlyxOux0dHROuusszxtkCsqKlRVVeV1Ts+ePRUTE3NSW+mG/PjHP5bNZtPf/vY3vf3227rmmmsUFRXlVavk/Y2VYRh64YUXTvvcsbGxSk5O1qpVq7yOv/jii14/22w2/fjHP9Y777zT4AXr8OHDp/w9o0ePliTNnj3b67h79OPqq68+ba1Nccstt8jpdOrpp58+6b7a2tqTLuxjxozRwYMH9ac//UkbN2486aLa0N+2uLhYr7/+eovqO3r06EnHBgwYIEmefxduueUWHThwQK+++upJ51ZWVnrt3xEVFXXSawIQOCorK7V48WJdc801uummm0663X///SotLfW0Pf7xj3+sjRs3NtiC2v0+9uMf/1iFhYUNjqi4z+nWrZtsNttprw2n0tRrU0pKii655BLNmzdPe/fubbAet+TkZI0aNUp//etftWDBAv3oRz9qUigYPXq01qxZ49V+u7y8XK+88ooyMjLUt2/fJr+uxjT3OhkfH6/s7GwtWrRIb731lsLCwnT99def9Jw//Bu8/fbbLV63c7rPKx06dNBll12ml19+WYcOHTrla3B/DuEa5LsYGUKTzJs3T8uXLz/p+C9/+Uv9+te/9uwJ8/Of/1whISF6+eWX5XA49Lvf/c5zbt++fXXZZZdp0KBBSkxM1Lp16/T3v/9d999/v6S6b3uuuOIK3XLLLerbt69CQkL0j3/8Q/n5+frJT35y2ho7dOigESNGaNasWSotLT3pA3ufPn3Us2dPPfzwwzpw4IBiY2P1zjvvNLlZw6RJk/Sb3/xGkyZN0uDBg7Vq1Sp9//33J533m9/8Rh999JGysrJ01113qW/fvjp69Kg2bNigDz74oMEP+m6ZmZmaMGGCXnnlFRUVFenSSy/VmjVr9MYbb+j666/3LAo+U5deeql+9rOfaebMmcrNzdVVV12l0NBQbdu2TW+//bZeeOEF3XTTTZ7z3XtiPPzww54L2YmuuuoqhYWFKScnRz/72c9UVlamV199VR06dGjwQnE6Tz31lFatWqWrr75a3bp1U0FBgV588UV16dLFs7D39ttv16JFi3TPPffoo48+0kUXXSSn06mtW7dq0aJFev/99z37IA0aNEgffPCBZs2apU6dOql79+6elu8A/N97772n0tJSXXvttQ3ef+GFF3o2YB0zZox+9atf6e9//7tuvvlm/fSnP9WgQYN09OhRvffee5o7d64yMzM1fvx4/fnPf9bkyZO1Zs0aXXzxxSovL9cHH3ygn//857ruuusUFxenm2++WX/4wx9ksVjUs2dP/etf/2rWepHmXJv+7//+T8OHD9f555+vu+++W927d9fu3bu1dOlS5ebmep07fvx4z/t4Q198NeTRRx/V3/72N40aNUq/+MUvlJiYqDfeeEO7du3SO++8c9IUv5Zq7nVyzJgxuu222/Tiiy8qOzvb02TH7ZprrtFTTz2liRMnatiwYdq0aZMWLFjQ6D55p3O6zyuSNGfOHA0fPlz9+vXTXXfdpR49eig/P1+rV6/W/v37PXscDRgwQDabTb/97W9VXFwsu93u2ZMPPqJ9m9fB37hbazd227dvn2EYhrFhwwYjOzvbiI6ONiIjI40RI0YYn3/+uddz/frXvzaGDBlixMfHGxEREUafPn2MZ555xqiurjYMwzAKCwuN++67z+jTp48RFRVlxMXFGVlZWcaiRYuaXO+rr75qSDJiYmKMysrKk+7fsmWLMXLkSCM6OtpITk427rrrLmPjxo0ntdr8YWttw6hrfXrnnXcacXFxRkxMjHHLLbcYBQUFJ7XWNgzDyM/PN+677z4jPT3dCA0NNdLS0owrrrjCeOWVV077Gmpqaownn3zS6N69uxEaGmqkp6cbU6ZM8WrfaRh1rbUbav3ZlNbVbq+88ooxaNAgIyIiwoiJiTH69etnPPLII8bBgwdPOvfWW281JBkjR45s8Lnee+89o3///kZ4eLiRkZFh/Pa3v/W0pt21a1ez6lu5cqVx3XXXGZ06dTLCwsKMTp06GWPHjjW+//57r/Oqq6uN3/72t8a5555r2O12IyEhwRg0aJDx5JNPGsXFxZ7ztm7dalxyySVGRESEIYk220CAycnJMcLDw43y8vJGz7njjjuM0NBQTzvnI0eOGPfff7/RuXNnIywszOjSpYsxYcIEr3bPFRUVxmOPPeZ5P05LSzNuuukmr60kDh8+bPz4xz82IiMjjYSEBONnP/uZsXnz5gZba0dFRTVYW1OvTYZhGJs3bzZuuOEGIz4+3ggPDzd69+5tTJ8+/aTndDgcRkJCghEXF9fg9bAxO3bsMG666SbP8w8ZMsT417/+5XWOu7X222+/7XW8qa2rDaN518mSkhLP+/df//rXk+6vqqoyHnroIaNjx45GRESEcdFFFxmrV68+6XrT1PpO93nFbceOHcb48eONtLQ0IzQ01OjcubNxzTXXGH//+9+9znv11VeNHj16GDabjTbbPshiGKdZaQ0AAAC/Ultbq06dOiknJ+ektTQAjmPNEAAAQIBZsmSJDh8+7NWUAcDJGBkCAAAIEF9++aW+/vprPf3000pOTm7RpqNAMGFkCAAAIEC89NJLuvfee9WhQwf9+c9/NrscwOcxMgQAAAAgKDEyBAAAACAoEYYAAAAABKWA2XTV5XLp4MGDiomJkcViMbscAAgahmGotLRUnTp1arVNGQMB1yUAME9Tr00BE4YOHjyo9PR0s8sAgKC1b98+denSxewyfAbXJQAw3+muTQEThmJiYiTVveDY2FiTqwGA4FFSUqL09HTP+zDqcF0CAPM09doUMGHIPQUhNjaWiw4AmICpYN64LgGA+U53bWJyNwAAAICgRBgCAAAAEJQIQwAAAACCEmEIAAAAQFAiDAEAAAAISoQhAAAAAEGJMAQAAAAgKBGGAAAAAAQlwhAAAACAoEQYAgAAABCUCEMAAAAAghJhCAAAAEBQIgwBAAAACEqEIQAAAABBiTAEAAAAICgRhgAAAAAEJcIQAAAAgKBEGAIAAAAQlAhDAAAAAIISYQgAAABAUCIMAQAAAAhKhCEAAAAAQYkwBAAAACAoEYYAAAAABCXCEAAAAICgRBgCAAAAEJQIQwAAAACCEmEIAAAAQFAiDAEAAAAISoQhAAAAAEGJMAQAAADApxiGIafLaPPfQxiS9M+NB3Xz3M/1wgfbzC4FAAAACFpVNU4tXLtXP5r9id7ZsL/Nf19Im/8GP3Csolprdx9TcrTd7FIAAACAoHO41KG/fLFHC77YoyPl1ZKkt9bs1S2D09v09xKGJCVEhkmS5w8PAAAAoO1tOViieZ/t0nu5B1XtdEmSOsdH6I5hGbrlgrYNQhJhSJKUFFUXho4RhgAAAIA25XIZ+ui7Ar326S59vuOI5/j5XeN15/Aeyj43VSG29lnNQxiSlBhdF4aOEoYAAACANlFRXau/r9+v1z/brV2F5ZIkm9WiUeel6c7h3TWwa0K710QYkpToHhmqqJbLZchqtZhcEQAAABAYDhZV6o3Vu/W3L/eqpKpWkhQTHqJxQ7pq/LAMdY6PMK02wpCOrxlyGVJRZY0nHAEAAABomdx9RXrt011atumQp012RlKkJl7UXTcN6qIou/lRxPwKfECozarY8BCVVNXqaHk1YQgAAABogVqnS//Zkq/XPt2l9XuOeY5f2CNRdw7vocv7dJDNh2ZhEYbqJUXbPWEIAAAAQNOVVNVo0dp9ev2z3TpQVClJCrVZlJPZSXcO765zO8WZXGHDCEP1EqPCtKuwXEfLHWaXAgAAAPiFvUcq9Prnu/T2uv0qc9StB0qMCtOtWV11+4Xd1CE23OQKT40wVM89NY69hgAAAIDGGYahtbuP6bVPd2rFlnzVLwfS2R2i9dPh3XXDwM4KD7WZW2QTEYbqJdY3UThaRhgCAAAAfqi61qVlmw7ptU93adOBYs/xS3ql6M7h3XXJ2cmyWHxnPVBTEIbqefYaqiAMAQAAAG5FFdVa8OVe/Xn1buWX1C0psYdYdeP5nfXTi7rr7NQYkytsOcJQvaQoNl4FAAAA3HYcLtO8T3fpnQ37VVXjkiR1iLFr/NBuGpfVLSA6MBOG6iUShgAAABDkDMPQZ9uP6LVPd+qj7w57jp/bKVZ3Du+ua/p3UliI1cQKWxdhqF6Cu4ECa4YAAAAQZKpqnHov96DmfbZLW/NKJUkWizTynFTdOby7sron+t16oKYgDNVzT5M7xpohAAAABLjKaqe+2ntMa3Yf1drdR7VhT5Eqa5ySpMgwm24ZnK47hmUoIznK5ErbFmGo3omttQ3DCMjkCwAAgOBUXFGjtfXBZ83uo9q0v1i17p7Y9TrHR2jCsG4ac0FXxUWEmlRp+yIM1UuKskuqaxlYXu1UtJ0/DQAAAPxTXnFV3ajPrroA9F1+qQzv7KO02HAN6Z6oC7onKqt7os5KiZbVGlwDAnzirxcRZlN4qFVVNS4dLasmDAEAAMAvGIah3UcqtHbXUX1ZH372Hq046bweyVF14ScjUUO6J6pLQkTQz4biE/8JkqLsOlBUqSPlDnVNijS7HAAAAOAkTpehrXklWrurbsrbml3HVFjm8DrHapHO6RirId0TNSQjUYMzEpUSYzepYt9FGDpBYlSYDhRV0kQBAAAAPsNR69Sm/cX1weeo1u85ptKqWq9zwmxWZabHeUZ+BnVLUEx4cKz7OROEoRMk0l4bAAAAJitz1GrDnmN1zQ52HVXuviI5al1e50TbQ3R+twQNyUjQkO5J6t8lTuGhNpMq9l+EoRMksfEqAAAA2tmRMofW7j7m6fb2zcESOX/Q6S0pKkwXZNQ1OxiSkahzOsYoxBY4m5+ahTB0ggTCEAAAANpYjdOlT7cVasW3+Vqz66i2F5SddE7n+Ahl1Xd6uyAjUT1TooK+2UFbIAydIJEwBAAAgDbgchlau/uo3tt4UMs2HdKxihqv+8/uEF3X7KA+/HSKjzCp0uBCGDoB0+QAAADQWgzD0DcHS/TexoP658aDOlRc5bkvOTpMo/t11PCzkjU4I9HzpTzaF2HoBJ4GCoQhAAAAtNDOw2V6b+NBvbfxoHYeLvccj7GHKPu8NF03oJOG9khizY8PIAydgGlyAAAAaIlDxZX618ZDenfjAW0+UOI5bg+xauQ5qcrJ7KTLeqfQ8c3HEIZOQBgCAABAUx0rr9ayzYf0bu5Brd19VEZ9Azib1aKLz07WtZmddGXfVPb78WGEoRMkRdXtylvmqJWj1il7CMkdAAAAx5U5arViS57eyz2oT7YVqvaEFthDMhKVM6CTRp+XpqRou4lVoqkIQyeIjQhRiNWiWpehY+U1SosjDAEAAAQ7R61T//3usN7deFArv81XVc3xDVDP7RSrazM76ZrMTupMBzi/Qxg6gcViUUJUmA6XOnSk3KG0uHCzSwIAAIAJnC5DX+w8ondzD2j55jyVVNV67uueHKWczE66NrOTzuoQbWKVOFOEoR9IjKwLQ6wbAgAACC6GYeirfUV6L/eglm46pMOlDs99abHhysnsqGszO+u8zrFsgBogCEM/QBMFAACA4PJdXqne23hA/9x4SHuPVniOx0eGanS/jro2s5OGZCTKaiUABZpmNzdftWqVcnJy1KlTJ1ksFi1ZsuSU5y9evFhXXnmlUlJSFBsbq6FDh+r9998/6bw5c+YoIyND4eHhysrK0po1a5pbWqtIjCYMAYA/au51ZPbs2erdu7ciIiKUnp6uBx98UFVVxzdEfOKJJ2SxWLxuffr0aeuXAaCd7DtaoTkfbdePZq9S9uxVmvPRDu09WqHIMJuuH9BJ8+4YrDVTR+rZG/rpwh5JBKEA1eyRofLycmVmZuqnP/2pbrzxxtOev2rVKl155ZV69tlnFR8fr9dff105OTn68ssvNXDgQEnSwoULNXnyZM2dO1dZWVmaPXu2srOz9d1336lDhw7Nf1VnIImRIQDwO829jrz55pt69NFHNW/ePA0bNkzff/+97rjjDlksFs2aNctz3rnnnqsPPvjA83NICBMqAH92uNShpV/XbYa6YW+R53iozaLLenfQtZmdNPKcVEWE0UQrWDT7XX3UqFEaNWpUk8+fPXu218/PPvus3n33Xf3zn//0hKFZs2bprrvu0sSJEyVJc+fO1dKlSzVv3jw9+uijzS3xjCRE1oWhI4QhAPAbzb2OfP7557rooos0btw4SVJGRobGjh2rL7/80uu8kJAQpaWltf0LANCmXC5Dr3++W79bvlWO2rpOcFaLNLRnkq7N7KQfndtRcZHsBRSMmj1N7ky5XC6VlpYqMTFRklRdXa3169dr5MiRx4uyWjVy5EitXr26vctTknuaXBlhCAD8QUuuI8OGDdP69es9U+l27typZcuWafTo0V7nbdu2TZ06dVKPHj106623au/evW33QgC0iQNFlbr1T1/q6X9tkaPWpX6d4/T4NX31xZQrtGDShRpzQVeCUBBr9/H+3//+9yorK9Mtt9wiSSosLJTT6VRqaqrXeampqdq6dWujz+NwOORwHO/wUVJS0ir1eRooVBCGAMAftOQ6Mm7cOBUWFmr48OEyDEO1tbW65557NHXqVM85WVlZmj9/vnr37q1Dhw7pySef1MUXX6zNmzcrJibmpOdsq+sSgJYxDEOLNxzQE+99o1JHrSJCbXrs6nN0a1ZXOsHBo11Hht588009+eSTWrRo0RmvBZo5c6bi4uI8t/T09FapkW5yABD4Pv74Yz377LN68cUXtWHDBi1evFhLly7V008/7Tln1KhRuvnmm9W/f39lZ2dr2bJlKioq0qJFixp8zra6LgFoviNlDt371w166O2NKnXUamDXeC375cW67cJuBCF4abcw9NZbb2nSpElatGiR11SG5ORk2Ww25efne52fn59/ynnaU6ZMUXFxsee2b9++VqkzKcouiTAEAP6iJdeR6dOn6/bbb9ekSZPUr18/3XDDDXr22Wc1c+ZMuVyuBh8THx+vXr16afv27Q3e31bXJQDNs/LbfGXP/kTLv8lTiNWiX2X31ts/G6ruyVFmlwYf1C5h6G9/+5smTpyov/3tb7r66qu97gsLC9OgQYO0cuVKzzGXy6WVK1dq6NChjT6n3W5XbGys1601JETVzRk9VlEtp8tolecEALSdllxHKioqZLV6XwJttrruUYbR8Ht/WVmZduzYoY4dOzZ4f1tdlwA0TZmjVo++87XufGOdCssc6pUarSX3XaT7RpylEFu7L5OHn2j2mqGysjKvb8V27dql3NxcJSYmqmvXrpoyZYoOHDigP//5z5LqpsZNmDBBL7zwgrKyspSXlydJioiIUFxcnCRp8uTJmjBhggYPHqwhQ4Zo9uzZKi8v93QFak/ubnKGIRVX1nimzQEAfNfpriPjx49X586dNXPmTElSTk6OZs2apYEDByorK0vbt2/X9OnTlZOT4wlFDz/8sHJyctStWzcdPHhQM2bMkM1m09ixY017nQAatmbXUT30dq72Ha2UxSJNGt5dD13VW+GhtMjGqTU7DK1bt04jRozw/Dx58mRJ0oQJEzR//nwdOnTIq9vOK6+8otraWt1333267777PMfd50vSmDFjdPjwYT3++OPKy8vTgAEDtHz58pMWw7aHUJtVcRGhKq6s0dFyB2EIAPzA6a4je/fu9RoJmjZtmiwWi6ZNm6YDBw4oJSVFOTk5euaZZzzn7N+/X2PHjtWRI0eUkpKi4cOH64svvlBKSkq7vz4ADXPUOjVrxfd6ZdVOGYbUOT5Cz92SqQt7JJldGvyExWhsPoCfKSkpUVxcnIqLi894asLlv/9YOwvLtfDuC5XFf0wAcEqt+f4bSPi7AG1ry8ESTV6Uq615pZKkmwd10eM5fRUTTptsNP09mK20G5AQFSYVltNEAQAAwMc4XYZeXrVDz6/4XjVOQ0lRYZp5Yz9ddS4bJKP5CEMNcE+NO0IYAgAA8Bl7j1Ro8qJcrdtzTJJ0Zd9Uzbyxn5Kj7SZXBn9FGGpAUn0YOkYYAgAAMJ1hGHpr7T49/a8tqqh2Ktoeohk5fXXToC7sG4QzQhhqACNDAAAAvqGgtEqPvrNJH24tkCRldU/U72/OVHpipMmVIRAQhhrgDkOsGQIAADDPvzcd0tR/bNKxihqFhVj1SHZv/fSi7rJaGQ1C6yAMNYAwBAAAYJ7iyho9+d43WvzVAUlS346xen7MAPVOizG5MgQawlADCEMAAADm+Gx7oX719kYdLK6S1SL9/LKz9IsrzlZYiPX0DwaaiTDUgKSouo4khCEAAID2UVXj1G+Xb9Xrn+2WJGUkReq5WwZoULcEcwtDQCMMNSAx+vjIkGEYdCkBAABoQ1/vL9KDC3O143C5JOnWrK567OpzFBnGR1W0Lf4Na0BiZF0Yqna6VOaoZSdjAACANlDjdOnFj3boDx9uU63LUIcYu357U3+N6N3B7NIQJAhDDYgIsyki1KbKGqeOldcQhgAAAFrZjsNlmrwwVxv3F0uSru7fUb++7jwl1K/dBtoDYagRiVFhOlBUqSPlDnVNoo89AABAa3C5DP3liz2a+e9vVVXjUmx4iJ6+/jxdm9mJpQlod4ShRiRF14UhmigAAAC0jkPFlfrV21/r0+2FkqSLz07W727qr45xESZXhmBFGGpEQv26oSOEIQAAgDNiGIbezT2o6e9uVmlVrcJDrZo6+hzdltWNDVRhKsJQI5LYawgAAOCMHSuv1rQlm7V00yFJUmZ6vGbdkqmeKdEmVwYQhhrl3nj1GGEIAACg2Ry1Ti3bdEgzl21VQalDIVaLfnHF2fr5ZT0VYmMDVfgGwlAj3HsNMU0OAACg6fYdrdCCL/fq7XX7PJ+jeqZE6fkxA9S/S7y5xQE/QBhqBNPkAAAAmsbpMvTR1gL99cs9+u/3h2UYdcfTYsN1a1ZX3XVJD4WH2swtEmgAYagRNFAAAAA4tYLSKi1au09/W7NPB4oqPccvPjtZt13YTVf06cCUOPg0wlAjkqJZMwQAAPBDhmHoi51H9dcv9+j9zXmqddUNAyVEhurmwekaN6SrMpKjTK4SaBrCUCMSo+ySmCYHAAAgScWVNVq8Yb8WfLlX2wvKPMfP7xqv2y7sptH9OjIVDn6HMNQIdze5MketHLVO2UP4jxsAAASfzQeK9dcv9ujd3IOqrHFKkiLDbLp+YGfdltVNfTvFmlwh0HKEoUbEhocoxGpRrcvQ0fJqdkYGAABBo6rGqX9uPKi/frlXG/cVeY73To3RbRd21fUDOysmPNS8AoFWQhhqhMViUUJUmA6XOnSkjDAEAAAC387DZVrw5V79ff1+FVfWSJJCbRaNOq+jbh/aTYO7JchisZhcJdB6CEOnkFQfho5VsG4IAAAEphqnSx9syddfv9yjz7Yf8RzvkhChcVlddcvgdCVH202sEGg7hKFTSGSvIQAAEKAOFVfqb2v26a01e1VQ6pAkWSzS5b076LYLu+mSXimyWRkFQmAjDJ2COwwdKSMMAQAA/+dyGfpsR6H+snqPVm4tkLO+LXZydJjGXJCusUO6qktCpMlVAu2HMHQKjAwBAIBAcKy8Wn9fv18Lvtyj3UcqPMezuifqtgu7KfvcNIWFsDkqgg9h6BQ8YYg1QwAAwM8YhqGv9hXpr1/s0b++PqTqWpckKcYeoh8P6qJbs7rq7NQYk6sEzEUYOoUkdxhimhwAAPAT5Y5avZt7UH/9Yo+2HCrxHD+3U6xuu7CbrhvQSZFhfAQEJMLQKSVG1XVOYZocAADwB5v2F+v2eV+qqKKuLbY9xKpr+nfSbRd21YD0eNpiAz9AGDqFhKi6zcSOlDtMrgQAAODUapwuPfz2RhVV1KhbUqRuv7CbbhrURfGRYWaXBvgswtApJDEyBAAA/MQrq3bqu/xSJUaF6R8/v8iz9hlA42gbcgruN5GiyhpP60kAAABfs6uwXC+s3CZJmn7NOQQhoIkIQ6eQEFk3Tc4wpCI6ygEAAB9kGIamLt6k6lqXLj47WdcP6Gx2SYDfIAydQojNqvj6QMRUOQAA4IveXr9fq3ceUXioVc9c348mCUAzEIZOI7F+0eERwhAAAPAxhWUOPbP0W0nSgyN7qWtSpMkVAf6FMHQa7jm3xwhDAADAxzz1zy0qrqxR346xunN4d7PLAfwOYeg03GGIkSEAAOBLPvquQO9tPCirRfrNj/spxMbHOqC5+K/mNJKi68IQa4YAAICvKHfUato/NkuSJl7UXf27xJtbEOCnCEOnkRBJGAIAAL7l+RXf60BRpTrHR2jylb3MLgfwW4Sh02CaHAAA8CVf7y/SvM92SZJ+fcN5irKHmFwR4L8IQ6fhniZHAwUAAGC2WqdLj76zSS5Dujazk0b07mB2SYBfIwydRmKUXRIjQwAAwHzzPtulLYdKFBcRqunX9DW7HMDvEYZOIynKvWbIYXIlAAAgmO09UqFZK76XJD129TlKibGbXBHg/whDp5EQdbyBgmEYJlcDAACCkWEYemzJJlXVuHRhj0TdPKiL2SUBAYEwdBrukaEap6EyR63J1QAAgGC0JPeAPtlWqLAQq2be2F8Wi8XskoCAQBg6jfBQmyLDbJJorw0AANrf0fJqPf2vbyVJv7zibHVPjjK5IiBwEIaagPbaAADALL9eukVHy6vVOzVGd13cw+xygIBCGGoCdxg6WkYYAgAA7efTbYVavOGALBZp5o/7KSyEj25Aa+K/qCZIPKGJAgAAQHuorHZq6j82SZLGX9hN53dNMLkiIPAQhprAE4YqCEMAAKB9vLBym/YerVDHuHD96kd9zC4HCEiEoSZIYmQIAAC0oy0HS/TqJzslSU9dd56i7SEmVwQEJsJQEyRG1W1qdoQ1QwAAoI05XYamLP5aTpeh0f3SdGXfVLNLAgIWYagJEqNCJUlHyx0mVwIAAALdG5/v1sb9xYoJD9ETOeeaXQ4Q0AhDTeAeGTpaUWNyJQAAIJDtP1ah3//nO0nSlFHnqENsuMkVAYGNMNQEx7vJMTIEAADahmEYevzdb1RR7dQFGQn6yQXpZpcEBDzCUBMksc8QAABoY0s3HdKHWwsUZrNq5o39ZLVazC4JCHiEoSZIqA9D5dVOVdU4Ta4GAAAEmuKKGj3x3hZJ0s9H9NRZHWJMrggIDoShJogND1Gore7bGdprAwCA1jbz39+qsMyhszpE697LeppdDhA0CENNYLFYlBDJXkMAAKD1fbHziN5au0+SNPPGfrKH2EyuCAgehKEmSmTjVQAA0MqqapyauniTJGlcVlddkJFockVAcCEMNRFhCAAAtLY5H23XzsJydYix6//9qI/Z5QBBhzDURO4wdIQwBAAAWsF3eaV66eMdkqQnrz1XcRGhJlcEBB/CUBO522sfIwwBAIAz5HIZmrL4a9W6DF3ZN1U/Oi/N7JKAoEQYaqLEKLskRoYAAMCZW/DlHm3YW6SoMJueuu5cWSzsKQSYgTDURInR7jVDDpMrAQAA/iyvuEq/Xf6dJOmRH/VRx7gIkysCghdhqIkSaa0NAABawePvblaZo1YDu8brtgu7mV0OENQIQ01ENzkAAHCmlm/O03+25CvEatHMG/vJZmV6HGAmwlATJUUThgAAQMuVVNXo8Xc3S5LuubSn+qTFmlwRAMJQE7lHhooqa+R0GSZXAwAA/M3vlm9VQalD3ZOjdP/lZ5ldDgARhposvr73v2FIxyoYHQIAAE23bvdR/fWLvZKkZ2/op/BQm8kVAZAIQ00WYrMqPrIuEDFVDgAANJWj1qkpizdJkm4Z3EVDeyaZXBEAN8JQM9BEAQAANNfL/92pbQVlSo4O09TR55hdDoATEIaaIYkwBAAAmmF7QZn++OF2SdLjOecqvn6rDgC+gTDUDO6RoSOEIQAAcBoul6Gpizep2unSiN4pyunf0eySAPwAYagZPNPkyghDAADg1Bat26c1u48qItSmp68/TxYLewoBvoYw1AzuMEQ3OQAAcCoFpVV6dtm3kqSHruqlLgmRJlcEoCGEoWZIjLJLYpocAAA4tSf/uUUlVbXq3yVOEy/qbnY5ABpBGGqG4w0UHCZXAgAAfNXKb/O19OtDslktmnljP9msTI8DfBVhqBkS3A0UWDMEAAAaUOao1fQlmyVJky7urnM7xZlcEYBTIQw1A621AQDAqfz+/e90sLhKXRMj9cAVvcwuB8BpEIaa4cQGCoZhmFwNAADwJbn7ivTG6t2SpGduOE8RYTZzCwJwWoShZnCHoRqnoVJHrcnVAAAAX1HjdOnRd76WYUg3Duysi89OMbskAE1AGGqG8FCbouq/5WGvIQAA4PbqJzu1Na9UCZGhmnZNX7PLAdBEzQ5Dq1atUk5Ojjp16iSLxaIlS5ac8vxDhw5p3Lhx6tWrl6xWqx544IGTzpk/f74sFovXLTw8vLmltQtPEwXWDQEAAEm7C8v1wgfbJEnTr+nrmUkCwPc1OwyVl5crMzNTc+bMadL5DodDKSkpmjZtmjIzMxs9LzY2VocOHfLc9uzZ09zS2oW7icIxwhAAAEHPMAxNW7JZjlqXLj47WTcM7Gx2SQCaIaS5Dxg1apRGjRrV5PMzMjL0wgsvSJLmzZvX6HkWi0VpaWnNLafdJdJRDgAA1PtkW6E+3V6osBCrfn39ebJY2FMI8Cc+s2aorKxM3bp1U3p6uq677jp98803ZpfUoMQouySmyQEAEOwMw9Dv//OdJGn8hd3ULSnK5IoANJdPhKHevXtr3rx5evfdd/XXv/5VLpdLw4YN0/79+xt9jMPhUElJidetPSRGhUqSjpY72uX3AQAA3/T+N/n6en+xosJsuveynmaXA6AFfCIMDR06VOPHj9eAAQN06aWXavHixUpJSdHLL7/c6GNmzpypuLg4zy09Pb1damVkCAAAOF2GnqsfFbpzeHclRdtNrghAS/hEGPqh0NBQDRw4UNu3b2/0nClTpqi4uNhz27dvX7vURgMFAADwbu4BbSsoU1xEqCZd0sPscgC0kE+GIafTqU2bNqljx46NnmO32xUbG+t1aw80UAAAILhV17o0u76V9j2X9lRseKjJFQFoqWaHobKyMuXm5io3N1eStGvXLuXm5mrv3r2S6kZsxo8f7/UY9/llZWU6fPiwcnNztWXLFs/9Tz31lP7zn/9o586d2rBhg2677Tbt2bNHkyZNOoOX1jYSo9lnCAB80Zw5c5SRkaHw8HBlZWVpzZo1pzx/9uzZ6t27tyIiIpSenq4HH3xQVVVVDZ77m9/8RhaLpcG98hB8Fq3bp71HK5QSY9eEYd3MLgfAGWh2a+1169ZpxIgRnp8nT54sSZowYYLmz5+vQ4cOeYKR28CBAz3/f/369XrzzTfVrVs37d69W5J07Ngx3XXXXcrLy1NCQoIGDRqkzz//XH37+t4OzomRjAwBgK9ZuHChJk+erLlz5yorK0uzZ89Wdna2vvvuO3Xo0OGk89988009+uijmjdvnoYNG6bvv/9ed9xxhywWi2bNmuV17tq1a/Xyyy+rf//+7fVy4MOqapz6w4d1o0L3jzhLkWHN/igFwIc0+7/gyy67TIZhNHr//PnzTzp2qvMl6fnnn9fzzz/f3FJM4R4Zqqh2qqrGqfBQm8kVAQBmzZqlu+66SxMnTpQkzZ07V0uXLtW8efP06KOPnnT+559/rosuukjjxo2TVLcn3tixY/Xll196nVdWVqZbb71Vr776qn7961+3/QuBz/vL6j3KL3Goc3yEfjKkfZo3AWg7PrlmyJfF2EMUaqvbUI3RIQAwX3V1tdavX6+RI0d6jlmtVo0cOVKrV69u8DHDhg3T+vXrPVPpdu7cqWXLlmn06NFe59133326+uqrvZ67MWZt+YD2U1pVoxc/rmvu9MDIs2UP4QtRwN8xtttMFotFiVFhyi9x6Gh5tTrFR5hdEgAEtcLCQjmdTqWmpnodT01N1datWxt8zLhx41RYWKjhw4fLMAzV1tbqnnvu0dSpUz3nvPXWW9qwYYPWrl3bpDpmzpypJ598suUvBD7vtU936VhFjXqkROmGgZ3NLgdAK2BkqAUSImmiAAD+7OOPP9azzz6rF198URs2bNDixYu1dOlSPf3005Kkffv26Ze//KUWLFig8PDwJj2nWVs+oH0cK6/Wnz7ZJUl66MreCrHxEQoIBIwMtUBStLuJgsPkSgAAycnJstlsys/P9zqen5+vtLS0Bh8zffp03X777Z6upf369VN5ebnuvvtuPfbYY1q/fr0KCgp0/vnnex7jdDq1atUq/fGPf5TD4ZDN5j1Fym63y25n481ANfe/O1TmqNW5nWI16ryG/70C4H/4WqMFEqPqLnZHy2tMrgQAEBYWpkGDBmnlypWeYy6XSytXrtTQoUMbfExFRYWsVu9LoDvcGIahK664Qps2bfJsDZGbm6vBgwfr1ltvVW5u7klBCIEtv6RK8z/fLUl6+Kreslot5hYEoNUwMtQCSVGMDAGAL5k8ebImTJigwYMHa8iQIZo9e7bKy8s93eXGjx+vzp07a+bMmZKknJwczZo1SwMHDlRWVpa2b9+u6dOnKycnRzabTTExMTrvvPO8fkdUVJSSkpJOOo7A98cPt8tR69Lgbgm6rHeK2eUAaEWEoRZIYK8hAPApY8aM0eHDh/X4448rLy9PAwYM0PLlyz1NFfbu3es1EjRt2jRZLBZNmzZNBw4cUEpKinJycvTMM8+Y9RLgo/YeqdDf1tTtn/ir7N6yWBgVAgKJxTjdJkB+oqSkRHFxcSouLlZsbGyb/q6/fLFH05ds1lV9U/XK+MFt+rsAwNe15/uvP+HvEhgmL8rV4g0HdPHZyfrLnVlmlwOgiZr6HsyaoRZwT5M7VsHIEAAAgWpbfqmWfHVAUt2oEIDAQxhqgcQoWmsDABDoZq34Xi5D+tG5aerfJd7scgC0AcJQCxxvoEAYAgAgEG3aX6x/b86TxSJNvqqX2eUAaCOEoRZIqA9DRRU1qnW6TK4GAAC0tt//5ztJ0g0DOqtXaozJ1QBoK4ShFkiIDJO7mcyxCvYaAgAgkHy584j++/1hhVgtemAko0JAICMMtYDNalF8RKgkmigAABBIDMPwjAqNuSBdXZMiTa4IQFsiDLWQp4lCGWEIAIBA8d/vD2vt7mOyh1j1P5efbXY5ANoYYaiFEmmiAABAQHG5DP3v+3WjQuOHdlNaXLjJFQFoa4ShFjoehhwmVwIAAFrD8m/y9M3BEkWF2XTvZWeZXQ6AdkAYaqHEKLsk6Wg5DRQAAPB3Tpeh5+rXCk26uIfnS08AgY0w1EJJjAwBABAw/vHVAe04XK74yFBNuri72eUAaCeEoRbyNFBgzRAAAH6tutal2R98L0m699KeigkPNbkiAO2FMNRCNFAAACAwLFy7V/uPVapDjF3jh2aYXQ6AdkQYaiHCEAAA/q+y2qn/+3C7JOl/Lj9LEWE2kysC0J4IQy1EGAIAwP+9sXq3Dpc61CUhQmMu6Gp2OQDaGWGohZKi68LQsYpqGYZhcjUAAKC5SqpqNPe/OyRJD47spbAQPhYBwYb/6lsoIbIuDNU4DZVU1ZpcDQAAaK4/fbJLRRU1OqtDtK4f2NnscgCYgDDUQuGhNkXVzytmqhwAAP7lSJlDr32yU5L00JW9ZLNaTK4IgBkIQ2cgMZp1QwAA+KO5/92h8mqn+nWO04/OSzO7HAAmIQydgcQouyTCEAAA/uRQcaXeWL1HkvTQVb1ksTAqBAQrwtAZSPJ0lHOYXAkAAGiqP3y4XdW1Lg3JSNSlvVLMLgeAiQhDZ8DdROEII0MAAPiFPUfKtWjtPknSw9m9GRUCghxh6Ay422sfLSMMAQDgD2Z/sE21LkOX9krRkO6JZpcDwGSEoTPg2Xi1gjAEAICv+y6vVEtyD0iSHr6qt8nVAPAFhKEz4AlDTJMDAMDnzVrxnQxDGnVemvp1iTO7HAA+gDB0BhIjCUMAAPiDjfuK9P43+bJapMlX9jK7HAA+gjB0Btz7DB1hzRAAAD7t9//5TpJ0w8AuOjs1xuRqAPgKwtAZcLfWPsaaIQAAfNbqHUf0ybZChdosemDk2WaXA8CHEIbOgHvNUEW1U1U1TpOrAQAAP2QYhmdU6CcXdFV6YqTJFQHwJYShMxBtD1GYre5PyF5DAAD4no++K9D6PccUHmrV/1x+ltnlAPAxhKEzYLFYlBAVKom9hgAA8DUul6Hfv/+9JGnC0Ax1iA03uSIAvoYwdIYSo+yS2GsIAABfs2zzIW05VKJoe4juubSn2eUA8EGEoTOU5NlryGFyJQAAwK3W6dKs/9SNCt11cQ8l1F+vAeBEhKEz5G6iQHttAAB8x+KvDmhnYbkSIkP10+EZZpcDwEcRhs5QYhQbrwIA4EsctU698ME2SdLPLztLMeGhJlcEwFcRhs4QYQgAAN/yty/36kBRpVJj7bp9aDezywHgwwhDZ4gwBACA76iortUfP9ouSfqfy89WeKjN5IoA+DLC0BlKIgwBAOAz5n++W4Vl1eqaGKlbBqebXQ4AH0cYOkMJhCEAAHxCcWWN5n68Q5L0wMizFRbCxxwAp8a7xBlyjwwdIQwBAGCqV1ftVElVrc7uEK3rBnQ2uxwAfoAwdIbca4aKK2tU63SZXA0AAMGpsMyheZ/tkiQ9dFVv2awWkysC4A8IQ2coPjJMlvr322MVNeYWAwBAkHrxox2qqHaqf5c4ZZ+banY5APwEYegM2awWJUSybggAALMcLKrUX7/YI0l6+KreslgYFQLQNIShVpAQWbeZ25Fyh8mVAAAQfP7w4TZVO13K6p6oi89ONrscAH6EMNQKkqLskhgZAgCgve0qLNeidfslSb/KZlQIQPMQhlqBu4nCMcIQAADt6vkV38vpMjSid4oGZySaXQ4AP0MYagWJ0bTXBgCgvX17qET//PqgpLoOcgDQXIShVpBIAwUAANrdc//5XoYhXd2vo87rHGd2OQD8EGGoFSSy8SoAAO1qw95j+uDbfFkt0oNX9jK7HAB+ijDUCpKiWTMEAEB7eu4/30mSfnx+F53VIdrkagD4K8JQK3CPDDFNDgCAtvf59kJ9tv2IQm0W/eKKs80uB4AfIwy1AqbJAQDQfpbkHpAk3Tw4XemJkSZXA8CfEYZawYmttQ3DMLkaAAAC27rdxyRJI8/pYHIlAPwdYagVuMNQrctQSWWtydUAABC4Cssc2llYLkka1JV9hQCcGcJQK7CH2BRtD5EkHa1gqhwAAG1l/Z66UaFeqdGKiww1uRoA/o4w1EqON1FwmFwJAACBa93uo5KkwRmMCgE4c4ShVpLgbqJQxsgQAABtZW39eqHB3RJMrgRAICAMtZIk2msDANCmKqud+uZgsSTpAkaGALQCwlAr8UyTY80QAABtYuP+ItU4DaXG2tUlIcLscgAEAMJQK/GMDDFNDgCANuFZL9QtURaLxeRqAAQCwlArSWSaHAAAbWpdfSe5wRmsFwLQOghDrcTTQIEwBABAq3O5DE9bbdYLAWgthKFWQgMFAADazvcFpSqtqlVkmE190mLMLgdAgCAMtRKmyQEA0HbcLbXP75qgEBsfXwC0Dt5NWklSlF0SYQgAgLbgbp4wiP2FALQiwlArSYgKlSRV1jhVWe00uRoAAALLut2sFwLQ+ghDrSTaHqKw+mH7I+UOk6sBACBwHCyq1IGiStmsFg3oGm92OQACCGGolVgsFs+6oWPlNSZXAwBA4HC31D6nY4yi7SEmVwMgkBCGWlGip702I0MAALSW9SdstgoArYkw1IqSoukoBwBAa1vLeiEAbYQw1IoSIglDAAC0ptKqGm3NK5EkDc6gkxyA1kUYakXHp8kRhgAAaA1f7S2Sy5DSEyOUGhtudjkAAgxhqBUleRooEIYAAGgN61gvBKANEYZaUWI0I0MAALQmdyc5psgBaAuEoVaUyJohAABaTY3Tpa/2FkmieQKAtkEYakXuNUOEIQAAztyWgyWqrHEqNjxEZ6VEm10OgABEGGpFtNYGAKD1HJ8ilyir1WJyNQACUbPD0KpVq5STk6NOnTrJYrFoyZIlpzz/0KFDGjdunHr16iWr1aoHHnigwfPefvtt9enTR+Hh4erXr5+WLVvW3NJMlxhllyQVV9aoxukyuRoAAPybp3kC64UAtJFmh6Hy8nJlZmZqzpw5TTrf4XAoJSVF06ZNU2ZmZoPnfP755xo7dqzuvPNOffXVV7r++ut1/fXXa/Pmzc0tz1RxEaGy1H9xdayC0SEAAFrKMIzjI0N0kgPQRkKa+4BRo0Zp1KhRTT4/IyNDL7zwgiRp3rx5DZ7zwgsv6Ec/+pF+9atfSZKefvpprVixQn/84x81d+7c5pZoGpvVooTIMB0tr9bR8mp1iGE/BAAAWmLv0QodLnUozGZV/y5xZpcDIED5xJqh1atXa+TIkV7HsrOztXr1apMqajlPE4UyRoYAAGiptbvrRoXO6xyr8FCbydUACFTNHhlqC3l5eUpNTfU6lpqaqry8vEYf43A45HA4PD+XlJS0WX3N4QlDTJMDAKDF1u+pWy9ES20AbcknRoZaYubMmYqLi/Pc0tPTzS5JkpREe20AAM6Ye2RoMGEIQBvyiTCUlpam/Px8r2P5+flKS0tr9DFTpkxRcXGx57Zv3762LrNJEurD0BGmyQEA0CLHyqu1vaBMkjSoG53kALQdnwhDQ4cO1cqVK72OrVixQkOHDm30MXa7XbGxsV43X8DIEAAAZ2Z9fRe5nilRnunnANAWmr1mqKysTNu3b/f8vGvXLuXm5ioxMVFdu3bVlClTdODAAf35z3/2nJObm+t57OHDh5Wbm6uwsDD17dtXkvTLX/5Sl156qZ577jldffXVeuutt7Ru3Tq98sorZ/jy2h9rhgAAODNrWS8EoJ00OwytW7dOI0aM8Pw8efJkSdKECRM0f/58HTp0SHv37vV6zMCBAz3/f/369XrzzTfVrVs37d69W5I0bNgwvfnmm5o2bZqmTp2qs88+W0uWLNF5553XktdkKrrJAQBwZtbXrxdiihyAttbsMHTZZZfJMIxG758/f/5Jx051vtvNN9+sm2++ubnl+JxEpskBANBiVTVOfb2/WBIjQwDank+sGQok7jB0hDAEAECzbTpQrGqnS8nRYeqWFGl2OQACHGGolSVF2SVJxyqqmzQiBgAAjlvnbqndLVEWi8XkagAEOsJQK0uICpUkOV2GSiprTa4GAAD/sm53XfOEwRmsFwLQ9ghDrcweYlOMvW4p1pFyh8nVAADgP1wuQ+v2sNkqgPZDGGoDCTRRAIB2N2fOHGVkZCg8PFxZWVlas2bNKc+fPXu2evfurYiICKWnp+vBBx9UVVWV5/6XXnpJ/fv39+xlN3ToUP373/9u65cR1HYcLlNxZY3CQ606t5Nv7B8IILARhtoATRQAoH0tXLhQkydP1owZM7RhwwZlZmYqOztbBQUFDZ7/5ptv6tFHH9WMGTP07bff6rXXXtPChQs1depUzzldunTRb37zG61fv17r1q3T5Zdfruuuu07ffPNNe72soLO2fr3QwPQEhdr4iAKg7fFO0waS6sPQMcIQALSLWbNm6a677tLEiRPVt29fzZ07V5GRkZo3b16D53/++ee66KKLNG7cOGVkZOiqq67S2LFjvUaTcnJyNHr0aJ199tnq1auXnnnmGUVHR+uLL75or5cVdNbtYb0QgPZFGGoDjAwBQPuprq7W+vXrNXLkSM8xq9WqkSNHavXq1Q0+ZtiwYVq/fr0n/OzcuVPLli3T6NGjGzzf6XTqrbfeUnl5uYYOHdrgOQ6HQyUlJV43NI+nkxzrhQC0k2ZvuorTY+NVAGg/hYWFcjqdSk1N9TqempqqrVu3NviYcePGqbCwUMOHD5dhGKqtrdU999zjNU1OkjZt2qShQ4eqqqpK0dHR+sc//qG+ffs2+JwzZ87Uk08+2TovKggVlFRp79EKWS3S+V3jzS4HQJBgZKgNEIYAwLd9/PHHevbZZ/Xiiy9qw4YNWrx4sZYuXaqnn37a67zevXsrNzdXX375pe69915NmDBBW7ZsafA5p0yZouLiYs9t37597fFSAoa7i1zvtFjFhIeaXA2AYMHIUBsgDAFA+0lOTpbNZlN+fr7X8fz8fKWlpTX4mOnTp+v222/XpEmTJEn9+vVTeXm57r77bj322GOyWuu+KwwLC9NZZ50lSRo0aJDWrl2rF154QS+//PJJz2m322W321vzpQWVtfX7C13AeiEA7YiRoTaQFE0YAoD2EhYWpkGDBmnlypWeYy6XSytXrmx0fU9FRYUn8LjZbDZJkmEYjf4ul8slh4M95NqCe73QoG6EIQDth5GhNpAYVffNIGEIANrH5MmTNWHCBA0ePFhDhgzR7NmzVV5erokTJ0qSxo8fr86dO2vmzJmS6jrFzZo1SwMHDlRWVpa2b9+u6dOnKycnxxOKpkyZolGjRqlr164qLS3Vm2++qY8//ljvv/++aa8zUJU7arXlUF3DiQtongCgHRGG2kBipLubHN8eAkB7GDNmjA4fPqzHH39ceXl5GjBggJYvX+5pqrB3716vkaBp06bJYrFo2rRpOnDggFJSUpSTk6NnnnnGc05BQYHGjx+vQ4cOKS4uTv3799f777+vK6+8st1fX6DL3Vckp8tQ5/gIdYqPMLscAEHEYpxqPoAfKSkpUVxcnIqLixUba+6u1WWOWp03o+6bwy1PZSsyjMwJIHD50vuvL+Hv0nQvfLBNz3/wva7N7KT/GzvQ7HIABICmvgezZqgNRIXZFBZS96dlqhwAAKfm3myV5gkA2hthqA1YLBYl0VEOAIDTqnW6tGEPm60CMAdhqI0keNYNEYYAAGjM1rxSlVc7FWMPUa/UGLPLARBkCENtxNNeu4wwBABAY9bV7y90frcE2awWk6sBEGwIQ23EvfHqsQrCEAAAjVnnniLH/kIATEAYaiPuMMQ0OQAAGmYYhtbWjwyxXgiAGQhDbcTTQIFpcgAANGj/sUrllzgUYrVoQHq82eUACEKEoTaSwMgQAACntL5+ity5neMUEWYzuRoAwYgw1EaOt9Z2mFwJAAC+yT1F7gLWCwEwCWGojSRG2SVJxypqTK4EAADftG43+wsBMBdhqI14GiiUMTIEAMAPFVfU6PuCUknSIEaGAJiEMNRG3GGopKpWNU6XydUAAOBbNuw9JsOQuidHKSXGbnY5AIIUYaiNxEeEyr133DGaKAAA4GXdnrr1QowKATATYaiNWK0WJUTWN1Fg41UAALysrV8vdEEGYQiAeQhDbSiRvYYAADhJda1LG/cVSaJ5AgBzEYbaUCJ7DQEAcJLNB4vlqHUpMSpMPZKjzC4HQBAjDLUhz8gQYQgAAI91u4+vF7JYLCZXAyCYEYbaECNDAACcjPVCAHwFYagNJdWHIbrJAQBQxzAMrd9TF4YGdWO9EABzEYbaENPkAADwtrOwXEfLq2UPseq8zrFmlwMgyBGG2lCCZ5qcw+RKAADwDevrp8hldomXPcRmcjUAgh1hqA0lRdXtqM3IEAAAddbWN08YzHohAD6AMNSGjk+TqzG5EgAAfMO6Pe7mCawXAmA+wlAbSoqub6BQUS2XyzC5GgAAzFVY5tCuwnJJ0vldGRkCYD7CUBuKjwyVJDldhkqqGB0CAAS3dfXrhXqnxiiu/hoJAGYiDLUhe4hNMfYQSew1BADAOtYLAfAxhKE2lhhNe20AAKTj64UIQwB8BWGojbHXEAAAUmW1U5sPFEuSBrPZKgAfQRhqY0mEIQAAtHF/kWpdhlJj7eqSEGF2OQAgiTDU5hIiCUMAABxfL5Qoi8VicjUAUIcw1Mbca4aOlBGGAADBa219J7kLurFeCIDvIAy1Mfc0uWMVhCEAQHByugxt2OtunsB6IQC+gzDUxhKj7JJorQ0ACF7f55eqtKpWUWE29UmLMbscAPAgDLWxxKi6TeWOljtMrgQAAHO41wud3y1BITY+egDwHbwjtTH3yNBR1gwBAIKUe3+hQawXAuBjCENtzL1m6Eh5tQzDMLkaAADa3zp38wTWCwHwMYShNubedNVR61JljdPkagAAaF8Hiyp1oKhSNqtFA9LjzS4HALwQhtpYZJhN9pC6PzPttQEAwcY9Ra5vx1hF2UNMrgYAvBGG2pjFYvGMDrHxKgAg2BzfbJX1QgB8D2GoHRCGAADByr1eaHA31gsB8D2EoXZAGAIABKOSqhptzSuRxMgQAN9EGGoHSYQhAEAQ+mpvkVyG1DUxUqmx4WaXAwAnIQy1g4QT2msDABAs1rvXC7G/EAAfRRhqB8dHhhwmVwIAQPtZ614vxP5CAHwUYagdJEbZJTFNDgAQPGqcLuXuK5LEeiEAvosw1A5ooAAACDZbDpaossapuIhQnZUSbXY5ANAgwlA7SIomDAEAgsvaE9YLWa0Wk6sBgIYRhtpBQiQNFAAAwWX9nrr1QoOYIgfAhxGG2oG7gUJpVa2qa10mVwMAQNsyDMPTPOECmicA8GGEoXYQFxEqW/0UgaIKRocAAIFtz5EKFZY5FGazql/nOLPLAYBGEYbagdVqUUJkqCSmygEAAt+6+ily/brEKTzUZnI1ANA4wlA7ca8bookCACDQrXM3T2C9EAAfRxhqJ+722owMAQACnXtk6IJurBcC4NsIQ+3E3V77GGEIABDAjpZXa3tBmSRpUDdGhgD4NsJQO2FkCAAQDNwttc/qEK2E+msfAPgqwlA7SYyyS5KOljtMrgQAgLazbs/xzVYBwNcRhtpJYn03ORooAAAC2br6/YUGs78QAD9AGGonidF1I0NHyghDAIDAVFXj1Kb9xZKkC+gkB8APEIbaSVL9vOljbLoKAAhQmw4Uq9rpUnK0XV0TI80uBwBOizDUTtwNFJgmBwAIVGvr9xe6ICNBFovF5GoA4PQIQ+0k0TMyVCOXyzC5GgAAWt961gsB8DOEoXaSEFkXhpwuQ8WVNSZXAwBA63K5DM9mq3SSA+AvCEPtJCzEqpjwEEnSUdYNAQACzPbDZSqurFFEqE19O8WaXQ4ANAlhqB0lsW4IABCg3C21B6THK9TGxwsA/oF3q3bk3omb9toAgECz7oTmCQDgLwhD7YiRIQBAoFq7py4M0TwBgD8hDLWj4+21HSZXAgBA68kvqdK+o5WyWqSBXePNLgcAmoww1I4So+ySpKPldJMDAAQO93qhPmmxigkPNbkaAGg6wlA7SmJkCAAQgNbtYb0QAP9EGGpHngYKrBkCAAQQ98jQINYLAfAzzQ5Dq1atUk5Ojjp16iSLxaIlS5ac9jEff/yxzj//fNntdp111lmaP3++1/1PPPGELBaL161Pnz7NLc3n0UABABBoyh212nKoRBIjQwD8T7PDUHl5uTIzMzVnzpwmnb9r1y5dffXVGjFihHJzc/XAAw9o0qRJev/9973OO/fcc3Xo0CHP7dNPP21uaT7P3UDhGGEIABAgcvcVyeky1Dk+Qh3jIswuBwCaJaS5Dxg1apRGjRrV5PPnzp2r7t2767nnnpMknXPOOfr000/1/PPPKzs7+3ghISFKS0trbjl+JfGEaXKGYchisZhcEQAAZ2btbndLbUaFAPifNl8ztHr1ao0cOdLrWHZ2tlavXu11bNu2berUqZN69OihW2+9VXv37m3r0tqdOww5al2qqHaaXA0AAGfOvV6I/YUA+KM2D0N5eXlKTU31OpaamqqSkhJVVlZKkrKysjR//nwtX75cL730knbt2qWLL75YpaWljT6vw+FQSUmJ183XRYbZZA+p+5OzbggA4O9qnS59tbc+DHVjZAiA//GJbnKjRo3SzTffrP79+ys7O1vLli1TUVGRFi1a1OhjZs6cqbi4OM8tPT29HStuGYvF4mmiQEc5AIC/25pXqvJqp2LCQ9QrNcbscgCg2do8DKWlpSk/P9/rWH5+vmJjYxUR0fBCy/j4ePXq1Uvbt29v9HmnTJmi4uJiz23fvn2tWndbSYymiQIAIDCsq18vNKhbgmxW1sEC8D9tHoaGDh2qlStXeh1bsWKFhg4d2uhjysrKtGPHDnXs2LHRc+x2u2JjY71u/iAxyi6JkSEAgP9bu4cpcgD8W7PDUFlZmXJzc5WbmyuprnV2bm6up+HBlClTNH78eM/599xzj3bu3KlHHnlEW7du1YsvvqhFixbpwQcf9Jzz8MMP67///a92796tzz//XDfccINsNpvGjh17hi/P9yRGhkqSjpY7TK4EAICWMwzDMzJE8wQA/qrZrbXXrVunESNGeH6ePHmyJGnChAmaP3++Dh065NUJrnv37lq6dKkefPBBvfDCC+rSpYv+9Kc/ebXV3r9/v8aOHasjR44oJSVFw4cP1xdffKGUlJQzeW0+iZEhAEAg2H+sUvklDoVYLcrsEm92OQDQIs0OQ5dddpkMw2j0/vnz5zf4mK+++qrRx7z11lvNLcNvJbFmCAAQANbtqRsVOq9znCLCbCZXAwAt4xPd5IKJe68hWmsDAPzZ2vr9hS5gs1UAfoww1M4SImmtDQDwf+vrw9CgbqwXAuC/CEPtzD1NjpEhAIC/Kq6o0Xf5dRujD2ZkCIAfIwy1M880uTLCEADAP23YWzcq1CM5SsnRdpOrAYCWIwy1s6T6MFTqqFV1rcvkagAAaL61J2y2CgD+jDDUzmLDQz27dB+rYHQIAOB/1nmaJ7BeCIB/Iwy1M6vVooT6jVePMFUOAOBnHLVObdxfJEkaxHohAH6OMGQC2msDAPzV5gMlctS6lBgVph7JUWaXAwBnhDBkAk8YYpocAMDPrKtfLzS4W4IsFovJ1QDAmSEMmSApqq7zztEyh8mVAADQPOv21K0XoqU2gEBAGDJBQlTdmiGmyQEA/IlhGFrvCUM0TwDg/whDJkisHxk6QhgCgFYzZ84cZWRkKDw8XFlZWVqzZs0pz589e7Z69+6tiIgIpaen68EHH1RVVZXn/pkzZ+qCCy5QTEyMOnTooOuvv17fffddW78Mn7azsFxHy6tlD7HqvE5xZpcDAGeMMGSCJBooAECrWrhwoSZPnqwZM2Zow4YNyszMVHZ2tgoKCho8/80339Sjjz6qGTNm6Ntvv9Vrr72mhQsXaurUqZ5z/vvf/+q+++7TF198oRUrVqimpkZXXXWVysvL2+tl+Rz3eqHM9HiFhfARAoD/CzG7gGBENzkAaF2zZs3SXXfdpYkTJ0qS5s6dq6VLl2revHl69NFHTzr/888/10UXXaRx48ZJkjIyMjR27Fh9+eWXnnOWL1/u9Zj58+erQ4cOWr9+vS655JI2fDW+a61nfyHWCwEIDHytYwJGhgCg9VRXV2v9+vUaOXKk55jVatXIkSO1evXqBh8zbNgwrV+/3jOVbufOnVq2bJlGjx7d6O8pLi6WJCUmNrxWxuFwqKSkxOsWaDzrhbqxXghAYGBkyAQJhCEAaDWFhYVyOp1KTU31Op6amqqtW7c2+Jhx48apsLBQw4cPl2EYqq2t1T333OM1Te5ELpdLDzzwgC666CKdd955DZ4zc+ZMPfnkk2f2YnzY4VKHdhWWy2KRzu/KyBCAwMDIkAncI0PHKqrlchkmVwMAwefjjz/Ws88+qxdffFEbNmzQ4sWLtXTpUj399NMNnn/fffdp8+bNeuuttxp9zilTpqi4uNhz27dvX1uVb4r1e+rWC/VOjVFcZKjJ1QBA62BkyATukSGXIRVX1nh+BgA0X3Jysmw2m/Lz872O5+fnKy0trcHHTJ8+XbfffrsmTZokSerXr5/Ky8t1991367HHHpPVevy7wvvvv1//+te/tGrVKnXp0qXROux2u+x2eyu8It+0rn690KBujAoBCByMDJkg1GZVbHhdDqW9NgCcmbCwMA0aNEgrV670HHO5XFq5cqWGDh3a4GMqKiq8Ao8k2Ww2SXV76bj/9/7779c//vEPffjhh+revXsbvQL/sHaPu3kC64UABA5GhkySGBWmkqpa1g0BQCuYPHmyJkyYoMGDB2vIkCGaPXu2ysvLPd3lxo8fr86dO2vmzJmSpJycHM2aNUsDBw5UVlaWtm/frunTpysnJ8cTiu677z69+eabevfddxUTE6O8vDxJUlxcnCIiIsx5oSaprHbqmwN1DSQG00kOQAAhDJkkMSpMu49U6Gi5w+xSAMDvjRkzRocPH9bjjz+uvLw8DRgwQMuXL/c0Vdi7d6/XSNC0adNksVg0bdo0HThwQCkpKcrJydEzzzzjOeell16SJF122WVev+v111/XHXfc0eavyZfsPVqhWpehuIhQdY4PriAIILARhkySGFU3r5xpcgDQOu6//37df//9Dd738ccfe/0cEhKiGTNmaMaMGY0+n3u6HKS8kipJUse4cFksFpOrAYDWw5ohk3g6yhGGAAA+Lr8+DHWIDTe5EgBoXYQhk7g7yDEyBADwdQX1YSgtNnC75QEIToQhkySx8SoAwE/kl9Stb01lZAhAgCEMmSSRMAQA8BN5TJMDEKAIQyZJjCYMAQD8g3uaXGoM0+QABBbCkEmYJgcA8BfuaXJpcYwMAQgshCGTJEQeb6BA+1YAgK9yugwdLmPNEIDARBgySVL9NLnqWpfKq50mVwMAQMOOlDnkdBmyWo7PagCAQEEYMklkWIjCQ+v+/Ow1BADwVe4pcsnRdoXY+NgAILDwrmaipKi6hajsNQQA8FXuDVdZLwQgEBGGTJQQFSpJOlruMLkSAAAall9a31Y7hjAEIPAQhkyU6B4ZKmNkCADgm/KL69tqx9JWG0DgIQyZiPbaAABf514zRCc5AIGIMGSiRHcYqiAMAQB8k3uaHCNDAAIRYchEnjDENDkAgI9iZAhAICMMmSiRaXIAAB9XUOIeGSIMAQg8hCETucMQrbUBAL7IUev0XKMIQwACEWHIRO4GCsdYMwQA8EGHS+umyIXZrEqIDDW5GgBofYQhE7FmCADgy9zrhTrE2mWxWEyuBgBaH2HIRO4wVOqolaPWaXI1AAB4Y70QgEBHGDJRbHiobNa6b9qOldeYXA0AAN7ySmirDSCwEYZMZLValBDpbqLgMLkaAAC8eabJxTAyBCAwEYZM5mmiwMgQAMDHuKfJpcURhgAEJsKQyY6312ZkCADgW/JLmSYHILARhkzGxqsAAF+VV1wfhpgmByBAEYZMRhgCAPiqAk9rbcIQgMBEGDIZYQgA4IvKHbUqddRKYs0QgMBFGDJZUjRhCADgewpK60aFosJsiraHmFwNALQNwpDJjrfWJgwBAHyHZ70QU+QABDDCkMmSmCYHAPBBBfWd5DrQSQ5AACMMmSyRaXIAAB+U795jiJEhAAGMMGQydwOFoopqOV2GydUAAFAnv76THNPkAAQywpDJ3GuGXIZUXFljcjUAANRxjwzRVhtAICMMmSzUZlVseF2XnqPlDpOrAQCgjjsMpbJmCEAAIwz5gKTougvNkTLWDQEAfAPT5AAEA8KQD3CvGzpWQRgCAJjPMAwaKAAICoQhH+AOQ+w1BADwBSWVtXLUuiRJKTFMkwMQuAhDPiCxvonCUabJAQB8QF79qFB8ZKjCQ20mVwMAbYcw5APcew0xMgQA8AWe5gkxTJEDENgIQz4gKYqNVwEAvsMThuIIQwACG2HIB9BAAQDgSwpK6zvJsV4IQIAjDPmABHcDBdYMAQB8QF6xe48hRoYABDbCkA9gmhwAwJew4SqAYEEY8gGJJ4QhwzBMrgYAEOzyS9lwFUBwIAz5gKSoum/eqp0ulVc7Ta4GABDsCkqYJgcgOBCGfEBEmE0R9fs4sNcQAMBMTpdxvIECYQhAgCMM+Qj3VLkj5Q6TKwEABLMj5Q45XYasFim5fh88AAhUhCEfkUgTBQCADygoqftSLjnarhAbHxMABDbe5XzE8ZEhwhAAwDz5rBcCEEQIQz7C3V77GGEIAGCi/BL3eiHaagMIfIQhH5HANDkAgA/Iqx8Z6sDIEIAgQBjyEUyTAwD4Andb7TTCEIAgQBjyEUmMDAEAfMDxNUNMkwMQ+AhDPoJucgAAX+BeM8Q0OQDBgDDkI5KiCUMAAPN5RoZiCEMAAh9hyEckRBKGAADmqq51edauMk0OQDAgDPmIpKi6i06Zo1aOWqfJ1QAAgtHhsropcqE2i2f6NgAEMsKQj4iNCFGI1SKJ0SEAgDncU+Q6xITLYrGYXA0AtD3CkI+wWCzsNQQAMFV+MZ3kAAQXwpAPSWTdEADARMfbatM8AUBwIAz5ENprAwDMlF9at2aIMAQgWBCGfEhifXvtI2WEIQBA+2NkCECwIQz5kKT6kaFjFYQhAED7Ox6GWDMEIDg0OwytWrVKOTk56tSpkywWi5YsWXLax3z88cc6//zzZbfbddZZZ2n+/PknnTNnzhxlZGQoPDxcWVlZWrNmTXNL83vuaXJHmCYHADBBfgnT5AAEl2aHofLycmVmZmrOnDlNOn/Xrl26+uqrNWLECOXm5uqBBx7QpEmT9P7773vOWbhwoSZPnqwZM2Zow4YNyszMVHZ2tgoKCppbnl/zrBlimhwAwARMkwMQbEKa+4BRo0Zp1KhRTT5/7ty56t69u5577jlJ0jnnnKNPP/1Uzz//vLKzsyVJs2bN0l133aWJEyd6HrN06VLNmzdPjz76aHNL9Fs0UAAAmKWiulalVbWSmCYHIHi0+Zqh1atXa+TIkV7HsrOztXr1aklSdXW11q9f73WO1WrVyJEjPecEi+PT5BwmVwIACDYF9VPkIsNsirY3+7tSAPBLbf5ul5eXp9TUVK9jqampKikpUWVlpY4dOyan09ngOVu3bm30eR0OhxyO46GhpKSkdQs3QVJU3TdxxypqTK4EABBs8k6YImexWEyuBgDah992k5s5c6bi4uI8t/T0dLNLOmMJUaGS6rrJOV2GydUAAIIJneQABKM2D0NpaWnKz8/3Opafn6/Y2FhFREQoOTlZNputwXPS0tIafd4pU6aouLjYc9u3b1+b1N+eEiLrpskZhlREe20AQDsqoJMcgCDU5mFo6NChWrlypdexFStWaOjQoZKksLAwDRo0yOscl8ullStXes5piN1uV2xsrNfN34XarIqLqBsdookCAKA90UkOQDBqdhgqKytTbm6ucnNzJdW1zs7NzdXevXsl1Y3YjB8/3nP+Pffco507d+qRRx7R1q1b9eKLL2rRokV68MEHPedMnjxZr776qt544w19++23uvfee1VeXu7pLhdMkugoBwAwgXvNUIcYpskBCB7NbqCwbt06jRgxwvPz5MmTJUkTJkzQ/PnzdejQIU8wkqTu3btr6dKlevDBB/XCCy+oS5cu+tOf/uRpqy1JY8aM0eHDh/X4448rLy9PAwYM0PLly09qqhAMEqPCtLOwnDAEAGhXTJMDEIyaHYYuu+wyGUbji/vnz5/f4GO++uqrUz7v/fffr/vvv7+55QScBE97bcIQAKD95JfWjQylxRGGAAQPv+0mF6iYJgcAaG+GYRxfMxRDGAIQPAhDPiaRMAQAaGcllbWqqnFJkjrQWhtAECEM+RjCEACgvbmnyMVFhCo81GZyNQDQfghDPoYwBABob+4pcmk0TwAQZAhDPiaRBgoAgHaWX99JjilyAIINYcjHJEXVXYiOljtMrgQAECzYcBVAsCIM+ZjE6LqRoWPlNadsYQ4AQGs5HoYYGQIQXAhDPiYxsi4MVTtdKnPUmlwNACAYsGYIQLAiDPmYiDCbIuo7+dBEAQDQHo6vGSIMAQguhCEf5G6icLiUdUMAgLZXwJohAEGKMOSDenaIliR9e6jE5EoAAIHO5TJUUP/lG2uGAAQbwpAPGtAlTpK0cX+xyZUAAALdkfJq1boMWSxSSjRhCEBwIQz5oP5d4iVJG/cVmVoHACDwuZsnJEfbFWLjYwGA4MK7ng/qn143MrT9cJlKq2pMrgYAEMgKSmmrDSB4EYZ8UIeYcHWOj5BhSJsOMFUOANB28orr1wvF0DwBQPAhDPmozPrRoa9ZNwQAaEOeDVfjCEMAgg9hyEexbggA0B480+QYGQIQhAhDPiqTMAQAaAfuDVdZMwQgGBGGfFS/LnGyWKSDxVWeb+0AAI2bM2eOMjIyFB4erqysLK1Zs+aU58+ePVu9e/dWRESE0tPT9eCDD6qq6vj77apVq5STk6NOnTrJYrFoyZIlbfwKzJFXzIarAIIXYchHRdtDdHb95qtf72PdEACcysKFCzV58mTNmDFDGzZsUGZmprKzs1VQUNDg+W+++aYeffRRzZgxQ99++61ee+01LVy4UFOnTvWcU15erszMTM2ZM6e9XoYp3F+4dWBkCEAQIgz5MM+6of1FptYBAL5u1qxZuuuuuzRx4kT17dtXc+fOVWRkpObNm9fg+Z9//rkuuugijRs3ThkZGbrqqqs0duxYr9GkUaNG6de//rVuuOGG9noZ7a7G6VJhWbUkKY2RIQBBiDDkwzLT4yVJuawbAoBGVVdXa/369Ro5cqTnmNVq1ciRI7V69eoGHzNs2DCtX7/eE3527typZcuWafTo0S2uw+FwqKSkxOvm6w6X1q0XCrVZlBAZZnI1AND+QswuAI0bUD8y9PX+YhmGIYvFYm5BAOCDCgsL5XQ6lZqa6nU8NTVVW7dubfAx48aNU2FhoYYPHy7DMFRbW6t77rnHa5pcc82cOVNPPvlkix9vhrz6ttodYsJltXKNARB8GBnyYb3TYhQWYlVxZY32HKkwuxwACBgff/yxnn32Wb344ovasGGDFi9erKVLl+rpp59u8XNOmTJFxcXFntu+fftaseK2UVDCeiEAwY2RIR8WFmLVuZ1i9dXeIm3cX6SM5CizSwIAn5OcnCybzab8/Hyv4/n5+UpLS2vwMdOnT9ftt9+uSZMmSZL69eun8vJy3X333XrsscdktTb/u0K73S673b9ChbutNuuFAAQrRoZ8nHu/IdYNAUDDwsLCNGjQIK1cudJzzOVyaeXKlRo6dGiDj6moqDgp8NhsNkmSYRhtV6yPyS+hrTaA4MbIkI/LTI+TxOarAHAqkydP1oQJEzR48GANGTJEs2fPVnl5uSZOnChJGj9+vDp37qyZM2dKknJycjRr1iwNHDhQWVlZ2r59u6ZPn66cnBxPKCorK9P27ds9v2PXrl3Kzc1VYmKiunbt2v4vsg24R4aYJgcgWBGGfJx7ZOibgyWqcboUamMwDwB+aMyYMTp8+LAef/xx5eXlacCAAVq+fLmnqcLevXu9RoKmTZsmi8WiadOm6cCBA0pJSVFOTo6eeeYZzznr1q3TiBEjPD9PnjxZkjRhwgTNnz+/fV5YG/OMDMUwMgQgOFmMAJkPUFJSori4OBUXFys2NtbsclqNy2VowFP/UUlVrf71P8N1Xuc4s0sCAC+B+v57pvzh73LlrP9qW0GZFkzK0kVnJZtdDgC0mqa+BzPM4OOsVgubrwIA2sTxNUNMkwMQnAhDfoB1QwCA1lZZ7VRJVa0kqQMNFAAEKcKQH3CvG9q4r9jcQgAAAcM9KhQRalOMnSXEAIITYcgPDEiPlyRtKyhVuaPW3GIAAAHBHYbS4sJlsVhMrgYAzEEY8gMdYsOVFhsulyFtPsDoEADgzOWX1rfVjmG9EIDgRRjyE551QzRRAAC0ggI2XAUAwpC/yKyfKse6IQBAa8grppMcABCG/MQA2msDAFqRe5ocI0MAghlhyE+c16Vumtz+Y5UqLHOYXA0AwN/lM00OAAhD/iI2PFQ9U6IkSV8zOgQAOEOsGQIAwpBfca8bymXdEADgDBiGobwS1gwBAGHIj7j3G2JkCABwJkqqalVV45LEyBCA4EYY8iP93U0U9hXJMAxziwEA+C33FLm4iFCFh9pMrgYAzEMY8iPndIxRqM2iYxU12ne00uxyAAB+Kr/E3UmOKXIAghthyI/YQ2zq2zFWkpTLVDkAQAvRSQ4A6hCG/Iy7icLX+4pMrQMA4L/czRM6xBCGAAQ3wpCf6c/mqwCAM+ReM5QWxzQ5AMGNMORnBqTXbb666UCxap0uk6sBAPij42uGGBkCENwIQ36mR3K0ou0hqqpx6fv8MrPLAQD4ofxSpskBgEQY8jtWq0X9u9SNDrHfEACgJfKL2XAVACTCkF9i3RAAoKVcLkMFpXXT5NLiGBkCENwIQ37IvW4od1+xyZUAAPzN0Ypq1boMWSxScjQjQwCCG2HID7nba3+fX6rKaqe5xQAA/Ip7j6GkKLtCbXwMABDceBf0Q2mx4eoQY5fTZeibg4wOAQCa7viGq4wKAQBhyA9ZLBbPuqFcNl8FADSDu612Gm21AYAw5K/c64Y27mdkCADQdO6RoQ6EIQAgDPkr97qhjYwMAQCa4fiGq0yTAwDCkJ/q3zlekrT3aIWOlVebWwwAwG8UeNYMMTIEAIQhPxUXGaruyVGS2G8IANB0eTRQAAAPwpAfy+xSv26I/YYAAE10fJocI0MAQBjyY551Q4wMAQCaoMbp0pFywhAAuBGG/Jg7DH29v0iGYZhbDADA5xWWOWQYUojVosTIMLPLAQDTEYb8WN+OsQqxWlRYVq0DRZVmlwMA8HF5xfVttWPsslotJlcDAOYjDPmx8FCb+nSMkcS6IQDA6XnWC8UxRQ4AJMKQ38vsEi+JdUMAgNMrKK3vJBdDGAIAiTDk99h8FQDQVPm01QYAL4QhPzegPgxtOlAsp4smCgCAxuUV102T60AnOQCQRBjyez1TohUZZlNFtVPbC8rMLgcA4MPc0+TSCEMAIIkw5PdsVov6dXZvvlpkbjEAAJ92fJocYQgAJMJQQBjA5qsAgCbwdJNjzRAASCIMBYRMwhAA4DSqapwqrqyRxJohAHAjDAWA/l3qpsltPVSqqhqnydUAAHyRe4pcRKhNseEhJlcDAL6BMBQAOsdHKDk6TLUuQ98cLDG7HACADzpxipzFYjG5GgDwDYShAGCxWDybr37NVDkAQAPcI0NMkQOA4whDAYLNVwEAp0InOQA4GWEoQLjXDW3cX2xyJQAAX+QJQzF0kgMAN8JQgHBPk9tVWK7iihpziwEA+Bz3mqG0OEaGAMCNMBQgEqLC1C0pUpL09YEic4sBAPgc1gwBwMkIQwHEPTrEuiEAwA8VlNZ3k2OaHAB4EIYCiHvdUO4+1g0BAI4zDEN5xTRQAIAfIgwFkAHujnL7i2QYhrnFAAB8RqmjVpX1m3IThgDgOMJQADm3U5xsVosOlzqUVz83HACAgvprQmx4iCLCbCZXAwC+gzAUQCLCbOqdGiOJdUMAgOPcneQYFQIAb4ShAJOZzrohAIA31gsBQMMIQwGGjnIAgB/KLyUMAUBDCEMBJrO+icKmA8VyuWiiAACQCjzT5GirDQAnIgwFmLM7RCsi1KYyR612FpaZXQ4AwAe4N1xlZAgAvLUoDM2ZM0cZGRkKDw9XVlaW1qxZ0+i5NTU1euqpp9SzZ0+Fh4crMzNTy5cv9zrniSeekMVi8br16dOnJaUFvRCbVed1jpXEuiEAQJ08TxhiZAgATtTsMLRw4UJNnjxZM2bM0IYNG5SZmans7GwVFBQ0eP60adP08ssv6w9/+IO2bNmie+65RzfccIO++uorr/POPfdcHTp0yHP79NNPW/aKwLohAICXArrJAUCDmh2GZs2apbvuuksTJ05U3759NXfuXEVGRmrevHkNnv+Xv/xFU6dO1ejRo9WjRw/de++9Gj16tJ577jmv80JCQpSWlua5JScnt+wVwbNu6Ov9RabWAQAwn8tlqIAGCgDQoGaFoerqaq1fv14jR448/gRWq0aOHKnVq1c3+BiHw6HwcO8334iIiJNGfrZt26ZOnTqpR48euvXWW7V3795T1uJwOFRSUuJ1Q50B9WFoy6ESOWqd5hYDADDVsYpq1TjrGuqkxDBNDgBO1KwwVFhYKKfTqdTUVK/jqampysvLa/Ax2dnZmjVrlrZt2yaXy6UVK1Zo8eLFOnTokOecrKwszZ8/X8uXL9dLL72kXbt26eKLL1ZpaWmjtcycOVNxcXGeW3p6enNeSkDrkhChhMhQ1TgNfXuo8b8hACDwuTdcTY4OU6iNvkkAcKI2f1d84YUXdPbZZ6tPnz4KCwvT/fffr4kTJ8pqPf6rR40apZtvvln9+/dXdna2li1bpqKiIi1atKjR550yZYqKi4s9t3379rX1S/EbFovFM1WOdUMAENzoJAcAjWtWGEpOTpbNZlN+fr7X8fz8fKWlpTX4mJSUFC1ZskTl5eXas2ePtm7dqujoaPXo0aPR3xMfH69evXpp+/btjZ5jt9sVGxvrdcNxniYKrBsCgKBGGAKAxjUrDIWFhWnQoEFauXKl55jL5dLKlSs1dOjQUz42PDxcnTt3Vm1trd555x1dd911jZ5bVlamHTt2qGPHjs0pDycYwMgQAEDHp8nRVhsATtbsaXKTJ0/Wq6++qjfeeEPffvut7r33XpWXl2vixImSpPHjx2vKlCme87/88kstXrxYO3fu1CeffKIf/ehHcrlceuSRRzznPPzww/rvf/+r3bt36/PPP9cNN9wgm82msWPHtsJLDE79u8RJknYcLldJVY3J1QAAzJJf30muQwwjQwDwQyHNfcCYMWN0+PBhPf7448rLy9OAAQO0fPlyT1OFvXv3eq0Hqqqq0rRp07Rz505FR0dr9OjR+stf/qL4+HjPOfv379fYsWN15MgRpaSkaPjw4friiy+UkpJy5q8wSCVF29UlIUL7j1Vq0/5iXXQWrcoBIBjlFzNNDgAa0+wwJEn333+/7r///gbv+/jjj71+vvTSS7Vly5ZTPt9bb73VkjJwGpnp8dp/rFIb9xcRhgAgSLlHhtLimCYHAD9Ej80ANsDdRIF1QwAQtNxrhpgmBwAnIwwFMPe6oY37ik2uBABghlqnS4Vl7gYKhCEA+CHCUAA7r3OcrBYpr6TK01oVABA8Dpc5ZBhSiNWipKgws8sBAJ9DGApgUfYQ9UqNkcRUOQAIRsenyNlltVpMrgYAfA9hKMCx+SoABC/3rIAOTJEDgAYRhgJcpmfzVdYNAUCwKShxt9WmkxwANIQwFOA8TRT2F8nlMkyuBgDQnvJK2GMIAE6FMBTgeqfFyB5iVWlVrXYfKTe7HABAO3KvGSIMAUDDCEMBLtRm1Xmdj48OAQCCRz4jQwBwSoShIOBposC6IQAIKgWekSHWDAFAQwhDQSAzvW5kKJf22gAQVPJLGRkCgFMhDAUB98jQlkMlqq51mVsMAKBdVNU4VVRRI4kwBACNIQwFgW5JkYqLCFV1rUvf5ZWaXQ4AoB24p8iFh1oVGx5icjUA4JsIQ0HAYrF49hvKpYkCAASFE6fIWSwWk6sBAN9EGAoSme79hlg3BABBwdNJLoYpcgDQGMJQkDjeUa7I1DoAAO0jr7g+DMURhgCgMYShING/vqPc9sNlKnPUmlwNAKCtFZTWt9WOoa02ADSGMBQkOsSEq3N8hAxD2rSf/YYAINCx4SoAnB5hKIj0d68bookCAAQ8dxjqwIarANAowlAQcXeUY90QAAS+/PrW2owMAUDjCENBxN1E4WumyQFAQDMMwzMylEYYAoBGEYaCSL8ucbJYpANFlSqo338CABB4yhy1qqh2SmKaHACcCmEoiETbQ3RWSrQk6et9jA4BQKByT5GLCQ9RZFiIydUAgO8iDAUZz7ohmigAQMCikxwANA1hKMgcD0OMDAEILHPmzFFGRobCw8OVlZWlNWvWnPL82bNnq3fv3oqIiFB6eroefPBBVVV5TyFu7nP6CtYLAUDTEIaCzID6Jgob9xXJMAxziwGAVrJw4UJNnjxZM2bM0IYNG5SZmans7GwVFBQ0eP6bb76pRx99VDNmzNC3336r1157TQsXLtTUqVNb/Jy+xD1NjvVCAHBqhKEg0zstRmE2q4ora7TnSIXZ5QBAq5g1a5buuusuTZw4UX379tXcuXMVGRmpefPmNXj+559/rosuukjjxo1TRkaGrrrqKo0dO9Zr5Ke5z+lLmCYHAE1DGAoyYSFW9e0UK4l1QwACQ3V1tdavX6+RI0d6jlmtVo0cOVKrV69u8DHDhg3T+vXrPeFn586dWrZsmUaPHt3i5/Ql7o6hqTGMDAHAqdBiJggNSI9X7r4ibdxXrOsGdDa7HAA4I4WFhXI6nUpNTfU6npqaqq1btzb4mHHjxqmwsFDDhw+XYRiqra3VPffc45km15LndDgccjgcnp9LSkrO5GWdkbzi+jVDcYwMAcCpMDIUhDLT4yQxMgQgeH388cd69tln9eKLL2rDhg1avHixli5dqqeffrrFzzlz5kzFxcV5bunp6a1YcfMcXzNEGAKAU2FkKAj1r2+isPlAsWqcLoXayMQA/FdycrJsNpvy8/O9jufn5ystLa3Bx0yfPl233367Jk2aJEnq16+fysvLdffdd+uxxx5r0XNOmTJFkydP9vxcUlJiSiAyDOP4NDnCEACcEp+Cg1D3pCjFhIfIUevSd3mlZpcDAGckLCxMgwYN0sqVKz3HXC6XVq5cqaFDhzb4mIqKClmt3pdAm80mqS5MtOQ57Xa7YmNjvW5mOFZRoxpnXbfQlGjWDAHAqTAyFISsVosyu8Tr0+2F+np/sc7rHGd2SQBwRiZPnqwJEyZo8ODBGjJkiGbPnq3y8nJNnDhRkjR+/Hh17txZM2fOlCTl5ORo1qxZGjhwoLKysrR9+3ZNnz5dOTk5nlB0uuf0Ve71QsnRYQoL4TtPADgVwlCQykyP06fbC7VxX5HGZXU1uxwAOCNjxozR4cOH9fjjjysvL08DBgzQ8uXLPQ0Q9u7d6zUSNG3aNFksFk2bNk0HDhxQSkqKcnJy9MwzzzT5OX1Vfv0UuQ4xTJEDgNOxGAGy82ZJSYni4uJUXFxs2tQEf/L+N3n62V/Wq09ajJY/cInZ5QDwY7z/Nsysv8vCtXv1/97ZpBG9U/T6xCHt9nsBwJc09T2Y8fMgNSA9XpL0fX6pKqprzS0GANBq3J3kaJ4AAKdHGApSqbHhSosNl8uQNh8wby8MAEDryiupnyZHGAKA0yIMBTHPfkP7iswtBADQagrqw1AaYQgAToswFMTc+w3lsvkqAASM49PkaKsNAKdDGApi7nVDjAwBQODIL2HDVQBoKsJQEOvXpW6a3P5jlTpS5jC5GgDAmap1ulRY/37egZEhADgtwlAQiw0PVc+UKEnS1/uLTa4GAHCmCsuq5TIkm9Wi5CjCEACcDmEoyGW61w0xVQ4A/J57ilyHGLusVovJ1QCA7yMMBblM97ohmigAgN/Lp602ADQLYSjIucPQ1/uLZRiGucUAAM5Ifml9J7kYpsgBQFMQhoLcOR1jFGqz6Gh5tfYfqzS7HADAGcgvrt9jKI6RIQBoCsJQkLOH2NS3Y6wk1g0BgL+jrTYANA9hCJ7NV9lvCAD8m3uaXAemyQFAkxCG4LVuCADgvwoYGQKAZiEMQQPS6zZf3XSgWLVOl8nVAABaKq+ENUMA0ByEIahHcrSi7SGqrHFqW0GZ2eUAAFqgqsapoooaSVJqDGEIAJqCMARZrRb161w3OsS6IQDwT4fr1wvZQ6yKjQgxuRoA8A+EIUg6cfNV1g0BgD86sZOcxWIxuRoA8A+EIUg6vm6IkSEA8E+e9UI0TwCAJiMMQdLxkaHv8ktVWe00txgAQLPll9S31Y6lrTYANBVhCJLqvklMibHL6TL0zUGmygGAv6GtNgA0H2EIkiSLxaJM9+arrBsCAL9zfM0QI0MA0FSEIXiwbggA/FceI0MA0GyEIXgc7yhXZGodAIDmK6hfM0QYAoCmIwzBo3/neEnSniMVOlZebW4xAIBmyWdkCACajTAEj7jIUHVPjpIkfX2AdUMA4C/KHLUqr+8E2iGGNUMA0FSEIXjJ7MK6IQDwN+5RoRh7iKLsISZXAwD+gzAEL551Q4QhAPAb+cX1U+TimCIHAM1BGIKX/p722kUyDMPcYgAATZJfSlttAGgJwhC8nNspViFWiwrLqnWw/ptGAIBvy3d3kothZAgAmoMwBC/hoTb16RgjialyAOAv3GuGOtBJDgCahTCEk2S6p8oRhgDAL7jDUBrT5ACgWQhDOIk7DOUShgDAL+Sz4SoAtAhhCCdxd5TbfKBYThdNFADA1zFNDgBahjCEk5zVIVqRYTaVVzu143CZ2eUAAE7BMAwVeEaGmCYHAM1BGMJJbFaL+nWu23yVqXIA4NuOVdSo2umSJHWgmxwANAthCA1i81UA8A/uKXJJUWEKC+GyDgDNwbsmGuRuovD1/mJzCwEAnBLrhQCg5QhDaFBmet00uW8PlaiqxmlyNQCAxrBeCABajjCEBnWOj1BydJhqXYa2HCoxuxwAQCPy6keGUlkvBADNRhhCgywWi/qz+SoA+Dz3NLnUOMIQADQXYQiNYt0QAPi+fKbJAUCLEYbQKPe6IUaGAMB3FZQyTQ4AWoowhEa5R4Z2FparuKLG3GIAAA3yTJOjmxwANBthCI1KiApT18RISdLXB4rMLQYAcJJap0uHS+unycUxTQ4AmoswhFNyb77KuiEA8D1HyqvlMiSb1aKkKMIQADQXYQinlNmlbt3QV3uPmVwJAOCH3FPkUqLtslktJlcDAP6HMIRTurBHkiTpw60F+np/kbnFAAC80EkOAM4MYQindF7nOOVkdpLLkB59Z5NqnS6zSwIA1MujeQIAnBHCEE7r8Wv6Ki4iVFsOlei1T3eZXQ4AoF4BYQgAzkiLwtCcOXOUkZGh8PBwZWVlac2aNY2eW1NTo6eeeko9e/ZUeHi4MjMztXz58jN6TrSvlBi7Hrv6HEnS8x98r71HKkyuCAAgndhWm2lyANASzQ5DCxcu1OTJkzVjxgxt2LBBmZmZys7OVkFBQYPnT5s2TS+//LL+8Ic/aMuWLbrnnnt0ww036Kuvvmrxc6L93Tyoi4b2SFJVjUuPLdkkwzDMLgkAgp57zVAHRoYAoEWaHYZmzZqlu+66SxMnTlTfvn01d+5cRUZGat68eQ2e/5e//EVTp07V6NGj1aNHD917770aPXq0nnvuuRY/J9qfxWLRszf2U1iIVZ9sK9Q/vjpgdkkAEPTcI0NphCEAaJFmhaHq6mqtX79eI0eOPP4EVqtGjhyp1atXN/gYh8Oh8HDvN+mIiAh9+umnLX5O9/OWlJR43dC2uidH6ZdXnC1JevpfW3SkzGFyRQAQ3PJZMwQAZ6RZYaiwsFBOp1Opqalex1NTU5WXl9fgY7KzszVr1ixt27ZNLpdLK1as0OLFi3Xo0KEWP6ckzZw5U3FxcZ5benp6c14KWujuS3qoT1qMjlXU6Jn/396dhzdV5m0cv5O0TVroBnSFsoqsZS1UUMfR6YCgCOOOKIiDKzhox3kHFGRcGR2Hl3kVwQXcF3REwY1R6riALFIoUtkFodA2ZetC9yZ5/4AGKi20kHLS5Pu5rlzgITn8cix5evd5nt/5dLPR5QCA3yqvcuhwSaUk9gwBwJlq9G5y//rXv9S5c2d17dpVQUFBmjRpksaPHy+z+ez+6qlTp6qgoMD9yMrK8lDFOJVAi1kzr06UySQtWr9P327bb3RJAOCX8o7tFwoKMCs8ONDgagCgaWpQImnVqpUsFovsdnuN43a7XbGxsbW+JioqSh999JGKi4u1e/dubdmyRc2bN1fHjh3P+JySZLVaFRYWVuOBc6Nv20iNG9RekvTQRxtVWuEwtiAA8EMn7hcymUwGVwMATVODwlBQUJD69++vtLQ09zGn06m0tDQNGjTolK+12Wxq3bq1qqqq9MEHH2jkyJFnfU4Y54GhXRQfblPWoVLNXrbN6HIAwO9Ud5JjiRwAnLkGr1VLTU3VSy+9pNdee02bN2/W3XffreLiYo0fP16SNHbsWE2dOtX9/NWrV2vRokXauXOnvvvuO11++eVyOp36n//5n3qfE96nuTVAj43qKUl6efkuZe4rMLgiAPAv1TNDtNUGgDMX0NAX3HDDDdq/f78efvhh5ebmqk+fPlq6dKm7AcKePXtq7AcqKyvTtGnTtHPnTjVv3lzDhw/XG2+8oYiIiHqfE97pd91idEWvOH36Y46mLtqoD+8ZrABLo29DAwBIshcd6yQXShgCgDNlcvnI3TMLCwsVHh6ugoIC9g+dQ3lFZUr55zcqLKvStCu6acLFHY0uCcA5xudv7Rr7uty/MEMfrt+nqcO66s5LOnn8/ADQlNX3M5gf4+OsRIfa9ODwbpKkf36xTVmHSgyuCAD8Q27BsQYK4cwMAcCZIgzhrN0wIEHJHVqotNKhaR9lykcmGwHAq1Uvk4tmmRwAnDHCEM6ayWTSk1cnKijArG+27deSDdlGlwQAPi+PbnIAcNYIQ/CITlHNde+l50mSHv14kw4XVxhcEQD4riPlVTpSXiVJiqGbHACcMcIQPObOSzrp/JjmOlhcoSc+22x0OQDgs6rbaodaA9TM2uDGsACAYwhD8JigALNmXt1LJpP07/S9WrHjgNElAYBPOn6PIZbIAcDZIAzBo/q3i9QtF7STJD344UaVVToMrggAfM/x/UIskQOAs0EYgsf9ZWgXxYbZtPtgif6Vtt3ocgDA51TPDBGGAODsEIbgcaG2QD06sock6cVvd2pTdqHBFQGAb8klDAGARxCG0CiG9IjVsJ6xcjhdmrroRzmc3HsIADyFttoA4BmEITSav13VQ6G2AG3YW6DXvv/F6HIAwGewTA4APIMwhEYTE2bTlGFdJUnPfLFVew+XGFwRAPgGe1F1GGJmCADOBmEIjWr0gLYa0D5SJRUOPbz4J7lcLJcDgLPhcrlkp5scAHgEYQiNymw2aebViQqymPXVljx98mOO0SUBQJOWX1KpiiqnJCkqlJkhADgbhCE0uvOiQ3XPpZ0kSY98/JPySyoMrggAmq7qJXItmgXJGmAxuBoAaNoIQzgn7v5tJ50X3VwHjlRo5mdbjC4HAJqs6iVy0cwKAcBZIwzhnLAGWDTz6kRJ0sK1WVr580GDKwKApolOcgDgOYQhnDMD2rfQmOS2kqQHP9yoskqHwRUBQNNjLzgahmIJQwBw1ghDOKf+OqyrokOt2nWgWM99tcPocgCgyaGtNgB4DmEI51SYLVCPjuwhSZr3zc/akltocEUA0LS49wwxMwQAZ40whHPu8p5xGtI9RlVOl6Z8sFEOJ/ceAoD6ymPPEAB4DGEIhnh0ZE81twYoIytfb67abXQ5ANBk5BayZwgAPIUwBEPEhtv018u7SJKeXrpF2fmlBlcEAN7P4XRpf9HRZXLsGQKAs0cYgmHGJLdT/3aRKq5w6OHFmXK5WC4HAKdy8Ei5nC7JbJJaNicMAcDZIgzBMGazSTOvTlSgxaRlm/P0eWau0SUBgFerbp4QFWqVxWwyuBoAaPoIQzDU+TGhuvuSTpKkGUt+UkFJpcEVAYD3Yr8QAHgWYQiGu+fS89Qxqpn2F5Xr70u3GF0OAHgt+7EwRFttAPAMwhAMZwu0aOYfEiVJ76zZo9U7DxpcEQB4p+NttdkvBACeQBiCV0ju2FKjByZIkqZ+uFFllQ6DKwIA71O9ZygmlJkhAPAEwhC8xpRh3RQVatXO/cV6/uufjS4HALxO9Z6hmHDCEAB4AmEIXiM8OFB/G9FDkjT36x3aZi8yuCIA8C529zI5whAAeAJhCF5leGKsUrpFq9Lh0tRFG+V0cu8hAPUzZ84ctW/fXjabTcnJyVqzZk2dz/3tb38rk8l00uOKK65wP8dut+vWW29VfHy8QkJCdPnll2v79u3n4q3UKY8brgKARxGG4FVMJpMeHdlTzYIsSt99WG+t2WN0SQCagIULFyo1NVUzZszQunXr1Lt3bw0dOlR5eXm1Pn/RokXKyclxPzIzM2WxWHTddddJklwul0aNGqWdO3dq8eLFWr9+vdq1a6eUlBQVFxefy7fmVl7l0KHiCknsGQIATyEMwevERwTrL0O7SJKe+nyLcgvKDK4IgLebNWuWbr/9do0fP17du3fXvHnzFBISogULFtT6/BYtWig2Ntb9+PLLLxUSEuIOQ9u3b9eqVas0d+5cDRgwQF26dNHcuXNVWlqqd95551y+Nbf9x2aFggLMiggJNKQGAPA1hCF4pVsGtVefhAgdKa/SjCWZRpcDwItVVFQoPT1dKSkp7mNms1kpKSlauXJlvc4xf/583XjjjWrWrJkkqbz8aPCw2Y7PwJjNZlmtVi1fvtyD1def/YS22iaTyZAaAMDXEIbglSxmk/5+TaICzCb95ye7lmbmGl0S4JOKy6u0OGOfFv7QdJekHjhwQA6HQzExMTWOx8TEKDf39J8da9asUWZmpiZMmOA+1rVrV7Vt21ZTp07V4cOHVVFRoaeeekp79+5VTk5OrecpLy9XYWFhjYcn0VYbADyPMASv1TU2THde0lGS9PDiTBWWVRpcEeAbKqqcWrbJrj+9s15Jjy/T5Hcz9MwX2+Tw04Yl8+fPV2JiogYOHOg+FhgYqEWLFmnbtm1q0aKFQkJC9N///lfDhg2T2Vz70Dlz5kyFh4e7HwkJCR6tk05yAOB5hCF4tXsv66wOrZopr6hcTy/dYnQ5QJPldLq08ueDmrroRw14YpkmvL5WSzZkq7TSofYtQzR6QILKq5rmzY5btWoli8Uiu91e47jdbldsbOwpX1tcXKx3331Xf/zjH0/6s/79+ysjI0P5+fnKycnR0qVLdfDgQXXs2LHWc02dOlUFBQXuR1ZW1pm/qVpUzwxF00kOADwmwOgCgFOxBVr05B8SNfqlVXpz1R6N6tNaSe1bGF0W0CS4XC5l7ivU4ox9+uTHHPcNOyUpOtSqK3vFa2SfePVqE96k96AEBQWpf//+SktL06hRoyRJTqdTaWlpmjRp0ilf+/7776u8vFw333xznc8JDw+XdLSpwtq1a/XYY4/V+jyr1SqrtfGCSvXMUCwzQwDgMYQheL1BnVrq+qQ2em/tXk1ZtFGf/ukiWQMsRpcFeK2f9x/RkoxsfbwhWzsPHG8DHWYL0LCecRrZJ17JHVvKYm66AejXUlNTNW7cOCUlJWngwIGaPXu2iouLNX78eEnS2LFj1bp1a82cObPG6+bPn69Ro0apZcuWJ53z/fffV1RUlNq2bauNGzdq8uTJGjVqlIYMGXJO3tOvsUwOADyPMIQm4cHh3fTVljztyDuieV/v1OSUzkaXBHiVnIJSfbIhR0s2ZGvjvgL3cVugWSndYnRV73hd0iXKZ3+QcMMNN2j//v16+OGHlZubqz59+mjp0qXupgp79uw5aa/P1q1btXz5cn3xxRe1njMnJ0epqamy2+2Ki4vT2LFjNX369EZ/L3WpDkMskwMAzzG5XC6f2DFbWFio8PBwFRQUKCwszOhy0Ag+3pCte99ZryCLWf++e5B6tYkwuiTAUIeLK/R5Zq4WZ+zTml8OqfrT3GI26TedW+mqPvH6ffdYNbc27s+9+PytnaevS+KM/6iovEppf75EnaKae6BCAPBd9f0MZmYITcaVveL00fp9StuSp5teWq2XxyXpgo4nL20BfFlJRZW+3GTXkoxsfbNtv6pO6AA3sH0LjegTr+E9Y9WyObMHvqS4vEpF5VWSWCYHAJ5EGEKTYTKZNPvGPprw2lqt3nVIYxes0Zyb+un33WNO/2KgCauocuq77fu1OCNbX26yq7TyeNe37nFhuqpPvEb0jlfriGADq0Rjql4i19wa0OgzfQDgT/hERZMSagvUa7cN1KS312vZZrvuejNdT13TS9f2b2N0aYBHOZ0urd51SEs2ZOvzzBzllxy/z1a7liG6qne8ruodr84xoQZWiXOFttoA0DgIQ2hybIEWzbu5n/76wUZ9sG6vHnh/g/JLKjTh4trv/QE0FdWtsJds2KePN9RshR0VatWVveI0sk9r9W7irbDRcHlFxzrJhbJEDgA8iTCEJinAYtY/ru2lyJBAvbx8lx7/dLMOl1TogSFd+CYRTc7O/Ue0ZEO2lmTUbIUdagvQ8J5xuqpPvC7wsVbYaJjcgmP3GAonDAGAJxGG0GSZzSY9dEU3RTYL0j/+s1Vz/vuzDpdU6rGRPfmmEV4vt6BMn/yYrcUZNVthWwPMSul+tBX2b324FTYahmVyANA4CENo0kwmkyZeep4iQ4L00Ecb9fbqPSooqdSsG3rzTSS8UlmlQzMW/6T30rNqtMK+uHMrjTxHrbDR9NhZJgcAjYIRFz7hpuS2Cg8O1H0L1+vTjTkqLKvUvJv7qxnfVMKL5BWW6Y430pWRlS9JGtA+Ulf1aU0rbJxW3rH9Y7TVBgDP4jtF+IwresUpLDhAd76Rru+2H9CYl1frlVsHKLJZkNGlAdqQla873lgre2G5woMD9dxNfXVx5yijy0ITUb1MLjac0AwAnmQ2ugDAky7uHKW3JiQrIiRQGVn5uv6Fle6Nx4BRFmfs0/UvrJS9sFznRTfXRxMvJAih3lwul7uzYDTL5ADAowhD8Dl920bqvTsHKSbMqu15R3TN3O+164QOXcC54nC69NTSLZr8bobKq5y6rGu0Ft0zWB1aNTO6NDQhBaWVqqhySqKBAgB4GmEIPun8mFD9+66j33Tuyy/VtXO/V+YJHbuAxlZUVqk7Xl+ruV//LEm665JOemlsksJsgQZXhqameolcZEggjWEAwMMIQ/BZCS1C9N6dg9QjPkwHiys0+sVVWr3zoNFlwQ/8cqBYf3j+e6VtyVNQgFmzb+ijKcO60vIdZ8RO8wQAaDSEIfi0qFCr3rnjAg3s0EJF5VUau2CNlm2yG10WfNiKHQc0cs4K7cg7opgwq96/c5BG9W1tdFlownIJQwDQaAhD8HlhtkC9fttApXSLUXmVU3e+ma4P0vcaXRZ8jMvl0qsrdmnsgjUqKK1U74QILZl0kXonRBhdGpq442212S8EAJ5GGIJfsAVaNO/mfrq6X2s5nC79+f0Nevm7nUaXBR9RUeXU1EUb9bePN8nhdOnqvq218I4L+Ek+PKJ6zxBfTwDgedxnCH4jwGLWM9f2VkRwkBas2KXHP92s/JJK/XnI+TKZ2MuBM3PgSLnufjNdP/xyWGaTNHVYN024uANfU/CY6j1D0YQhAPA4whD8itls0vQru6ll8yD94z9b9dx/d+hwSYUeHdmTze1osJ+yC3TH6+nal1+qUGuA/u+mvrq0S7TRZcHHVIehWMIQAHgcYQh+x2QyaeKl5ykiJFDTPsrUW6v3KL+0Uv97fR8FBbByFPXz2cYc/fm9DSqtdKhDq2Z6aWySzotubnRZ8EHHl8mxZwgAPI0wBL81JrmdwoMDdf/CDH36Y44KSys17+b+amblnwXq5nS6NDttu/4vbbsk6eLOrfTc6H4KD+H+QfA8h9Ol/UfYMwQAjYUfg8OvXdkrXi+PG6DgQIu+235AN89frfySCqPLgpcqLq/SPW+tcwehCRd10Cu3DiAIodEcLC6Xw+mS2SS1bBZkdDkA4HMIQ/B7l5wfpbduT1Z4cKDW78nX9S+sVG5BmdFlwctkHSrRNXO/19KfchVkMevpa3tp2pXdFWDhYxSNx15wdFYoKtTK1xoANAI+WQFJ/dpG6v27BikmzKpt9iO6dt732nWg2Oiy4CVW7zyokXNWaEtukVo1t+qdO5J1fVKC0WXBD9i54SoANCo2RwDHnB8Tqn/fNVi3zF+tXw6W6Lp53+vV8QPVs3W40aV5vUqHU/bCMmXnlyk7v1T78kuVnV8qa4BFQ3rEaED7Fk22W9/bq/fo4cWZqnK6lNg6XC/c0l/xEcFGlwU/YS861lY7lDAEAI2BMAScIKFFiN6/a7DGLVijTTmFGv3iKr08LknJHVsaXZphXC6XCkur3AEnu6A67BwNPtn5pbIXlsnpqv31C1bsUlSoVcN6xuqKxDglNZFgVOlw6rFPNun1lbslSSN6x+vpa3opOMhicGXwJ3SSA4DGRRgCfiUq1Kp377xAE15bqzW7DmnsgjWac1M/pXSPMbq0RlFRdXRWxx128ku174Sgk51fquIKx2nPE2QxKy7CpvjwYMVHBCs+wqacgjJ98VOu9heV6/WVu/X6yt2KCrVqeM9YDffiYHS4uEL3vLVOK3celMkkPTCki+75bSdupIpzLo97DAFAoyIMAbUIswXq9dsGatLb67Rsc57ufDNd/7i2l67u18bo0hrE5XIpv6SyRtDJLqgZfPKKyuWqY1bnRC2bBblDTnxEsFpHVIeeo8daNbPKXEuwqfhDolbsOKBPN+boP8eC0Wsrd+u1lbsVHWrV8MS4o8GoXWStrz/XtuYWacLrPyjrUKmaBVk0+8a++r2PBmF4v1z2DAFAoyIMAXWwBVo09+b++uu/f9Si9fuU+t4G5ZdU6raLOhhdmpvD6VJeUZmyDpUq61CJ9h6uuZQtJ79MpZX1mNUJMB8LN8dndlqfEHTiwoPPeHlYUIBZl3aN1qVdo/XksWD0yY85+mJTrvKKyvXq97/o1e9/UUyYVcN6xumKXnHq39aYYPTlJrvue3e9iiscatsiRC+PS9L5MaHnvA6gWvUyuWiWyQFAoyAMAacQaDHrmet6KyIkSAtW7NKjn2zS4ZIKpf7+/HOyZMrlOnrDxb2Hj4edvYdLlHXo6K/78ktV6Tj9tE6r5la1PjajU/048b9bNgs6J+/nxGBUXtXz6IzRj7n6YlOu7IXGBSOXy6Xnv/5Zz3yxVS6XNKhjSz0/pp8iua8LDJbHzBAANCrCEHAaZrNJ06/sphbNAvXMF9v07Fc7dLikQo9c1fOs97tUL2PLOlxSI/BkHS5x/768ynnKc1jMJsVH2JQQGaI2kcFqHRGi+Aibe2YnNtwmW6D3bfq3Blh0WdcYXdY1RuVVPbV8+9GldF9ustcIRrFhNl3eM1ZX9opTv0YIRqUVDv3l3xv0yY85kqRxg9pp2pXdFcg9XWCwiiqnDhYfvQk0e4YAoHEQhoB6MJlMmnRZZ0WEBGn64ky9uWqP8ksqNev6PgoKOPU3zUVlle6ZnKxfzfDsPVyqI+VVp/m7pbgwm9pEhqhNi2C1iQxRQuSxX1sEKzbM1uRvxmgNsOh33WL0u24xKq9yHA9GP9mVW1hWIxgNSzzalc4TwSg7v1R3vLFWmfsKFWA26dGRPXVTclsPvSvg7OQda6sdZDErIiTQ4GoAwDcRhoAGuPmCdgoPDlTqexn65MccFZZV6X+v761DxRU1ZneyDpVqb/7RXwtKK0973qhQq9pEBivhWMA5GniOzvTERwSfNnD5kl8Ho++2HdBnx2aMcgvL9MqKX/TKiqPBaHhinK7oFau+CQ0PRum7D+vON9J14Ei5WjQL0twx/fy6hTq8z4n7hehkCACNgzAENNCI3vEKCw7UXW+k69tt+9X/8WWnfU1kSKASWoS4A0+bE38fGeyVy9i8gTXAopTuMUrpfjwYfboxR8uOBaMFK3ZpwYpdigu3ufcY9U2IOG0wen9tlh76MFMVDqe6xobqpbFJSmgRco7eFVA/7BcCgMZHGALOwCXnR+nNCcnumYVQa4DatKi5fC3hhGVtza38UztbJwajskqHvtt+fMYop6BmMKpu1/3rYFTlcGrm51s0f/kuSdKwnrF65rreasb/H3ghuzsM0UkOABoL3wEAZ6h/u0itmnqZissdCmc9/zllC7To991j9PsTgtGnP2Zr2eY85RSUaf7yXZq/fJfiw20alnh0xqhjq2a69531+m77AUnSfSmd9afLOnvFvY2A2uQeWybHzBAANB7CEHAWAixmhYf4z34eb/TrYPTttv3uGaPsE4JRoMWkSodLwYEWzbq+t4YlxhldOnBKLJMDgMZHGALgM2yBFg3pEashPWLdwah6j1FxhUOtI4L10tgkdY8PM7pU4LTsRSyTA4DGRhgC4JN+HYzW7TmsHvHhCg9mSSOahimXd9MvB4vVr12k0aUAgM8iDAHwebZAiwZ3amV0GUCDJLYJV2KbcKPLAACfxmYHAAAAAH6JMAQAAADALxGGAAAAAPglwhAAAAAAv3RGYWjOnDlq3769bDabkpOTtWbNmlM+f/bs2erSpYuCg4OVkJCg+++/X2VlZe4//9vf/iaTyVTj0bVr1zMpDQAAAADqpcHd5BYuXKjU1FTNmzdPycnJmj17toYOHaqtW7cqOjr6pOe//fbbmjJlihYsWKDBgwdr27ZtuvXWW2UymTRr1iz383r06KFly5YdLyyARncAAAAAGk+DZ4ZmzZql22+/XePHj1f37t01b948hYSEaMGCBbU+//vvv9eFF16om266Se3bt9eQIUM0evTok2aTAgICFBsb6360akUbXAAAAACNp0FhqKKiQunp6UpJSTl+ArNZKSkpWrlyZa2vGTx4sNLT093hZ+fOnfrss880fPjwGs/bvn274uPj1bFjR40ZM0Z79uw5ZS3l5eUqLCys8QAAAACA+mrQWrQDBw7I4XAoJiamxvGYmBht2bKl1tfcdNNNOnDggC666CK5XC5VVVXprrvu0oMPPuh+TnJysl599VV16dJFOTk5euSRR3TxxRcrMzNToaGhtZ535syZeuSRRxpSPgAAAAC4NXo3ua+//lpPPvmknn/+ea1bt06LFi3Sp59+qscee8z9nGHDhum6665Tr169NHToUH322WfKz8/Xe++9V+d5p06dqoKCAvcjKyursd8KAAAAAB/SoJmhVq1ayWKxyG631zhut9sVGxtb62umT5+uW265RRMmTJAkJSYmqri4WHfccYceeughmc0n57GIiAidf/752rFjR521WK1WWa3WhpQPAAAAAG4NmhkKCgpS//79lZaW5j7mdDqVlpamQYMG1fqakpKSkwKPxWKRJLlcrlpfc+TIEf3888+Ki4trSHkAAAAAUG8N7l+dmpqqcePGKSkpSQMHDtTs2bNVXFys8ePHS5LGjh2r1q1ba+bMmZKkESNGaNasWerbt6+Sk5O1Y8cOTZ8+XSNGjHCHogceeEAjRoxQu3btlJ2drRkzZshisWj06NEefKsAAAAAcFyDw9ANN9yg/fv36+GHH1Zubq769OmjpUuXupsq7Nmzp8ZM0LRp02QymTRt2jTt27dPUVFRGjFihJ544gn3c/bu3avRo0fr4MGDioqK0kUXXaRVq1YpKirKA28RAAAAAE5mctW1Vq2JKSwsVHh4uAoKChQWFmZ0OQDgN/j8rR3XBQCMU9/P4EbvJgcAAAAA3ogwBAAAAMAvEYYAAAAA+CXCEAAAAAC/RBgCAAAA4JcIQwAAAAD8EmEIAAAAgF8iDAEAAADwS4QhAAAAAH6JMAQAAADALxGGAAAAAPglwhAAAAAAv0QYAgAAAOCXCEMAAAAA/BJhCAAAAIBfIgwBAAAA8EsBRhfgKS6XS5JUWFhocCUA4F+qP3erP4dxFOMSABinvmOTz4ShoqIiSVJCQoLBlQCAfyoqKlJ4eLjRZXgNxiUAMN7pxiaTy0d+lOd0OpWdna3Q0FCZTKYGv76wsFAJCQnKyspSWFhYI1TYdHFt6sa1qRvXpm6+dm1cLpeKiooUHx8vs5nV19UYlxoX16duXJu6cW3q5mvXpr5jk8/MDJnNZrVp0+aszxMWFuYTXwCNgWtTN65N3bg2dfOla8OM0MkYl84Nrk/duDZ149rUzZeuTX3GJn6EBwAAAMAvEYYAAAAA+CXC0DFWq1UzZsyQ1Wo1uhSvw7WpG9emblybunFtUB98nZwa16duXJu6cW3q5q/XxmcaKAAAAABAQzAzBAAAAMAvEYYAAAAA+CXCEAAAAAC/RBgCAAAA4JcIQ5LmzJmj9u3by2azKTk5WWvWrDG6JMPNnDlTAwYMUGhoqKKjozVq1Cht3brV6LK80t///neZTCbdd999RpfiNfbt26ebb75ZLVu2VHBwsBITE7V27VqjyzKcw+HQ9OnT1aFDBwUHB6tTp0567LHHRB8b1Iax6WSMTfXH2FQT41Ld/H1s8vswtHDhQqWmpmrGjBlat26devfuraFDhyovL8/o0gz1zTffaOLEiVq1apW+/PJLVVZWasiQISouLja6NK/yww8/6IUXXlCvXr2MLsVrHD58WBdeeKECAwP1+eefa9OmTfrnP/+pyMhIo0sz3FNPPaW5c+fqueee0+bNm/XUU0/p6aef1rPPPmt0afAyjE21Y2yqH8ammhiXTs3fxya/b62dnJysAQMG6LnnnpMkOZ1OJSQk6N5779WUKVMMrs577N+/X9HR0frmm2/0m9/8xuhyvMKRI0fUr18/Pf/883r88cfVp08fzZ492+iyDDdlyhStWLFC3333ndGleJ0rr7xSMTExmj9/vvvYNddco+DgYL355psGVgZvw9hUP4xNJ2NsOhnj0qn5+9jk1zNDFRUVSk9PV0pKivuY2WxWSkqKVq5caWBl3qegoECS1KJFC4Mr8R4TJ07UFVdcUePrB9KSJUuUlJSk6667TtHR0erbt69eeuklo8vyCoMHD1ZaWpq2bdsmSdqwYYOWL1+uYcOGGVwZvAljU/0xNp2MselkjEun5u9jU4DRBRjpwIEDcjgciomJqXE8JiZGW7ZsMagq7+N0OnXffffpwgsvVM+ePY0uxyu8++67WrdunX744QejS/E6O3fu1Ny5c5WamqoHH3xQP/zwg/70pz8pKChI48aNM7o8Q02ZMkWFhYXq2rWrLBaLHA6HnnjiCY0ZM8bo0uBFGJvqh7HpZIxNtWNcOjV/H5v8OgyhfiZOnKjMzEwtX77c6FK8QlZWliZPnqwvv/xSNpvN6HK8jtPpVFJSkp588klJUt++fZWZmal58+b5/aDz3nvv6a233tLbb7+tHj16KCMjQ/fdd5/i4+P9/toADcXYVBNjU90Yl07N38cmvw5DrVq1ksVikd1ur3HcbrcrNjbWoKq8y6RJk/TJJ5/o22+/VZs2bYwuxyukp6crLy9P/fr1cx9zOBz69ttv9dxzz6m8vFwWi8XACo0VFxen7t271zjWrVs3ffDBBwZV5D3+8pe/aMqUKbrxxhslSYmJidq9e7dmzpzpFwMO6oex6fQYm07G2FQ3xqVT8/exya/3DAUFBal///5KS0tzH3M6nUpLS9OgQYMMrMx4LpdLkyZN0ocffqivvvpKHTp0MLokr/G73/1OGzduVEZGhvuRlJSkMWPGKCMjw28Hm2oXXnjhSa1ut23bpnbt2hlUkfcoKSmR2VzzY9discjpdBpUEbwRY1PdGJvqxthUN8alU/P3scmvZ4YkKTU1VePGjVNSUpIGDhyo2bNnq7i4WOPHjze6NENNnDhRb7/9thYvXqzQ0FDl5uZKksLDwxUcHGxwdcYKDQ09aX16s2bN1LJlS9atS7r//vs1ePBgPfnkk7r++uu1Zs0avfjii3rxxReNLs1wI0aM0BNPPKG2bduqR48eWr9+vWbNmqXbbrvN6NLgZRibasfYVDfGproxLp2a349NLrieffZZV9u2bV1BQUGugQMHulatWmV0SYaTVOvjlVdeMbo0r3TJJZe4Jk+ebHQZXuPjjz929ezZ02W1Wl1du3Z1vfjii0aX5BUKCwtdkydPdrVt29Zls9lcHTt2dD300EOu8vJyo0uDF2JsOhljU8MwNh3HuFQ3fx+b/P4+QwAAAAD8k1/vGQIAAADgvwhDAAAAAPwSYQgAAACAXyIMAQAAAPBLhCEAAAAAfokwBAAAAMAvEYYAAAAA+CXCEAAAAAC/RBgCAAAA4JcIQwAAAAD8EmEIAAAAgF8iDAEAAADwS/8P0JjbTknU/6MAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, axs = plt.subplots(1, 2, figsize=(10, 10))\n", + "axs[0].set_title(\"Loss value on eval set\")\n", + "axs[0].plot(eval_metrics_history[\"test_loss\"])\n", + "axs[1].set_title(\"Accuracy on eval set\")\n", + "axs[1].plot(eval_metrics_history[\"test_accuracy\"])" + ] + }, + { + "cell_type": "markdown", + "id": "a3f7b0ad-ddfa-4ab3-b56f-6ea99385ff6a", + "metadata": {}, + "source": [ + "## Use Model for Inference\n", + "After all that, the product of what we were working for: a trained model we can save and load for inference. For people using LLMs recently, this pattern may look rather familiar: an input sentence tokenized into an array and computed 'next' token-by-token. While many recent LLMs are decoder-only, this was an encoder/decoder architecture with the very specific english-to-spanish pattern baked in.\n", + "\n", + "We've changed a couple things from the source 'use' function, here - because of the tokenizer used, things like `[start]` and `[end]` are no longer single tokens - instead `[start]` is `[29563, 60] = \"[start\" + \"]\"` and `[end]` is `[58308, 60] = \"[end\" + \"]\"` - thus we start with only a single token `[start` and can't only test on `last_token = \"[end\"]`. Otherwise, the main change here is that the input is assumed a single sentence, rather than batch inference." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "e4589706-cfd6-4efb-9975-bfa0df75d4f0", + "metadata": {}, + "outputs": [], + "source": [ + "def decode_sequence(input_sentence):\n", + " input_sentence = custom_standardization(input_sentence)\n", + " tokenized_input_sentence = tokenize_and_pad(input_sentence, tokenizer, sequence_length)\n", + " encoder_input = jnp.array([tokenized_input_sentence])\n", + "\n", + " emb_enc = model.positional_embedding(encoder_input)\n", + " encoder_outputs = model.encoder(emb_enc, mask=None)\n", + "\n", + " dummy_input_shape = (1, 40, model.config.embed_dim)\n", + " model.init_cache(dummy_input_shape)\n", + "\n", + " decoded_sentence = \"[start\"\n", + " current_token_id = tokenizer.encode(\"[start\")\n", + " current_input = jnp.array([current_token_id])\n", + "\n", + " for i in range(sequence_length):\n", + " logits = model.decode_step(current_input, encoder_outputs, step_index=i)\n", + "\n", + " sampled_id = np.argmax(logits[0, 0, :]).item()\n", + " sampled_token = tokenizer.decode([sampled_id])\n", + "\n", + " decoded_sentence += \"\" + sampled_token\n", + "\n", + " clean_token = sampled_token.strip()\n", + "\n", + " if sampled_token == \"[end]\" or clean_token == \"end\":\n", + " decoded_sentence += \"]\"\n", + " break\n", + "\n", + " # Update input for next loop\n", + " current_input = jnp.array([[sampled_id]])\n", + "\n", + " return decoded_sentence" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "554c2f72-0bd3-4ed1-804b-5f1a4cc13851", + "metadata": {}, + "outputs": [], + "source": [ + "test_eng_texts = [pair[0] for pair in test_pairs]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "c1d6edbb-af89-42c9-90c3-d61612b75da3", + "metadata": {}, + "outputs": [], + "source": [ + "test_result_pairs = []\n", + "for _ in range(10):\n", + " input_sentence = random.choice(test_eng_texts)\n", + " translated = decode_sequence(input_sentence)\n", + "\n", + " test_result_pairs.append(f\"[Input]: {input_sentence} [Translation]: {translated}\")" + ] + }, + { + "cell_type": "markdown", + "id": "258c2172-5a0f-4dee-9b21-f65433183c62", + "metadata": {}, + "source": [ + "## Test Results\n", + "For the model and the data, not too shabby - It's definitely spanish-ish. Though when 'making' friends, please don't confuse 'hacer' (to make) with 'comer' (to eat)." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "4f0ae018-b7cd-4849-b245-c5c647ad1a95", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Input]: This is the same necklace that I lost yesterday. [Translation]: [start] este es el mismo que perdí ayer [end]\n", + "[Input]: Tom has a house with two rooms. [Translation]: [start] tom tiene una casa de dos habitación [end]\n", + "[Input]: Tom is a schemer. [Translation]: [start] tom es un confunduista [end]\n", + "[Input]: You must study much harder. [Translation]: [start] debes estudiar más [end]\n", + "[Input]: Why isn't this working? [Translation]: [start] por qué no funciona esto [end]\n", + "[Input]: Do you like oranges? [Translation]: [start] te gustan las naranjas [end]\n", + "[Input]: I started writing a book. [Translation]: [start] empezé a escribir un libro [end]\n", + "[Input]: He's watching me. [Translation]: [start] él me está mirando [end]\n", + "[Input]: You speak Russian, don't you? [Translation]: [start] hablas ruso verdad [end]\n", + "[Input]: I didn't have any desire to do that. [Translation]: [start] no tenía nada de ganas de hacer eso [end]\n" + ] + } + ], + "source": [ + "for i in test_result_pairs:\n", + " print(i)" + ] + }, + { + "cell_type": "markdown", + "id": "5ca18d4c-b3c0-4abb-b5fc-fc96a2264b53", + "metadata": {}, + "source": [ + "Example output from the above cell:\n", + "\n", + " [Input]: Take this medicine after meals. [Translation]: [start] toma esta medicina después de comer [end]\n", + " [Input]: The English are said to be conservative. [Translation]: [start] el inglés son dijo que sería conservadores [end]\n", + " [Input]: Tom might call Mary tonight. [Translation]: [start] tom quizás podría llamar a mary esta noche [end]\n", + " [Input]: I have not finished lunch. [Translation]: [start] no he finalmente terminado [end]\n", + " [Input]: Are you ready to start? [Translation]: [start] estás listo para empezar [end]\n", + " [Input]: Tom worked as a lifeguard during the summer. [Translation]: [start] tom trabajó como un salvavidas durante el verano [end]\n", + " [Input]: Can I pay later? [Translation]: [start] puedo pagar más tarde [end]\n", + " [Input]: They went hand in hand. [Translation]: [start] ellos se fueron [end]\n", + " [Input]: You look like a baboon. [Translation]: [start] parecés como un papión [end]\n", + " [Input]: A cloud floated across the sky. [Translation]: [start] una sola nubeió en el cielo [end]" + ] + }, + { + "cell_type": "markdown", + "id": "cd25c648", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "jupytext": { + "formats": "ipynb,md:myst" + }, + "kernelspec": { + "display_name": "env (3.11.14)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}