diff --git a/examples/11_hwa_lm_eval.ipynb b/examples/11_hwa_lm_eval.ipynb new file mode 100644 index 0000000..901c762 --- /dev/null +++ b/examples/11_hwa_lm_eval.ipynb @@ -0,0 +1,470 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "547ae66e-5da6-43bb-8bb5-7ebdee8697ac", + "metadata": {}, + "source": [ + "# XBTorch::Example 09: Hardware-Aware Language Model Evaluation\n", + "\n", + "## Introduction\n", + "\n", + "In this example, we demonstrate how to use **XBTorch** to evaluate large language models (LLMs)\n", + "under simulated hardware noise and quantization effects.\n", + "This notebook uses the [`lm_eval`](https://github.com/EleutherAI/lm-evaluation-harness)\n", + "framework [1] for standardized evaluation, while wrapping the model with XBTorch's inference accelerator.\n", + "\n", + "Contrary to Example 07 on hardware-aware inference, here we will use the stateless mode of operation for the inference accelerator. This is because maintaining a stateful representation of a crossbar that is large enough to fit all parameters of an LLM would be cost-prohibitive.\n", + "\n", + "This enables researchers to **quantify degradation in performance** due to things like ADC/DAC bit precision limits\n", + "and analog crossbar variability.\n", + "\n", + "---\n", + "**Key Concepts**\n", + "- Integrating XBTorch into transformer model evaluation\n", + "- Simulating low-precision analog inference with `SimpleFixedPoint`\n", + "- Measuring task accuracy using `lm_eval`\n", + "\n", + "**We’ll demonstrate this using a Hugging Face model on the PIQA benchmark.**\n", + "---\n", + "\n", + "## Getting Started\n", + "\n", + "Let's begin by importing the required dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "4e143fb6-baec-4ad0-bc47-b71d390ed508", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/mnt/osama.yousuf1/xbtorch/env/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "W1010 23:09:48.035000 2851489 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. \n", + "W1010 23:09:48.035000 2851489 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures.\n" + ] + } + ], + "source": [ + "# General imports\n", + "import torch\n", + "import numpy as np\n", + "from transformers import AutoTokenizer, AutoModelForCausalLM\n", + "\n", + "# LM Evaluation Harness imports\n", + "from lm_eval import evaluator\n", + "from lm_eval.models.huggingface import HFLM\n", + "\n", + "# XBTorch imports\n", + "import xbtorch\n", + "from xbtorch.patches import xbtorch_model\n", + "from xbtorch.deployment import SimpleFixedPoint" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "200c8554-103a-45c5-91dc-944231f52be1", + "metadata": {}, + "outputs": [], + "source": [ + "def noisy_lm_evaluate(\n", + " model_path,\n", + " eval_tasks,\n", + " acc=None,\n", + " eval_limit=None,\n", + " num_fewshot=0,\n", + " exclude=None,\n", + " batch_size=32,\n", + " **config_kwargs\n", + "):\n", + " \"\"\"\n", + " Evaluate a Hugging Face causal LM under simulated noisy inference conditions\n", + " using XBTorch's SimpleFixedPoint accelerator and lm-evaluation-harness.\n", + "\n", + " Parameters\n", + " ----------\n", + " model_path : str\n", + " Path or Hugging Face Hub ID of the pretrained model.\n", + " eval_tasks : list[str]\n", + " Evaluation tasks (e.g., [\"piqa\", \"hellaswag\"]).\n", + " eval_limit : int, optional\n", + " Number of examples to limit evaluation to.\n", + " num_fewshot : int, optional\n", + " Number of few-shot examples per evaluation (default: 0).\n", + " exclude : list[str], optional\n", + " Layer names to exclude from XBTorch patching (default: [\"lm_head\"]).\n", + " batch_size : int, optional\n", + " Batch size used during evaluation.\n", + " config_kwargs : dict\n", + " Extra keyword arguments forwarded to model loading.\n", + "\n", + " Returns\n", + " -------\n", + " dict\n", + " A dictionary containing task results (accuracy, perplexity, etc.)\n", + " \"\"\"\n", + " if exclude is None:\n", + " exclude = []\n", + "\n", + " device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n", + "\n", + " # --- Step 1: Initialize inference accelerator ---\n", + " if (acc is None):\n", + " acc = SimpleFixedPoint(\n", + " g_min=100,\n", + " g_max=200,\n", + " adc_bits=8,\n", + " dac_bits=8,\n", + " device=device,\n", + " stateful=False # This is critical! LLMs can be too big for the stateful mode of operation.\n", + " )\n", + "\n", + " xbtorch.initialize(\n", + " inference_accelerator=acc,\n", + " pytorch_device=device\n", + " )\n", + "\n", + " # --- Step 2: Load pretrained model ---\n", + " default_config_kwargs = {\n", + " \"device_map\": device,\n", + " \"trust_remote_code\": True,\n", + " }\n", + " config_kwargs = {**default_config_kwargs, **config_kwargs}\n", + "\n", + " model = AutoModelForCausalLM.from_pretrained(model_path, **config_kwargs)\n", + " model = xbtorch_model(model, replace_all=True, exclude=exclude)\n", + " model.xb_eval(enable=True)\n", + "\n", + " tokenizer = AutoTokenizer.from_pretrained(model_path)\n", + "\n", + " # --- Step 3: Prepare evaluation harness ---\n", + " patched_lm_eval_model = HFLM(\n", + " pretrained=model,\n", + " tokenizer=tokenizer,\n", + " batch_size=batch_size\n", + " )\n", + "\n", + " # --- Step 4: Run evaluation ---\n", + " patched_results = evaluator.simple_evaluate(\n", + " model=patched_lm_eval_model,\n", + " tasks=eval_tasks,\n", + " num_fewshot=num_fewshot,\n", + " limit=eval_limit\n", + " )\n", + "\n", + " return patched_results[\"results\"]" + ] + }, + { + "cell_type": "markdown", + "id": "542a5e73-6be4-4d07-a7e0-0121699b694e", + "metadata": {}, + "source": [ + "## Hardware-Aware Evaluation Example\n", + "\n", + "Let’s now apply this function to a transformer model on the **PIQA** task. We will use [SpectraSuite's TriLM models](https://huggingface.co/SpectraSuite) [2] for this purpose.\n", + "You can substitute any Hugging Face model and benchmark supported by `lm_eval`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "342c7092-7bba-470a-9ec6-c36ee65aa28e", + "metadata": {}, + "outputs": [], + "source": [ + "eval_tasks = [\"piqa\"] # Task(s) to evaluate\n", + "num_fewshot = 0 # Zero-shot evaluation\n", + "eval_limit = 200#None # Limit evaluation samples for speed\n", + "batch_size = 40\n", + "\n", + "models = [\n", + " \"SpectraSuite/TriLM_830M_Unpacked\", \n", + " \"SpectraSuite/TriLM_3.9B_Unpacked\"\n", + " ]\n", + "\n", + "# Optionally, you can define a sweep for ADC/DAC precision bits\n", + "layer_sweep_params = {\n", + " \"adc_dac_bits\": [4, 8, 16]\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "f5866a73-83cd-4b7a-90b7-6bbcd17cdddc", + "metadata": {}, + "source": [ + "## Performing the Evaluation\n", + "\n", + "We can now loop over models (and optionally over ADC/DAC precision values)\n", + "to observe how quantization affects performance." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "445e2679-9002-45bd-b6b0-f1db068ee249", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluating model: SpectraSuite/TriLM_830M_Unpacked\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1158.99it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:33<00:00, 11.92it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'piqa': {'alias': 'piqa', 'acc,none': 0.635, 'acc_stderr,none': 0.03412767927155776, 'acc_norm,none': 0.675, 'acc_norm_stderr,none': 0.033202212797844806}}\n", + "Evaluating model: SpectraSuite/TriLM_3.9B_Unpacked\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████| 2/2 [00:10<00:00, 5.37s/it]\n", + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1153.23it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:45<00:00, 8.71it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'piqa': {'alias': 'piqa', 'acc,none': 0.67, 'acc_stderr,none': 0.03333249580187342, 'acc_norm,none': 0.695, 'acc_norm_stderr,none': 0.03263741725420572}}\n" + ] + } + ], + "source": [ + "results_summary = {}\n", + "\n", + "for model_path in models:\n", + " print(f\"Evaluating model: {model_path}\")\n", + " result = noisy_lm_evaluate(\n", + " model_path=model_path,\n", + " eval_limit=eval_limit,\n", + " batch_size=batch_size,\n", + " num_fewshot=num_fewshot,\n", + " eval_tasks=eval_tasks,\n", + " exclude=[\"lm_head\"]\n", + " )\n", + "\n", + " results_summary[model_path] = result\n", + " print(result)" + ] + }, + { + "cell_type": "markdown", + "id": "bde5edfd-75e2-4c51-a299-23c78daaa885", + "metadata": {}, + "source": [ + "## (Optional) Parameter Sweep\n", + "\n", + "To simulate progressively more aggressive quantization,\n", + "we can vary the ADC/DAC bit precision of the inference accelerator.\n", + "\n", + "Note: this section can be time-consuming for large models." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f8500d83-4432-49db-8ecf-4bf0b35418ec", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Testing with ADC/DAC precision = 4 bits\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1165.72it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:07<00:00, 51.70it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Testing with ADC/DAC precision = 8 bits\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1186.42it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:06<00:00, 62.97it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Testing with ADC/DAC precision = 16 bits\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1193.11it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:11<00:00, 34.24it/s]\n" + ] + } + ], + "source": [ + "sweep_results = {}\n", + "\n", + "for bits in layer_sweep_params[\"adc_dac_bits\"]:\n", + " print(f\"Testing with ADC/DAC precision = {bits} bits\")\n", + "\n", + " acc = SimpleFixedPoint(\n", + " g_min=100,\n", + " g_max=200,\n", + " adc_bits=bits,\n", + " dac_bits=bits,\n", + " device=\"cuda\" if torch.cuda.is_available() else \"cpu\",\n", + " stateful=False\n", + " )\n", + "\n", + " xbtorch.initialize(inference_accelerator=acc)\n", + "\n", + " result = noisy_lm_evaluate(\n", + " model_path=models[0], # using the 830M model for this experiment\n", + " acc=acc,\n", + " eval_limit=eval_limit,\n", + " batch_size=batch_size,\n", + " num_fewshot=num_fewshot,\n", + " eval_tasks=eval_tasks,\n", + " exclude=[\"lm_head\"]\n", + " )\n", + "\n", + " sweep_results[bits] = result" + ] + }, + { + "cell_type": "markdown", + "id": "05722f0c-8349-4f5b-8d0a-230126bc8a29", + "metadata": {}, + "source": [ + "## Visualization\n", + "\n", + "Let's visualize the effect of ADC/DAC bit precision on task accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "848f07b0-3fca-4119-989b-975d70b30f97", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm4AAAGJCAYAAAAzAb+0AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAajxJREFUeJzt3XlYVOXbB/DvDDvIvoNs4oK4AIKSuyYuaS6pueWaqbkvZWa9rpmWlZqamqVZuWtuWZpKmvpTMcF9QVQElUWRVRAYZp73D2RynEFBB4aB7+e6uGrOPOec+zwOcHPOue8jEUIIEBEREVGFJ9V1AERERERUMkzciIiIiPQEEzciIiIiPcHEjYiIiEhPMHEjIiIi0hNM3IiIiIj0BBM3IiIiIj3BxI2IiIhITzBxIyIiItITTNyIqiCJRILZs2eXer3bt29DIpFg3bp1Wo+J6Fne3t4YOnSorsOodNq0aYM2bdqUap1169ZBIpHg9u3bZRITlRwTN3quom/WM2fO6DqUSqdobiUSCY4fP672vhACHh4ekEgkePPNN3UQoXb8+eefkEgkcHNzg0Kh0HU49AISiQTjxo3T+B5/Hryc2bNnK7/XJRIJzM3N4e/vj//7v/9DZmamrsMjPcPEjUjHTE1NsXHjRrXl//zzD+7evQsTExMdRKU9GzZsgLe3NxITE/H333/rOhwinVm5ciV+/fVXLFq0CH5+fvj888/RqVMnlPcjww8cOIADBw6Uap1Bgwbh8ePH8PLyKqOoqKSYuBHpWOfOnbFt2zYUFBSoLN+4cSOCg4Ph4uKio8heXXZ2Nnbv3o0pU6YgKCgIGzZs0HVIxcrOztZ1CFWOPsy5NmPs3bs3Bg4ciPfffx87duxAz549cfLkSZw6darYdXJycrS2/yLGxsYwNjYu1ToGBgYwNTWFRCLRejxUOkzc6JXl5+dj5syZCA4OhrW1NSwsLNCyZUscPnxYZVzR/VFff/01Vq9eDV9fX5iYmKBx48b4999/1ba7bds2+Pv7w9TUFPXr18fOnTsxdOhQeHt7K8ccOXIEEokER44c0bivp+/FunDhAoYOHYoaNWrA1NQULi4uePfdd/Hw4UO1fR85cgQhISEwNTWFr68vvv/+e+XljmetX78ewcHBMDMzg52dHfr164c7d+6UeP769++Phw8f4uDBg8pl+fn52L59OwYMGKBxnezsbHzwwQfw8PCAiYkJ6tSpg6+//lrtL/e8vDxMnjwZjo6OsLS0RLdu3XD37l2N27x37x7effddODs7w8TEBPXq1cPatWtLfBya7Ny5E48fP8bbb7+Nfv36YceOHcjNzVUbl5ubi9mzZ6N27dowNTWFq6srevbsiZs3byrHKBQKfPvtt2jQoAFMTU3h6OiITp06KS/bPe/+u2fv6Sv6t7xy5QoGDBgAW1tbtGjRAkDpPif37t3D8OHD4ebmBhMTE/j4+GD06NHIz8/HrVu3IJFIsHjxYrX1Tpw4AYlEgk2bNmmct+TkZBgaGmLOnDlq70VHR0MikWD58uUAAJlMhjlz5qBWrVowNTWFvb09WrRoofJ5Kkslna/nzbkQAvPmzUP16tVhbm6Otm3b4vLlyyrrp6enw8DAAEuXLlUuS0lJgVQqhb29vcpnf/To0Sp/8Bw7dgxvv/02PD09YWJiAg8PD0yePBmPHz9W2cfQoUNRrVo13Lx5E507d4alpSXeeecdAIWfvyVLlqBevXowNTWFs7MzRo0ahbS0tJeeu9dffx0AEBsbC6Dw3rP69esjMjISrVq1grm5OT755BMAhd/Ls2bNQs2aNZXH8NFHHyEvL09tu+vXr0eTJk1gbm4OW1tbtGrVSuUMm6Z73JYtW4Z69eop1wkJCVG5ElDcPW4rVqxAvXr1YGJiAjc3N4wdOxbp6ekqY4qO68qVK2jbti3Mzc3h7u6OhQsXvuzUVWmGug6A9F9mZiZ+/PFH9O/fHyNGjEBWVhbWrFmDjh074vTp0wgMDFQZv3HjRmRlZWHUqFGQSCRYuHAhevbsiVu3bsHIyAgA8Mcff6Bv375o0KABFixYgLS0NAwfPhzu7u4vHefBgwdx69YtDBs2DC4uLrh8+TJWr16Ny5cv49SpU8qk7OzZs+jUqRNcXV0xZ84cyOVyzJ07F46Ojmrb/PzzzzFjxgz06dMH7733Hh48eIBly5ahVatWOHv2LGxsbF4Yl7e3N5o2bYpNmzbhjTfeAADs27cPGRkZ6Nevn8ovKqDwl1y3bt1w+PBhDB8+HIGBgfjrr78wdepU3Lt3TyVReO+997B+/XoMGDAAzZo1w99//40uXbqoxZCcnIzXXntNeX+To6Mj9u3bh+HDhyMzMxOTJk0qxUz/Z8OGDWjbti1cXFzQr18/fPzxx/j999/x9ttvK8fI5XK8+eabCA8PR79+/TBx4kRkZWXh4MGDuHTpEnx9fQEAw4cPx7p16/DGG2/gvffeQ0FBAY4dO4ZTp04hJCTkpeJ7++23UatWLcyfP1/5i7+kn5OEhAQ0adIE6enpGDlyJPz8/HDv3j1s374dOTk5qFGjBpo3b44NGzZg8uTJavNiaWmJ7t27a4zL2dkZrVu3xtatWzFr1iyV97Zs2QIDAwPlHM6ePRsLFizAe++9hyZNmiAzMxNnzpxBVFQU2rdv/1Lzkpubi5SUFLXljx49UltW0vkqomnOZ86ciXnz5qFz587o3LkzoqKi0KFDB+Tn5yvXs7GxQf369XH06FFMmDABAHD8+HFIJBKkpqbiypUrqFevHoDCRK1ly5bKdbdt24acnByMHj0a9vb2OH36NJYtW4a7d+9i27ZtKvEVFBSgY8eOaNGiBb7++muYm5sDAEaNGoV169Zh2LBhmDBhAmJjY7F8+XKcPXsW//vf/5Q/u0qj6A8Te3t75bKHDx/ijTfeQL9+/TBw4EA4OztDoVCgW7duOH78OEaOHIm6devi4sWLWLx4Ma5fv45du3Yp158zZw5mz56NZs2aYe7cuTA2NkZERAT+/vtvdOjQQWMcP/zwAyZMmIDevXtj4sSJyM3NxYULFxAREVHsH49A4Wdvzpw5CAsLw+jRoxEdHY2VK1fi33//VZuTtLQ0dOrUCT179kSfPn2wfft2TJs2DQ0aNFD+3KMSEkTP8dNPPwkA4t9//y12TEFBgcjLy1NZlpaWJpydncW7776rXBYbGysACHt7e5Gamqpcvnv3bgFA/P7778plDRo0ENWrVxdZWVnKZUeOHBEAhJeXl3LZ4cOHBQBx+PBhlf0X7eunn35SLsvJyVGLfdOmTQKAOHr0qHJZ165dhbm5ubh3755yWUxMjDA0NBRPf8vcvn1bGBgYiM8//1xlmxcvXhSGhoZqy5/19NwuX75cWFpaKmN8++23Rdu2bYUQQnh5eYkuXboo19u1a5cAIObNm6eyvd69ewuJRCJu3LghhBDi3LlzAoAYM2aMyrgBAwYIAGLWrFnKZcOHDxeurq4iJSVFZWy/fv2EtbW1Mi5N81qc5ORkYWhoKH744QflsmbNmonu3burjFu7dq0AIBYtWqS2DYVCIYQQ4u+//xYAxIQJE4od87zYnj3eWbNmCQCif//+amNL+jkZPHiwkEqlGr83imL6/vvvBQBx9epV5Xv5+fnCwcFBDBkyRG29pxWte/HiRZXl/v7+4vXXX1e+DggIUPl8vCoAL/x6+phLOl/Fzfn9+/eFsbGx6NKli3LehBDik08+EQBU5mns2LHC2dlZ+XrKlCmiVatWwsnJSaxcuVIIIcTDhw+FRCIR33777XNjXLBggZBIJCIuLk65bMiQIQKA+Pjjj1XGHjt2TAAQGzZsUFm+f/9+jcufVXTs0dHR4sGDByI2NlZ8//33wsTERDg7O4vs7GwhhBCtW7cWAMSqVatU1v/111+FVCoVx44dU1m+atUqAUD873//E0IU/pySSqXirbfeEnK5XGXs03PbunVr0bp1a+Xr7t27i3r16j33GIp+XsXGxgoh/vt369Chg8q+li9fLgCItWvXquwPgPjll1+Uy/Ly8oSLi4vo1avXc/dL6niplF6ZgYGB8n4JhUKB1NRUFBQUICQkBFFRUWrj+/btC1tbW+Xror+Mb926BaDwTMbFixcxePBgVKtWTTmudevWaNCgwUvHaWZmpvz/ojMKr732GgAo45TL5Th06BB69OgBNzc35fiaNWuq/VW4Y8cOKBQK9OnTBykpKcovFxcX1KpVS+1S8fP06dMHjx8/xt69e5GVlYW9e/cW+5fun3/+CQMDA+VZhyIffPABhBDYt2+fchwAtXHPnj0TQuC3335D165dIYRQOZaOHTsiIyND47/ji2zevBlSqRS9evVSLuvfvz/27duncnnpt99+g4ODA8aPH6+2jaKzNb/99hskEona2aenx7yM999/X21ZST4nCoUCu3btQteuXTWe7SuKqU+fPjA1NVW5t++vv/5CSkoKBg4c+NzYevbsCUNDQ2zZskW57NKlS7hy5Qr69u2rXGZjY4PLly8jJiamJIdcIt27d8fBgwfVvqZOnao2tiTz9bRn5/zQoUPIz8/H+PHjVf4tNZ3lbdmyJZKTkxEdHQ2g8Mxaq1at0LJlSxw7dgxA4Vk4IYTKGbenY8zOzkZKSgqaNWsGIQTOnj2rtp/Ro0ervN62bRusra3Rvn17le+P4OBgVKtWrcTf63Xq1IGjoyN8fHwwatQo1KxZE3/88YfyrB4AmJiYYNiwYWr7r1u3Lvz8/FT2X3SptWj/u3btgkKhwMyZMyGVqv56f973iY2NDe7evavxlpXiFP27TZo0SWVfI0aMgJWVFf744w+V8dWqVVP5zBsbG6NJkybKn/tUcrxUSlrx888/45tvvsG1a9cgk8mUy318fNTGenp6qrwuSuKKfpnHxcUBKEyWnlWzZs2XSiIAIDU1FXPmzMHmzZtx//59lfcyMjIAAPfv38fjx4+L3ffTYmJiIIRArVq1NO6vNJdOHB0dERYWho0bNyInJwdyuRy9e/fWODYuLg5ubm6wtLRUWV63bl3l+0X/lUqlykuNRerUqaPy+sGDB0hPT8fq1auxevVqjft8dr5Koug+m4cPHyrvdwoKCkJ+fj62bduGkSNHAii8XFSnTh0YGhb/4+jmzZtwc3ODnZ1dqeN4Hk2fz5J8Th48eIDMzEzUr1//udu3sbFB165dsXHjRnz22WcACi+Turu7K3/pFsfBwQHt2rXD1q1bletu2bIFhoaG6Nmzp3Lc3Llz0b17d9SuXRv169dHp06dMGjQIDRs2PDFE1CM6tWrIywsTG25pvsjSzJfT3t2zos+r89+Hzk6Oqr8gQf890fesWPHUL16dZw9exbz5s2Do6Mjvv76a+V7VlZWCAgIUK4XHx+PmTNnYs+ePWr3pD0bo6GhIapXr66yLCYmBhkZGXByclI7HqDk3x+//fYbrKysYGRkhOrVq6t9bwKAu7u7WuFATEwMrl69qvF2jaf3f/PmTUilUvj7+5coniLTpk3DoUOH0KRJE9SsWRMdOnTAgAED0Lx582LXKfp3e/bnibGxMWrUqKF8v0j16tXVkkdbW1tcuHChVLESEzfSgvXr12Po0KHo0aMHpk6dCicnJxgYGGDBggUqN5cXMTAw0Lgd8RIl8cX9FSmXy9WW9enTBydOnMDUqVMRGBiIatWqQaFQoFOnTi/VX0yhUEAikWDfvn0aj+nps4UlMWDAAIwYMQJJSUl44403SnR/nDYUHfvAgQMxZMgQjWNKmwTExMQo/3rXlNhu2LBBmbhpS2k+C0WePhNTRNufk8GDB2Pbtm04ceIEGjRogD179mDMmDFqZ0Q06devH4YNG4Zz584hMDAQW7duRbt27eDg4KAc06pVK9y8eRO7d+/GgQMH8OOPP2Lx4sVYtWoV3nvvvVLHW1qlnS9Nc15Sbm5u8PHxwdGjR+Ht7Q0hBJo2bQpHR0dMnDgRcXFxOHbsGJo1a6acX7lcjvbt2yM1NRXTpk2Dn58fLCwscO/ePQwdOlQtRhMTE7V/G4VCAScnp2KrootLqJ7VqlUrlX87TTTNj0KhQIMGDbBo0SKN63h4eJRo/8WpW7cuoqOjsXfvXuzfvx+//fYbVqxYgZkzZ2oskHkZ2vy5X9UxcaNXtn37dtSoUQM7duxQ+eWp6bJWSRT1Cbpx44bae88uK/qL/Nkqpmf/2ktLS0N4eDjmzJmDmTNnKpc/e3nJyckJpqamJdq3r68vhBDw8fFB7dq1X3BUL/bWW29h1KhROHXqlMrlsWd5eXnh0KFDyMrKUjnrdu3aNeX7Rf9VKBTKM1pFii4zFSmqOJXL5RrPsryMDRs2wMjICL/++qvaD+zjx49j6dKliI+Ph6enJ3x9fREREQGZTFbsWUpfX1/89ddfSE1NLfasW0k/C89T0s+Jo6MjrKyscOnSpRdus1OnTnB0dMSGDRsQGhqKnJwcDBo0qETx9OjRA6NGjVJ+Hq5fv47p06erjbOzs8OwYcMwbNgwPHr0CK1atcLs2bPLPHEr6Xw9T9HnNSYmBjVq1FAuf/DggcaKzZYtW+Lo0aPw8fFBYGAgLC0tERAQAGtra+zfvx9RUVEqycbFixdx/fp1/Pzzzxg8eLByeWmqbn19fXHo0CE0b978lRLPl+Xr64vz58+jXbt2z73k6evrC4VCgStXrqgVhb2IhYUF+vbti759+yI/Px89e/bE559/junTp8PU1FRtfNG/W3R0tMq/W35+PmJjY7X2s4TU8R43emVFv5if/sspIiICJ0+efKntubm5oX79+vjll19Uqtj++ecfXLx4UWWsl5cXDAwMcPToUZXlK1aseGGMALBkyRK1cWFhYdi1axcSEhKUy2/cuKG8d6xIz549YWBggDlz5qhtVwihsX3E81SrVg0rV67E7Nmz0bVr12LHde7cGXK5XNkOosjixYshkUiU9+IV/ffZqlRNx9yrVy/89ttvGhORBw8elOo4gMLErWXLlujbty969+6t8lV0n1RRK4xevXohJSVF7XiA//69evXqBSGExr/+i8ZYWVnBwcHhhZ+F5ynp50QqlaJHjx74/fffNT5F4On1DQ0N0b9/f2zduhXr1q1DgwYNSnwG08bGBh07dsTWrVuxefNmGBsbo0ePHipjnv2cVatWDTVr1lRpE5GRkYFr165pvHT5Kko6X88TFhYGIyMjLFu2TGU7xW2jZcuWuH37NrZs2aK8dCqVStGsWTMsWrQIMplM5f42TTEKIfDtt9+WOMY+ffpALpcrL1k/raCgQO2PBW3r06cP7t27hx9++EHtvcePHyt7zfXo0QNSqRRz585VO5P4vDNbz36GjI2N4e/vDyGEyq0vTwsLC4OxsTGWLl2qsu01a9YgIyNDY/U6aQfPuFGJrF27Fvv371dbPnHiRLz55pvYsWMH3nrrLXTp0gWxsbFYtWoV/P39NbYPKIn58+eje/fuaN68OYYNG4a0tDQsX74c9evXV9mmtbU13n77bSxbtgwSiQS+vr7Yu3ev2j0nVlZWaNWqFRYuXAiZTAZ3d3ccOHBA2T/pabNnz8aBAwfQvHlzjB49Wpkk1a9fH+fOnVOO8/X1xbx58zB9+nTcvn0bPXr0gKWlJWJjY7Fz506MHDkSH374YamOu7hLlU/r2rUr2rZti08//RS3b99GQEAADhw4gN27d2PSpEnK+2YCAwPRv39/rFixAhkZGWjWrBnCw8M1nk384osvcPjwYYSGhmLEiBHw9/dHamoqoqKicOjQIaSmppb4GCIiInDjxo1iH5vk7u6ORo0aYcOGDZg2bRoGDx6MX375BVOmTMHp06fRsmVLZGdn49ChQxgzZgy6d++Otm3bYtCgQVi6dCliYmKUl+GOHTuGtm3bKvf13nvv4YsvvsB7772HkJAQHD16FNevXy9x7KX5nMyfPx8HDhxA69atlS0aEhMTsW3bNhw/flzlUvfgwYOxdOlSHD58GF9++WWJ4wEKi3kGDhyIFStWoGPHjmqX0P39/dGmTRsEBwfDzs4OZ86cwfbt21Xmf+fOnRg2bBh++uknrT77szTzVRxHR0d8+OGHWLBgAd5880107twZZ8+exb59+zReVixKyqKjozF//nzl8latWmHfvn3K3pBF/Pz84Ovriw8//BD37t2DlZUVfvvtt1L1X2vdujVGjRqFBQsW4Ny5c+jQoQOMjIwQExODbdu24dtvvy32nlRtGDRoELZu3Yr3338fhw8fRvPmzSGXy3Ht2jVs3boVf/31F0JCQlCzZk18+umn+Oyzz9CyZUv07NkTJiYm+Pfff+Hm5oYFCxZo3H6HDh3g4uKC5s2bw9nZGVevXsXy5cvRpUsXtXtpizg6OmL69OmYM2cOOnXqhG7duiE6OhorVqxA48aNX1h8Q6+gvMpXST8VlYAX93Xnzh2hUCjE/PnzhZeXlzAxMRFBQUFi7969YsiQISqtO4raNXz11Vdq+8Ez7RqEEGLz5s3Cz89PmJiYiPr164s9e/aIXr16CT8/P5VxDx48EL169RLm5ubC1tZWjBo1Sly6dEmtNcTdu3fFW2+9JWxsbIS1tbV4++23RUJCgsZ9h4eHi6CgIGFsbCx8fX3Fjz/+KD744ANhamqqFvtvv/0mWrRoISwsLISFhYXw8/MTY8eOFdHR0SWa2+e1WhFCvR2IEEJkZWWJyZMnCzc3N2FkZCRq1aolvvrqK5WSfyGEePz4sZgwYYKwt7cXFhYWomvXruLOnTsajzk5OVmMHTtWeHh4CCMjI+Hi4iLatWsnVq9erRxTknYg48ePFwDEzZs3ix0ze/ZsAUCcP39eCFHYruHTTz8VPj4+yn337t1bZRsFBQXiq6++En5+fsLY2Fg4OjqKN954Q0RGRirH5OTkiOHDhwtra2thaWkp+vTpI+7fv19sO5AHDx6oxVaaz0lcXJwYPHiwcHR0FCYmJqJGjRpi7Nixau1xhBCiXr16QiqVirt37xY7L5pkZmYKMzMzAUCsX79e7f158+aJJk2aCBsbG2FmZib8/PzE559/LvLz85Vjij5rJWnjAkCMHTtW43uaPrMlna/nzblcLhdz5swRrq6uwszMTLRp00ZcunRJeHl5aWyb4uTkJACI5ORk5bLjx48LAKJly5Zq469cuSLCwsJEtWrVhIODgxgxYoQ4f/682pwMGTJEWFhYFDs3q1evFsHBwcLMzExYWlqKBg0aiI8++kgkJCQUu86Ljv1prVu3LrYtR35+vvjyyy9FvXr1hImJibC1tRXBwcFizpw5IiMjQ2Xs2rVrRVBQkHJc69atxcGDB1X283Q7kO+//160atVK2NvbCxMTE+Hr6yumTp2qst1n24EUWb58ufDz8xNGRkbC2dlZjB49WqSlpZXouJ79HUElIxGCdwaS/ggMDISjo2O5dYV/Wo8ePbTedoGqjqCgINjZ2SE8PFzXoRCRHuM9blQhyWQytWd3HjlyBOfPn1d7VEtZePZRODExMfjzzz/LZd9U+Zw5cwbnzp1TuTmeiOhl8IwbVUi3b99GWFgYBg4cCDc3N1y7dg2rVq2CtbU1Ll26pPKImLLg6uqqfP5iXFwcVq5ciby8PJw9e7bYvm1Ez7p06RIiIyPxzTffICUlBbdu3dJYoUdEVFIsTqAKydbWFsHBwfjxxx/x4MEDWFhYoEuXLvjiiy/KPGkDCls4bNq0CUlJSTAxMUHTpk0xf/58Jm1UKtu3b8fcuXNRp04dbNq0iUkbEb2yCnHG7bvvvsNXX32FpKQkBAQEYNmyZWjSpInGsW3atME///yjtrxz587KR2wIITBr1iz88MMPSE9PR/PmzbFy5Ur+0iUiIiK9pvN73LZs2YIpU6Zg1qxZiIqKQkBAADp27FjsI0R27NiBxMRE5delS5dgYGCAt99+Wzlm4cKFWLp0KVatWoWIiAhYWFigY8eOyM3NLa/DIiIiItI6nZ9xCw0NRePGjZXNNxUKBTw8PDB+/Hh8/PHHL1x/yZIlmDlzJhITE2FhYQEhBNzc3PDBBx8oe2hlZGTA2dkZ69atQ79+/cr0eIiIiIjKik7vccvPz0dkZKTKI1ykUinCwsJK3HV/zZo16NevHywsLAAAsbGxSEpKUnnchrW1NUJDQ3Hy5EmNiVteXp5Kl3GFQoHU1FTY29s/9/EiRERERNoghEBWVhbc3Nye+yxjnSZuKSkpkMvlcHZ2Vlnu7OysfO7i85w+fRqXLl3CmjVrlMuSkpKU23h2m0XvPWvBggVae5AuERER0cu6c+cOqlevXuz7el1VumbNGjRo0KDYQoaSmj59OqZMmaJ8nZGRAU9PT8TGxhb7uI9XJZPJcPjwYbRt27bYB2vTi3EetYPzqB2cR+3gPGoP51I7ymMes7Ky4OPj88K8Q6eJm4ODAwwMDJCcnKyyPDk5GS4uLs9dNzs7G5s3b8bcuXNVlhetl5ycDFdXV5VtBgYGatyWiYkJTExM1Jbb2dnBysqqJIdSajKZDObm5rC3t+c30yvgPGoH51E7OI/awXnUHs6ldpTHPBZt90W3aOm0qtTY2BjBwcEqj4BRKBQIDw9H06ZNn7vutm3bkJeXp/YgWx8fH7i4uKhsMzMzExERES/cJhEREVFFpvNLpVOmTMGQIUMQEhKCJk2aYMmSJcjOzsawYcMAAIMHD4a7uzsWLFigst6aNWvQo0cPtWasEokEkyZNwrx581CrVi34+PhgxowZcHNzQ48ePcrrsIiIiIi0TueJW9++ffHgwQPMnDkTSUlJCAwMxP79+5XFBfHx8WrVFdHR0Th+/DgOHDigcZsfffQRsrOzMXLkSKSnp6NFixbYv38/u5YTERGRXtN54gYA48aNw7hx4zS+d+TIEbVlderUwfPaz0kkEsydO1ft/jciIiIifabzJycQERERUckwcSMiIiLSE0zciIiIiIohVwhExKYiMkWCiNhUyBU6fVJoxbjHjYiIiKii2X8pEXN+v4LEjFwABvgl5gxcrU0xq6s/OtV3feH6ZYFn3IiIiIiesf9SIkavj3qStP0nKSMXo9dHYf+lRJ3ExcSNiIiI6ClyhcCc369A00XRomVzfr+ik8umTNyIiIiIAAghcDctB4sPRqudaVMZByAxIxenY1PLL7gneI8bERERVUkyuQJXEjIRGZem/ErKLD5he9b9rJKP1RYmbkRERFQlpOfkIyo+DWduFyZp5++mI1emUBljKJXAy94cNx9kv3B7Tpbl/0QmJm5ERERU6QghcCslu/BM2u00RMan4cb9R2rjbMyNEOxpi0ZetgjxskXD6jYwNpSixZd/IykjV+N9bhIALtamaOJjV+bH8SwmbkRERKT3cmVyXLib8eSSZyoi49KQliNTG1fD0QIhXrYI9rJFsJcdajhYQCqVqI2b1dUfo9dHQQKoJG+Sp9430LBeWWPiRkRERHrnflZu4Zm0uDSciUvD5YQMyOSq58dMDKUIqG6DYG9b5Vk1OwvjEm2/U31XrBzY6Kk+boVcdNzHjYkbERERVWhyhcD15CyciUtDVFwazsSl4k7qY7VxTpYmCPG2RSPPwjNq9dysYWz48g00OtV3RXt/F5y8cR8HjkWgQ8tQNK3ppJMzbUWYuBEREVGF8iivAOfi03HmySXPc/HpyMorUBkjlQB1XKyeuuxpi+q2ZpBItJtUGUglCPWxw8OrAqE+djpN2gAmbkRERKRDhb3THqtUe15LysSzvW2rmRgiyNMGjTxtEeJti0APG1iaGukmaB1i4kZERETlRiZX4LKyd1rhGbXkzDy1cR52Zgj2tEWwtx2CPW1Rx8VS52e7KgImbkRERFRm0rILe6cVFRFc0NA7zchAgnpu1gh+0pKjkZctnK3Kv0eaPmDiRkRERFohhMDNB9mIiitK1FI1NrK1NTdCsFdR7zQ7NKxuDVMjAx1ErH+YuBEREdFLyZXJcf5OOiLjC5vcRsVr7p3m62iBEC+7wiICb1vUcLDQehFBVcHEjYiIiErkfmYuzsQ91TvtXgYKFBp6p3nYKKs9G3nawraEvdPoxZi4ERERkRq5QiA6KUtZQHAmLg1304rvnRb85Iyav6vVK/VOo+dj4kZERETIypXh3J10nHlyyfNsfDoeaeid5udiVVhE8KTRbVn0TqPiMXEjIiKqYop6pxUVEETGpSP6Ob3Tgp8UEQR4WFfJ3mkVCRM3IiKiSi6/QIFLiWlPeqcVft3P0tw7LcTL7km1py1qO7N3WkXDxI2IiKiSScvOR2RcGv6NfYhDlwzw0b9/I69Ac++0px8Z5cTeaRUeEzciIiI9VtQ77ekiglsqvdMkABTK3mlFRQTsnaafmLgRERHpkcf5cly4m44zcWmFjW7j05CuoXdaTadqaORhDYP0eAzt0gq1Xa1ZRFAJMHEjIiKqwJIzc5UPX4+M19w7zdRIioDqNk/achRWe9qYG0Mmk+HPP+NQw5ENbysLJm5EREQVhFwhcC0pU1lAcOZ2Gu6lq/dOc7YyUSki8HezgpEBe6dVBUzciIiIdCQzV4Zz8f9d9jwbn4bsfLnKGKkEqOtqpSwgCPayhbsNe6dVVUzciIiIyoEQAndSHyMyPlV56TM6OQvimd5pliaGCPKyRbBnYZPbAA8bVDPhr2sqxE8CERFRGcgvUOBSQgainlzyjIxPwwMNvdM87cwR4mVbeNnT2xa1nNg7jYrHxI2IiEgLUp/0Tiv8SsX5uxnI19A7rb77f73TGnnZwsmSvdOo5Ji4ERERlZJCIXAr5dF/1Z5xabiVkq02zs7CGI2eXPIM9rJFA3f2TqNXw8SNiIjoBR7ny3H+broySYsqpndaLadqKkUEPg5sw0HaxcSNiIjoGUkZucoHsEfFpeFyQqbG3mmBHv89gD3I0wY25sY6ipiqCiZuRERUpRXIFbiWlIWo+DTlpU9NvdNcrEwR7P1ftWddV/ZOo/LHxI2IiKqUzFwZzsanK4sIzsWnF9s77b9qTzu4WZvysifpHBM3IiKqtIQQiE/Neara8/m904qqPQM9bGDB3mlUAfFTSURElUZegRyX7mUW9k6LS0VkXDpSHqn3TvOyN0ewp23hpU8vW9R2soSUvdNIDzBxIyIivfXwUZ7y4euRt9Nw4Z567zRjAynqu1shxNsOjTxt0cjLhr3TSG8xcSMiIr2gUAjcfPAIZ5667BmroXeavYWx8uHrwV62qM/eaVSJMHEjIqIKKSe/AFfiMxEZl/qkd1o6Mh6r906r7VzUO80OwV628LY3ZxEBVVpM3IiIqEJIzHiMyLg0nL71EIcvGmBKxGHIn+mdZmZkoOydFuxti0YetrA2N9JRxETlj4kbERGVu6LeaYVNbtMQpdY7TQJAwNXaVPkUghAvO/i5WrJ3GlVpTNyIiKjMZTyW4Wx82pNqzzScu5OOnGd6pxlIJajraokgDxtIH8ZiePe28HSw1FHERBUTEzciItKqot5pZ27/V+15/b6G3mmmhoUPYH9yRi3gSe80mUyGP/+8BVdrVn4SPYuJGxERvZKi3mlFRQTF9U7ztjd/Uu1ZWERQy6kae6cRlRITNyIiKpWUJ73Top605Ciud1qD6tbK+9MaedrC0dJERxETVR5M3IiIqFgKhcCNB4+UD1+PjEvF7Yc5auPsLYz/KyLwtkU9N/ZOIyoLTNyIiEgpJ78A5+6kI/LJ/WlRcWnIzC1QG1fYO83uSbWnLbzYO42oXDBxIyKqwhLSH6s8gP1KYmaxvdNCvG3RyIu904h0iYkbEVEVUSBX4GpiFiLjUpW90xIyctXGuVmbPvXIKDvUdbWEIXunEVUITNyIiCqpjMcyRBX1Trtd2DvtsUy9d5q/q5Xy/rRgL1u42ZjpKGIiehEmbkRElYAQAnEPc556AHsqYu4/UuudZmVqiEZetgj2LHxkVED1wt5pRKQf+N1KRKSHcmVyXE7IUFZ7RsWnIeVRvto4b3tzBHvZIcS78GxaTUf2TiPSZ0zciIj0wIOsPETF/1dEcPFuBvLl6r3TGj7dO83LFg7V2DuNqDLReeL23Xff4auvvkJSUhICAgKwbNkyNGnSpNjx6enp+PTTT7Fjxw6kpqbCy8sLS5YsQefOnQEAs2fPxpw5c1TWqVOnDq5du1amx0FEpC0KhUDM/UdPHsCeiqi4NI290xyqGT91b5od6rtbwcSQvdOIKjOdJm5btmzBlClTsGrVKoSGhmLJkiXo2LEjoqOj4eTkpDY+Pz8f7du3h5OTE7Zv3w53d3fExcXBxsZGZVy9evVw6NAh5WtDQ53np0RExcrOK8D5O+lPErXCy55Zz/ROk0iA2k6WCPYuvD8txNsWnnbsnUZU1eg0o1m0aBFGjBiBYcOGAQBWrVqFP/74A2vXrsXHH3+sNn7t2rVITU3FiRMnYGRU2EPI29tbbZyhoSFcXFzKNHYiopd170nvtKgnZ9SuJmap9U4zN37SO+3JJc8gT1tYm7F3GlFVp7PELT8/H5GRkZg+fbpymVQqRVhYGE6ePKlxnT179qBp06YYO3Ysdu/eDUdHRwwYMADTpk2DgcF/lwdiYmLg5uYGU1NTNG3aFAsWLICnp2exseTl5SEv778HImdmZgIAZDIZZDLZqx6qRkXbLavtVxWcR+3gPGqHpnmUyRWITnpU+BSC+HRExacjKVP9Aexu1qYI8rRBsKcNGnnaoI5zNbXeaVXl34efR+3hXGpHecxjSbctEeLZYvHykZCQAHd3d5w4cQJNmzZVLv/oo4/wzz//ICIiQm0dPz8/3L59G++88w7GjBmDGzduYMyYMZgwYQJmzZoFANi3bx8ePXqEOnXqIDExEXPmzMG9e/dw6dIlWFpaaoxF031xALBx40aYm5tr6YiJqCrIKQBisyS4nSXBrSwg/pEE+QrVy5lSCLhbAD6WAjUsBXwsBWxYQ0BUpeXk5GDAgAHIyMiAlZVVseP0KnGrXbs2cnNzERsbqzzDtmjRInz11VdITEzUuJ/09HR4eXlh0aJFGD58uMYxms64eXh4ICUl5bmT9ypkMhkOHjyI9u3bKy/7UulxHrWD8/hyhBCIS81BZFw6zt5Jx5m4NNx8oF5EYGVqiCBPGzTysEGwlw0auFvB3Jj33haHn0ft4VxqR3nMY2ZmJhwcHF6YuOnsJ4eDgwMMDAyQnJyssjw5ObnY+9NcXV1hZGSkclm0bt26SEpKQn5+PoyNjdXWsbGxQe3atXHjxo1iYzExMYGJifqfu0ZGRmX+QS+PfVQFnEft4Dw+X65Mjkv3MpRNbqPi0vAwW3PvtBDv/x7A7sveaS+Fn0ft4VxqR1nOY0m3q7PEzdjYGMHBwQgPD0ePHj0AAAqFAuHh4Rg3bpzGdZo3b46NGzdCoVBAKi289+P69etwdXXVmLQBwKNHj3Dz5k0MGjSoTI6DiCqvB1l5yqcQRMal4dK9TPXeaYZSNHS3RrC3LQLdrZB6/Qz6dG/BX5JEVCZ0eq5+ypQpGDJkCEJCQtCkSRMsWbIE2dnZyirTwYMHw93dHQsWLAAAjB49GsuXL8fEiRMxfvx4xMTEYP78+ZgwYYJymx9++CG6du0KLy8vJCQkYNasWTAwMED//v11coxEpB8UCoHr97MKE7XbaYiMT0Ocxt5pJgj2skGIlx0aedmq9E6TyWT4M7a8IyeiqkSniVvfvn3x4MEDzJw5E0lJSQgMDMT+/fvh7OwMAIiPj1eeWQMADw8P/PXXX5g8eTIaNmwId3d3TJw4EdOmTVOOuXv3Lvr374+HDx/C0dERLVq0wKlTp+Do6Fjux0dEFVd2XgHOPdU77WwxvdPqOFui0ZNLnsFe7J1GRLql87tjx40bV+yl0SNHjqgta9q0KU6dOlXs9jZv3qyt0IiokhBCICEjF2dupz7pnZaGq4mZeKZ1GsyNDZ605LBFsLcdAj1s2DuNiCoUnSduRETaJpMrcDUxs/AB7PGFlz6TMnPVxrnbmD31yChb+LlYqvVOIyKqSJi4EZHeS8/Jx9n4dJx5UkRw/k4GHsvkKmMMpBLUc7N6Uulph0ZeNnC1NtNRxEREL4eJGxHpFSEEYlOyC5/p+eSy5437j9TGWZsZqZxNa1jdmr3TiEjv8acYEVVouTI5Lt7LKLzs+eQB7KkaeqfVcLBQJmkh3rao4cDeaURU+TBxI6IK5X5WbuGZtCf3p126lwGZXLWKwNhQioDq1gj2Kmxy28jTBvbV+MwoIqr8mLgRkc7IFQLXk5/0TnvyFZ+quXdayJMzaY28bFHfzRrGhiwiIKKqh4kbEZWbR3kFOBdf1DstFefi05GVp7l3WtElz2BPO3jYmbF3GhERmLgRURkRQuBe+mPlmbQzt9NwLUm9d5qFsQGCPG2VTW4DPW1gZcreaUREmjBxIyKtkMkVuJKQ+VS1ZyqSM/PUxrnbmBWeSXtSSFDHmb3TiIhKiokbURUnVwhExKYiMkUC+9hUNK3pBIMSVGOm5+QjKj5NWe15/m46cmWqD2A3VPZOs1Mmai7WpmV1KERElR4TN6IqbP+lRMz5/QoSM3IBGOCXmDNwtTbFrK7+6FTfVTlOCIFbKdkqD2AvSe+0gOo2MDM2KMcjIiKq3Ji4EVVR+y8lYvT6KDxzyxmSMnIxen0UprSvDUMDKSKfPI0gLUemto0ajhYI9rRVXvpk7zQiorLFxI2oCpIrBOb8fkUtaQOgXPbNwesqy00MpQiobqMsImjkZQs7C+Myj5WIiP7DxI2oCjodm/rk8ujzNfG2RYd6Lgj2skU99k4jItI5Jm5EVdD9rBcnbQDwzmte6B7oXsbREBFRSfHPZ6IqyMmyZJWdJR1HRETlg4kbURXUxMcO1mbFN7mVAHC1NkUTH7vyC4qIiF6IiRtRFXT+bjqyn3nUVJGimtBZXf1L1M+NiIjKDxM3oirmfmYu3v81EgUKgUAPG7hYqV4OdbE2xcqBjVT6uBERUcXA4gSiKiSvQI7310fiflYeajlVw/r3QmFmZICTN+7jwLEIdGgZWuInJxARUflj4kZUhczecwVR8emwMjXED4NDUM2k8EdAqI8dHl4VCPWxY9JGRFSB8VIpURWxISIOm07HQyIBlvYPgreDha5DIiKiUmLiRlQFnLmditl7LgMApnasgzZ1nHQcERERvQwmbkSVXFJGLkZviIJMLtClgStGt/bVdUhERPSSmLgRVWJFxQgPsvLg52KJhb0bQiLhPWxERPqKiRtRJSWEwIxdl3DuTjqszYzw/aBgWJiwHomISJ8xcSOqpNafisPWM3chlQDL+gfBy57FCERE+o6JG1EldDo2FXN+vwIAmNbJD61qO+o4IiIi0gYmbkSVTGLGY4zZUPhkhK4BbhjZqoauQyIiIi1h4kZUieTK5Hj/10ikPMqHn4slvuzVgMUIRESVCBM3okpCCIFPd17C+bsZsDE3wg+DQ2BuzGIEIqLKhIkbUSXx84nb+C2qsBhhef9G8LAz13VIRESkZUzciCqBU7ce4rM/rgIAPulcFy1qOeg4IiIiKgtM3Ij03L30xxi7IQpyhUCPQDcMb+Gj65CIiKiMMHEj0mO5MjlG/XoGD7PzUc/NCgt68skIRESVGRM3Ij0lhMD0HRdx6V4m7CyM8f2gYJgZG+g6LCIiKkNM3Ij01Nr/3cbOs/dgIJVg+YAgVLdlMQIRUWXHxI1ID524kYL5fxYWI3zauS6a+bIYgYioKmDiRqRn7qTmYOzGwmKEno3cMay5t65DIiKicsLEjUiPPM6XY9SvkUjLkaGBuzXmv8UnIxARVSVM3Ij0hBACH++4gCuJmbB/UoxgasRiBCKiqoSJG5Ge+PFYLHafS4ChVILv3mkENxszXYdERETljIkbkR44HpOCBfsKixFmvOmP12rY6zgiIiLSBSZuRBXcndQcjNsUBYUAegdXx+CmXroOiYiIdISJG1EFlpNfgBG/nEF6jgwB1a0xr0d9FiMQEVVhTNyIKighBD7afgHXkrLgUM0Yq1iMQERU5TFxI6qgvj96C3svJMJQKsGKd4Lhas1iBCKiqo6JG1EF9M/1B1i4/xoAYFa3emjiY6fjiIiIqCJg4kZUwcQ9zMb4jYXFCH1DPDAw1FPXIRERUQXBxI2oAsnOK8DIXyKRmVuAIE8bzO1Rj8UIRESkVOrEzdvbG3PnzkV8fHxZxENUZQkhMHX7eUQnZ8HR0gSrBgbDxJDFCERE9J9SJ26TJk3Cjh07UKNGDbRv3x6bN29GXl5eWcRGVKWsOHITf15MgpGBBKsGNoKzlamuQyIiogrmpRK3c+fO4fTp06hbty7Gjx8PV1dXjBs3DlFRUWURI1Gldzj6Pr4+EA0AmNOtPoK9WIxARETqXvoet0aNGmHp0qVISEjArFmz8OOPP6Jx48YIDAzE2rVrIYTQZpxElVZsSjYmbDoLIYD+TTwxgMUIRERUDMOXXVEmk2Hnzp346aefcPDgQbz22msYPnw47t69i08++QSHDh3Cxo0btRkrUaXzKK8AI385g6zcAgR72WJ2N39dh0RERBVYqRO3qKgo/PTTT9i0aROkUikGDx6MxYsXw8/PTznmrbfeQuPGjbUaKFFlo1AIfLD1HGLuP4KzlQlWvtOIxQhERPRcpU7cGjdujPbt22PlypXo0aMHjIyM1Mb4+PigX79+WgmQqLL67vAN/HU5GcYGUqwcGAwnFiMQEdELlDpxu3XrFry8vJ47xsLCAj/99NNLB0VU2YVfTcaiQ9cBAJ/1qIdGnrY6joiIiPRBqYsT7t+/j4iICLXlEREROHPmTKkD+O677+Dt7Q1TU1OEhobi9OnTzx2fnp6OsWPHwtXVFSYmJqhduzb+/PPPV9omUXm6+eARJm0+ByGAga95om9jFiMQEVHJlDpxGzt2LO7cuaO2/N69exg7dmyptrVlyxZMmTIFs2bNQlRUFAICAtCxY0fcv39f4/j8/Hy0b98et2/fxvbt2xEdHY0ffvgB7u7uL71NovKUlSsrLEbIK0Bjb1vMfLOerkMiIiI9UurE7cqVK2jUqJHa8qCgIFy5cqVU21q0aBFGjBiBYcOGwd/fH6tWrYK5uTnWrl2rcfzatWuRmpqKXbt2oXnz5vD29kbr1q0REBDw0tskKi8KhcDkLedx80E2XKxMseKdYBgb8qlzRERUcqW+x83ExATJycmoUaOGyvLExEQYGpZ8c/n5+YiMjMT06dOVy6RSKcLCwnDy5EmN6+zZswdNmzbF2LFjsXv3bjg6OmLAgAGYNm0aDAwMXmqbAJCXl6fy9IfMzEwAhS1PZDJZiY+pNIq2W1bbryr0aR6X/X0Th64mw9hQiu/6B8DGVFph4taneazIOI/awXnUHs6ldpTHPJZ026VO3Dp06IDp06dj9+7dsLa2BlB439knn3yC9u3bl3g7KSkpkMvlcHZ2Vlnu7OyMa9euaVzn1q1b+Pvvv/HOO+/gzz//xI0bNzBmzBjIZDLMmjXrpbYJAAsWLMCcOXPUlh84cADm5uYlPqaXcfDgwTLdflVR0efxYqoEP0YXtvro7SXD3Qv/w90LOg5Kg4o+j/qC86gdnEft4VxqR1nOY05OTonGlTpx+/rrr9GqVSt4eXkhKCgIAHDu3Dk4Ozvj119/Le3mSkWhUMDJyQmrV6+GgYEBgoODce/ePXz11VeYNWvWS293+vTpmDJlivJ1ZmYmPDw80KFDB1hZWWkjdDUymQwHDx5E+/btNbZUoZLRh3m8+SAbn3x/CoAcg0I9MPPNuroOSY0+zKM+4DxqB+dReziX2lEe81h0te9FSp24ubu748KFC9iwYQPOnz8PMzMzDBs2DP379y/VwTg4OMDAwADJyckqy5OTk+Hi4qJxHVdXVxgZGcHA4L8mpXXr1kVSUhLy8/NfaptA4eVfExMTteVGRkZl/kEvj31UBRV1HjNzZRiz8Ryy8+Ro4mOHmd3qw8ig4t7XVlHnUd9wHrWD86g9nEvtKMt5LOl2X+qRVxYWFhg5cuTLrKpkbGyM4OBghIeHo0ePHgAKz6iFh4dj3LhxGtdp3rw5Nm7cCIVCAam08Jff9evX4erqCmNjYwAo9TaJyopCITB58zncSsmGm7UpVrzTqEInbUREVPG99LNKr1y5gvj4eOTn56ss79atW4m3MWXKFAwZMgQhISFo0qQJlixZguzsbAwbNgwAMHjwYLi7u2PBggUAgNGjR2P58uWYOHEixo8fj5iYGMyfPx8TJkwo8TaJysuSQ9cRfu0+TAyl+H5QCByqqZ/VJSIiKo2XenLCW2+9hYsXL0IikUAIAQCQSCQAALlcXuJt9e3bFw8ePMDMmTORlJSEwMBA7N+/X1lcEB8frzyzBgAeHh7466+/MHnyZDRs2BDu7u6YOHEipk2bVuJtEpWH/ZcSsfTvGwCABT0boEF1ax1HRERElUGpE7eJEyfCx8cH4eHh8PHxwenTp/Hw4UN88MEH+Prrr0sdwLhx44q9jHnkyBG1ZU2bNsWpU6deeptEZS0mOQsfbD0PAHi3uQ96Nqqu44iIiKiyKHXidvLkSfz9999wcHCAVCqFVCpFixYtsGDBAkyYMAFnz54tiziJ9ELGYxlG/HIG2flyNK1hj086++k6JCIiqkRKfae0XC6HpaUlgMLK0ISEBACAl5cXoqOjtRsdkR6RKwQmbj6L2w9z4G5jhuUDgmDIYgQiItKiUp9xq1+/Ps6fPw8fHx+EhoZi4cKFMDY2xurVq9WepkBUlSw6GI0j0Q9gaiTF94OCYc9iBCIi0rJSJ27/93//h+zsbADA3Llz8eabb6Jly5awt7fHli1btB4gkT7482Iivjt8EwDwZa+GqO/OYgQiItK+UiduHTt2VP5/zZo1ce3aNaSmpsLW1lZZWUpUlUQnZeHDbYXFCCNa+qB7oLuOIyIiosqqVDfgyGQyGBoa4tKlSyrL7ezsmLRRlZSek48Rv5xBTr4czWvaY1onFiMQEVHZKVXiZmRkBE9Pz1L1aiOqrOQKgfGbziI+NQfVbc2wvH8jFiMQEVGZKvVvmU8//RSffPIJUlNTyyIeIr3x1V/ROBaTAlMjKVYPCoGthbGuQyIiokqu1Pe4LV++HDdu3ICbmxu8vLxgYWGh8n5UVJTWgiOqqH4/n4BV/xQWIyzsHQB/NysdR0RERFVBqRO3ooe3E1VVVxIy8dH2CwCAUa1roFuAm44jIiKiqqLUidusWbPKIg4ivZCWnY+Rv57BY5kcLWs54KOOLEYgIqLywzupiUqoQK7A+E1ncTftMTztzLGsfxAMpKymJiKi8lPqM25SqfS5rT9YcUqV1cK/onH8RgrMjQ2wenAwbMxZjEBEROWr1Inbzp07VV7LZDKcPXsWP//8M+bMmaO1wIgqkt3n7mH10VsAgK96B8DPhcUIRERU/kqduHXv3l1tWe/evVGvXj1s2bIFw4cP10pgRBXFpXsZmPZbYTHCmDa+6NLQVccRERFRVaW1e9xee+01hIeHa2tzRBVCanY+Rv0aiVyZAm3qOOKDDnV0HRIREVVhWkncHj9+jKVLl8Ldnc9opMqjQK7AuI1RuJf+GN725vi2L4sRiIhIt0p9qfTZh8kLIZCVlQVzc3OsX79eq8ER6dKCfddw4uZDWBgbYPXgEFibG+k6JCIiquJKnbgtXrxYJXGTSqVwdHREaGgobG1ttRocka7siLqLNcdjAQDf9AlAbWdLHUdERET0Eonb0KFDyyAMoorj4t0MTN9xEQAw/vWa6FSfxQhERFQxlPoet59++gnbtm1TW75t2zb8/PPPWgmKSFdSHuVh1K9nkFegwOt+TpgcVlvXIRERESmVOnFbsGABHBwc1JY7OTlh/vz5WgmKSBdkcgXGbohCQkYuajhYYHHfQEhZjEBERBVIqRO3+Ph4+Pj4qC338vJCfHy8VoIi0oXP/7iKiNhUVDMxxOrBwbA2YzECERFVLKVO3JycnHDhwgW15efPn4e9vb1WgiIqb9vO3MG6E7cBAIv6BKCmE4sRiIio4il14ta/f39MmDABhw8fhlwuh1wux99//42JEyeiX79+ZREjUZk6fycdn+66BACY2K4WOtRz0XFEREREmpW6qvSzzz7D7du30a5dOxgaFq6uUCgwePBg3uNGeudBVh5G/RqJ/AIFwuo6Y2K7WroOiYiIqFilTtyMjY2xZcsWzJs3D+fOnYOZmRkaNGgALy+vsoiPqMzkFygwZkMkkjJz4etogcV9A1iMQEREFVqpE7citWrVQq1aPDtB+uuzvVfw7+00WJoYYvXgEFiashiBiIgqtlLf49arVy98+eWXassXLlyIt99+WytBEZW1rf/ewa+n4iCRAEv6BcLXsZquQyIiInqhUiduR48eRefOndWWv/HGGzh69KhWgiIqS2fj0/B/T4oRJofVRru6zjqOiIiIqGRKnbg9evQIxsbGasuNjIyQmZmplaCIysr9rFy8vz4S+XIFOvg7Y1zbmroOiYiIqMRKnbg1aNAAW7ZsUVu+efNm+Pv7ayUoorKQX6DA6PVRSM7MQy2naljEJyMQEZGeKXVxwowZM9CzZ0/cvHkTr7/+OgAgPDwcGzduxPbt27UeIJG2zP79MiLj0mBpWliMUM3kpWtziIiIdKLUv7m6du2KXbt2Yf78+di+fTvMzMwQEBCAv//+G3Z2dmURI9Er23Q6Hhsj4iGRAEv7BcHHwULXIREREZXaS51y6NKlC7p06QIAyMzMxKZNm/Dhhx8iMjIScrlcqwESvarIuFTM3F1YjPBhhzpo6+ek44iIiIheTqnvcSty9OhRDBkyBG5ubvjmm2/w+uuv49SpU9qMjeiVJWfm4v31UZDJBd6o74IxbXx1HRIREdFLK9UZt6SkJKxbtw5r1qxBZmYm+vTpg7y8POzatYuFCVTh5BXI8f76SDzIykMdZ0t8/XYAJBIWIxARkf4q8Rm3rl27ok6dOrhw4QKWLFmChIQELFu2rCxjI3ppQgjM2n0ZZ+PTYWVqiNWDg2HBYgQiItJzJf5Ntm/fPkyYMAGjR4/mo66owtsQEY/N/96BVAIsG9AIXvYsRiAiIv1X4jNux48fR1ZWFoKDgxEaGorly5cjJSWlLGMjein/3k7F7D2XAQBTO/qhdW1HHUdERESkHSVO3F577TX88MMPSExMxKhRo7B582a4ublBoVDg4MGDyMrKKss4iUokMeMxRq+PQoFCoEtDV7zfuoauQyIiItKaUleVWlhY4N1338Xx48dx8eJFfPDBB/jiiy/g5OSEbt26lUWMRCWSK5Pj/fVRSHmUBz8XS3zVuyGLEYiIqFJ56XYgAFCnTh0sXLgQd+/exaZNm7QVE1GpCSEwY9clnL+TDmszI6weFAJzYxYjEBFR5fJKiVsRAwMD9OjRA3v27NHG5ohKbcPpO9gWeRdSCbB8QBA87c11HRIREZHW8ZQE6b0bmcDKiGgAwMdv+KFlLRYjEBFR5aSVM25EupKYkYufog1QoBDoFuCGES1ZjEBERJUXEzfSW7kyOcZsPIdHBRLUdbHEl71YjEBERJUbEzfSS0IIfLLzIi4lZMLCUGDFgECYGRvoOiwiIqIyxcSN9NK6E7exI+oeDKQSDKmtQHVbM12HREREVOaYuJHeOXEzBfP+uAoAmNaxNupYCx1HREREVD6YuJFeuZuWg3Ebz0KuEHgryB1Dm3rqOiQiIqJyw8SN9MbjfDlG/RqJ1Ox81He3woKeDViMQEREVQoTN9ILQghM33EBlxMyYWdhjO8HhcDUiMUIRERUtTBxI72w5ngsdp1LgIFUgu8GNIK7DYsRiIio6mHiRhXe/26kYP6fhcUI/9elLpr62us4IiIiIt1g4kYV2p3UHIzbGAWFAHo1qo6hzbx1HRIREZHOMHGjCutxvhwjf41EWo4MDatb4/O36rMYgYiIqrQKkbh999138Pb2hqmpKUJDQ3H69Olix65btw4SiUTly9TUVGXM0KFD1cZ06tSprA+DtEgIgY9+u4CriZlwqGaMVQODWYxARERVnqGuA9iyZQumTJmCVatWITQ0FEuWLEHHjh0RHR0NJycnjetYWVkhOjpa+VrTWZhOnTrhp59+Ur42MTHRfvBUZn44dgu/n0+AoVSCFe8Ew43FCERERLo/47Zo0SKMGDECw4YNg7+/P1atWgVzc3OsXbu22HUkEglcXFyUX87OzmpjTExMVMbY2tqW5WGQFh2LeYAv9l0DAMzs6o8mPnY6joiIiKhi0OkZt/z8fERGRmL69OnKZVKpFGFhYTh58mSx6z169AheXl5QKBRo1KgR5s+fj3r16qmMOXLkCJycnGBra4vXX38d8+bNg7295mrEvLw85OXlKV9nZmYCAGQyGWQy2ascYrGKtltW29dX8U8VI/Ru5I5+wW7PnSPOo3ZwHrWD86gdnEft4VxqR3nMY0m3LRFC6OxBjwkJCXB3d8eJEyfQtGlT5fKPPvoI//zzDyIiItTWOXnyJGJiYtCwYUNkZGTg66+/xtGjR3H58mVUr14dALB582aYm5vDx8cHN2/exCeffIJq1arh5MmTMDBQv09q9uzZmDNnjtryjRs3wtzcXItHTM+TJwcWXzJAYo4EXtUExteTw0jn54SJiIjKXk5ODgYMGICMjAxYWVkVO07vErdnyWQy1K1bF/3798dnn32mccytW7fg6+uLQ4cOoV27dmrvazrj5uHhgZSUlOdO3quQyWQ4ePAg2rdvDyMjozLZhz4RQmDilgvYdzkZDtWMsXP0a3CxMn3hepxH7eA8agfnUTs4j9rDudSO8pjHzMxMODg4vDBx0+mlUgcHBxgYGCA5OVlleXJyMlxcXEq0DSMjIwQFBeHGjRvFjqlRowYcHBxw48YNjYmbiYmJxuIFIyOjMv+gl8c+9MGKIzew73IyjAwkWDUwGB72lqVan/OoHZxH7eA8agfnUXs4l9pRlvNY0u3q9EKUsbExgoODER4erlymUCgQHh6ucgbueeRyOS5evAhXV9dix9y9excPHz587hjSnSPR9/HVX4VVwrO61kOIN4sRiIiINNH5HURTpkzBDz/8gJ9//hlXr17F6NGjkZ2djWHDhgEABg8erFK8MHfuXBw4cAC3bt1CVFQUBg4ciLi4OLz33nsACgsXpk6dilOnTuH27dsIDw9H9+7dUbNmTXTs2FEnx0jFu52SjQmbzkIIoH8TD7wT6qnrkIiIiCosnfdx69u3Lx48eICZM2ciKSkJgYGB2L9/v7LFR3x8PKTS//LLtLQ0jBgxAklJSbC1tUVwcDBOnDgBf39/AICBgQEuXLiAn3/+Genp6XBzc0OHDh3w2WefsZdbBfMorwAjfz2DzNwCNPK0wexu9fhkBCIioufQeeIGAOPGjcO4ceM0vnfkyBGV14sXL8bixYuL3ZaZmRn++usvbYZHZUAIgQ+3nsf15EdwsjTByoHBMDHkkxGIiIieR+eXSqlq+u7wDey/nAQjAwlWDgyGcwkqSImIiKo6Jm5U7v6+loxvDl4HAMztXh/BXnyqBRERUUkwcaNydevBI0zcfA5CAO+EeqJ/ExYjEBERlRQTNyo3WbkyjPw1Elm5BQjxssWsrvVevBIREREpMXGjcqFQCHyw9Txu3H8EZysTrBjYCMaG/PgRERGVBn9zUrlY9vcNHLiSDGMDKVYNDIaTJYsRiIiISouJG5W5Q1eSsfhQYTHCvLfqI8iTxQhEREQvg4kblakb9x9h8pZzAIDBTb3QJ8RDtwERERHpMSZuVGYyc2UY+esZZOUVoIm3HWa86a/rkIiIiPQaEzcqEwqFwJQt53DrQTZcrU3x3TuNYGTAjxsREdGr4G9SKhNLwmNw6Op9GBtK8f2gYDha8jmxREREr4qJG2ndX5eTsDQ8BgCw4K0GaFjdRrcBERERVRJM3EirYpKzMOVJMcLQZt7oFVxdtwERERFVIkzcSGsyHhc+GSE7X45QHzt82qWurkMiIiKqVJi4kVbIFQKTNp9FbEo23G3MsILFCERERFrH36ykFYsPXsfh6AcweVKMYF+NxQhERETaxsSNXtm+i4lYfvgGAOCLXg1Q391axxERERFVTkzc6JVEJ2Xhg23nAQDDW/jgrSAWIxAREZUVJm700jJyCp+MkJMvRzNfe0x/w0/XIREREVVqTNzopcgVAhM2n0Xcwxy425hh+YBGMGQxAhERUZnib1p6KV8fiMY/1x/A1EiK1YODYWdhrOuQiIiIKj0mblRqey8kYOWRmwCAL3s1RD03FiMQERGVByZuVCpXEzMxddsFAMDIVjXQPdBdxxERERFVHUzcqMTSc/Ix8tczeCyTo2UtB3zUsY6uQyIiIqpSmLhRiRTIFRi/6SzupD6Gh50ZlvUPYjECERFROeNvXiqRr/6KxrGYFJgZGWD1oBDYmLMYgYiIqLwxcaMX2nM+Ad8fvQUA+OrthqjraqXjiIiIiKomJm70XJcTMvDR9sInI7zf2hdvNnTTcURERERVFxM3KlZqdj5G/RqJXJkCrWo7YiqLEYiIiHSKiRtpVCBXYNzGKNxNewwve3Ms6xcEA6lE12ERERFVaUzcSKMv9l3DiZsPYW5cWIxgbW6k65CIiIiqPCZupGbX2Xv48XgsAOCbtwNQx8VSxxERERERwMSNnnHpXgam/Vb4ZISxbX3xRgNXHUdERERERZi4kdLDR3kY9Wsk8goUaFvHEVPasxiBiIioImHiRgAAmVyBsRujcC/9MXwcLLCExQhEREQVDhM3AgDM//MqTt1KhYWxAVYPCoa1GYsRiIiIKhomboTfIu/ip//dBgAs6huIWs4sRiAiIqqImLhVcRfupmP6zosAgAmv10THei46joiIiIiKw8StCnuQVViMkF+gQDs/J0wKq63rkIiIiOg5mLhVUUXFCIkZuajhaIHF/QIhZTECERFRhcbErYqat/cKTsemopqJIVYPCoGVKYsRiIiIKjomblXQ1jN38PPJOADA4r6BqOlUTccRERERUUkwcatizt1Jx//tvAQAmBRWC+39nXUcEREREZUUE7cq5H5WLt7/NRL5cgXa+ztjwuu1dB0SERERlQITtyoiv0CBMeujkJSZC19HCyzqE8BiBCIiIj3DxK2KmLv3Ms7EpcHSxBA/DA6BJYsRiIiI9A4Ttypg8+l4rD8VD4kE+LZ/IGo4shiBiIhIHzFxq+Si4tMwc/dlAMAH7WvjdT8WIxAREekrJm6V2P3M/4oROtVzwdi2NXUdEhEREb0CJm6VVF6BHO+vj8T9rDzUcqqGr/sEQCJhMQIREZE+Y+JWSc3ecwVR8emwMi0sRqhmYqjrkIiIiOgVMXGrhDZExGHT6cJihKX9g+DtYKHrkIiIiEgLmLhVMmdup2L2nsJihKkd66BNHScdR0RERETawsStEknKyMX766Mgkwt0aeCK0a19dR0SERERaRETt0qiqBgh5VEe/FwssbB3QxYjEBERVTJM3CoBIQRm7LqEc3fSYW1mhO8HBcOCxQhERESVDhO3SmD9qThsPXMXUgmwrH8QvOxZjEBERFQZVYjE7bvvvoO3tzdMTU0RGhqK06dPFzt23bp1kEgkKl+mpqYqY4QQmDlzJlxdXWFmZoawsDDExMSU9WHoxOnYVMz5/QoAYFonP7Sq7ajjiIiIiKis6Dxx27JlC6ZMmYJZs2YhKioKAQEB6NixI+7fv1/sOlZWVkhMTFR+xcXFqby/cOFCLF26FKtWrUJERAQsLCzQsWNH5ObmlvXhlKuE9McYsyESBQqBrgFuGNmqhq5DIiIiojKk88Rt0aJFGDFiBIYNGwZ/f3+sWrUK5ubmWLt2bbHrSCQSuLi4KL+cnf97/qYQAkuWLMH//d//oXv37mjYsCF++eUXJCQkYNeuXeVwROUjVybH6PWRSHmUDz8XS3zZqwGLEYiIiCo5nd7Bnp+fj8jISEyfPl25TCqVIiwsDCdPnix2vUePHsHLywsKhQKNGjXC/PnzUa9ePQBAbGwskpKSEBYWphxvbW2N0NBQnDx5Ev369VPbXl5eHvLy8pSvMzMzAQAymQwymeyVj1OTou2+zPaFEJi+8zLO382AjZkRVgwIgJFElFmsFdmrzCP9h/OoHZxH7eA8ag/nUjvKYx5Lum2dJm4pKSmQy+UqZ8wAwNnZGdeuXdO4Tp06dbB27Vo0bNgQGRkZ+Prrr9GsWTNcvnwZ1atXR1JSknIbz26z6L1nLViwAHPmzFFbfuDAAZibm7/MoZXYwYMHS73O0UQJdt42gAQCA7xzcfHkEVwsg9j0ycvMI6njPGoH51E7OI/aw7nUjrKcx5ycnBKN07ueEU2bNkXTpk2Vr5s1a4a6devi+++/x2efffZS25w+fTqmTJmifJ2ZmQkPDw906NABVlZWrxyzJjKZDAcPHkT79u1hZGRU4vUiYlOxKyISgMDHnerg3ebeZRKfvnjZeSRVnEft4DxqB+dReziX2lEe81h0te9FdJq4OTg4wMDAAMnJySrLk5OT4eLiUqJtGBkZISgoCDdu3AAA5XrJyclwdXVV2WZgYKDGbZiYmMDExETjtsv6g16afdxLf4wJWy5ArhDoEeiGka1r8r62J8rj36oq4DxqB+dROziP2sO51I6ynMeSblenxQnGxsYIDg5GeHi4cplCoUB4eLjKWbXnkcvluHjxojJJ8/HxgYuLi8o2MzMzERERUeJtVkS5MjlG/XoGqdn5qOdmhQU9+WQEIiKiqkbnl0qnTJmCIUOGICQkBE2aNMGSJUuQnZ2NYcOGAQAGDx4Md3d3LFiwAAAwd+5cvPbaa6hZsybS09Px1VdfIS4uDu+99x6AworTSZMmYd68eahVqxZ8fHwwY8YMuLm5oUePHro6zFcihMD0HRdx6V4m7CyM8f2gYJgZG+g6LCIiIipnOk/c+vbtiwcPHmDmzJlISkpCYGAg9u/frywuiI+Ph1T634nBtLQ0jBgxAklJSbC1tUVwcDBOnDgBf39/5ZiPPvoI2dnZGDlyJNLT09GiRQvs379frVGvvlj7v9vYefYeDKQSLB8QhOq2ZVswQURERBWTzhM3ABg3bhzGjRun8b0jR46ovF68eDEWL1783O1JJBLMnTsXc+fO1VaIOnPiRgrm/3kVAPBp57po5uug44iIiIhIV3TegJeKdyc1B2M3RkGuEOjZyB3DqngFKRERUVXHxK2Cepwvx6hfI5GWI0MDd2vMf4tPRiAiIqrqmLhVQEIITPvtAq4kZsL+STGCqRGLEYiIiKo6Jm4V0I/HYrHnfAIMpRJ8904juNmY6TokIiIiqgCYuFUwx2NSsGBfYTHCjDf98VoNex1HRERERBUFE7cK5E5qDsZtioJCAL2Dq2NwUy9dh0REREQVCBO3CiInvwAjfjmD9BwZAqpbY16P+ixGICIiIhVM3CoAIQSmbr+Aa0lZcKhmjFUsRiAiIiINmLhVAN8fvYU/LiTCUCrByoHBcLVmMQIRERGpqxBPTqhq5AqBiNhURKZIcO9YLL46EAMAmNWtHhp72+k4OiIiIqqomLiVs/2XEjHn9ytIzMgFYADEFCZtzX3tMTDUU7fBERERUYXGS6XlaP+lRIxeH/UkaVN14uZD/HU5SQdRERERkb5g4lZO5AqBOb9fgXjOmDm/X4Fc8bwRREREVJUxcSsnp2NTNZ5pKyIAJGbk4nRsavkFRURERHqFiVs5uZ9VfNL2MuOIiIio6mHiVk6cLE21Oo6IiIiqHiZu5aSJjx1crU1R3LMQJABcrU3RxIftQIiIiEgzJm7lxEAqwayu/gCglrwVvZ7V1R8GUj7mioiIiDRj4laOOtV3xcqBjeBirXo51MXaFCsHNkKn+q46ioyIiIj0ARvwlrNO9V3R3t8FJ2/cx4FjEejQMhRNazrxTBsRERG9EBM3HTCQShDqY4eHVwVCfeyYtBEREVGJ8FIpERERkZ5g4kZERESkJ5i4EREREekJJm5EREREeoKJGxEREZGeYOJGREREpCfYDkQDIQQAIDMzs8z2IZPJkJOTg8zMTBgZGZXZfio7zqN2cB61g/OoHZxH7eFcakd5zGNRzlGUgxSHiZsGWVlZAAAPDw8dR0JERERVSVZWFqytrYt9XyJelNpVQQqFAgkJCbC0tIREUjbNcTMzM+Hh4YE7d+7AysqqTPZRFXAetYPzqB2cR+3gPGoP51I7ymMehRDIysqCm5sbpNLi72TjGTcNpFIpqlevXi77srKy4jeTFnAetYPzqB2cR+3gPGoP51I7ynoen3emrQiLE4iIiIj0BBM3IiIiIj3BxE1HTExMMGvWLJiYmOg6FL3GedQOzqN2cB61g/OoPZxL7ahI88jiBCIiIiI9wTNuRERERHqCiRsRERGRnmDiRkRERKQnmLgRERER6Qkmbjr0xRdfQCKRYNKkSboORe/cu3cPAwcOhL29PczMzNCgQQOcOXNG12HpHblcjhkzZsDHxwdmZmbw9fXFZ5999sJn5VV1R48eRdeuXeHm5gaJRIJdu3apvC+EwMyZM+Hq6gozMzOEhYUhJiZGN8FWYM+bR5lMhmnTpqFBgwawsLCAm5sbBg8ejISEBN0FXEG96PP4tPfffx8SiQRLliwpt/j0SUnm8urVq+jWrRusra1hYWGBxo0bIz4+vtxiZOKmI//++y++//57NGzYUNeh6J20tDQ0b94cRkZG2LdvH65cuYJvvvkGtra2ug5N73z55ZdYuXIlli9fjqtXr+LLL7/EwoULsWzZMl2HVqFlZ2cjICAA3333ncb3Fy5ciKVLl2LVqlWIiIiAhYUFOnbsiNzc3HKOtGJ73jzm5OQgKioKM2bMQFRUFHbs2IHo6Gh069ZNB5FWbC/6PBbZuXMnTp06BTc3t3KKTP+8aC5v3ryJFi1awM/PD0eOHMGFCxcwY8YMmJqall+QgspdVlaWqFWrljh48KBo3bq1mDhxoq5D0ivTpk0TLVq00HUYlUKXLl3Eu+++q7KsZ8+e4p133tFRRPoHgNi5c6fytUKhEC4uLuKrr75SLktPTxcmJiZi06ZNOohQPzw7j5qcPn1aABBxcXHlE5QeKm4e7969K9zd3cWlS5eEl5eXWLx4cbnHpm80zWXfvn3FwIEDdRPQEzzjpgNjx45Fly5dEBYWputQ9NKePXsQEhKCt99+G05OTggKCsIPP/yg67D0UrNmzRAeHo7r168DAM6fP4/jx4/jjTfe0HFk+is2NhZJSUkq39/W1tYIDQ3FyZMndRiZ/svIyIBEIoGNjY2uQ9ErCoUCgwYNwtSpU1GvXj1dh6O3FAoF/vjjD9SuXRsdO3aEk5MTQkNDn3tpuiwwcStnmzdvRlRUFBYsWKDrUPTWrVu3sHLlStSqVQt//fUXRo8ejQkTJuDnn3/WdWh65+OPP0a/fv3g5+cHIyMjBAUFYdKkSXjnnXd0HZreSkpKAgA4OzurLHd2dla+R6WXm5uLadOmoX///nxYeil9+eWXMDQ0xIQJE3Qdil67f/8+Hj16hC+++AKdOnXCgQMH8NZbb6Fnz574559/yi0Ow3LbE+HOnTuYOHEiDh48WL7XwysZhUKBkJAQzJ8/HwAQFBSES5cuYdWqVRgyZIiOo9MvW7duxYYNG7Bx40bUq1cP586dw6RJk+Dm5sa5pApDJpOhT58+EEJg5cqVug5Hr0RGRuLbb79FVFQUJBKJrsPRawqFAgDQvXt3TJ48GQAQGBiIEydOYNWqVWjdunW5xMEzbuUoMjIS9+/fR6NGjWBoaAhDQ0P8888/WLp0KQwNDSGXy3Udol5wdXWFv7+/yrK6deuWa1VPZTF16lTlWbcGDRpg0KBBmDx5Ms8IvwIXFxcAQHJyssry5ORk5XtUckVJW1xcHA4ePMizbaV07Ngx3L9/H56ensrfO3Fxcfjggw/g7e2t6/D0ioODAwwNDXX++4dn3MpRu3btcPHiRZVlw4YNg5+fH6ZNmwYDAwMdRaZfmjdvjujoaJVl169fh5eXl44i0l85OTmQSlX/fjMwMFD+ZUml5+PjAxcXF4SHhyMwMBAAkJmZiYiICIwePVq3wemZoqQtJiYGhw8fhr29va5D0juDBg1Su5+6Y8eOGDRoEIYNG6ajqPSTsbExGjdurPPfP0zcypGlpSXq16+vsszCwgL29vZqy6l4kydPRrNmzTB//nz06dMHp0+fxurVq7F69Wpdh6Z3unbtis8//xyenp6oV68ezp49i0WLFuHdd9/VdWgV2qNHj3Djxg3l69jYWJw7dw52dnbw9PTEpEmTMG/ePNSqVQs+Pj6YMWMG3Nzc0KNHD90FXQE9bx5dXV3Ru3dvREVFYe/evZDL5cp7BO3s7GBsbKyrsCucF30en014jYyM4OLigjp16pR3qBXei+Zy6tSp6Nu3L1q1aoW2bdti//79+P3333HkyJHyC1KnNa3EdiAv6ffffxf169cXJiYmws/PT6xevVrXIemlzMxMMXHiROHp6SlMTU1FjRo1xKeffiry8vJ0HVqFdvjwYQFA7WvIkCFCiMKWIDNmzBDOzs7CxMREtGvXTkRHR+s26AroefMYGxur8T0A4vDhw7oOvUJ50efxWWwHUrySzOWaNWtEzZo1hampqQgICBC7du0q1xglQrBFOhEREZE+YHECERERkZ5g4kZERESkJ5i4EREREekJJm5EREREeoKJGxEREZGeYOJGREREpCeYuBERERHpCSZuRERERHqCiRsRURkYOnSo3jzi6vbt25BIJDh37pxWxxKR9jFxI6ISOXnyJAwMDNClSxe194p+mRd9WVpaol69ehg7dixiYmLUxufn52PhwoUICAiAubk5HBwc0Lx5c/z000+QyWQqY4cNG4b/+7//AwCVfVhYWKBWrVoYOnQoIiMjNcZ89+5dGBsbF/ssYCEEVq9ejdDQUFSrVg02NjYICQnBkiVLkJOTo3GdZ4/V2NgYNWvWxLx58/D0g2i+/fZbrFu3Tvm6TZs2mDRpksZtPq1NmzbKbZuamsLf3x8rVqx44XqvwsPDA4mJiSV6ZnJpxhKR9jFxI6ISWbNmDcaPH4+jR48iISFB45hDhw4hMTER58+fx/z583H16lUEBAQgPDxcOSY/Px8dO3bEF198gZEjR+LEiRM4ffo0xo4di2XLluHy5cvKsXK5HHv37kW3bt2Uy3766SckJibi8uXL+O677/Do0SOEhobil19+UYtn3bp16NOnDzIzMxEREaH2/qBBgzBp0iR0794dhw8fxrlz5zBjxgzs3r0bBw4ceO58FB1rTEwM5syZg88//xxr165Vvm9tbQ0bG5vnbqM4I0aMQGJiIq5cuYI+ffpg7Nix2LRpk8ax+fn5L7WPpxkYGMDFxQWGhoZaHUtEZaBcn4xKRHopKytLVKtWTVy7dk307dtXfP755yrvFz0Q/OzZsyrL5XK5aNOmjfDy8hIFBQVCCCG+/PJLIZVKRVRUlNp+8vPzxaNHj5Svjx49KlxdXYVCoRBCCAFA7Ny5U229wYMHC0tLS5GamqpcplAoRI0aNcT+/fvFtGnTxIgRI1TW2bJliwCg8QHRCoVCpKena5yL4o61Xbt2YsyYMcrXQ4YMEd27d1f+P555aHVsbKzG7bdu3VpMnDhRZVmtWrVEv379lO+PHTtWTJw4Udjb24s2bdoIIYS4ePGi6NSpk7CwsBBOTk5i4MCB4sGDB8ptyOVy8eWXXwpfX19hbGwsPDw8xLx58zQeU2pqqhgwYIBwcHAQpqamombNmmLt2rXFHv+RI0dE48aNhbGxsXBxcRHTpk0TMplM5ZjGjx8vpk6dKmxtbYWzs7OYNWuWxuMnoufjGTcieqGtW7fCz88PderUwcCBA7F27VqVy4LFkUqlmDhxIuLi4pSXMzds2ICwsDAEBQWpjTcyMoKFhYXy9Z49e9C1a1dIJJLn7mfy5MnIysrCwYMHlcsOHz6MnJwchIWFYeDAgdi8eTOys7OV72/YsAF16tRB9+7d1bYnkUhgbW39wuMrcubMGURGRiI0NFTj+99++y2aNm2qPJOWmJgIDw+PEm/fzMxM5czazz//DGNjY/zvf//DqlWrkJ6ejtdffx1BQUE4c+YM9u/fj+TkZPTp00e5zvTp0/HFF19gxowZuHLlCjZu3AhnZ2eN+ysas2/fPly9ehUrV66Eg4ODxrH37t1D586d0bhxY5w/fx4rV67EmjVrMG/ePJVxP//8MywsLBAREYGFCxdi7ty5Kv9eRFQyPNdNRC+0Zs0aDBw4EADQqVMnZGRk4J9//kGbNm1euK6fnx+AwnvDmjRpgpiYmBKtBwC7d+/G4sWLS7WPp2Pu168fDAwMUL9+fdSoUQPbtm3D0KFDAQAxMTGoU6dOieLQpFmzZpBKpcjPz4dMJsPIkSMxePBgjWOtra1hbGwMc3NzuLi4lHgfcrkcmzZtwoULFzBy5Ejl8lq1amHhwoXK1/PmzUNQUBDmz5+vXLZ27Vp4eHjg+vXrcHV1xbfffovly5djyJAhAABfX1+0aNFC437j4+MRFBSEkJAQAIC3t3exMa5YsQIeHh5Yvnw5JBIJ/Pz8kJCQgGnTpmHmzJmQSgvPDzRs2BCzZs1Sxr98+XKEh4ejffv2JZ4PIuI9bkT0AtHR0Th9+jT69+8PADA0NETfvn2xZs2aEq1fdGau6KxZSc7UAcDVq1eRkJCAdu3alXof6enp2LFjhzLZBICBAweqxFzSOIqzZcsWnDt3DufPn8fWrVuxe/dufPzxx6+0zSIrVqxAtWrVYGZmhhEjRmDy5MkYPXq08v3g4GCV8efPn8fhw4dRrVo15VdRMnvz5k1cvXoVeXl5JZpLABg9ejQ2b96MwMBAfPTRRzhx4kSxY69evYqmTZuqnBVt3rw5Hj16hLt37yqXNWzYUGU9V1dX3L9/v0TxENF/eMaNiJ5rzZo1KCgogJubm3KZEAImJiZYvnz5Cy8pXr16FQDg4+MDAKhduzauXbv2wv3u2bMH7du3h6mp6QvHPruPjRs3Ijc3V+XSpRACCoUC169fR+3atUscR3E8PDxQs2ZNAEDdunVx8+ZNzJgxA7Nnzy5RzM/zzjvv4NNPP4WZmRlcXV2VZ62KPH05GQAePXqErl274ssvv1TblqurK27dulWq/b/xxhuIi4vDn3/+iYMHD6Jdu3YYO3Ysvv7669IfzBNGRkYqryUSCRQKxUtvj6iq4hk3IipWQUEBfvnlF3zzzTc4d+6c8uv8+fNwc3MrttKxiEKhwNKlS+Hj46O8p23AgAE4dOgQzp49qzZeJpMp70PbvXu3xvvPNFmyZAmsrKwQFhYGoDDZ/OCDD9RibtmypbLyc8CAAbh+/Tp2796ttj0hBDIyMkq07yIGBgYoKCgotsrT2NgYcrm8RNuytrZGzZo14e7urpa0adKoUSNcvnwZ3t7eqFmzpspXUdsUMzMzlereF3F0dMSQIUOwfv16LFmyBKtXr9Y4rm7dujh58qTKGcz//e9/sLS0RPXq1Uu8PyIqGSZuRFSsvXv3Ii0tDcOHD0f9+vVVvnr16qV2ufThw4dISkrCrVu3sGfPHoSFheH06dNYs2YNDAwMAACTJk1C8+bN0a5dO3z33Xc4f/48bt26ha1bt+K1115DTEwM7t+/jzNnzuDNN99Uiyk9PR1JSUmIi4vDwYMH0bt3b2zcuBErV66EjY0Nzp07h6ioKLz33ntqMffv3x8///wzCgoK0KdPH/Tt2xf9+/fH/PnzcebMGcTFxWHv3r0ICwvD4cOHnzs3Rcd69+5d7Nu3D99++y3atm0LKysrjeO9vb0RERGB27dvIyUlRatnm8aOHYvU1FT0798f//77L27evIm//voLw4YNg1wuh6mpKaZNm4aPPvoIv/zyC27evIlTp04Ve7l75syZ2L17N27cuIHLly9j7969qFu3rsaxY8aMwZ07dzB+/Hhcu3YNu3fvxqxZszBlypQSJZ1EVEo6q2clogrvzTffFJ07d9b4XkREhAAgzp8/r2wRUfRlbm4u6tatK8aMGSNiYmLU1s3NzRULFiwQDRo0EKampsLOzk40b95crFu3TshkMvHjjz+K5s2bq6339D5MTU2Fr6+vGDJkiIiMjFSOGTdunPD399cYc2JiopBKpWL37t1CiMIWGStXrhSNGzcW5ubmwsrKSgQHB4tvv/1W5OTkaNzGs8dqYGAgqlevLkaMGCHu37+vHPd0OxAhhIiOjhavvfaaMDMzK3U7kJK8f/36dfHWW28JGxsbYWZmJvz8/MSkSZOUrVTkcrmYN2+e8PLyEkZGRsLT01PMnz9f5ZiKWnx89tlnom7dusLMzEzY2dmJ7t27i1u3bmkcK0TJ2oE8G3P37t3FkCFDij1OItJMIsQr3qFLRKRl3bp1Q4sWLfDRRx/pOhQiogqF57GJqMJp0aKFsoqViIj+wzNuRERERHqCZ9yIiIiI9AQTNyIiIiI9wcSNiIiISE8wcSMiIiLSE0zciIiIiPQEEzciIiIiPcHEjYiIiEhPMHEjIiIi0hNM3IiIiIj0xP8DdI5gXI/7YAwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "bit_values = list(sweep_results.keys())\n", + "accuracies = [\n", + " list(sweep_results[b].values())[0][\"acc,none\"] for b in bit_values\n", + "]\n", + "\n", + "plt.figure(figsize=(7, 4))\n", + "plt.plot(bit_values, accuracies, marker='o')\n", + "plt.xlabel(\"ADC/DAC Bit Precision\")\n", + "plt.ylabel(\"Accuracy\")\n", + "plt.title(\"Language Model Accuracy vs. Hardware Precision\")\n", + "plt.grid(True)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "c9c0b5a4-6f0c-474b-acfd-41083ca9f024", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "We demonstrated how XBTorch can be integrated into the LM Evaluation Harness\n", + "to simulate low-precision inference in large language models.\n", + "\n", + "**Key Takeaways:**\n", + "- The `SimpleFixedPoint` accelerator enables simulation of analog crossbars and quantization. For large models, the stateful mode of operation can be used.\n", + "- Language model evaluation can be done seamlessly using `lm_eval`.\n", + "- Reducing ADC/DAC precision degrades model performance, providing a direct measure of hardware sensitivity.\n", + "\n", + "---\n", + "**References**\n", + "1. Gao et al., *\"The Language Model Evaluation Harness*, EleutherAI, 2021. \n", + "2. Kaushal et al., *\"Spectra: Surprising effectiveness of pretraining ternary language models at scale\"*, 2024." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/12_input_encoding_modes.ipynb b/examples/12_input_encoding_modes.ipynb new file mode 100644 index 0000000..b223772 --- /dev/null +++ b/examples/12_input_encoding_modes.ipynb @@ -0,0 +1,621 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "07154d42-19ec-4b85-a1c9-fa32fcf3b903", + "metadata": {}, + "source": [ + "# XBTorch::Example 12:Input Encoding Modes\n", + "\n", + "## Introduction\n", + "\n", + "In this example, we will utilize hardware-aware inference capabilities provided by XBTorch and study the impact of input encoding modes. We will focus on the same two-layer perceptron network using the MNIST dataset that we've seen in the previous hardware-aware training examples. Like last time, any of these parts can be extended for other applications as needed." + ] + }, + { + "cell_type": "markdown", + "id": "8d79269c-e90b-4bd4-b0bd-7a42d74eb642", + "metadata": {}, + "source": [ + "## Getting Started\n", + "\n", + "Let's import necessary dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "929afff3-4807-47c1-ab90-fa9b993c9418", + "metadata": {}, + "outputs": [], + "source": [ + "# General imports\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import random\n", + "import time\n", + "import pickle\n", + "from pathlib import Path\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "\n", + "from torchvision import datasets, transforms\n", + "from torch.utils.data import DataLoader, ConcatDataset\n", + "\n", + "from functools import partial\n", + "import torch.optim as optim" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "16d35280-9a4e-4dad-a47b-95b97d91186a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "W0528 23:15:19.468000 3163875 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. \n", + "W0528 23:15:19.468000 3163875 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures.\n", + "/mnt/osama.yousuf1/xbtorch/env/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "# XBTorch imports\n", + "import xbtorch\n", + "\n", + "from xbtorch.patches import xbtorch_model\n", + "\n", + "from xbtorch.nn.utils import test_classifier\n", + "\n", + "from xbtorch.deployment import SimpleFixedPoint, map_random, encode_simple_binary, encode_MAO, encode_LEA1, encode_LEA2, compute_error\n", + "\n", + "from nets.mlp import SimpleMLP" + ] + }, + { + "cell_type": "markdown", + "id": "d12c9e5b-9e1d-4d98-ba11-7a51ebdfb7d8", + "metadata": {}, + "source": [ + "Let's fix the seed for reproducibility, and initialize network and XBTorch-specific parameters." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "1d082657-2f38-4468-8793-9d2f5ffc7fe7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using device: cuda:5\n" + ] + } + ], + "source": [ + "seed = 0\n", + "\n", + "torch.manual_seed(seed)\n", + "np.random.seed(seed)\n", + "random.seed(seed) # controls weight updatejump table stochasticity\n", + "\n", + "# Check if CUDA is available and select the device\n", + "device = torch.device(\"cuda:5\" if torch.cuda.is_available() else \"cpu\")\n", + "print(f\"Using device: {device}\")" + ] + }, + { + "cell_type": "markdown", + "id": "779524a9-dd83-4443-baec-2b1392b449a1", + "metadata": {}, + "source": [ + "## Initialize XBTorch" + ] + }, + { + "cell_type": "markdown", + "id": "c4e1bf6a-b7ba-40ee-80e9-59d3d86507e4", + "metadata": {}, + "source": [ + "Like last time, let's initialize the XBTorch library." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "174e1100-568a-43d7-9e1f-56123ff9f572", + "metadata": {}, + "outputs": [], + "source": [ + "xb_size = (2500, 2500)\n", + "g_min = 133\n", + "g_max = 233\n", + "read_noise = 20\n", + "write_noise = 50\n", + "weight_encoding_scheme = encode_simple_binary\n", + "mapping_scheme = map_random\n", + "output_polling_mode = \"avg\"\n", + "adc_precision = dac_precision = 8\n", + "\n", + "# Parameters not previously shown\n", + "stateful = False\n", + "input_encoding_scheme = \"instant\"" + ] + }, + { + "cell_type": "markdown", + "id": "eb618857-d3ca-4c8c-aa27-745c4a6d3b79", + "metadata": {}, + "source": [ + "In the cell above, we initialize various XBTorch parameters pertaining to hardware-aware inference. Here's a quick summary of what thew new ones mean:\n", + "\n", + "| Parameter | Purpose |\n", + "| -------- | ------- |\n", + "| `stateful` | Whether a physical representation of the entire crossbar should be maintained |\n", + "| `input_encoding_scheme` | Mode for how network inputs should be applied to the crossbar |\n", + "\n", + "The primary parameter we are interested in here is the `input_encoding_scheme`. Two modes are supported:\n", + "- `instant`: Similar to voltage-amplitude encoding. The DAC applies multi-bit input voltages directly on the crossbar.\n", + "- `linear`: Linear bit-sliced temporal encoding capturing representative pulse-domain accumulation behavior. The \"DAC\" decomposes the input into bit slices, each slice performs a separate VMM, and partial sums are accumulated with binary weighting.\n", + "\n", + "Note that for this example, ADC quantization happens after accumulation. For more details on the API, view the full XBTorch documentation." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3ccbdd9e-ac09-460f-88b0-70701346c219", + "metadata": {}, + "outputs": [], + "source": [ + "inference_accelerator = SimpleFixedPoint(g_min=g_min, \n", + " g_max=g_max, \n", + " adc_bits=adc_precision, \n", + " dac_bits=dac_precision, \n", + " read_noise=read_noise, \n", + " write_noise=write_noise, \n", + " xb_size=xb_size,\n", + " weight_encoding_scheme=weight_encoding_scheme, \n", + " input_encoding_scheme=input_encoding_scheme,\n", + " xb_mapping_scheme=mapping_scheme,\n", + " stateful=stateful,\n", + " device=device)" + ] + }, + { + "cell_type": "markdown", + "id": "3352e60f-309e-44be-acb6-83f3934a97d2", + "metadata": {}, + "source": [ + "We can now initialize XBTorch. Recall that if a parameter is not provided, default values are utilized. The reader is directed to `XBParams.__init__()` for the underlying implementation." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "07caa63f-2170-4457-9842-4cc01021c2b1", + "metadata": {}, + "outputs": [], + "source": [ + "wage_params = { \"wl_weight\": 2, # 2 = ternary weights\n", + " \"wl_grad\": 8,\n", + " \"wl_activation\": 8,\n", + " \"wl_error\": 8,\n", + " \"rounding_weight\" : \"nearest\",\n", + " \"rounding_activation\" : \"nearest\",\n", + " \"rounding_grad\" : \"nearest\",\n", + " \"rounding_error\" : \"nearest\",\n", + " }\n", + "\n", + "# Init xbtorch\n", + "xbtorch.initialize(pytorch_device=device,\n", + " inference_accelerator=inference_accelerator,\n", + " wage_quantize=True, # since the trained solution utilized wage quantization\n", + " wage_params=wage_params\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "91ed9cf6-646f-491c-bc37-6c81880b3898", + "metadata": {}, + "source": [ + "## Prepare the dataset\n", + "\n", + "We can now prepare the dataset for our neural network." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "be1c9077-2138-4b17-82ee-38d0187aa229", + "metadata": {}, + "outputs": [], + "source": [ + "# Define transforms to apply to the data\n", + "transform = transforms.Compose([\n", + " transforms.ToTensor(), # Convert images to tensors\n", + " transforms.Normalize((0.1307,), (0.3081,)) # Normalize the image data\n", + "])\n", + "\n", + "# Load the MNIST training and test datasets\n", + "train_dataset = datasets.MNIST(root='./data', train=True, download=True, transform=transform)\n", + "test_dataset = datasets.MNIST(root='./data', train=False, download=True, transform=transform)\n", + "\n", + "# Create data loaders for batching and shuffling\n", + "train_loader = DataLoader(train_dataset, batch_size=4096, shuffle=True, generator=torch.Generator(), num_workers=4)\n", + "test_loader = DataLoader(test_dataset, batch_size=10000, shuffle=False, generator=torch.Generator(), num_workers=4)" + ] + }, + { + "cell_type": "markdown", + "id": "2ef1caab-9b88-415c-86a5-f3d41cf1396f", + "metadata": {}, + "source": [ + "## Prepare the model\n", + "\n", + "We simply instantiate and load the state dictionary of our HWA pre-trained simple 2-layer perceptron network. " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e367f430-0b49-46b4-bbcb-bdbc4466866e", + "metadata": {}, + "outputs": [], + "source": [ + "# Define the model\n", + "input_size = 28 * 28\n", + "hidden_size = 150\n", + "output_size = 10\n", + "\n", + "# Create Model\n", + "model = SimpleMLP(input_size, hidden_size, output_size).to(device)\n", + "model = xbtorch_model(model)\n", + "\n", + "# Override state dict. \n", + "# If we had dumped the final state_dict directly, model.load_state_dict(torch.load(..)) could be used instead\n", + "with open(\"checkpoints/hwa_train_mlp_hwa_example.pkl\", \"rb\") as f:\n", + " full_weights_hwa = pickle.load(f)\n", + " \n", + "epoch = -1 # load last epoch's weights\n", + "\n", + "for name, param in model.named_parameters():\n", + " new_tensor = torch.from_numpy(full_weights_hwa[name][epoch, ...]).to(\n", + " device=param.device,\n", + " dtype=param.dtype,\n", + " )\n", + " param.data = new_tensor" + ] + }, + { + "cell_type": "markdown", + "id": "bdf4684e-77ba-4578-b0d9-9212ab5f6034", + "metadata": {}, + "source": [ + "## Baseline performance\n", + "\n", + "We can firstly compute the baseline test accuracy of our network. This is the accuracy before activating the inference accelerator (i.e. all computations of the network happen without the accelerator primitive's intervention)." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "5f6953d2-d3d7-441a-a707-5614cc4f4a86", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "89.7\n" + ] + } + ], + "source": [ + "baseline_acc, _ = test_classifier(test_loader, model, device)\n", + "\n", + "print(baseline_acc)" + ] + }, + { + "cell_type": "markdown", + "id": "839b6144-9133-43d5-a73e-160bcaee4e59", + "metadata": {}, + "source": [ + "## Network Performance w/ Instant Encoding\n", + "\n", + "We can now study the HWA inference performance with instant encoding of the inputs. The linear mode introduces extra quantization events, so it is interesting to study how the two fare against one another under the presence of other non-idealities. For example, one could sweep read noise as a representative non-ideality and compare the overall network performance. For now, we will keep things simple and compare the two while keeping other parameters unchanged." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "b602c2fb-5b6d-4d6d-88a9-fde89306b2d4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Iteration 1\n", + "\n", + "Iteration 2\n", + "\n", + "Iteration 3\n", + "\n", + "Iteration 4\n", + "\n", + "Iteration 5\n", + "\n", + "Iteration 6\n", + "\n", + "Iteration 7\n", + "\n", + "Iteration 8\n", + "\n", + "Iteration 9\n", + "\n", + "Iteration 10\n" + ] + } + ], + "source": [ + "iterations = 10\n", + "\n", + "# Step 1: Enable evaluation on the inference accelerator\n", + "model.xb_eval()\n", + "\n", + "instant_encoding_accs = np.zeros((iterations,))\n", + "\n", + "for i in range(iterations):\n", + " print(f\"\\nIteration {i+1}\")\n", + " # Step 2: Clear previous mappings\n", + " inference_accelerator.initialize_chip()\n", + " # Step 2: Initialize array mappings\n", + "\n", + " if stateful:\n", + " model.initialize_array_mappings(output_polling_mode=output_polling_mode, \n", + " additional_args={}, \n", + " existing_mappings=[]\n", + " )\n", + "\n", + " # Step 3: Test the performance like before!\n", + " acc, _ = test_classifier(test_loader, model, device)\n", + " instant_encoding_accs[i] = acc" + ] + }, + { + "cell_type": "markdown", + "id": "f925ac76-b8c6-4e49-87bd-d970b5a34f63", + "metadata": {}, + "source": [ + "## Network Performance w/ Linear Encoding\n", + "\n", + "Let's repeat the same network's performance experiment with linear encoding." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "5fe122ee-2eaa-4471-b17b-5edd82cf6503", + "metadata": {}, + "outputs": [], + "source": [ + "# Instantiate new accelerator\n", + "\n", + "input_encoding_scheme = \"linear\"\n", + "\n", + "inference_accelerator = SimpleFixedPoint(g_min=g_min, \n", + " g_max=g_max, \n", + " adc_bits=adc_precision, \n", + " dac_bits=dac_precision, \n", + " read_noise=read_noise, \n", + " write_noise=write_noise, \n", + " xb_size=xb_size,\n", + " weight_encoding_scheme=weight_encoding_scheme, \n", + " input_encoding_scheme=input_encoding_scheme,\n", + " xb_mapping_scheme=mapping_scheme,\n", + " stateful=stateful,\n", + " device=device)\n", + "\n", + "# Reload model\n", + "\n", + "model = SimpleMLP(input_size, hidden_size, output_size).to(device)\n", + "model = xbtorch_model(model)\n", + "\n", + "with open(\"checkpoints/hwa_train_mlp_hwa_example.pkl\", \"rb\") as f:\n", + " full_weights_hwa = pickle.load(f)\n", + " \n", + "epoch = -1 # load last epoch's weights\n", + "\n", + "for name, param in model.named_parameters():\n", + " new_tensor = torch.from_numpy(full_weights_hwa[name][epoch, ...]).to(\n", + " device=param.device,\n", + " dtype=param.dtype,\n", + " )\n", + " param.data = new_tensor" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "4ad14868-62dd-4541-b5a5-ced1db74f898", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Iteration 1\n", + "\n", + "Iteration 2\n", + "\n", + "Iteration 3\n", + "\n", + "Iteration 4\n", + "\n", + "Iteration 5\n", + "\n", + "Iteration 6\n", + "\n", + "Iteration 7\n", + "\n", + "Iteration 8\n", + "\n", + "Iteration 9\n", + "\n", + "Iteration 10\n" + ] + } + ], + "source": [ + "# Step 1: Enable evaluation on the inference accelerator\n", + "model.xb_eval()\n", + "\n", + "linear_encoding_accs = np.zeros((iterations,))\n", + "\n", + "for i in range(iterations):\n", + " print(f\"\\nIteration {i+1}\")\n", + " # Step 2: Clear previous mappings\n", + " inference_accelerator.initialize_chip()\n", + " # Step 2: Initialize array mappings\n", + "\n", + " if stateful:\n", + " model.initialize_array_mappings(output_polling_mode=output_polling_mode, \n", + " additional_args={}, \n", + " existing_mappings=[]\n", + " )\n", + "\n", + " # Step 3: Test the performance like before!\n", + " acc, _ = test_classifier(test_loader, model, device)\n", + " linear_encoding_accs[i] = acc" + ] + }, + { + "cell_type": "markdown", + "id": "fe7e0449-1dcc-4e73-9feb-d9d746dddfe3", + "metadata": {}, + "source": [ + "## Visualization\n", + "\n", + "Let's visualize the effect of input encoding mode on network performance." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "33344a75-61f3-4ac4-a775-da67a9daef4a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_3163875/3332972988.py:6: MatplotlibDeprecationWarning: The 'labels' parameter of boxplot() has been renamed 'tick_labels' since Matplotlib 3.9; support for the old name will be dropped in 3.11.\n", + " plt.boxplot(\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxYAAAHqCAYAAACZcdjsAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAoX9JREFUeJzs3XdYU9f/B/B3EiBsZA9FWSpIceEexY3WumqHtlVxV+uqra22dW+titVWq7Vq1Q611X6rP+uqYt2r4pbhqsoQUIZAgNzz+yMlNQaUMAzg+/U8PDHnnnvyuZdLzCf3DJkQQoCIiIiIiKgE5MYOgIiIiIiIKj4mFkREREREVGJMLIiIiIiIqMSYWBARERERUYkxsSAiIiIiohJjYkFERERERCXGxIKIiIiIiEqMiQUREREREZUYEwsiIiIiIioxJhZEL5CMjAwMGTIEbm5ukMlkGDdunLFDqlRu3rwJmUyGL774wtihEJWZsLAweHl56ZTJZDJMmzbNKPEYS5s2bdCmTRtjh0FUrjCxIKpA1q1bB5lMhtOnTxdr/zlz5mDdunUYMWIENmzYgH79+pVyhBWLTCbT+bG1tUVISAh27txp7NAMcvnyZUybNg03b940diil5v/+7/8gk8ng4eEBSZKMHU6ZCAsL07sG83/Mzc2NHV6FkJ/My2QyzJo1q8A677zzDmQyGaytrZ9zdEQvHhNjB0BEz8+ff/6JZs2aYerUqcYOpdzo2LEj+vfvDyEEbt26hRUrVqBbt27YtWsXQkNDjR1ekVy+fBnTp09HmzZt9L5Jrqg2bdoELy8v3Lx5E3/++Sc6dOhg7JDKhFKpxLfffqtXrlAojBBN8WVlZcHExHgfKczNzfHjjz/i888/1yl/9OgRfvvtNyZqRM8JEwuiF0hiYiLq1KlTau1JkoScnJwK/Z92rVq18O6772qf9+7dG3Xq1MHSpUsrTGJR2eR/GJw7dy7Wrl2LTZs2lVpikZeXB0mSYGZmVirtlZSJiYnO9VdRGfs94JVXXsGvv/6KyMhI1KtXT1v+22+/IScnB507d8aff/5pxAiJXgzsCkVUwYWFhcHa2hp3795Fz549YW1tDWdnZ3z00UdQq9UAgIMHD0Imk+HGjRvYuXOntutAftcZlUqFqVOnws/PD0qlEp6envj444+hUql0Xksmk2HUqFHYtGkTAgMDoVQq8ccffwAA7t69i0GDBsHV1RVKpRKBgYH47rvvdPbPj2Pz5s2YPXs2qlWrBnNzc7Rv3x4xMTF6x3bixAm88sorsLe3h5WVFerWrYulS5fq1Ll69Spef/11ODg4wNzcHI0aNcL//ve/Yp/PgIAAODk5ITY2Vqc8MTERgwcPhqurK8zNzVGvXj2sX7++0HaWLFmCGjVqwMLCAiEhIbh48aLO9sL6ZxfUf/2nn35CcHAwbGxsYGtri6CgIO15WLduHd544w0AQNu2bbW/24MHDwIAvLy88Oqrr+Lw4cNo0qQJzM3N4ePjg++//17vtR8+fIhx48bB09MTSqUSfn5+mD9/vl5XpKfFAwC5ubmYPn06atasCXNzczg6OqJVq1bYu3dvoefrcdu2bUNWVhbeeOMN9OnTB7/++iuys7P16mVnZ2PatGmoVasWzM3N4e7ujtdee037u3t8zEt4eDh8fX2hVCpx+fJlAJo7eK1bt4aVlRWqVKmCHj164MqVKzqvkZ6ejnHjxsHLywtKpRIuLi7o2LEjzp49q60THR2N3r17w83NDebm5qhWrRr69OmD1NTUIh3vs+R3gTxy5AjGjx8PZ2dnWFlZoVevXrh//75e/V27diEkJET7+2ncuDF++OEHnTpbtmxBcHAwLCws4OTkhHfffRd3797Va2v79u146aWXYG5ujpdeegnbtm0rMMYnx1hMmzYNMpkMMTExCAsLQ5UqVWBnZ4eBAwciMzNTZ9+srCyMGTMGTk5OsLGxQffu3XH37l2Dxm00b94c3t7eese5adMmdO7cGQ4ODgXu9/XXX2vfyzw8PPD+++/j4cOHevVWrVoFX19fWFhYoEmTJvjrr78KbK+o76V79+5Fq1atUKVKFVhbW6N27dr49NNPi3SsROUZ71gQVQJqtRqhoaFo2rQpvvjiC+zbtw+LFi2Cr68vRowYgYCAAGzYsAEffPABqlWrhg8//BAA4OzsDEmS0L17dxw+fBjDhg1DQEAALly4gCVLliAqKgrbt2/Xea0///wTmzdvxqhRo+Dk5AQvLy8kJCSgWbNm2sTD2dkZu3btwuDBg5GWlqY3SHzevHmQy+X46KOPkJqaigULFuCdd97BiRMntHX27t2LV199Fe7u7hg7dizc3Nxw5coV7NixA2PHjgUAXLp0CS1btkTVqlUxceJEWFlZYfPmzejZsyd++eUX9OrVy+BzmZqaigcPHsDX11dblpWVhTZt2iAmJgajRo2Ct7c3tmzZgrCwMDx8+FAbT77vv/8e6enpeP/995GdnY2lS5eiXbt2uHDhAlxdXQ2KZ+/evejbty/at2+P+fPnAwCuXLmCI0eOYOzYsXj55ZcxZswYfPnll/j0008REBAAANpHAIiJicHrr7+OwYMHY8CAAfjuu+8QFhaG4OBgBAYGAgAyMzMREhKCu3fvYvjw4ahevTqOHj2KSZMmIS4uDuHh4UWKB9B8qJw7dy6GDBmCJk2aIC0tDadPn8bZs2fRsWPHZx7zpk2b0LZtW7i5uaFPnz6YOHEifv/9d20CBWiu+VdffRX79+9Hnz59MHbsWKSnp2Pv3r24ePGizu9v7dq1yM7OxrBhw6BUKuHg4IB9+/ahS5cu8PHxwbRp05CVlYVly5ahZcuWOHv2rDa5e++997B161aMGjUKderUQXJyMg4fPowrV66gYcOGyMnJQWhoKFQqFUaPHg03NzfcvXsXO3bswMOHD2FnZ/fM401KStIrMzMzg62trU7Z6NGjYW9vj6lTp+LmzZsIDw/HqFGj8PPPP2vrrFu3DoMGDUJgYCAmTZqEKlWq4O+//8Yff/yBt99+W1tn4MCBaNy4MebOnYuEhAQsXboUR44cwd9//40qVaoAAPbs2aO9gzd37lwkJydj4MCBqFat2jOPKd+bb74Jb29vzJ07F2fPnsW3334LFxcX7bUDaJLpzZs3o1+/fmjWrBkiIiLQtWvXIr9Gvr59+2Ljxo2YN28eZDIZkpKSsGfPHmzYsEH7Bcjjpk2bhunTp6NDhw4YMWIErl27hhUrVuDUqVM4cuQITE1NAQBr1qzB8OHD0aJFC4wbNw7Xr19H9+7d4eDgAE9PT217RX0vvXTpEl599VXUrVsXM2bMgFKpRExMDI4cOWLwMROVO4KIKoy1a9cKAOLUqVPasgEDBggAYsaMGTp1GzRoIIKDg3XKatSoIbp27apTtmHDBiGXy8Vff/2lU75y5UoBQBw5ckRbBkDI5XJx6dIlnbqDBw8W7u7uIikpSae8T58+ws7OTmRmZgohhDhw4IAAIAICAoRKpdLWW7p0qQAgLly4IIQQIi8vT3h7e4saNWqIBw8e6LQpSZL23+3btxdBQUEiOztbZ3uLFi1EzZo1xbMAEIMHDxb3798XiYmJ4vTp06Jz584CgFi4cKG2Xnh4uAAgNm7cqC3LyckRzZs3F9bW1iItLU0IIcSNGzcEAGFhYSHu3LmjrXvixAkBQHzwwQfaspCQEBESEqIX04ABA0SNGjW0z8eOHStsbW1FXl5eocexZcsWAUAcOHBAb1uNGjUEAHHo0CFtWWJiolAqleLDDz/Uls2cOVNYWVmJqKgonf0nTpwoFAqFuH37dpHjqVevnt51VlQJCQnCxMRErF69WlvWokUL0aNHD5163333nQAgFi9erNdG/jWS//uwtbUViYmJOnXq168vXFxcRHJysrYsMjJSyOVy0b9/f22ZnZ2deP/99wuN9++//xYAxJYtWww6TiH++9st6Cc0NFRbL//vvkOHDjrX/wcffCAUCoV4+PChEEKIhw8fChsbG9G0aVORlZVV4DnJyckRLi4u4qWXXtKps2PHDgFATJkyRVtWv3594e7urm1fCCH27NkjAOhco0Jo/pamTp2qfT516lQBQAwaNEinXq9evYSjo6P2+ZkzZwQAMW7cOJ16YWFhem0WJP93vHDhQnHx4kUBQPte9tVXXwlra2vx6NEjMWDAAGFlZaXdLzExUZiZmYlOnToJtVqtLV++fLkAIL777jud81W/fn2d96xVq1YJADp/w0V9L12yZIkAIO7fv//UYyOqiNgViqiSeO+993Set27dGtevX3/mflu2bEFAQAD8/f2RlJSk/WnXrh0A4MCBAzr1Q0JCdMZpCCHwyy+/oFu3bhBC6LQRGhqK1NRUnW4jADBw4ECdPu6tW7cGAG28f//9N27cuIFx48Zpvz3NJ5PJAAApKSn4888/8eabbyI9PV37msnJyQgNDUV0dHSBXTuetGbNGjg7O8PFxQWNGjXC/v378fHHH2P8+PHaOv/3f/8HNzc39O3bV1tmamqKMWPGICMjAxERETpt9uzZE1WrVtU+b9KkCZo2bYr/+7//e2Y8T6pSpQoePXpU5G5EBalTp472HAOaO1W1a9fWuT62bNmC1q1bw97eXud32KFDB6jVahw6dKjI8VSpUgWXLl1CdHS0wbH+9NNPkMvl6N27t7asb9++2LVrFx48eKAt++WXX+Dk5ITRo0frtZF/jeTr3bs3nJ2dtc/j4uJw7tw5hIWF6XSRqVu3Ljp27Kjze6pSpQpOnDiBe/fuFRhv/h2J3bt363XxKQpzc3Ps3btX72fevHl6dYcNG6ZzbK1bt4ZarcatW7cAaO4mpaenY+LEiXpjHvL3O336NBITEzFy5EidOl27doW/v792RrT8czRgwACduy4dO3Y0aJxWQe9LycnJSEtLAwDtnYSRI0fq1Cvo9/osgYGBqFu3Ln788UcAwA8//IAePXrA0tJSr+6+ffuQk5ODcePGQS7/76PQ0KFDYWtrqz0P+efrvffe03nPCgsL07sbVdT30vz3tN9++63SznhGLy4mFkSVgLm5uc4HJwCwt7fX+SBWmOjoaFy6dAnOzs46P7Vq1QKgGVvwOG9vb53n9+/fx8OHD7Fq1Sq9NgYOHFhgG9WrV9eLFYA23vw+8i+99FKhccfExEAIgcmTJ+u9bv6sV0++bkF69OiBvXv3YufOndp+4ZmZmTofNm7duoWaNWvqlAH/dTfK/2CXr2bNmnqvU6tWrWJNBzty5EjUqlULXbp0QbVq1TBo0KACu3U8zZPnG9C/PqKjo/HHH3/oncv8QdP557Io8cyYMQMPHz5ErVq1EBQUhAkTJuD8+fNFinXjxo1o0qQJkpOTERMTg5iYGDRo0AA5OTnYsmWLtl5sbCxq165dpJmInrxm839ftWvX1qsbEBCApKQkPHr0CACwYMECXLx4EZ6enmjSpAmmTZumk5B5e3tj/Pjx+Pbbb+Hk5ITQ0FB89dVXRR5foVAo0KFDB72f+vXr69Utjb+bpx27v7+/dnv+Y0HXckH7FuZZMd+6dQtyuVzvd+Tn51fk13jc22+/jS1btiAmJgZHjx7Vdv96UmHnwczMDD4+Ps88D6ampvDx8dEpK+p76VtvvYWWLVtiyJAhcHV1RZ8+fbB582YmGVQpcIwFUSVQkqkpJUlCUFAQFi9eXOD2x/sQA4CFhYXe/gDw7rvvYsCAAQW2UbduXZ3nhcUrhChSzI+/7kcffVTo7E1F+XBSrVo17YfnV155BU5OThg1ahTatm2L1157rcjxGEomkxV4vPkD7vO5uLjg3Llz2L17N3bt2oVdu3Zh7dq16N+//1MHjz+uKOdbkiR07NgRH3/8cYF18z8cFSWel19+GbGxsfjtt9+wZ88efPvtt1iyZAlWrlyJIUOGFBpndHQ0Tp06BaDgD7SbNm3CsGHDinTMj3vymjXEm2++idatW2Pbtm3Ys2cPFi5ciPnz5+PXX39Fly5dAACLFi1CWFiY9njHjBmDuXPn4vjx4waNR3iW0vi7ed6ed8x9+/bFpEmTMHToUDg6OqJTp05l8joFKep7qYWFBQ4dOoQDBw5g586d+OOPP/Dzzz+jXbt22LNnT4WbapjocUwsiF5wvr6+iIyMRPv27fW6kBSFs7MzbGxsoFarS21K0PyBtxcvXiy0zfxvC01NTUt1jYPhw4djyZIl+Pzzz9GrVy/IZDLUqFED58+fhyRJOnctrl69CgCoUaOGThsFdQGKiorSme3J3t6+wK5qT979ADTfonbr1g3dunWDJEkYOXIkvvnmG0yePBl+fn7F+r09ydfXFxkZGUU6l8+KBwAcHBwwcOBADBw4EBkZGXj55Zcxbdq0pyYWmzZtgqmpKTZs2KD34erw4cP48ssvcfv2bVSvXh2+vr44ceIEcnNztYNsiyr/93Xt2jW9bVevXoWTkxOsrKy0Ze7u7hg5ciRGjhyJxMRENGzYELNnz9YmFgAQFBSEoKAgfP755zh69ChatmyJlStXFrpoW1l4/O+msKT68WPP76KT79q1a9rt+Y8FXcsFnbfiqlGjBiRJwo0bN3SSyYJmiSuK6tWro2XLljh48CBGjBhR6B2tx8/D43cecnJycOPGDe3fwePn4fHzlZubixs3buhMbWvIe6lcLkf79u3Rvn17LF68GHPmzMFnn32GAwcOVNo1W+jFwK5QRC+4N998E3fv3sXq1av1tmVlZWm7hBRGoVCgd+/e+OWXX/SmVAVQ4HSYz9KwYUN4e3sjPDxcb+rH/G86XVxc0KZNG3zzzTeIi4srldcFNOsKfPjhh7hy5Qp+++03AJo7GfHx8Tqz7+Tl5WHZsmWwtrZGSEiIThvbt2/XGd9x8uRJnDhxQueDqK+vL65evaoTZ2RkpN7MMMnJyTrP5XK59g5Q/hSW+R+CC5oms6jefPNNHDt2DLt379bb9vDhQ+Tl5RU5nifrWFtbw8/PT2/KzSdt2rQJrVu3xltvvYXXX39d52fChAkAoO0/37t3byQlJWH58uV67Tzr23B3d3fUr18f69ev1zlnFy9exJ49e/DKK68A0Nw9erJLk4uLCzw8PLTHkpaWpj03+YKCgiCXy595vKWtU6dOsLGxwdy5c/Wm580/J40aNYKLiwtWrlypE9+uXbtw5coV7WxMj5+jx8/B3r17tdP1lob8u41ff/21TvmyZcuK3easWbMwderUp47T6NChA8zMzPDll1/qXC9r1qxBamqq9jw0atQIzs7OWLlyJXJycrT11q1bp/f3VtT30pSUFL3t+V3fnvc1Q1TaeMeC6AXXr18/bN68Ge+99x4OHDiAli1bQq1W4+rVq9i8eTN2796NRo0aPbWNefPm4cCBA2jatCmGDh2KOnXqICUlBWfPnsW+ffsK/I/0aeRyuXYF7Pr162PgwIFwd3fH1atXcenSJe2H36+++gqtWrVCUFAQhg4dCh8fHyQkJODYsWO4c+cOIiMji3VOwsLCMGXKFMyfPx89e/bEsGHD8M033yAsLAxnzpyBl5cXtm7diiNHjiA8PBw2NjY6+/v5+aFVq1YYMWIEVCoVwsPD4ejoqNPNaNCgQVi8eDFCQ0MxePBgJCYmYuXKlQgMDNQObAWAIUOGICUlBe3atUO1atVw69YtLFu2DPXr19eO8ahfvz4UCgXmz5+P1NRUKJVKtGvXDi4uLkU+5gkTJuB///sfXn31Ve1UtI8ePcKFCxewdetW3Lx5E05OTkWKp06dOmjTpg2Cg4Ph4OCA06dPa6dsLcyJEye00/kWpGrVqmjYsCE2bdqETz75BP3798f333+P8ePH4+TJk2jdujUePXqEffv2YeTIkejRo8dTj3fhwoXo0qULmjdvjsGDB2unm7Wzs9OunZCeno5q1arh9ddfR7169WBtbY19+/bh1KlTWLRoEQDN9MujRo3CG2+8gVq1aiEvL097x+XxAeiFycvLw8aNGwvc1qtXL507J89ia2uLJUuWYMiQIWjcuDHefvtt2NvbIzIyEpmZmVi/fj1MTU0xf/58DBw4ECEhIejbt692ulkvLy988MEH2vbmzp2Lrl27olWrVhg0aBBSUlKwbNkyBAYGIiMjo8hxPU1wcDB69+6N8PBwJCcna6ebjYqKAqA/EL8oQkJC9JL9Jzk7O2PSpEmYPn06OnfujO7du+PatWv4+uuv0bhxY+2ihaamppg1axaGDx+Odu3a4a233sKNGzewdu1avTEWRX0vnTFjBg4dOoSuXbuiRo0aSExMxNdff41q1aqhVatWBh8vUblinMmoiKg4Cptu9vFpFPPlT/f4uIKmmxVCM6Xi/PnzRWBgoFAqlcLe3l4EBweL6dOni9TUVG09AIVOvZmQkCDef/994enpKUxNTYWbm5to3769WLVqlbZO/nSzT07NmT9l5Nq1a3XKDx8+LDp27ChsbGyElZWVqFu3rli2bJlOndjYWNG/f3/h5uYmTE1NRdWqVcWrr74qtm7dWmCcj3va8UybNk1nCteEhAQxcOBA4eTkJMzMzERQUJBevI9Pfblo0SLh6ekplEqlaN26tYiMjNR7jY0bNwofHx9hZmYm6tevL3bv3q033ezWrVtFp06dhIuLizAzMxPVq1cXw4cPF3FxcTptrV69Wvj4+AiFQqETd2G/84Kmu01PTxeTJk0Sfn5+wszMTDg5OYkWLVqIL774QuTk5BQ5nlmzZokmTZqIKlWqCAsLC+Hv7y9mz56tbaMgo0ePFgBEbGxsoXXyfyf55zIzM1N89tlnwtvbW3vNvf7669o2Hv99FGTfvn2iZcuWwsLCQtja2opu3bqJy5cva7erVCoxYcIEUa9ePe01WK9ePfH1119r61y/fl0MGjRI+Pr6CnNzc+Hg4CDatm0r9u3bV+hx5HvadLMAxI0bN4QQBf/dC/Hf39OT0wz/73//Ey1atNAeV5MmTcSPP/6oU+fnn38WDRo0EEqlUjg4OIh33nlHZ4rkfL/88osICAgQSqVS1KlTR/z6669616gQhU83++SUqvnHkn9sQgjx6NEj8f777wsHBwdhbW0tevbsKa5duyYAiHnz5j31HD7rd5yvsPfJ5cuXC39/f2FqaipcXV3FiBEj9Ka4FkKIr7/+Wnh7ewulUikaNWokDh06VODfUFHeS/fv3y969OghPDw8hJmZmfDw8BB9+/bVm+qZqCKSCVGOR30RERHRC+fcuXNo0KABNm7ciHfeecfY4RBREXGMBRERERlNVlaWXll4eDjkcjlefvllI0RERMXFMRZERERkNAsWLMCZM2fQtm1bmJiYaKcxHjZsmN5010RUvrErFBERERnN3r17MX36dFy+fBkZGRmoXr06+vXrh88++6xICyASUfnBxIKIiIiIiEqMYyyIiIiIiKjEmFgQEREREVGJsfNiASRJwr1792BjY1OsxXmIiIiIiCoDIQTS09Ph4eEBufzp9ySYWBTg3r17nImCiIiIiOhf//zzD6pVq/bUOkZNLNLT0zF58mRs27YNiYmJaNCgAZYuXYrGjRsD0GRIU6dOxerVq/Hw4UO0bNkSK1asQM2aNZ/a7ldffYWFCxciPj4e9erVw7Jly9CkSZMix2VjYwNAcwJtbW2Lf4BERiRJEu7fvw9nZ+dnfsNARERlg+/FVNGlpaXB09NT+/n4aYyaWAwZMgQXL17Ehg0b4OHhgY0bN6JDhw64fPkyqlatigULFuDLL7/E+vXr4e3tjcmTJyM0NBSXL1+Gubl5gW3+/PPPGD9+PFauXImmTZsiPDwcoaGhuHbtGlxcXIoUV373J1tbWyYWVGFJkoTs7GzY2tryPzMiIiPhezFVFkUZHmC06WazsrJgY2OD3377DV27dtWWBwcHo0uXLpg5cyY8PDzw4Ycf4qOPPgIApKamwtXVFevWrUOfPn0KbLdp06Zo3Lgxli9fDkDzB+3p6YnRo0dj4sSJRYotLS0NdnZ2SE1NZWJBFZYkSUhMTISLiwv/MyMiMhK+F1NFZ8jnYqPdscjLy4Narda782BhYYHDhw/jxo0biI+PR4cOHbTb7Ozs0LRpUxw7dqzAxCInJwdnzpzBpEmTtGVyuRwdOnTAsWPHCo1FpVJBpVJpn6elpQHQvBlIklTsYyQyJkmSIITgNUxEZER8L6aKzpBr12iJhY2NDZo3b46ZM2ciICAArq6u+PHHH3Hs2DH4+fkhPj4eAODq6qqzn6urq3bbk5KSkqBWqwvc5+rVq4XGMnfuXEyfPl2v/P79+8jOzjb00IjKBUmSkJqaCiEEvyUjIjISvhdTRZeenl7kukYdY7FhwwYMGjQIVatWhUKhQMOGDdG3b1+cOXPmucYxadIkjB8/Xvs8f5CKs7Mzu0JRhSVJEmQyGQcMEhEZEd+LqaIrbFxzQYyaWPj6+iIiIgKPHj1CWloa3N3d8dZbb8HHxwdubm4AgISEBLi7u2v3SUhIQP369Qtsz8nJCQqFAgkJCTrlCQkJ2vYKolQqoVQq9crlcjnfBKhCk8lkvI6JiIyM78VUkRly3ZaLK9zKygru7u548OABdu/ejR49esDb2xtubm7Yv3+/tl5aWhpOnDiB5s2bF9iOmZkZgoODdfaRJAn79+8vdB8iIiIiIio5o96x2L17N4QQqF27NmJiYjBhwgT4+/tj4MCBkMlkGDduHGbNmoWaNWtqp5v18PBAz549tW20b98evXr1wqhRowAA48ePx4ABA9CoUSM0adIE4eHhePToEQYOHGikoyQiIiIiqvyMmlikpqZi0qRJuHPnDhwcHNC7d2/Mnj0bpqamAICPP/4Yjx49wrBhw/Dw4UO0atUKf/zxh05fr9jYWCQlJWmfv/XWW7h//z6mTJmC+Ph41K9fH3/88YfegG4iIiIiIio9RlvHojzjOhZUGXDudCIi4+N7MVV0hnwu5hVOREREREQlxsSCiIiIiIhKjIkFERERERGVmFEHbxNR2ZIkgYt3U/EgKw8OlmYI9LCFXC4zdlhERERUCTGxIKqkjsUm47fjUThyJxc5asBUIYOvizVGhPiihZ+TscMjIiKiSoZdoYgqoaMxSfh8+wX8k5IJK6UJXGyUsFKa4EpcOj7ddgFHY5Ke3QgRERGRAZhYEFUykiSwIiIWGao82FuawdxUAblcBnNTBdxslchQqbEiIhaSxJmmiYiIqPSwKxRREWRmZuLq1avGDqNIohPSEfn3FZjL8hB98x4ybapBYarUbpfnSYiMV2OLRzZqutoYMdKC+fv7w9LS0thhEBERkYGYWBAVwdWrVxEcHGzsMEpVn6+MHUHBzpw5g4YNGxo7DCIiIjIQEwuiIvD398eZM2eMHUaRRCekY/bOKxAP7uDiD7PRoP/nsHb10m7PzpOQnaPGZ10Dyu0dCyIiIqp4mFgQFYGlpWWF+Ra9viSw4545zpyRAADWrjVg51kbACCEQGaaCvXcbfBGaBNOPUtERESlhoO3iSoZuVyGESG+sDTV/Hmr8iRIkkBWrhrxaSpYKxUYEeLLpIKIiIhKFRMLokqohZ8TRratCQDIzlEjMUOFTFUeAtxtMKdXENexICIiolLHrlBElVTdanYAgE9fCYCbXyBX3iYiIqIyxcSCqJLzdbFGo1rOxg6DiIiIKjl2hSIiIiIiohJjYkFERERERCXGrlBEFZwkCVy6l4aUzByOoyAiIiKjYWJBVIEdjUnCiohYxCZmIFctYKqQwdfFGiNCfGFm7OCIiIjohcLEgqiCOhqThE+3XUCGKg/2lmYwU8iRo5ZwJS4dn267gP41JWOHSERERC8QJhZEFZAkCayIiEWGKg9utuaQyTRdn8zlCrjZyhGfpsIvZ+8aOUoiIiJ6kXDwNlEFdOleGmITM2BvaaZNKvLJZDJUsTTFnZRMI0VHRERELyImFkQVUEpmDnLVAmaKgv+ElQo58oR4zlERERHRi4yJBVEF5GBpBlOFDDnqgsdRqNQSTGScGYqIiIieHyYWRBVQoIctfF2s8SAzF+KJOxNCCDzMzEU1B0sjRUdEREQvIiYWRBWQXC7DiBBfWCsViE9TIStXDUkSyMpVIz5NBWulAr0bVjN2mERERPQCYWJBVEG18HPCnF5BCHC3QaYqD4kZKmSq8hDgboM5vYJQt5qdsUMkIiKiFwinmyWqwFr4OaGZj2OBK2+fPn3T2OERERHRC4SJBVEFJ5fLEMS7E0RERGRk7ApFREREREQlxsSCiIiIiIhKjIkFERERERGVGBMLIiIiIiIqMSYWRERERERUYkwsiIiIiIioxJhYEBERERFRiTGxICIiIiKiEuMCeUSVXGxiBh5F3ddZlZuIiIiotDGxIKqkzt9JBQDM+b8rMHPLg6lCBl8Xa4wI8UULPycjR0dERESVDbtCEVVCR2OS8PWBaACAhZkCLjZKWClNcCUuHZ9uu4CjMUlGjpCIiIgqGyYWRJWMJAmsiIhFZq4aAGBmIodcLoO5qQJutkpkqNRYERELSRJGjpSI6DmTJODe30DMPs2jJBk7IqJKhV2hiCqZS/fSEJuYAVtzU71tMpkMVSxNEZuYgUv30hBUzc4IERIRGcH1CODwEiApGpByAbkp4FQTaPUB4BNi7OiIKgXesSCqZFIyc5CrFjCVF/znrVTIkSsJpGTmPOfIiIiM5HoEsGMckHAJMLMCrF01jwmXNOXXI4wdIVGlwMSCqJJxsDSDqUKG3EJu8avUEkzlMjhYmj3nyIiIjECSNHcqVBmAjTtgagHI5JpHG3dN+eEl7BZFVAqM2hVKrVZj2rRp2LhxI+Lj4+Hh4YGwsDB8/vnnkMk0U2LmPz5pwYIFmDBhQoHbpk2bhunTp+uU1a5dG1evXi3dAyAqhwI9bOHrYo0z/+TpbRNC4GFmLgLcbRDoYWuE6IiIii8zM9Pw/8vvXwXOndckEg8z9LfnmQFx5wHXnwBn/9IJ9DGZmZmIjIxEvXr1YGlpWertlxV/f/8KFS+VD0ZNLObPn48VK1Zg/fr1CAwMxOnTpzFw4EDY2dlhzJgxAIC4uDidfXbt2oXBgwejd+/eT207MDAQ+/bt0z43MeFwkvImOjoa6enpxg6jUmrnlIEzD+4AAFLu3oQkATmSQEZ2HixM5WgX5Idz5/42cpQVn42NDWrWrGnsMIheGFevXkVwcHDZNL7knbJpt4I6c+YMGjZsaOwwqIIx6qfto0ePokePHujatSsAwMvLCz/++CNOnjyprePm5qazz2+//Ya2bdvCx8fnqW2bmJjo7UvlR3R0NGrVqmXsMF4Il36YrVc26EsjBFJJRUVFMbkgek78/f1x5swZw3a6fxXY/bnmjoWJUn97ngrIzQJCZ5XJHYvLly+jX79+2LBhA+rUqVPq7ZcVf//SPxdU+Rk1sWjRogVWrVqFqKgo1KpVC5GRkTh8+DAWL15cYP2EhATs3LkT69evf2bb0dHR8PDwgLm5OZo3b465c+eievXqBdZVqVRQqVTa52lpaQAASZIgsc9lmUhN1Sze9v333yMgIMDI0VROmZmZOH/+PGzcfaGSmcLO3ATeTlZcebuUXLlyBf3790dqairfJ4ieE3Nzc9SvX9+wnaS6QMIWIOEyYOMEPN7FWggg/RHgWg9o/yZQyKQXJZGXp+mWWqtWLcNjNzK+txFg2HVg1MRi4sSJSEtLg7+/PxQKBdRqNWbPno133in4duT69ethY2OD11577antNm3aFOvWrUPt2rURFxeH6dOno3Xr1rh48SJsbGz06s+dO1dvTAYA3L9/H9nZ2cU7OHqqlJQUAICrqyuqVatm5GgqJ0mS4OzsDDs7O8jL4D/LF13+NZySkoLExEQjR0NET1X3feDUaiA3GzCzBhQmgDoPyMkAqrgDdYcASWWzcOiDBw+0j3yvoIrIkG7rRk0sNm/ejE2bNuGHH35AYGAgzp07h3HjxsHDwwMDBgzQq//dd9/hnXfegbm5+VPb7dKli/bfdevWRdOmTVGjRg1s3rwZgwcP1qs/adIkjB8/Xvs8LS0Nnp6ecHZ2hq0tB7iWBQcHB+2ji4uLkaOpnCRJgkwmg7OzMxOLMsBrmKgCcWkH2JgAR5YCSecfW8fCD2j5HuD9cpm9tL29vfaR7xVUET3rc/fjjJpYTJgwARMnTkSfPn0AAEFBQbh16xbmzp2rl1j89ddfuHbtGn7++WeDX6dKlSqoVasWYmJiCtyuVCqhVOr3u5TL5fxAVkbyzyvPcdmSyWQ8x2WE1zBRBePbRpNAxEcCmcmApSPgVq9Muj89ju8VVNEZct0a9QrPzMzUC1ahUBTYl2vNmjUIDg5GvXr1DH6djIwMxMbGwt3dvdixEhERUQUnlwMeDQC/DppHftAnKlVG/Yvq1q0bZs+ejZ07d+LmzZvYtm0bFi9ejF69eunUS0tLw5YtWzBkyJAC22nfvj2WL1+uff7RRx8hIiICN2/exNGjR9GrVy8oFAr07du3TI+HiIiIiOhFZdSuUMuWLcPkyZMxcuRIJCYmwsPDA8OHD8eUKVN06v30008QQhSaGMTGxiLpsUFXd+7cQd++fZGcnAxnZ2e0atUKx48fh7Ozc5keD1FlJ0kCl+6lISUzBw6WZgj0sOUsU0RERATAyImFjY0NwsPDER4e/tR6w4YNw7BhwwrdfvPmTZ3nP/30UylER0SPOxqThBURsYhNzECuWsBUIYOvizVGhPiihZ+TscMjIiIiI2PnQiJ6pqMxSfh02wVciUuDldIELjZKWClNcCUuHZ9uu4CjMWUzTSMRERFVHEwsiOipJElgRUQsMlR5cLM1h7mpAnK5DOamCrjZKpGhUmNFRCwkSRg7VCIiIjIiJhZE9FSX7qUhNjED9pZmkMl0x1PIZDJUsTRFbGIGLt1LM1KEREREVB4wsSCip0rJzEGuWsBMUfDbhVIhR64kkJKZ85wjIyIiovKEiQURPZWDpRlMFTLkqPXXlwEAlVqCqVwGB0uz5xwZERERlSdMLIjoqQI9bOHrYo0HmbkQQncchRACDzNz4etijUAPWyNFSEREROUBEwsieiq5XIYRIb6wVioQn6ZCVq4akiSQlatGfJoK1koFRoT4cj0LIiKiFxwTCyJ6phZ+TpjTKwgB7jbIVOUhMUOFTFUeAtxtMKdXENexICIiIuMukEdEFUcLPyc083HkyttERERUICYWRFRkcrkMQdXsjB0GERERlUNMLIheQJIkeOeBiIiIShUTC6IXzNGYJKyIiEVsYgZy1QKmChl8XawxIsSXYyWIiIio2Dh4m+gFcjQmCZ9uu4ArcWmwUprAxUYJK6UJrsSl49NtF3A0JsnYIRIREVEFxcSC6AUhSQIrImKRocqDm605zE0VkMtlMDdVwM1WiQyVGisiYiFJ4tmNERERET2BiQXRC+LSvTTEJmbA3tIMMpnueAqZTIYqlqaITczApXtpRoqQiIiIKjImFkQviJTMHOSqBcwUBf/ZKxVy5EoCKZk5zzkyIiIiqgyYWBC9IBwszWCqkCFHLRW4XaWWYCqXwcHS7DlHRkRERJUBEwuiF0Sghy18XazxIDMXQuiOoxBC4GFmLnxdrBHoYWukCImIiKgiY2JB9IKQy2UYEeILa6UC8WkqZOWqIUkCWblqxKepYK1UYESIL9ezICIiomJhYkH0Amnh54Q5vYIQ4G6DTFUeEjNUyFTlIcDdBnN6BXEdCyIiIio2LpBH9IJp4eeEZj6OXHmbiIiIShUTC6IXkFwuQ1A1O2OHQURERJUIEwsiIiKiikKSgPhIIDMZsHQE3OoBcvZsp/KBiQURERFRRXA9Aji8BEiKBqRcQG4KONUEWn0A+IQYOzoiJhZkPG7WMlg8jALu8ZuWMiEETFJSAHUcIOP4idJm8TAKbtY8r0T0nFyPAHaMA1QZgIU9YKIE8lRAwiVN+avhTC7I6JhYkNEMDzZDwKHhwCFjR1I5yQFwjqeyEwDNNUxEVOYkSXOnQpUB2Lj/92WRqQUgJODRfWD/DKDGbkDBj3ZkPLz6yGi+OZODt6asQ4C/v7FDqZQkIZCSkgIHBwfIecei1F25ehXfLHob3Y0dCBGVb5KkebxzEvBQFG9MRHykpvuThf1/SUVOBpCeAORla5KLe38D34UC7afwzgUZDRMLMpr4DIGsKrUAj/rGDqVSOnT7IDZe+xHvNnkXbaq3MXY4lU5WvIT4DPHsikT04roeAfwxTfPvgwuA61bFGxORmawZU2Gi1DzPyQAe3tYkFHIFIBSa7UnR7BZFRsXO7USVUMQ/ERh/cDweqh5i/MHxiPgnwtghERG9WPLHRCRf1zy3dADMrP4bE3HdgPdlS0fNQO08leZ5esK/SYUpIJMDMqF5tHbRdJc6vOS/OyVEzxHvWBBVMhH/RGDsgbHAv1+mS0LC2ANjsbTtUoR4lu43WJIkuNAeUQUXHR2N9PR0Y4dRuUgS8H9TgeQkXE23AgBcTciCXGYBCBvNmIiNU4FXvihatyhJAjKcgeRYQGkDpGUAchkgywWEACQ1oFAC2RKgNgPizgOuPwHOL0ZXYxsbG9SsWdPYYRCYWBBVKvlJhSQkyKD5gC8gIIR4anJRnAThaEwSVkTEIjYxA7lqAVOFDL4u1hgR4osWfhw2TlQRREdHo1atWsYO44XQ77urT5TcAaY2LuVXuf/fP5e8U8ptl29RUVFMLsoBJhZElcTjSYWA0CYWgCa5KOzORXEShKMxSfh02wVkqPJgb2kGM4UcOWoJV+LS8em2C5jTK4jJBVEFkH+nYuPGjQgICDByNJXIPyeAA3MBS0dk5gGRqTaoZ5cOS9N/35eF0IybaDsJ8Gxa9HbvnAFOrgbu/5ukyOSAwuzfblaWmrI8FZCbBYTOeiHuWFy5cgXvvvsu77qVE0wsiCqBJ5OKghSUXBQnQZAkgRURschQ5cHN1hyyf2coMZcr4GYrR3yaCisiYtHMx5HdoogqiICAADRs2NDYYVQebjIg1gYwM4Vkagk/pTdcVDcgz39/zs0CcmyAxs0AjwZFb7dhQ6DrQM3sT0nRmjEVppb/zRQlBJCeDrjWBTr24Yrc9NzxiiOq4IqSVOR7PLk4cPugToJgbqqAXC6DuakCbrZKZKjUWBERC0nSbfPSvTTEJmbA3tJMm1Tkk8lkqGJpitjEDFy6l1bqx0pEVCG41dPM/pT1QPNh/3FCaMqdamrqGUphoplS1soRyE79b7rZ3CwgPU4zBqPVB0wqyCh41RFVYIYkFfnyk4sPDo7DtdQTBicIKZk5yFULmCkKfvtQKuTIlQRSMnOKd1BERM+L9O/6DzH7NI+lNZOSXK75cK+0BtLjgbyc0v3w7xOimVLWNRDIeQRkJGgeXQOBV5dwqlkyGnaFIqqgipNU5BMQUAsJWQ5rYP1oOCDV1aujVMiRWkCC4GBpBlOFDDlqCeZyhd5+KrUEU7kMDpZclZqIyrHrEZppWZOiNWtAyE2Lt8ZEYfI//B8OBzKzgYxEzZoTroGl8xo+IYBXa83ieZnJmilpi7P4HlEpYmJBVEHNPD4TaqEuQQsCgECyxY+weqSfWBSWIAR62MLXxRpX4tLhZivXudshhMDDzFwEuNsg0MO2BLEREZWh/DUmVBma1axNlJpBz/lrTJTWAnM+IUD1lkDUCcA0E7ByKt0P/3K5YWM0iMoY01qiCmpys8lQyBQ6sz8ZQrOfHPKU1yGe6AOcnyD4uljrJQhyuQwjQnxhrVQgPk2FrFw1JEkgK1eN+DQVrJUKjAjx5cBtIiqfJElzp0KVAdi4A6YWmtmVTC00z0t7gTm5HHDwBXzba5IA3lGgSoxXN1EFFeIZgqVtl0IukxucXMggg1wmx/t1ZsFO1DU4QWjh54Q5vYIQ4G6DTFUeEjNUyFTlIcDdhlPNElH5Fh+p6f5kYf/fbEr5ZDJNeVK0ph4RGYRdoYgqsPzkwpCxFvlJRf6Us3Ud/lvHIlUSMJXLEOBu88yF7lr4OaGZjyNX3iaiiiUzWTOmwkRZ8HYTJZD9UFOPiAzCxIKogjMkuXgyqQBKliDI5TIEVbMrtWMhIipzlo6agdp5Kk33pyflqTTbLR2ff2xEFRy7QhFVAkXpFlVQUpEvP0EIqeWMoGp2vOtARJVXWa4xQfSCY2JBVEk8Lbl4WlJBRPRC0VljIk6ztgQXmCMqFfyrIapECkoumFQQET2BC8wRlQmjJhZqtRqTJ0+Gt7c3LCws4Ovri5kzZ+pMfRkWFgaZTKbz07lz52e2/dVXX8HLywvm5uZo2rQpTp48WZaHQlRuPJ5cAGBSQURUEJ8Q4N1fgT4bgZ5fax7f/ZVJBVEJGHXw9vz587FixQqsX78egYGBOH36NAYOHAg7OzuMGTNGW69z585Yu3at9rlSWchMDv/6+eefMX78eKxcuRJNmzZFeHg4QkNDce3aNbi4uJTZ8RCVFyGeIVjcZjE2ntyIxU0WM6kgIioIF5gjKlVGTSyOHj2KHj16oGvXrgAALy8v/Pjjj3p3F5RKJdzc3Irc7uLFizF06FAMHDgQALBy5Urs3LkT3333HSZOnFh6B0BUjr1c7WX4m/kzmSYiIqLnwqiJRYsWLbBq1SpERUWhVq1aiIyMxOHDh7F48WKdegcPHoSLiwvs7e3Rrl07zJo1C46OBU8Dl5OTgzNnzmDSpEnaMrlcjg4dOuDYsWMF7qNSqaBSqbTP09LSAACSJEEqrZU3SUf+eeU5LjuSJEEIwfNbRngNU2XA67js8b24bPEaLnuGnFejJhYTJ05EWloa/P39oVAooFarMXv2bLzzzjvaOp07d8Zrr70Gb29vxMbG4tNPP0WXLl1w7NgxKBQKvTaTkpKgVqvh6uqqU+7q6oqrV68WGMfcuXMxffp0vfL79+8jOzu7hEdJBUlJSdE+JiYmGjmaykmSJKSmpkIIATlnNyl1vIapMuB1XPb4Xly2eA2XvfT09CLXNWpisXnzZmzatAk//PADAgMDce7cOYwbNw4eHh4YMGAAAKBPnz7a+kFBQahbty58fX1x8OBBtG/fvlTimDRpEsaPH699npaWBk9PTzg7O8PW1rZUXoN0OTg4aB/ZVadsSJIEmUwGZ2dn/mdWBngNU2XA67js8b24bPEaLnvm5uZFrmvUxGLChAmYOHGiNnkICgrCrVu3MHfuXG1i8SQfHx84OTkhJiamwMTCyckJCoUCCQkJOuUJCQmFjtNQKpUFDgiXy+V8Eygj+eeV57hsyWQynuMywmuYKgNex88H34vLDq/hsmfIeTXqbyAzM1MvWIVC8dS+XHfu3EFycjLc3d0L3G5mZobg4GDs379fWyZJEvbv34/mzZuXTuBERERERKTDqIlFt27dMHv2bOzcuRM3b97Etm3bsHjxYvTq1QsAkJGRgQkTJuD48eO4efMm9u/fjx49esDPzw+hoaHadtq3b4/ly5drn48fPx6rV6/G+vXrceXKFYwYMQKPHj3SzhJFRERERESly6hdoZYtW4bJkydj5MiRSExMhIeHB4YPH44pU6YA0Ny9OH/+PNavX4+HDx/Cw8MDnTp1wsyZM3W6LsXGxiIpKUn7/K233sL9+/cxZcoUxMfHo379+vjjjz/0BnQTERG96NysZbB4GAXcYzeSMiEETFJSAHUcIJMZO5pKx+JhFNyseV7LC6MmFjY2NggPD0d4eHiB2y0sLLB79+5ntnPz5k29slGjRmHUqFEljJCIiKhyGx5shoBDw4FDxo6kcpIDcDJ2EJVYADTXMJUPRk0siIiIyLi+OZODt6asQ4C/v7FDqZQkIZCSkgIHBwfIecei1F25ehXfLHob3Y0dCAFgYkFERPRCi88QyKpSC/Cob+xQKidJQp4iEXBxAThrUanLipcQnyGMHQb9i1c4ERERERGVGBMLIiIiIiIqMSYWRERERERUYkwsiIiIiIioxJhYEBERERFRiTGxICIiIiKiEuN0s0RERPTikSQgPhLITAYsHQG3epwOlqiEmFgQERHRi+V6BHB4CZAUDUi5gNwUcKoJtPoA8AkxdnREFRZTcyIiInpxXI8AdowDEi4BZlaAtavmMeGSpvx6hLEjJKqwipVY5Obm4p9//sG1a9eQkpJS2jERERERlT5J0typUGUANu6AqQUgk2sebdw15YeXaOoRkcGKnFikp6djxYoVCAkJga2tLby8vBAQEABnZ2fUqFEDQ4cOxalTp8oyViIiIqLii4/UdH+ysAdkMt1tMpmmPClaU4+IDFakxGLx4sXw8vLC2rVr0aFDB2zfvh3nzp1DVFQUjh07hqlTpyIvLw+dOnVC586dER0dXdZxExERERkmM1kzpsJEWfB2E6Vme2by842LqJIo0uDtU6dO4dChQwgMDCxwe5MmTTBo0CCsXLkSa9euxV9//YWaNWuWaqBUuWRmZgIAzp49a+RIKq/MzExERkaiXr16sLS0NHY4lc6VK1eMHQIRGcrSUTNQO0+l6f70pDyVZrul4/OPjagSKFJi8eOPPxapMaVSiffee69EAdGL4erVqwCAoUOHGjkSopKxsbExdghEVFRu9TSzPyVcAkzMdbtDCQFkPQBcAzX1iMhgJZpuNjc3F1FRUVCr1ahduzaUykJuLRI9oWfPngAAf39/fpteRi5fvox+/fphw4YNqFOnjrHDqZRsbGx4d5aoIpHLNVPK7hgHpMdpxlSYKDV3KrIeAEobzXauZ0FULMVOLP766y/06dMHubm5yMvLg4mJCb7//nt07ty5NOOjSsrJyQlDhgwxdhiVmvTvrCb+/v5o2LChkaMhIionfEKAV8P/W8ci+6Gm+5NrINexICqhIicWkiRB/lgGP27cOGzatAlt2rQBAKxatQojRozAjRs3Sj1IIiIiolLjEwJ4tebK20SlrMh/QU2bNtUZaJuTk4Pq1atrn1evXh3Z2dmlGx0RERFRWZDLAY8GgF8HzSOTCqISK/Idi+XLl2PIkCEICQnBrFmzMHXqVAQHB6N27drIzc3F1atXsWzZsrKMlYiIiIiIyqkiJxZNmzbFqVOnsGDBAgQHB2PBggW4du0aTpw4AbVajcaNG6Nq1aplGSsREREREZVTBg3eVigUmDRpEt5880289957WL9+PZYtWwYPD4+yio+ISokkCVy6l4aUzBw4WJoh0MMWcrns2TsSERERFYFBicWlS5dw9epVBAUFYe/evVi/fj1at26NDz/8ECNHjiyrGImohI7GJGFFRCxiEzOQqxYwVcjg62KNESG+aOHnZOzwiIiIqBIo8kilxYsXo3Hjxli4cCGaN2+O1atXY8CAAThx4gSOHz+O5s2b48KFC2UZKxEVw/k7qfh02wVciUuDldIELjZKWClNcCUuHZ9uu4CjMUnGDpGIiIgqgSInFgsWLMDOnTtx/PhxnD17FosXLwagWY/g+++/x4wZM/Dmm2+WWaBEVDy/nL2DDFUe3GzNYW6qgFwug7mpAm62SmSo1FgREQtJEsYOk4iIiCq4IicWQgjtOhYKhQJC6H4Q6dixI/7+++/SjY6ISuxOSibsLc0gk+mOp5DJZKhiaYrYxAxcupdmpOiIiIiosijyGIsJEybglVdeQb169RAVFYU5c+bo1TE3Ny/V4Iio5PIkATNFwd8hKBVypEoCKZk5zzkqIiIiqmyKnFh89NFHCA0N1Q7e9vf3L8u4iKiUmMhlyFFLMJcr9Lap1BJM5TI4WJoZITIiIiKqTAyaFSooKAhBQUFlFQsRlYFqDpZIyMyFm61cpzuUEAIPM3MR4G6DQA9bI0ZIRERElUGRxljMmzcPmZmZRWrwxIkT2LlzZ4mCIqLS07thNVgrFYhPUyErVw1JEsjKVSM+TQVrpQIjQny5ngURERGVWJESi8uXL6NGjRoYOXIkdu3ahfv372u35eXl4fz58/j666/RokULvPXWW7CxsSmzgInIMHWr2WFOryAEuNsgU5WHxAwVMlV5CHC3wZxeQVzHgojoeZAk4N7fQMw+zaMkGTsiolJXpK5Q33//PSIjI7F8+XK8/fbbSEtLg0KhgFKp1N7JaNCgAYYMGYKwsDAO4iYqZ1r4OaGZjyNX3iYiMob4i8Cer4CkKEDKBeSmgFNNoNUHgE+IsaMjKjVFHmNRr149rF69Gt988w3Onz+PW7duISsrC05OTqhfvz6cnPitJ1F5JpfLEFTNzthhEBG9WG4cAk6tBh5eBizsABMlkKcCEi4BO8YBr4YzuaBKw6DB2wAgl8tRv3591K9fvwzCISIiIqokJAk4shTIlQAbNyD/JrGpBWBiDqTHAYeXAF6tAXmRlxYjKrd4FRMRERGVhfhIICkGMLMGnlikFDIZYGEPJEVr6hFVAkwsiIiIiMpCZrJmTIWikA4iJkrN9szk5xsXURlhYkFERERUFiwdNQO11XkFb89TabZbOj7fuIjKCBMLIiIiorLgVg9w8gNyMgAhdLcJAWQ90MwO5VbPOPERlTKDE4u1a9cWebE8IiIioheWXI5003a495cC6ZcTgNwsQEiax/Q4QGmjmXKWA7epkjD4Sp44cSLc3NwwePBgHD16tCxiIiIiIqrw0g8cwN2530KdDdz9U4H06AwgIwHIeQS4BgKvLuFUs1SpGJxY3L17F+vXr0dSUhLatGkDf39/zJ8/H/Hx8WURHxEREVGFk37gAO6MHgOo1ZoCIcOdfXKke4wE+mwE3v2VSQVVOgYnFiYmJujVqxd+++03/PPPPxg6dCg2bdqE6tWro3v37vjtt98gcZl6IiIiekHpJBX5YyuEACQJd+asQfq1h+z+RJVSia5qV1dXtGrVCs2bN4dcLseFCxcwYMAA+Pr64uDBg6UUIhEREVHFUGBSkU8IQK3GndFjkH7ggHECJCpDxUosEhIS8MUXXyAwMBBt2rRBWloaduzYgRs3buDu3bt48803MWDAgGe2o1arMXnyZHh7e8PCwgK+vr6YOXMmxL9/iLm5ufjkk08QFBQEKysreHh4oH///rh3795T2502bRpkMpnOj7+/f3EOlYiIiKhInppU5GNyQZVYISu2FK5bt27YvXs3atWqhaFDh6J///5wcHDQbreyssKHH36IhQsXPrOt+fPnY8WKFVi/fj0CAwNx+vRpDBw4EHZ2dhgzZgwyMzNx9uxZTJ48GfXq1cODBw8wduxYdO/eHadPn35q24GBgdi3b99/B2pi8KESERERFUmRkop8jyUX1ZZ9CZu2bZ9PkERlzOBP2y4uLoiIiEDz5s0LrePs7IwbN248s62jR4+iR48e6Nq1KwDAy8sLP/74I06ePAkAsLOzw969e3X2Wb58OZo0aYLbt2+jevXqhbZtYmICNze3ohwSERERUbEZlFTkY3JBlZDBicWaNWueWUcmk6FGjRrPrNeiRQusWrUKUVFRqFWrFiIjI3H48GEsXry40H1SU1Mhk8lQpUqVp7YdHR0NDw8PmJubo3nz5pg7d26hiYhKpYJKpdI+T0tLAwBIksSB6FRh5V+7vI6JqDB8nyi59IMHcXfcB5pEQSbT/DxGyOUQMhlEYYO1hcA/Y8ehavgS2LRpU/YBVzK8hsueIefV4MRizJgx8PPzw5gxY3TKly9fjpiYGISHhxe5rYkTJyItLQ3+/v5QKBRQq9WYPXs23nnnnQLrZ2dn45NPPkHfvn1ha2tbaLtNmzbFunXrULt2bcTFxWH69Olo3bo1Ll68CBsbG736c+fOxfTp0/XK79+/j+zs7CIfD1F58uDBA+1jYmKikaMhovIoJSVF+8j3ieK5t3Ej1L6+hW4XMhlUVasCAGRPuZsRu3EjPOrUKfX4Kjtew2UvPT29yHUNTix++eUX/O9//9Mrb9GiBebNm2dQYrF582Zs2rQJP/zwAwIDA3Hu3DmMGzcOHh4eeoO/c3Nz8eabb0IIgRUrVjy13S5dumj/XbduXTRt2hQ1atTA5s2bMXjwYL36kyZNwvjx47XP09LS4OnpCWdn56cmMETlhSQJXI5LQ0pmLhwsTVHH3Rb29vYAAHt7e7i4uBg5QiIqj/LHSDo4OPB9opgs3n1Xc8eikG5Q+XcqrKKiICvom1+ZDFAoNHcs+DswGK/hsmdubl7kugYnFsnJybCzs9Mrt7W1RVJSkkFtTZgwARMnTkSfPn0AAEFBQbh16xbmzp2rk1jkJxW3bt3Cn3/+afCH/SpVqqBWrVqIiYkpcLtSqYRSqdQrl8vlkHOeaSrnjsYkYUVELGITM5CrFjBVyODrYo22jhkAeB0TUeHy3xv4PlF8du3aQb40XDPGQpIKTC5kQkAmSfqJhUwGyOWotjScYyyKiddw2TPkvBr8G/Dz88Mff/yhV75r1y74+PgY1FZmZqZesAqFQqcvV35SER0djX379sHR0dHQkJGRkYHY2Fi4u7sbvC9ReXY0JgmfbruAK3FpsFKawMVGCSulCa7EpePrA9HGDo+I6IVg07Ytqi37ElAo9MZYFOrfOxUcuE2VicF3LMaPH49Ro0bh/v37aNeuHQBg//79WLRokUHdoADN1LWzZ89G9erVERgYiL///huLFy/GoEGDAGiSitdffx1nz57Fjh07oFarER8fD0Bzy8vMzAwA0L59e/Tq1QujRo0CAHz00Ufo1q0batSogXv37mHq1KlQKBTo27evoYdLVG5JksCKiFhkqPLgZmsO2b//mZnLFXCzleP6PUlbj4iIylZ+clGk2aGYVFAlZXBiMWjQIKhUKsyePRszZ84EoJkmdsWKFejfv79BbS1btgyTJ0/GyJEjkZiYCA8PDwwfPhxTpkwBANy9e1c7nqN+/fo6+x44cABt/p09ITY2Vqcb1p07d9C3b18kJyfD2dkZrVq1wvHjx+Hs7Gzo4RKVW5fupSE2MQP2lmbapCKfTCaDjbnmz/tG0iM0MUaAREQvmCIlF0wqqBIr1qpxI0aMwIgRI3D//n1YWFjA2tq6WC9uY2OD8PDwQu90eHl5aVfhfpqbN2/qPP/pp5+KFQ9RRZKSmYNctYCZouAejWb/djNMzc57nmEREb3Q9JKLxzGpoEquRKNcnJ2di51UEFHJOFiawVQhQ4664Pmlc/4dq2RnzlXniYiepwLHXDCpoBdAsT5xbN26FZs3b8bt27eRk5Ojs+3s2bOlEhgRPV2ghy18XaxxJS4dbrZyne5QQgik/3unwtvJylghEhG9sPKTi3/GjtMUKBSc/YkqPYPvWHz55ZcYOHAgXF1d8ffff6NJkyZwdHTE9evXddaPIKKyJZfLMCLEF9ZKBeLTVMjKVUOSBLJy1YhPU8HSVK6tR0REz59N27aoGr4ECvsqmnUqmFRQJWdwYvH1119j1apVWLZsGczMzPDxxx9j7969GDNmDFJTU8siRiIqRAs/J8zpFYQAdxtkqvKQmKFCpioPAe42GNm2prHDIyJ64dm0aQOPOXNg8++EM0SVmcFdoW7fvo0WLVoAACwsLLTLfPfr1w/NmjXD8uXLSzdCInqqFn5OaObjiEv30pCSmQMHSzMEetji7Nkzxg6NiIiIXiAG37Fwc3NDSkoKAKB69eo4fvw4AODGjRtFmsGJiEqfXC5DUDU7hNRyRlA1O3Z/IiIioufO4MSiXbt22rUlBg4ciA8++AAdO3bEW2+9hV69epV6gEREREREVP4Z3BVq1apVkP6dxvL999+Ho6Mjjh49iu7du2P48OGlHiAREREREZV/BiUWeXl5mDNnDgYNGoRq1aoBAPr06YM+ffqUSXBERERERFQxGNQVysTEBAsWLEBeHlfyJSIiIiKi/xg8xqJ9+/aIiIgoi1iIiIiIiKiCMniMRZcuXTBx4kRcuHABwcHBsLLSXdW3e/fupRYcERERERFVDAYnFiNHjgQALF68WG+bTCaDWq0ueVRERERERFShGJxY5M8IRURERERElM/gMRZERERERERPMviOxYwZM566fcqUKcUOhoiIiIiIKiaDE4tt27bpPM/NzcWNGzdgYmICX19fJhZERERERC8ggxOLv//+W68sLS0NYWFh6NWrV6kERUREREREFUupjLGwtbXF9OnTMXny5NJojoiIiIiIKphSG7ydmpqK1NTU0mqOiIiIiIgqEIO7Qn355Zc6z4UQiIuLw4YNG9ClS5dSC4yIiIiIiCoOgxOLJUuW6DyXy+VwdnbGgAEDMGnSpFILjIiIiIiIKg6DE4sbN26URRxERERERFSBGTzGIjU1FSkpKXrlKSkpSEtLK5WgiIiIiIioYjE4sejTpw9++uknvfLNmzejT58+pRIUERERERFVLAYnFidOnEDbtm31ytu0aYMTJ06USlBERERERFSxGJxYqFQq5OXl6ZXn5uYiKyurVIIiIiIiIqKKxeDEokmTJli1apVe+cqVKxEcHFwqQRERERERUcVi8KxQs2bNQocOHRAZGYn27dsDAPbv349Tp05hz549pR4gERERERGVfwbfsWjZsiWOHTsGT09PbN68Gb///jv8/Pxw/vx5tG7duixiJCIiIiKics7gOxYAUL9+fWzatKm0YyEiIiIiogrK4DsW//d//4fdu3frle/evRu7du0qlaCIiIiIiKhiMTixmDhxItRqtV65EAITJ04slaCIiIiIiKhiMTixiI6ORp06dfTK/f39ERMTUypBERERERFRxWJwYmFnZ4fr16/rlcfExMDKyqpUgiIiIiIioorF4MSiR48eGDduHGJjY7VlMTEx+PDDD9G9e/dSDY6IiIiIiCoGgxOLBQsWwMrKCv7+/vD29oa3tzcCAgLg6OiIhQsXlkWMRERERERUzhk83aydnR2OHj2KvXv3IjIyEhYWFqhbty5efvnlsoiPiIiIiIgqgGKtYyGTydCpUyd06tQJgGZGqF27dmHNmjXYunVrqQZIRERERETln8FdoR5348YNTJ48GdWrV0evXr2QnZ1dWnEREREREVEFYvAdC5VKha1bt2LNmjU4fPgw1Go1vvjiCwwePBi2trZlESMREREREZVzRb5jcebMGYwcORJubm4IDw9Hz5498c8//0AulyM0NJRJBRERERHRC6zIdyyaNm2K0aNH4/jx46hdu3ZZxkRERERERBVMke9YtG/fHmvWrMGMGTPwxx9/QAhR4hdXq9WYPHkyvL29YWFhAV9fX8ycOVOnbSEEpkyZAnd3d1hYWKBDhw6Ijo5+ZttfffUVvLy8YG5ujqZNm+LkyZMljpeIiIiIiApW5MRi9+7duHTpEmrXro0RI0bA3d0dY8eOBaCZJao45s+fjxUrVmD58uW4cuUK5s+fjwULFmDZsmXaOgsWLMCXX36JlStX4sSJE7CyskJoaOhTB4r//PPPGD9+PKZOnYqzZ8+iXr16CA0NRWJiYrHiJCIiIiKipzNoVihPT09MmTIFN27cwIYNG3D//n2YmJigR48e+PTTT3H27FmDXvzo0aPo0aMHunbtCi8vL7z++uvo1KmT9u6CEALh4eH4/PPP0aNHD9StWxfff/897t27h+3btxfa7uLFizF06FAMHDgQderUwcqVK2FpaYnvvvvOoPiIiIiIiKhoij3dbMeOHfHDDz/g3r17GD16NHbt2oXGjRsb1EaLFi2wf/9+REVFAQAiIyNx+PBhdOnSBYBmOtv4+Hh06NBBu4+dnR2aNm2KY8eOFdhmTk4Ozpw5o7OPXC5Hhw4dCt2HiIiIiIhKplgL5D3O3t4eo0ePxujRow2+YzFx4kSkpaXB398fCoUCarUas2fPxjvvvAMAiI+PBwC4urrq7Ofq6qrd9qSkpCSo1eoC97l69WqB+6hUKqhUKu3ztLQ0AIAkSZAkyaBjIiov8q9dXsdEVBi+T5Q9SZIghOD5LSO8hsueIee1xInF4xo2bGhQ/c2bN2PTpk344YcfEBgYiHPnzmHcuHHw8PDAgAEDSjO0p5o7dy6mT5+uV37//n0u+kcV1oMHD7SPHF9ERAW5e/cuACAiIgIpKSlGjqZyysrKQlRUFGrVqgULCwtjh1Pp5E/ok5KSwv/rykh6enqR65ZqYmGoCRMmYOLEiejTpw8AICgoCLdu3cLcuXMxYMAAuLm5AQASEhLg7u6u3S8hIQH169cvsE0nJycoFAokJCTolCckJGjbe9KkSZMwfvx47fO0tDR4enrC2dmZ63NQhWVvb699dHFxMXI0RFQe5f9f+dFHHxk5EqKSqVGjBv+vKyPm5uZFrmvUxCIzMxNyue4wD4VCob3l4u3tDTc3N+zfv1+bSKSlpeHEiRMYMWJEgW2amZkhODgY+/fvR8+ePQFobuHs378fo0aNKnAfpVIJpVKpVy6Xy/XiI6oo8q9dXsdEVJjXXnsNcrkc/v7+sLS0NHY4ldLly5fRr18/bNiwAXXq1DF2OJWSjY0NatasaewwKi1DPkMYNbHo1q0bZs+ejerVqyMwMBB///03Fi9ejEGDBgHQTGM7btw4zJo1CzVr1oS3tzcmT54MDw8PbdIAaNbY6NWrlzZxGD9+PAYMGIBGjRqhSZMmCA8Px6NHjzBw4EBjHCYREVG55OTkhCFDhhg7jEot/8tSf39/g7uME1U0xUos8vLycPDgQcTGxuLtt9+GjY0N7t27B1tbW1hbWxe5nWXLlmHy5MkYOXIkEhMT4eHhgeHDh2PKlCnaOh9//DEePXqEYcOG4eHDh2jVqhX++OMPndsysbGxSEpK0j5/6623cP/+fUyZMgXx8fGoX78+/vjjD70B3UREREREVDpkwsAltG/duoXOnTvj9u3bUKlUiIqKgo+PD8aOHQuVSoWVK1eWVazPTVpaGuzs7JCamsoxFlRhnT59Go0bN8apU6fQqFEjY4dDRPRC4nsxVXSGfC42uOP12LFj0ahRIzx48EBndoNevXph//79hkdLREREREQVnsFdof766y8cPXoUZmZmOuVeXl7aaeuIiIiIiOjFYnBiIUkS1Gq1XvmdO3dgY2NTKkER0bNJksCle2lIycyBg6UZAj1sIZfLjB0WERERvaAMTiw6deqE8PBwrFq1CoBm5qaMjAxMnToVr7zySqkHSET6jsYkYUVELGITM5CrFjBVyODrYo0RIb5o4edk7PCIiIjoBWTwGItFixbhyJEjqFOnDrKzs/H2229ru0HNnz+/LGIkosccjUnCp9su4EpcGqyUJnCxUcJKaYIrcen4dNsFHI1JenYjRERERKXM4DsW1apVQ2RkJH7++WdERkYiIyMDgwcPxjvvvMOl6onKmCQJrIiIRYYqD2625pDJNF2fzOUKuNnKEZ+mwoqIWDTzcTRypERERPSiKdY6FiYmJnjnnXfwzjvvlHY8ROVSZmYmrl69auwwEJ2Qjsi/r8DCTIG0NP0bjvI8CZHxamzxyEZu8j8AgKtXr1aolbe5AjAREVHFZHBiMXfuXLi6umpXx8733Xff4f79+/jkk09KLTii8uLq1asIDg42dhhF1uer//7dr18/4wVSDGfOnOHqtERERBWQwYnFN998gx9++EGvPDAwEH369GFiQZWSv78/zpw5Y+wwEJ2Qjtk7NXcslCb6dyGy8yRk56jxWdcAVLVRIDIyEvXq1atQdwD8/f2NHQIREREVg8GJRXx8PNzd3fXKnZ2dERcXVypBEZU3lpaW5eJb9PqSwI575rgSlw5bW6V2jAUACCGQmaZCPXcbvBHaBICAn58fXFxcKlRXKCIiIqqYDP604enpiSNHjuiVHzlyBB4eHqUSFBEVTC6XYUSIL6yVCsSnqZCVq4YkCWTlqhGfpoK1UoERIb5cz4KIiIieO4PvWAwdOhTjxo1Dbm4u2rVrBwDYv38/Pv74Y3z44YelHiAR6Wrh54Q5vYK061ikSgKmchkC3G24jgUREREZjcGJxYQJE5CcnIyRI0ciJycHAGBubo5PPvkEkyZNKvUAiUhfCz8nNPNx5MrbRESVmSQB8ZFAZjJg6Qi41QPYtZXKMYMTC5lMhvnz52Py5Mm4cuUKLCwsULNmTSiVyrKIj4gKIZfLEFTNzthhEBFRWbgeARxeAiRFA1IuIDcFnGoCrT4AfEKMHR1RgYq1jgUAWFtbo3HjxqUZCxERERFdjwB2jANUGYCFPWCiBPJUQMIlTfmr4UwuqFwyOLF49OgR5s2bh/379yMxMRGSJOlsv379eqkFR0RERPRCkSTNnQpVBmDjDuTP/mdqAZiYA+lxmu1erdktisodgxOLIUOGICIiAv369YO7u7vOdJdEREREVALxkZruTxb2/yUV+WQyTXlStKaeRwPjxEhUCIMTi127dmHnzp1o2bJlWcRDRERE9OLKTNaMqTApZOyqiRLIfqipR1TOGHwPzd7eHg4ODmURCxEREdGLzdJRM1A7T1Xw9jyVZrul4/ONi6gIDE4sZs6ciSlTpiAzM7Ms4iEiIiJ6cbnV08z+lPUAEEJ3mxCacqeamnpE5YzBXaEWLVqE2NhYuLq6wsvLC6ampjrbz549W2rBEREREb1Q5HLNlLI7xmkGaj8+K1TWA0Bpo9nOgdtUDhmcWPTs2bMMwiAiIiIiAJqpZF8N/28di+yHmu5ProFcx4LKNYMTi6lTp5ZFHERERESUzydEM6UsV96mCqRYC+Q9fPgQW7duRWxsLCZMmAAHBwecPXsWrq6uqFq1amnHSERERPTikcs5pSxVKAYnFufPn0eHDh1gZ2eHmzdvYujQoXBwcMCvv/6K27dv4/vvvy+LOImIiIiIqBwz+H7a+PHjERYWhujoaJibm2vLX3nlFRw6dKhUgyMiIiIioorB4MTi1KlTGD58uF551apVER8fXypBERERERFRxWJwYqFUKpGWlqZXHhUVBWdn51IJioiIiIiIKhaDE4vu3btjxowZyM3NBQDIZDLcvn0bn3zyCXr37l3qARIRERERUflncGKxaNEiZGRkwMXFBVlZWQgJCYGfnx9sbGwwe/bssoiRiIiIiIjKOYNnhbKzs8PevXtx5MgRREZGIiMjAw0bNkSHDh3KIj4iIiIiIqoADEoscnNzYWFhgXPnzqFly5Zo2bJlWcVFREREREQViEFdoUxNTVG9enWo1eqyioeIiIiIiCogg8dYfPbZZ/j000+RkpJSFvEQEREREVEFZPAYi+XLlyMmJgYeHh6oUaMGrKysdLafPXu21IIjIiIiIqKKweDEomfPnmUQBhERERERVWQGJxZTp04tiziIiIiIiKgCM3iMBQA8fPgQ3377LSZNmqQda3H27FncvXu3VIMjIiIiIqKKweA7FufPn0eHDh1gZ2eHmzdvYujQoXBwcMCvv/6K27dv4/vvvy+LOImIiIiIqBwz+I7F+PHjERYWhujoaJibm2vLX3nlFRw6dKhUgyMiIiIioorB4MTi1KlTGD58uF551apVER8fXypBERERERFRxWJwYqFUKpGWlqZXHhUVBWdn51IJioiIiIiIKhaDE4vu3btjxowZyM3NBQDIZDLcvn0bn3zyCXr37l3qARIRERERUflncGKxaNEiZGRkwMXFBVlZWQgJCYGfnx9sbGwwe/bssoiRiIiIiIjKOYMTCzs7O+zduxc7duzAl19+iVGjRuH//u//EBERobcK97N4eXlBJpPp/bz//vu4efNmgdtkMhm2bNlSaJthYWF69Tt37mzoYRIRERERkQGKNN2sg4MDoqKi4OTkhEGDBmHp0qVo2bIlWrZsWaIXP3XqFNRqtfb5xYsX0bFjR7zxxhvw9PREXFycTv1Vq1Zh4cKF6NKly1Pb7dy5M9auXat9rlQqSxQnERERERE9XZESi5ycHKSlpcHJyQnr16/H/PnzYWNjU+IXf3Kw97x58+Dr64uQkBDIZDK4ubnpbN+2bRvefPNNWFtbP7VdpVKpty8REREREZWdIiUWzZs3R8+ePREcHAwhBMaMGQMLC4sC63733XfFCiQnJwcbN27E+PHjIZPJ9LafOXMG586dw1dfffXMtg4ePAgXFxfY29ujXbt2mDVrFhwdHYsVFxERERERPVuREouNGzdiyZIliI2NhUwmQ2pqKrKzs0s1kO3bt+Phw4cICwsrcPuaNWsQEBCAFi1aPLWdzp0747XXXoO3tzdiY2Px6aefokuXLjh27BgUCkWB+6hUKqhUKu3z/Ol0JUmCJEnFOyAiI5MkCUIIXsNEREaU/x7MzxRUURly3cqEEMKQxr29vXH69OlSvwMQGhoKMzMz/P7773rbsrKy4O7ujsmTJ+PDDz80qN3r16/D19cX+/btQ/v27QusM23aNEyfPl2vPCoqqlS6fBEZgyRJSE1NhZ2dHeRyg+dpICKiUhAZGYnOnTvjjz/+QL169YwdDpHB0tPTUatWLaSmpsLW1vapdYt0x+JxN27cKHZghbl16xb27duHX3/9tcDtW7duRWZmJvr3729w2z4+PnByckJMTEyhicWkSZMwfvx47fO0tDR4enrC2dn5mSeQqLySJAkymQzOzs5MLIiIjMTe3l776OLiYuRoiAxnbm5e5LoGJxYAsH//fuzfvx+JiYl6t0eKM8Zi7dq1cHFxQdeuXQvcvmbNGnTv3r1YK3vfuXMHycnJcHd3L7SOUqkscOYouVzOD2RUoclkMl7HRERGlP/+y/diqqgMuW4NvsKnT5+OTp06Yf/+/UhKSsKDBw90fgwlSRLWrl2LAQMGwMREP8+JiYnBoUOHMGTIkAL39/f3x7Zt2wAAGRkZmDBhAo4fP46bN29i//796NGjB/z8/BAaGmpwbEREREREVDQG37FYuXIl1q1bh379+pVKAPv27cPt27cxaNCgArd/9913qFatGjp16lTg9mvXriE1NRUAoFAocP78eaxfvx4PHz6Eh4cHOnXqhJkzZ3ItCyIiIiKiMmRwYpGTk/PMmZkM0alTJzxt/PicOXMwZ86cQrc/vq+FhQV2795darEREREREVHRGNwVasiQIfjhhx/KIhYiIiIiIqqgDL5jkZ2djVWrVmHfvn2oW7cuTE1NdbYvXry41IIjIiIiIqKKweDE4vz586hfvz4A4OLFizrbCloxm4iIiIiIKj+DE4sDBw6URRxERERERFSBcUJlIiIiIiIqsSLfsXjttdeKVK+w1bOJiIiIiKjyKnJiYWdnV5ZxEBERERFRBVbkxGLt2rVlGQcREREREVVgHGNBREREREQlZvCsUERERERUxiQJiI8EMpMBS0fArR4g5/fBVL4xsSAiIiIqT65HAIeXAEnRgJQLyE0Bp5pAqw8AnxBjR0dUKKa+REREROXF9Qhgxzgg4RJgZgVYu2oeEy5pyq9HGDtCokIxsSAiIiIqDyRJc6dClQHYuAOmFoBMrnm0cdeUH16iqUdUDjGxICIiIioP4iM13Z8s7AGZTHebTKYpT4rW1CMqh5hYEBEREZUHmcmaMRUmyoK3myg12zOTn29cREXExIKIiIioPLB01AzUzlMVvD1Ppdlu6fh84yIqIiYWREREROWBWz3N7E9ZDwAhdLcJoSl3qqmpR1QOMbEgIiIiKg/kcs2UskprID0OyM0ChKR5TI8DlDaa7VzPgsopXplERERE5YVPCPBqOOAaCOQ8AjISNI+ugcCrS7iOBZVrXCCPiIiIqDzxCQG8WnPlbapwmFgQERERlTdyOeDRwNhREBmEqS8REREREZUYEwsiIiIiIioxJhZERERERFRiTCyIiIiIiKjEmFgQEREREVGJMbEgIiIiIqISY2JBREREREQlxsSCiIiIiIhKjIkFERERERGVGBMLIiIiIiIqMSYWRERERERUYkwsiIiIiIioxJhYEBERERFRiTGxICIiIiKiEmNiQUREREREJcbEgoiIiIiISoyJBRERERERlRgTCyIiIiIiKjEmFkREREREVGJMLIiIiIiIqMSYWBARERERUYmZGDuAikytViM3N9fYYRAVSJIk5ObmIjs7G3J5+fgOwczMrNzEQkRERKWLiUUxCCEQHx+Phw8fGjsUokIJISBJEtLT0yGTyYwdDgBALpfD29sbZmZmxg6FiIiISplREwsvLy/cunVLr3zkyJH46quv0KZNG0REROhsGz58OFauXFlom0IITJ06FatXr8bDhw/RsmVLrFixAjVr1iy1uPOTChcXF1haWpabD21EjxNCIC8vDyYmJuXiGpUkCffu3UNcXByqV69eLmIiIiKi0mPUxOLUqVNQq9Xa5xcvXkTHjh3xxhtvaMuGDh2KGTNmaJ9bWlo+tc0FCxbgyy+/xPr16+Ht7Y3JkycjNDQUly9fhrm5eYljVqvV2qTC0dGxxO0RlZXyllgAgLOzM+7du4e8vDyYmpoaOxwiIiIqRUZNLJydnXWez5s3D76+vggJCdGWWVpaws3NrUjtCSEQHh6Ozz//HD169AAAfP/993B1dcX27dvRp0+fEsecP6biWQkOEenL7wKlVquZWBAREVUy5WYUZU5ODjZu3IhBgwbpfLu6adMmODk54aWXXsKkSZOQmZlZaBs3btxAfHw8OnTooC2zs7ND06ZNcezYsVKNt7x8A0xUkfDvhoiIqPIqN4O3t2/fjocPHyIsLExb9vbbb6NGjRrw8PDA+fPn8cknn+DatWv49ddfC2wjPj4eAODq6qpT7urqqt1WEJVKBZVKpX2elpYGQNMnXJIknbqSJEEIof0hKs/yr9Hycq3m/90U9LdFRFQZ5b/X8X2PKipDrttyk1isWbMGXbp0gYeHh7Zs2LBh2n8HBQXB3d0d7du3R2xsLHx9fUvttefOnYvp06frld+/fx/Z2dk6Zbm5uZAkCXl5ecjLyyu1GKhy+f777/Hhhx/i/v37RotBCKEdw1Re7hTk5eVBkiQkJyezKxQRvRAePHigfUxMTDRyNESGS09PL3LdcpFY3Lp1C/v27Sv0TkS+pk2bAgBiYmIKTCzyx2IkJCTA3d1dW56QkID69esX2u6kSZMwfvx47fO0tDR4enrC2dkZtra2OnWzs7ORnp4OExMTmJiUi9NXJAMHDsTDhw+xbdu2UmnP29sbY8eOxbhx40qlvcfJ5XL8+uuv6Nmz51PrRUREYMaMGTh37hyys7NRtWpVtGjRAqtWrTL6dKb5azWUh2ukPH2ANzExgVwuh6OjY6lMpkBEVN7Z29trH11cXIwcDZHhDPn/2vifegCsXbsWLi4u6Nq161PrnTt3DgB0kobHeXt7w83NDfv379cmEmlpaThx4gRGjBhRaLtKpRJKpVKvXC6X6y3mJZfLIZPJtD/FJUkCl+6lISUzBw6WZgj0sIVcXvbfKpfmN9clPQclafvy5cvo0qULRo8ejS+//BIWFhaIjo7GL7/8AkmSjP4Nff7rGzMOIUS5iONx+b/Xgv62iIgqo/z3Or7vUUVlyHVr9CtckiSsXbsWAwYM0Pl2NzY2FjNnzsSZM2dw8+ZN/O9//0P//v3x8ssvo27dutp6/v7+2m/hZTIZxo0bh1mzZuF///sfLly4gP79+8PDw+OZ334/T0djkjBg7UkM33AaH22OxPANpzFg7UkcjUl6bjG0adMGY8aMwccffwwHBwe4ublh2rRp2u1CCEybNg3Vq1eHUqmEh4cHxowZo9331q1b+OCDD3QSgOTkZPTt2xdVq1aFpaUlgoKC8OOPPxr0ul5eXgCAXr16QSaTaZ8/ac+ePXBzc8OCBQvw0ksvwdfXF507d8bq1athYWFhUDyjR4/GuHHjYG9vD1dXV6xevRqPHj3CwIEDYWNjAz8/P+zatUu7z8GDByGTybBz507UrVsX5ubmaNasGS5evPjUc/7bb7+hYcOGMDc3h4+PD6ZPn67tTve0801ERERUERg9sdi3bx9u376NQYMG6ZSbmZlh37596NSpE/z9/fHhhx+id+/e+P3333XqXbt2DampqdrnH3/8MUaPHo1hw4ahcePGyMjIwB9//FFuul0cjUnCp9su4EpcGqyUJnCxUcJKaYIrcen4dNuF55pcrF+/HlZWVjhx4gQWLFiAGTNmYO/evQCAX375BUuWLME333yD6OhobN++HUFBQQCAX3/9FdWqVcOMGTMQFxeHuLg4AJpuYsHBwdi5cycuXryIYcOGoV+/fjh58mSRX/fUqVMANHex4uLitM+f5Obmhri4OBw6dKjQ4zMkHicnJ5w8eRKjR4/GiBEj8MYbb6BFixY4e/YsOnXqhH79+unNSDZhwgQsWrQIp06dgrOzM7p166adjvhJf/31F/r374+xY8fi8uXL+Oabb7Bu3TrMnj37meebiIiIqEIQpCc1NVUAEKmpqXrbsrKyxOXLl0VWVpbB7arVknj32+MieOYe0XXpIfHql39pf7ouPSSCZ+4V7357XKjVUmkcho4BAwaIHj16aJ+HhISIVq1a6dRp3Lix+OSTT4QQQixatEjUqlVL5OTkFNhejRo1xJIlS575ul27dhUffvhhkV9XCCEAiG3btj213by8PBEWFiYACDc3N9GzZ0+xbNmyAn9nhsSTl5cnrKysRL9+/bRlcXFxAoA4duyYEEKIAwcOCADip59+0tZJTk4WFhYW4ueffxZCCLF27VphZ2en3d6+fXsxZ84cnVg2bNgg3N3dhRDPPt/FIUmSyMnJEZJU+tdTcZXk74eIqCI6deqUACBOnTpl7FCIiuVpn4ufZPQ7Fi+SS/fSEJuYAXtLM70+7zKZDFUsTRGbmIFL99KeSzyPdykDNGNX8meseOONN5CVlQUfHx8MHToU27Zte+YsWGq1GjNnzkRQUBAcHBxgbW2N3bt34/bt20V+3aJSKBRYu3Yt7ty5gwULFqBq1aqYM2cOAgMDtXdQihOPQqGAo6Ojzt2C/OmLn4yxefPm2n87ODigdu3auHLlSoHxRkZGYsaMGbC2ttb+DB06FHFxccjMzCzW+SYiIiIqT5hYPEcpmTnIVQuYKQo+7UqFHLmSQEpmznOJ58nZgmQymXauYk9PT1y7dg1ff/01LCwsMHLkSLz88suFdvUBgIULF2Lp0qX45JNPcODAAZw7dw6hoaHIydE9nqe9rqGqVq2Kfv36Yfny5bh06RKys7OxcuXKEsfzeFl+EliS+cczMjIwffp0nDt3Tvtz4cIFREdHw9zcvFjnm4iIiKg8KRezQr0oHCzNYKqQIUctwVyu0NuuUkswlcvgYGncqVLzWVhYoFu3bujWrRvef/99+Pv748KFC2jYsCHMzMy0ayTkO3LkCHr06IF3330XgOaDeFRUFOrUqWPQ65qamuq1XRT29vZwd3fHo0ePSjWewhw/fhzVq1cHoJmfPCoqCgEBAQXWbdiwIa5duwY/P79C23va+SYiIiIq75hYPEeBHrbwdbHGlbh0uNnKdbpDCSHwMDMXAe42CPSwfUorz8e6deugVqvRtGlTWFpaYuPGjbCwsECNGjUAaGZvOnToEPr06QOlUgknJyfUrFkTW7duxdGjR2Fvb4/FixcjISHB4A/yXl5e2L9/P1q2bAmlUqmdA/xx33zzDc6dO4devXrB19cX2dnZ+P7773Hp0iUsW7YMAEotnsLMmDEDjo6OcHV1xWeffQYnJ6dCZx+bMmUKXn31VVSvXh2vv/465HI5IiMjcfHiRcyaNeuZ55uIiIiovGNXqOdILpdhRIgvrJUKxKepkJWrhiQJZOWqEZ+mgrVSgREhvs9lPYtnqVKlClavXo2WLVuibt262LdvH37//Xc4OjoC0HyovnnzJnx9feHs7AwA+Pzzz9GwYUOEhoaiTZs2cHNzK9Y0v4sWLcLevXvh6emJBg0aFFinSZMmyMjIwHvvvYfAwECEhITg+PHj2L59O0JCQko1nsLMmzcPY8eORXBwMOLj4/H7778XujBfaGgoduzYgT179qBx48Zo1qwZlixZok0cnnW+iYiIiMo7mRBCGDuI8iYtLQ12dnZITU0tcOXtGzduwNvbu9hT2B6NScKKiFjEJmYgVxIwlcvg62KNESG+aOHnVBqHQGXo4MGDaNu2LR48eIAqVaoYO5xCCSGQl5cHExOTcrNAXmn8/RARVSSnT59G48aNcerUKTRq1MjY4RAZ7Gmfi5/ErlBG0MLPCc18HI2y8jYRERERUVlgYmEkcrkMQdXsjB0GEREREVGpYGJBZKA2bdqAPQiJiIiIdHHwNhERERERlRgTCyIiIiIiKjEmFkREREREVGJMLIiIiIiIqMSYWBARERERUYkxsSAiIiIiohJjYkEAAJlMhu3btxs7jApp3bp15XoFbiIiIqLngYnFCyIsLAw9e/YsdHtcXBy6dOny/AIyUEREBNq1awcHBwdYWlqiZs2aGDBgAHJycowdGhERERGBiYXxSBJw728gZp/mUZKMGo6bmxuUSqVRYxBCIC8vT6/88uXL6Ny5Mxo1aoRDhw7hwoULWLZsGczMzKBWq40QKRERERE9iYmFMVyPADa+Bvz0LrB9pOZx42uaciN5vCvUzZs3IZPJ8Ouvv6Jt27awtLREvXr1cOzYMZ19Dh8+jNatW8PCwgKenp4YM2YMHj16pN2+YcMGNGrUCDY2NnBzc8Pbb7+NxMRE7faDBw9CJpNh165dCA4OhlKpxOHDh/Vi27NnD9zc3LBgwQK89NJL8PX1RefOnbF69WpYWFgAAJKTk9G3b19UrVoVlpaWCAoKwo8//qjTTps2bTB69GiMGzcO9vb2cHV1xerVq/Ho0SMMHDgQNjY28PPzw65du/Ri3LlzJ+rWrQtzc3M0a9YMFy9efOr5/O2339CwYUOYm5vDx8cH06dP1yZNQghMmzYN1atXh1KphIeHB8aMGVOE3xIRERFR+cXE4nm7HgHsGAckXALMrABrV81jwiVNuRGTiyd99tln+Oijj3Du3DnUqlULffv21X44jo2NRefOndG7d2+cP38eP//8Mw4fPoxRo0Zp98/NzcXMmTMRGRmJ7du34+bNmwgLC9N7nYkTJ2LevHm4cuUK6tatq7fdzc0NcXFxOHToUKGxZmdnIzg4GDt37sTFixcxbNgw9OvXDydPntSpt379ejg5OeHkyZMYPXo0RowYgTfeeAMtWrTA2bNn0alTJ/Tr1w+ZmZk6+02YMAGLFi3CqVOn4OzsjG7duiE3N7fAWP766y/0798fY8eOxeXLl/HNN99g3bp1mD17NgDgl19+wZIlS/DNN98gOjoa27dvR1BQUKHHRkRERFQhCNKTmpoqAIjU1FS9bVlZWeLy5csiKyvL8IbVaiHW9xBigZ8QK1oLsfLl/35WtNaUr++hqVfKBgwYIHr06FHodgBi27ZtQgghbty4IQCIb7/9Vrv90qVLAoC4cuWKEEKIwYMHi2HDhum08ddffwm5XF7ouTl16pQAINLT04UQQhw4cEAAENu3b39q7Hl5eSIsLEwAEG5ubqJnz55i2bJlBf5+Hte1a1fx4Ycfap+HhISIVq1a6bRrZWUl+vXrpy2Li4sTAMSxY8d0Yvzpp5+0dZKTk4WFhYX4+eefhRBCrF27VtjZ2Wm3t2/fXsyZM0cnlg0bNgh3d3chhBCLFi0StWrVEjk5OU+Nv6QkSRI5OTlCkqQyfR1DlOjvh4ioAsr/v+/UqVPGDoWoWJ72ufhJvGPxPMVHAknRgIU9IJPpbpPJNOVJ0Zp65cDjdw/c3d0BQNuVKTIyEuvWrYO1tbX2JzQ0FJIk4caNGwCAM2fOoFu3bqhevTpsbGwQEhICALh9+7bO6zRq1OipcSgUCqxduxZ37tzBggULULVqVcyZMweBgYGIi4sDAKjVasycORNBQUFwcHCAtbU1du/erfdajx+TQqGAo6Ojzt0CV1dXnePM17x5c+2/HRwcULt2bVy5cqXAeCMjIzFjxgydczN06FDExcUhMzMTb7zxBrKysuDj44OhQ4di27ZtBY4tISIiIqpImFg8T5nJgJQLmBQySNpEqdmemfx84yqEqamp9t+yfxMh6d9B5hkZGRg+fDjOnTun/YmMjER0dDR8fX3x6NEjhIaGwtbWFps2bcKpU6ewbds2ANCbycnKyqpI8VStWhX9+vXD8uXLcenSJWRnZ2PlypUAgIULF2Lp0qX45JNPcODAAZw7dw6hoaF6r/X4MeUf19OOszgyMjIwffp0nXNz4cIFREdHw9zcHJ6enrh27Rq+/vprWFhYYOTIkXj55ZcL7VpFREREVBGYGDuAF4qlIyA3BfJUgKmF/vY8lWa7pePzj81ADRs2xOXLl+Hn51fg9gsXLiA5ORnz5s2Dp6cnAOD06dOl9vr29vZwd3fXDhY/cuQIevTogXfffReAJjGIiopCnTp1SuX1jh8/jurVqwMAHjx4gKioKAQEBBRYt2HDhrh27Vqh5wYALCws0K1bN3Tr1g3vv/8+/P39ceHCBTRs2LBU4iUiIiJ63phYPE9u9QCnmpqB2ibmut2hhACyHgCugZp6ZSA1NRXnzp3TKXN0dNR+8DfEJ598gmbNmmHUqFEYMmQIrKyscPnyZezduxfLly9H9erVYWZmhmXLluG9997DxYsXMXPmzGLF/c033+DcuXPo1asXfH19kZ2dje+//x6XLl3CsmXLAAA1a9bE1q1bcfToUdjb22Px4sVISEgotcRixowZcHR0hKurKz777DM4OTkVui7IlClT8Oqrr6J69ep4/fXXIZfLERkZiYsXL2LWrFlYt24d1Go1mjZtCktLS2zcuBEWFhaoUaNGqcRKREREZAzsCvU8yeVAqw8ApTWQHgfkZgFC0jymxwFKG812edn8Wg4ePIgGDRro/EyfPr1YbdWtWxcRERGIiopC69at0aBBA0yZMgUeHh4AAGdnZ6xbtw5btmxBnTp1MG/ePHzxxRfFeq0mTZogIyMD7733HgIDAxESEoLjx49j+/bt2nEbn3/+ORo2bIjQ0FC0adMGbm5uT10Q0FDz5s3D2LFjERwcjPj4ePz+++8wMzMrsG5oaCh27NiBPXv2oHHjxmjWrBmWLFmiTRyqVKmC1atXo2XLlqhbty727duH33//HY6O5f9OFREREVFhZEIIYewgypu0tDTY2dkhNTUVtra2Otuys7Nx48YNeHt7w9zcvHgvcD0COLxEM1BbytV0f3KqqUkqfEJK4QiotBw8eBBt27bFgwcPUKVKFWOHYxDx74KDJiYm2rEjxlYqfz9ERBXI6dOn0bhxY5w6deqZk5UQlUdP+1z8JHaFMgafEMCrtWb2p8xkzZgKt3pldqeCiIiIiKisMbEwFrkc8Ghg7CiIiIiIiEoFEwuip2jTpg3YW5CIiIjo2dj3hoiIiIiISoyJBRERERERlRgTCyIiIiIiKjEmFkREREREVGJMLIwo4p8IdNjSARH/RBg7FCIiIiKiEmFiYSQR/0Rg7IGxSMhMwNgDY5lcEBEREVGFxsTCCPKTCklIAABJSEwuKrlr167Bzc0N6enpxg6lzPTp0weLFi0ydhhERERkJEwsnrPHkwoBzfoIAqLMk4uwsDD07Nmz1Nrz8vJCeHh4qbX3OJlMhu3btxepnkwmw/Hjx3XKVSoVHB0dIZPJcPDgwTKJ0VCTJk3C6NGjYWNjoy3bvXs3mjVrBhsbGzg7O6N37964efOmdvvBgwe1x/j4T3x8fKGvc/PmTW09uVwOMzMzyOVynXPUpk2bAtvt2rWrts4XX3wBFxcXuLi46CULJ06cQHBwMPLy8nTKP//8c8yePRupqanFPU1ERERUgTGxeI4KSiryPY/kojLy9PTE2rVrdcq2bdsGa2trI0Wk7/bt29ixYwfCwsK0ZTdu3ECPHj3Qrl07nDt3Drt370ZSUhJee+01vf2vXbuGuLg47Y+Li8szX3Pfvn24d+8ebt++jXv37iE4OFi77ddff9Vp7+LFi1AoFHjjjTcAAOfPn8eUKVPw008/4ccff8Tnn3+OCxcuAADy8vLw3nvvYeXKlTAx0V1f86WXXoKvry82btxYnNNEREREFRwTi+fkaUlFvueZXLRp0wZjxozBxx9/DAcHB7i5uWHatGn/xSIEpk2bhurVq0OpVMLDwwNjxozR7nvr1i188MEH2m+7ASA5ORl9+/ZF1apVYWlpiaCgIPz4448Gva6XlxcAoFevXpDJZNrnhRkwYAB++uknZGVlacu+++47DBgwQK/uP//8gzfffBNVqlSBg4MDevTooXOH4NSpU+jYsSOcnJxgZ2eHkJAQnD17VqcNmUyGb7/9Fr169YKlpSVq1qyJ//3vf0+NcfPmzahXrx6qVq2qLTtz5gzUajVmzZoFX19fNGzYEB999BHOnTuH3Nxcnf1dXFzg5uam/ZHLn/1n6+joqLOPqampdlv+ec//2bt3LywtLbWJxdWrV1G3bl20a9cO7du3R926dXH16lUAwMKFC/Hyyy+jcePGBb5ut27d8NNPPz0zPiIiIqp8mFg8B0VJKvI9z+Ri/fr1sLKywokTJ7BgwQLMmDEDe/fuBQD88ssvWLJkCb755htER0dj+/btCAoKAqD5xrtatWqYMWOG9ltvAMjOzkZwcDB27tyJixcvYtiwYejXrx9OnjxZ5Nc9deoUAGDt2rWIi4vTPi9McHAwvLy88MsvvwDQ3B04dOgQ+vXrp1MvNzcXoaGhsLGxwV9//YUjR47A2toanTt3Rk5ODgAgPT0dAwYMwOHDh3H8+HHUrFkTr7zyit64iOnTp+PNN9/E+fPn8corr+Cdd95BSkpKoTH+9ddfaNSokV7ccrkca9euhVqtRmpqKjZs2IAOHTroJAEAUL9+fbi7u6Njx444cuTIU89Hvu7du8PV1RVt2rR5ZuKzZs0a9OnTB1ZWVgCAoKAgREVF4fbt27h16xaioqLw0ksvITY2FmvXrsWsWbMKbatJkyY4efIkVCpVkeIkIiKiyoOJRRkzJKnI97ySi7p162Lq1KmoWbMm+vfvj0aNGmH//v0ANB/Q3dzc0KFDB1SvXh1NmjTB0KFDAWi+8VYoFLCxsdF+6w0AVatWxUcffYT69evDx8cHo0ePRufOnbF58+Yiv66zszMAoEqVKnBzc9M+f5pBgwbhu+++AwCsW7cOr7zyit5+P//8MyRJwrfffougoCAEBARg7dq1uH37tnYcRrt27fDuu+/C398fAQEBWLVqFTIzMxERofs7CAsLQ9++feHn54c5c+YgIyNDL3l63K1bt+Dh4aFT5u3tjT179uDTTz+FUqlElSpVcOfOHZ1z5e7ujpUrV+KXX37BL7/8Ak9PT7Rp00bvLsrjrK2tsWjRImzZsgU7duxAixYt0KtXr0KTi5MnT+LixYsYMmSItiwgIABz5sxBx44d0alTJ8ydOxcBAQEYPnw4FixYgN27d+Oll15CgwYNcOjQIZ32PDw8kJOT89RxIERERFQ5mTy7ChVXcZKKfI8nF0vbLkWIZ0ipx1e3bl2d5+7u7khMTAQAvPHGGwgPD4ePjw86d+6MV155Bd26ddPrV/84tVqNOXPmYPPmzbh79y5ycnKgUqlgaWlZ5NctjnfffRcTJ07E9evXsW7dOnz55Zd6dSIjIxETE6MzeBrQ3GWJjY0FACQkJODzzz/HwYMHkZiYCLVajczMTNy+fbvQ+K2srGBra/vU+LOysmBubq5TFh8fj6FDh2LAgAHo27cv0tPTMWXKFLz++uvYu3cvZDIZateujdq1a2v3adGiBWJjY7FkyRJs2LChwNdycnLC+PHjAWi6szVo0AAJCQlYuHAhunfvrld/zZo1CAoKQpMmTXTK33vvPbz33nva5+vXr4eNjQ2aN2+O2rVr49SpU7hz5w769OmDGzduQKlUAgAsLCwAAJmZmYWeDyIiIqqcmFiUoZnHZ0It1MXeX0BALdSYeXxmmSQWT3a5kclkkCTNFLienp64du0a9u3bh71792LkyJFYuHAhIiIi9PbLt3DhQixduhTh4eEICgqClZUVxo0bp+1qVJTXLQ5HR0e8+uqrGDx4MLKzs9GlSxe97ksZGRkIDg7Gpk2b9PbPv7sxYMAAJCcnY+nSpahRowaUSiWaN29e4vidnJzw4MEDnbKvvvoKdnZ2WLBggbZs48aN8PT0xIkTJ9CsWbMC22rSpAkOHz5c6GsVtk9+V7PHPXr0CD/99BNmzJjx1P2TkpIwffp0HDp0CCdOnECtWrVQs2ZN1KxZE7m5uYiKitJ2k8vvElaUO01ERERUuRi1K5SXl1eB016+//77SElJwejRo1G7dm1YWFigevXqGDNmzDOnsgwLC9Nrr3Pnzs/piHRNbjYZCpkCMsiKtb8MMihkCkxuNrmUIysaCwsLdOvWDV9++SUOHjyIY8eOaWcHMjMzg1qtmzQdOXIEPXr0wLvvvot69erBx8cHUVFRBr+uqampXtvPMmjQIBw8eBD9+/eHQqHQ296wYUNER0fDxcUFfn5+Oj92dnba+MeMGYNXXnkFgYGBUCqVSEpKMjj+JzVo0ACXL1/WKcvMzNQbhJ0f99OSlHPnzsHd3d2g1y9sny1btkClUuHdd9996v4ffPABPvjgA1SrVg1qtVpncHleXp7O7+rixYuoVq0anJycDIqRiIiIKj6j3rE4deqU3oeSjh074o033sC9e/dw7949fPHFF6hTpw5u3bqF9957D/fu3cPWrVuf2m7nzp11piDN76bxvIV4hmBp26XF6g4lgwxymbzMukE9y7p166BWq9G0aVNYWlpi48aNsLCwQI0aNQBoksJDhw6hT58+UCqVcHJyQs2aNbF161YcPXoU9vb2WLx4MRISElCnTh2DXtvLywv79+9Hy5YtoVQqYW9v/8x9OnfujPv378PW1rbA7e+88w4WLlyIHj16YMaMGahWrRpu3bqFX3/9FR9//DGqVauGmjVrYsOGDWjUqBHS0tIwYcIEbdeekggNDcWQIUOgVqu1yUPXrl2xZMkSzJgxQ9sV6tNPP0WNGjXQoEEDAEB4eDi8vb0RGBiI7OxsfPvtt/jzzz+xZ88ebdvLly/Htm3btGNU1q9fDzMzMzRo0ABCCGzduhVr167Ft99+qxfXmjVr0LNnTzg6OhYa+969exEVFYX169cDABo3boyrV69i165d+Oeff6BQKHS6a/3111/o1KlTic8ZERERVTxGvWPh7OysM+3ljh074Ovri5D/b+/Oo5o68/+Bv4NAZBOUxUAJKosFKzCAVhkXFMWA1mLtjNWqB1yqjlVaW6utOpZFLUdlXNB2aq3gqC1qC6lOay10VMRRXFpQq0hVFv0JblUJIgrk/v5gcr9E9s0Avl/n5HDy3O3DPcmT+7nPcv380LdvX3z77bcYO3YsnJyc4O/vj5UrV2L//v3VHsz1NKlUqrXfhlyYthZNcqEn0Wtwy4WukwqgcvD0F198gUGDBsHDwwMpKSnYv3+/eBEaGRmJ3NxcODk5id1eli1bBm9vbygUCgwbNgwymaxJD+WLiYlBcnIy5HK5eJFdH4lEAisrKxgaGta43NjYGKmpqXBwcMD48ePh5uYmdp3SJCNffvkl7t27B29vb0ydOhVhYWENemZEfYKCgqCvr4+UlBSxzN/fH1999RWUSiW8vLwQGBgIqVSKH3/8UUxmnjx5gvfffx/u7u7w8/NDZmYmUlJSMGLECHE/d+7cEceIaERFRcHHxwcDBw7E/v37kZCQgGnTpmmtc+nSJaSlpWHGjBm1xv3o0SPMmzcPn3/+udi6Ym9vj9jYWEybNg0rV67E9u3bxXhLS0uhVCrFQf5ERET0fJEIgtC4UcWt5MmTJ7Czs8N7772HJUuW1LjO1q1b8dFHH+H27du17ic0NBRKpRKGhobo2rUr/P39sWLFijrvyj6tqKgI5ubmePDgQbU74KWlpcjJyUGvXr2qDcitS0MHcreFpIJa3ubNm7Fv3z4cPHjwmR1TEASUl5dDX19ffNZIa/rss8+QlJSk1aLytKZ+f4iI2qvTp0+jf//+OHXqVLWpx4nag7qui5/WZgZvK5VK3L9/X+vpxFXduXMHUVFRmDVrVp37CQwMxPjx49GrVy9cuXIFS5YsQVBQEI4fP15j33sAePz4sda8+0VFRQAq+7o/3d9drVZDEATx1VBD7Ydi/fD1ePfQu7UmF5qkYv3w9RhqP7RR+6e2bdasWbh37x6KioqqzUzVmjSfoWfxWdLX18fGjRvrPJbme1PTd4uIqCPS1HWs96i9asznts20WCgUChgaGmL//v3VlhUVFSEgIADdunXDvn37ap2VqCZXr16Fk5NTtS4kVYWHhyMiIqJaeXZ2drWLwLKyMjx48AA9evRo0h3X1P+XivdT36+WXGiSipihMRj6wtBG75foaYIgiOM6nkWLRUOUlpYiLy8P5ubmjfoeExG1V5mZmQgMDMSPP/4IT09PXYdD1GgqlQq9e/duPy0WeXl5SElJQWJiYrVlKpUKgYGBMDMzQ1JSUqMvRhwdHWFlZYXLly/Xmlh89NFH4tz/QGUiI5fLYW1tXWNXKJVKBX19/Tqf6VAb/x7+1VouqrZU+Nmz+xO1rLZ0Aa+vrw89PT1YWlqyKxQRPRc04zy7du3aIuP2iJ61xvxet4nEIi4uDjY2NhgzZoxWeVFRERQKBaRSKfbt29ekC5Hr16/j7t27dU7RKZVKa5w5Sk9Pr9qUoHp6elpT2TbFMPkwcbaoCqGCYyqoVQiCIH5G20qLheZ7U9N3i4ioI9LUdaz3qL1qzOdW559wtVqNuLg4hISEaLUAFBUVYdSoUXj48CG+/PJLFBUVobCwEIWFhVpT1Lq6uiIpKQlA5UPQPvjgA5w4cQK5ubn4+eefERwcDGdnZygUimf+v9VFM1tUd+PuTCqIiIiIqN3TeYtFSkoK8vPzMX36dK3yX375Benp6QAAZ2dnrWU5OTno2bMngMppMzUPzevUqRPOnj2L7du34/79+7Czs8OoUaMQFRWls2dZ1MVP7seEgoiIiIg6BJ0nFqNGjapxFplhw4Y1aCabqusYGRk90+k8iYiIiIioks67Qj3PVIcO4fdhw6A6dEjXoRARERERNQsTCx1RHTqE6/PDUF54E9fnhzG5ICIiIqJ2jYmFDmiSCmgGoVdU6Dy5kEgkUCqVOjt+R3Lp0iXIZDKoVCpdh9JqJk6ciJiYGF2HQURERG0IE4tnTCup0IwPEYRWTy5CQ0Mxbty4WpcXFBQgKCioVY7dEjTTlJ44cUKr/PHjx7C0tIREIsHhw4d1E9xTPvroI8yfP1/r4YoHDx7EwIEDYWZmBmtra7z++uvIzc0VlycmJiIgIEB8doqvr2+944Vyc3O1pj6u7Rzt3bsXrq6u6Ny5M9zd3fHDDz9oLV+7di1sbGxgY2NTLVlIT0+Hj48PysvLtcqXLVuGlStXihMnEBERETGxeIZqTCo0nkFyUReZTKbzmbMEQah2AVuVXC5HXFycVllSUhJMTU1bO7QGy8/Px7///W+EhoaKZTk5OQgODoa/vz8yMjJw8OBB3LlzB+PHjxfXSU1NRUBAAH744QecOXMGw4cPx9ixY/Hrr7/We8yUlBQUFBSILx8fH3HZ8ePH8eabb2LGjBn49ddfMW7cOIwbNw7nz58HAJw9exbLly9HQkICvv76ayxbtgznzp0DAJSXl2POnDn45z//We1hkH379oWTkxN27tzZnNNFREQNoVYDN34FLqdU/lWrdR0RUY2YWDwjdSYVGjpMLqp2hdLcCU9MTMTw4cNhbGwMT09PHD9+XGubtLQ0DBkyBEZGRpDL5QgLC8PDhw/F5Tt27EC/fv1gZmYGmUyGN998E7du3RKXHz58GBKJBAcOHICPjw+kUinS0tJqjTEkJAQJCQl49OiRWLZt2zaEhIRUW/fatWuYMGECLCws0K1bNwQHB2u1EJw6dQoBAQGwsrKCubk5/Pz88Msvv1Q7J1u3bsVrr70GY2NjuLi4YN++fXWexz179sDT0xMvvPCCWHbmzBlUVFRgxYoVcHJygre3NxYuXIiMjAyUlZUBANavX49Fixahf//+cHFxwapVq+Di4oL9+/fXeTwAsLS0hEwmE19Vn7QdGxuLwMBAfPDBB3Bzc0NUVBS8vb2xadMmAEBWVhY8PDzg7++PESNGwMPDA1lZWQCANWvWYOjQoejfv3+Nxx07diwSEhLqjY+IiJrh6hFg53ggYQqgnFv5d+f4ynKiNoaJxTPQoKRCQ8ctF1UtXbpUvADu3bs3Jk2aJLYoXLlyBYGBgXj99ddx9uxZ7N69G2lpaZg3b564fVlZGaKiopCZmQmlUonc3FytO/kaH374IaKjo3Hx4kV4eHjUGo+Pjw969uyJb7/9FkBl60BqaiqmTp2qtV5ZWRkUCgXMzMxw9OhRHDt2DKampggMDMSTJ08AACqVCiEhIUhLS8OJEyfg4uKC0aNHVxsXERERgQkTJuDs2bMYPXo0Jk+ejD/++KPWGI8ePYp+/fpVi1tPTw9xcXGoqKjAgwcPsGPHDowcOVIrCahKrVZDpVKhW7dutR5L49VXX4WNjQ0GDx5cLfFJT0/HiBEjtMoUCoWYJLq7uyM7Oxv5+fnIy8tDdnY2+vbtiytXriAuLg4rVqyo9bgvv/wyTp48icePH9cbIxERNcHVI8C/3wVu/gYYmgCm3Sv/3vytspzJBbUxTCxaWaOSCo02klwsXLgQY8aMQe/evREREYG8vDxcvnwZAPDJJ59g8uTJePfdd+Hi4oI///nP2LhxI/71r3+htLQUADB9+nQEBQXB0dERAwcOxMaNG3HgwAEUFxdrHScyMhIBAQFwcnKq90J6+vTp2LZtGwAgPj4eo0ePhrW1tdY6u3fvhlqtxtatW+Hu7g43NzfExcUhPz9fHIfh7++PKVOmwNXVFW5ubtiyZQtKSkpw5Ih2JR0aGopJkybB2dkZq1atQnFxMU6ePFlrfHl5ebCzs9Mq69WrF3766ScsWbIEUqkUFhYWuH79Ovbs2VPrftauXYvi4mJMmDCh1nVMTU0RExODvXv34vvvv8fgwYMxbtw4reSisLAQ3bt319que/fuKCwsBAC4ublh1apVCAgIwKhRo/DJJ5/Azc0Ns2fPxurVq3Hw4EH07dsXXl5eSE1N1dqPnZ0dnjx5Iu6LiIhakFoNpK0DHhcDZraAgREg0av8a2ZbWZ62jt2iqE3R+QPyOrImJRUaVZIL+9iNMBs+vHWCrEPV1gNbW1sAwK1bt+Dq6orMzEycPXsWu3btqhKyALVajZycHLi5ueHMmTMIDw9HZmYm7t27B/X/Kr/8/Hz06dNH3O7pO/x1mTJlCj788ENcvXoV8fHx2LhxY7V1MjMzcfnyZa3B0wBQWlqKK1euAABu3ryJZcuW4fDhw7h16xYqKipQUlKC/Pz8Ws+BiYkJunTpotWd62mPHj1C586dtcoKCwvx1ltvISQkBJMmTYJKpcLy5cvxl7/8BcnJyZBIJFrrf/XVV4iIiMB3330HGxubWo9lZWWF9957T3zfv39/3LhxA2vWrMGrr75a63ZPmzNnDubMmSO+3759O8zMzODr64sXX3wRp06dwvXr1zFx4kTk5OSIY3GMjIwAACUlJQ0+FhFRc5SUlIjdNdsLTbxZWVnQ02vE/dzbWUDG2cpE4n5x9eXlhkDBWaB7AmDt2kLR/h9XV1cYGxu3+H6pY2Ni0YoKIyKAOgYj10sQgPJyFEZE6CSxqNpNR3Pxq0kOiouLMXv2bISFhVXbzsHBAQ8fPoRCoYBCocCuXbtgbW2N/Px8KBQKsTuShomJSYNjsrS0xCuvvIIZM2agtLQUQUFB1bovFRcXw8fHRyvp0dC0boSEhODu3bvYsGEDevToAalUCl9f32qxPd1VSSKRiOegJlZWVrh3755W2ebNm2Fubo7Vq1eLZTt37oRcLkd6ejoGDhwolickJGDmzJnYu3cvRo4cWc/ZqG7AgAFITk4W38tkMty8eVNrnZs3b0Imk9W4/Z07dxAREYHU1FSkp6ejd+/ecHFxgYuLC8rKypCdnQ13d3cAELuEPd1iRETUWrKysrQmqGhPnu6222LWTW6V3Z45cwbe3t6tsm/quJhYtCLZxx83vcUCACQSoFMnyD7+uOWDayZvb29cuHABzs7ONS4/d+4c7t69i+joaMjlcgDA6dOnW+TY06dPx+jRo7F48WJ06tSpxth2794NGxsbdOnSpcZ9HDt2DJ9++ilGjx4NoHKw9507d5odm5eXFy5cuKBVVlJSUu0ulSbuqknK119/jenTpyMhIQFjxoxp0vEzMjLE1iWgMtH4z3/+gwULFohlycnJ8PX1rXH7BQsWYMGCBbC3t8epU6fEweVA5SxRFZpnrwA4f/487O3tYWVl1aRYiYgay9XVFWfOnNF1GI1SUlKCzMxMeHp6Nq4F4HYWcHBZZYuFfg2zNpY/BsoeAYoVrdZiQdRYTCxakdnw4bCP3di05OJ/SUVLdoN68OABMjIytMosLS3FC//GWLx4MQYOHIh58+Zh5syZMDExwYULF5CcnIxNmzbBwcEBhoaGiI2NxZw5c3D+/HlERUW1yP8RGBiI27dv15o0TJ48GWvWrEFwcDAiIyNhb2+PvLw8JCYmYtGiRbC3t4eLi4s4a1VRURE++OADsWtPcygUCsycORMVFRVi8jBmzBisW7cOkZGRYleoJUuWoEePHvDy8gJQ2f0pJCQEGzZswIABA8RxC0ZGRjA3NwcAbNq0CUlJSfj5558BVHZZMjQ0FPeRmJiIbdu2YevWrWI88+fPx4gRIxATE4MxY8YgISEBp0+fxpYtW6rFnpycjOzsbGzfvh1AZdeqrKwsHDhwANeuXUOnTp3w4osviusfPXoUo0aNavY5IyJqKGNj43Z3F12tVsPZ2Rk2NjaN6wql/hNwc0/lQG0zy8rrAg1BAFQqoLsHEDARaMx+iVoRP4mtTJNcoFMn7UqhLq2QVACV07t6eXlpvSIiIpq0Lw8PDxw5cgTZ2dkYMmQIvLy8sHz5cnHgsrW1NeLj47F371706dMH0dHRWLt2bYv8HxKJBFZWVjA0NKxxubGxMVJTU+Hg4IDx48fDzc1N7DqlSUa+/PJL3Lt3D97e3pg6dSrCwsLqHM/QUEFBQdDX10dKSopY5u/vj6+++gpKpRJeXl4IDAyEVCrFjz/+KCYzW7ZsQXl5Od5++23Y2tqKr3feeUfcz507d8QxIhpRUVHw8fHBgAED8N1332H37t2YNm2auNzX1xe7du3Cli1b4OnpiW+++QZKpRJ9+/bV2s+jR48wb948fP755+IPn729PWJjYzFt2jSsXLkS27dvF+MtLS2FUqnEW2+91exzRkRENdDTAwYvAKSmgKqgsnVCUFf+VRUAUrPK5UwqqA2RCEJT+uh0bEVFRTA3N8eDBw+q3RUvLS1FTk4OevXqVW2Qbl0aPJC7lZIKenY2b96Mffv21fvk7NameeCgvr5+tQHizfXZZ58hKSkJP/30U6O2a+r3h4iovVKr1bh161bjWyw0rh6pnP3pzu+AugzQMwCsXCqTCke/lg+Y6Cl1XRc/jV2hnpEGdYtiUtEhzJ49G/fv34dKpao2M1VHYWBggNjYWF2HQUTU8Tn6AT2HAIWZQMldwNgSkHmypYLaJCYWz1CdyQWTig5DX18fS5cu1XUYrWrmzJm6DoGI6PmhpwfYeek6CqJ6Md19xmocc8GkgoiIiIjaOSYWOqCVXABMKoiIiIio3WNioSOa5EJf1p1JBRERERG1exxj0UR1PX25ocyGD2dCQc8VTkJHRETUcTGxaCRDQ0Po6enhxo0bsLa2hqGhYYtP5UnUElpzutmmxnP79m1IJBIYGBjoOhwiIiJqYUwsGklPTw+9evVCQUEBbty4oetwiGolCALUajX09PTaRGIBVD7c0N7eXnwqOREREXUcTCyawNDQEA4ODigvL0dFRYWuwyGqkVqtxt27d2Fpadm0hzK1AgMDAyYVREREHRQTiybSdOdglw5qq9RqNQwMDNC5c+c2k1gQERFRx8WrDSIiIiIiajYmFkRERERE1GxMLIiIiIiIqNk4xqIGmrn2i4qKdBwJUdOp1WqoVCqOsSAi0iHWxdTeaa6HG/IsKiYWNVCpVAAAuVyu40iIiIiIiHRPpVLB3Ny8znUkAh+FW41arcaNGzdgZmbWZub/J2qsoqIiyOVyXLt2DV26dNF1OEREzyXWxdTeCYIAlUoFOzu7elvd2GJRAz09Pdjb2+s6DKIW0aVLF/6YERHpGOtias/qa6nQYGc/IiIiIiJqNiYWRERERETUbEwsiDooqVSKjz/+GFKpVNehEBE9t1gX0/OEg7eJiIiIiKjZ2GJBRERERETNxsSCiIiIiIiajYkFERERERE1GxMLImpV4eHh+NOf/iS+Dw0Nxbhx43QWDxG1TxKJBEqlUtdhtDusg+lZYmJBz6WWrlh79uyJ9evXt9j+qmroj6lEIqnxlZCQ0CpxNdWGDRsQHx+v6zCIqI2pr14uKChAUFDQswuokVgHE/HJ20QdSlxcHAIDA7XKLCwsdBNMLRr69E4ioqpkMpmuQ4AgCKioqIC+fs2XT6yD6XnHFgsiAMOGDUNYWBgWLVqEbt26QSaTITw8XFwuCALCw8Ph4OAAqVQKOzs7hIWFidvm5eVhwYIF4h0qALh79y4mTZqEF154AcbGxnB3d8fXX3/dqOP27NkTAPDaa69BIpGI72tjYWEBmUym9ercuTMAID4+HhYWFjh48CDc3NxgamqKwMBAFBQUaO1j27ZteOmllyCVSmFra4t58+aJy/Lz8xEcHAxTU1N06dIFEyZMwM2bN7W2j46ORvfu3WFmZoYZM2agtLRUa/nTdyXrOwcAkJWVhcGDB6Nz587o06cPUlJS2C2C6DlT9Tufm5sLiUSCxMREDB8+HMbGxvD09MTx48e1tklLS8OQIUNgZGQEuVyOsLAwPHz4UFy+Y8cO9OvXD2ZmZpDJZHjzzTdx69Ytcfnhw4chkUhw4MAB+Pj4QCqVIi0trdYYWQfT846JBdH/bN++HSYmJkhPT8fq1asRGRmJ5ORkAMC3336LdevW4fPPP8fvv/8OpVIJd3d3AEBiYiLs7e0RGRmJgoIC8UeitLQUPj4++P7773H+/HnMmjULU6dOxcmTJxt83FOnTgGovAtWUFAgvm+qkpISrF27Fjt27EBqairy8/OxcOFCcflnn32Gt99+G7NmzcK5c+ewb98+ODs7AwDUajWCg4Pxxx9/4MiRI0hOTsbVq1fxxhtviNvv2bMH4eHhWLVqFU6fPg1bW1t8+umn9cZV1zmoqKjAuHHjYGxsjPT0dGzZsgVLly5t1nkgoo5h6dKlWLhwITIyMtC7d29MmjQJ5eXlAIArV64gMDAQr7/+Os6ePYvdu3cjLS1N60K9rKwMUVFRyMzMhFKpRG5uLkJDQ6sd58MPP0R0dDQuXrwIDw+PJsfLOpg6PIHoORQSEiIEBweL7/38/ITBgwdrrdO/f39h8eLFgiAIQkxMjNC7d2/hyZMnNe6vR48ewrp16+o97pgxY4T333+/wccVBEEAICQlJdW7bwBC586dBRMTE61XXl6eIAiCEBcXJwAQLl++LG6zefNmoXv37uJ7Ozs7YenSpTXu/6effhI6deok5Ofni2W//fabAEA4efKkIAiC4OvrK8ydO1druwEDBgienp7i+8ae+wMHDgj6+vpCQUGBuDw5ObnB54WI2oen64anVf3O5+TkCACErVu3iss19dHFixcFQRCEGTNmCLNmzdLax9GjRwU9PT3h0aNHNR7j1KlTAgBBpVIJgiAIhw4dEgAISqWy3vhZBxMJAlssiP7n6btQtra2YpP4X//6Vzx69AiOjo546623kJSUJN4Vq01FRQWioqLg7u6Obt26wdTUFAcPHkR+fn6Dj9tY69atQ0ZGhtbLzs5OXG5sbAwnJ6caj3Xr1i3cuHEDI0aMqHHfFy9ehFwuh1wuF8v69OkDCwsLXLx4UVxnwIABWtv5+vrWG3dd5+DSpUuQy+Va/atffvnlevdJRB1f1brD1tYWAMS6IzMzE/Hx8TA1NRVfCoUCarUaOTk5AIAzZ85g7NixcHBwgJmZGfz8/ACgWj3dr1+/BsXDOpiedxy8TfQ/BgYGWu8lEgnUajUAQC6X49KlS0hJSUFycjLmzp2LNWvW4MiRI9W201izZg02bNiA9evXw93dHSYmJnj33Xfx5MmTBh+3sWQymdhsXpOajiUIAgDAyMioScdsCS15Dojo+VG17tCMb9PUHcXFxZg9e7Y4Hq4qBwcHPHz4EAqFAgqFArt27YK1tTXy8/OhUCiq1dMmJiYNiod1MD3v2GJB1EBGRkYYO3YsNm7ciMOHD+P48eM4d+4cAMDQ0BAVFRVa6x87dgzBwcGYMmUKPD094ejoiOzs7EYf18DAoNq+W4OZmRl69uyJn3/+ucblbm5uuHbtGq5duyaWXbhwAffv30efPn3EddLT07W2O3HiRLPievHFF3Ht2jWtAYrNHWtCRB2ft7c3Lly4AGdn52ovQ0NDZGVl4e7du4iOjsaQIUPg6ura5NbilsA6mDoCtlgQNUB8fDwqKiowYMAAGBsbY+fOnTAyMkKPHj0AVM7elJqaiokTJ0IqlcLKygouLi745ptv8N///hddu3bFP/7xD9y8eVP8AWgozQ/NoEGDIJVK0bVr11rXvX//PgoLC7XKzMzMGny3LTw8HHPmzIGNjQ2CgoKgUqlw7NgxzJ8/HyNHjoS7uzsmT56M9evXo7y8HHPnzoWfn5/YTeCdd95BaGgo+vXrh0GDBmHXrl347bff4Ojo2Kj/uaqAgAA4OTkhJCQEq1evhkqlwrJlywD83x1KIuoYHjx4gIyMDK0yS0tLre4/DbV48WIMHDgQ8+bNw8yZM2FiYoILFy4gOTkZmzZtgoODAwwNDREbG4s5c+bg/PnziIqKalb8rIPpeccWC6IGsLCwwBdffIFBgwbBw8MDKSkp2L9/PywtLQEAkZGRyM3NhZOTE6ytrQEAy5Ytg7e3NxQKBYYNGwaZTNakh/LFxMQgOTkZcrkcXl5eda47bdo02Nraar1iY2MbfKyQkBCsX78en376KV566SW88sor+P333wFU/oB899136Nq1K4YOHYqRI0fC0dERu3fvFrd/44038Pe//x2LFi2Cj48P8vLy8Le//a3R/3NVnTp1glKpRHFxMfr374+ZM2eKM5JopnEkoo7h8OHD8PLy0npFREQ0aV8eHh44cuQIsrOzMWTIEHh5eWH58uXimAdra2vEx8dj79696NOnD6Kjo7F27dpmxc86mJ53EkHTuY+IqJ04duwYBg8ejMuXL2sNhCQiotbHOphqw8SCiNq8pKQkmJqawsXFBZcvX8Y777yDrl271vmgKiIiahmsg6mhOMaCiNo8lUqFxYsXIz8/H1ZWVhg5ciRiYmJ0HRYR0XOBdTA1FFssiIiIiIio2Th4m4iIiIiImo2JBRERERERNRsTCyIiIiIiajYmFkRERERE1GxMLIiIiIiIqNmYWBARERERUbMxsSAiIiIiomZjYkFERERERM3GxIKIiIiIiJrt/wMNBrIJ3FXjGgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "xs = np.arange(1, iterations + 1)\n", + "\n", + "plt.figure(figsize=(8, 5))\n", + "\n", + "# Boxplots\n", + "plt.boxplot(\n", + " [instant_encoding_accs, linear_encoding_accs],\n", + " positions=[1, 2],\n", + " widths=0.5,\n", + " labels=[\"Instant Encoding\", \"Linear Encoding\"]\n", + ")\n", + "\n", + "# Scatter overlay\n", + "plt.scatter(\n", + " np.ones_like(instant_encoding_accs) + np.random.uniform(-0.08, 0.08, len(instant_encoding_accs)),\n", + " instant_encoding_accs,\n", + " alpha=0.8,\n", + " label=\"Instant Samples\"\n", + ")\n", + "\n", + "plt.scatter(\n", + " 2 * np.ones_like(linear_encoding_accs) + np.random.uniform(-0.08, 0.08, len(linear_encoding_accs)),\n", + " linear_encoding_accs,\n", + " alpha=0.8,\n", + " label=\"Linear Samples\"\n", + ")\n", + "\n", + "# Mean markers\n", + "plt.scatter(1, np.mean(instant_encoding_accs), marker='D', s=80,\n", + " label=f'Instant Mean ({np.mean(instant_encoding_accs):.2f}%)')\n", + "\n", + "plt.scatter(2, np.mean(linear_encoding_accs), marker='D', s=80,\n", + " label=f'Linear Mean ({np.mean(linear_encoding_accs):.2f}%)')\n", + "\n", + "plt.ylabel(\"Inference Accuracy (%)\")\n", + "plt.title(\"Inference Robustness Across Encoding Modes\")\n", + "plt.grid(True, alpha=0.3)\n", + "plt.legend()\n", + "\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "e2849886-4210-401b-a21e-118c9e96c61a", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "This notebook demonstrates that input encoding strategy can materially influence inference robustness under hardware nonidealities. In this particular example, instantaneous encoding achieves higher average accuracy than linear bit-sliced encoding; however, this outcome is strongly dependent on the underlying hardware assumptions and operating conditions.\n", + "\n", + "Specifically, the present experiments assume device operation within the linear regime and use identical `V_read` values for both encoding modes. Under these conditions, the temporal decomposition introduced by linear encoding primarily introduces additional quantization overhead without providing compensating benefits. Furthermore, the `SimpleFixedPoint` inference accelerator employs dynamic output quantization, which prevents ADC saturation and clipping effects that could otherwise favor temporally encoded execution.\n", + "\n", + "Different modeling assumptions may lead to different conclusions. For example, if the device model incorporated voltage-dependent nonlinearities such that larger instantaneous voltages pushed devices outside their linear operating region, linear bit-sliced encoding would likely offer improved robustness. Similarly, architectures with fixed ADC ranges, saturation behavior, or explicit clipping could benefit from temporal accumulation strategies.\n", + "\n", + "These experiments therefore highlight not only the importance of encoding choice, but also the flexibility of XBTorch’s modular accelerator interface. Existing parameters can be swept to explore alternate operating regimes, and users can readily implement custom inference accelerator profiles with more complex transfer functions, enabling evaluation of architecture-specific encoding schemes tailored to their target IMC hardware." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 5af2652..44915f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,8 @@ dependencies = [ "ninja>=1.11", "qtorch>=0.3", "smt>=2.9.4", + "transformers>=4.57.0", + "lm_eval>=0.4.9" ] # List additional groups of dependencies here (e.g. development diff --git a/src/xbtorch/deployment/__init__.py b/src/xbtorch/deployment/__init__.py index cc919ed..74c6de4 100644 --- a/src/xbtorch/deployment/__init__.py +++ b/src/xbtorch/deployment/__init__.py @@ -1,9 +1,9 @@ """ Deployment (mapping, encoding, etc.) of solutions to inference accelerators """ -from .base import Daffodil, SimpleFixedPoint +from .base import Daffodil, SimpleFixedPoint, register_accelerator from .mapping import map_random -from .encoding import encode_simple_binary, encode_MAO, encode_LEA1, encode_LEA2 +from .weight_encoding import encode_simple_binary, encode_MAO, encode_LEA1, encode_LEA2 from .metrics import compute_error from .correction import train_collaborative, test_collaborative, CollaborativeLoss, add_collaborative_logistic_classifiers, dnn_favorable_searching_code from .committee import test_committee \ No newline at end of file diff --git a/src/xbtorch/deployment/base.py b/src/xbtorch/deployment/base.py index 4311486..e472de6 100644 --- a/src/xbtorch/deployment/base.py +++ b/src/xbtorch/deployment/base.py @@ -23,8 +23,18 @@ import numpy as np from xbtorch.deployment.mapping import map_random -from xbtorch.deployment.encoding import encode_simple_binary, encode_LEA1, encode_LEA2 +from xbtorch.deployment.weight_encoding import encode_simple_binary, encode_LEA1, encode_LEA2 +ACCELERATOR_REGISTRY = {} + +def register_accelerator(name: str): + """Decorator to register a custom layer under a string name.""" + def decorator(cls): + ACCELERATOR_REGISTRY[name] = cls + return cls + return decorator + +@register_accelerator("Generic") class GenericAccelerator(metaclass=abc.ABCMeta): """ Abstract base class for hardware accelerator models in XBTorch. @@ -46,14 +56,21 @@ class GenericAccelerator(metaclass=abc.ABCMeta): Amplitude of uniform read noise applied during chip readout. write_noise : float Standard deviation of Gaussian noise applied during weight writes. + stateful: bool, optional + In stateful mode, a physical representation of the entire crossbar is maintained, and weights are mapped to these limited devices. + In stateless mode, weights are mapped and VMM is performed on the fly. This is more memory-efficient. Essentially behaves like an infinite size stateful crossbar. xb_size : tuple of int, optional - Dimensions of the crossbar array (columns, rows). Default: (2500, 2500). + Dimensions of the crossbar array (columns, rows). Default: (2500, 2500). Utilized only when stateful is True. stuck_percentage : float, optional Fraction of devices randomly stuck at high or low values. Default: 0.0. stuck_mode : {"ideal", "real"}, optional Mode for stuck-at defect modeling: - "ideal": stuck devices are fixed at g_min or g_max. - "real": stuck devices are fixed at predefined realistic values. + input_encoding_scheme : {"instant", "linear"}, optional + Mode for stuck-at defect modeling: + - "instant": Instantaneous voltage-amplitude encoding of inputs. + - "linear": Linear, bit-sliced encoding of inputs. weight_encoding_scheme : callable, optional Function used to encode weights into conductance matrices. Default: :func:`encode_simple_binary`. @@ -80,17 +97,36 @@ class GenericAccelerator(metaclass=abc.ABCMeta): read noise. """ - def __init__(self, g_min, g_max, v_read, read_noise, write_noise, xb_size=(2500, 2500), stuck_percentage=0.0, stuck_mode='real', weight_encoding_scheme=encode_simple_binary, xb_mapping_scheme=map_random, device="cpu"): + def __init__(self, + g_min, + g_max, + v_read, + read_noise, + write_noise, + stateful=True, + xb_size=(2500, 2500), + stuck_percentage=0.0, + stuck_mode='real', + input_encoding_scheme='instant', + weight_encoding_scheme=encode_simple_binary, + xb_mapping_scheme=map_random, + device="cpu"): + self.read_noise = read_noise self.write_noise = write_noise self.g_min = g_min self.g_max = g_max self.v_read = v_read + self.input_encoding_scheme = input_encoding_scheme self.weight_encoding_scheme = weight_encoding_scheme self.xb_mapping_scheme = xb_mapping_scheme self.stuck_percentage = stuck_percentage + self.stateful = stateful + + if (self.stuck_percentage > 0 and not self.stateful): + raise ValueError("Stuck devices can not be simulated without a stateful representation of a crossbar. See examples for usage.") - self.columns, self.rows = xb_size + self.columns, self.rows = xb_size if stateful else (-1, -1) # self.stuck_low = 0 # self.stuck_high = self.g_max * 2 self.stuck_mode = stuck_mode @@ -104,26 +140,31 @@ def __init__(self, g_min, g_max, v_read, read_noise, write_noise, xb_size=(2500, else: raise ValueError(f"Stuck mode {stuck_mode} not implemented") + if input_encoding_scheme not in ["instant", "linear"]: + raise ValueError(f"IO Encoding mode {input_encoding_scheme} not implemented") # Create defect map - # TODO: Separate out defect maps + # TODO: Separate out defect maps; - self.name = f'cols_{self.columns}_row_{self.rows}_stuck_{self.stuck_percentage}' + self.name = f'stateful_{self.stateful}_cols_{self.columns}_row_{self.rows}_stuck_{self.stuck_percentage}' self.device = device - self.initialize_chip() + if (self.stateful): + self.initialize_chip() def initialize_chip(self): """ - Initialize the simulated chip state. + Initialize the simulated chip state, assuming stateful mode. + If stateless, defect maps will be patched on dynamically during operation. - Fills the array with uninitialized values (-1). - Generates a defect map based on the specified stuck percentage. """ - self._chip = torch.ones((self.columns, self.rows)).to(self.device) * -1 # uninitialized devices - self.defect_map = self.gen_defect_map(self.stuck_percentage) # defect map is a paired list of (defective indices, defective conductance states) - self._chip[self.defect_map[0]] = self.defect_map[1] + if (self.stateful): + self._chip = torch.ones((self.columns, self.rows)).to(self.device) * -1 # uninitialized devices + self.defect_map = self.gen_defect_map(self.stuck_percentage) # defect map is a paired list of (defective indices, defective conductance states) + self._chip[self.defect_map[0]] = self.defect_map[1] def get_xb_size(self): """ @@ -135,6 +176,9 @@ def read_chip(self, row, n_rows, col, n_cols, fast_mode=True): """ Read a subarray of the chip, optionally with read noise. + This method requires the object to be in a stateful mode. + If `self.stateful` is False, a RuntimeError is raised. + Parameters ---------- row : int @@ -152,12 +196,55 @@ def read_chip(self, row, n_rows, col, n_cols, fast_mode=True): ------- torch.Tensor Subarray with applied read noise (if configured). + + + Raises + ------ + RuntimeError + If `self.stateful` is False. + ValueError + If `fast_mode` is False (not implemented yet). + """ + + if not self.stateful: + raise RuntimeError("Cannot read chip when self.stateful is False.") + subarray = self._chip[row:row+n_rows, col:col+n_cols] noise = torch.empty_like(subarray).uniform_(-self.read_noise, self.read_noise) if (not fast_mode): raise ValueError("Not implemented") if (self.read_noise > 0): subarray = subarray + noise return subarray + + def read_chip_stateless(self, subarray): + """ + Read a subarray of the chip, optionally with read noise. + + This method requires the object to be in a stateful mode. + If `self.stateful` is False, a RuntimeError is raised. + + Parameters + ---------- + G + + Returns + ------- + torch.Tensor + Subarray with applied read noise (if configured). + + + Raises + ------ + RuntimeError + If `self.stateful` is False. + ValueError + If `fast_mode` is False (not implemented yet). + + """ + + noise = torch.empty_like(subarray).uniform_(-self.read_noise, self.read_noise) + if (self.read_noise > 0): subarray = subarray + noise + return subarray def gen_defect_map(self, stuck_percentage): """ @@ -175,6 +262,9 @@ def gen_defect_map(self, stuck_percentage): - indices are tensor indices of defective devices, - values are their fixed conductances (stuck_high or stuck_low). """ + if (not self.stateful): + return + num_elements = int(stuck_percentage * self._chip.numel()) defect_indices = np.unravel_index( np.random.choice(self._chip.shape[0] * self._chip.shape[1], num_elements, replace=False), (self._chip.shape[0], self._chip.shape[1]) @@ -184,6 +274,29 @@ def gen_defect_map(self, stuck_percentage): defect_values[defect_values == 1] = self.stuck_high return defect_indices, defect_values.to(self.device) + def map_weights_to_array_stateless(self, sw_weight): + + if (self.stateful): + return + + encoded_return = self.weight_encoding_scheme(self, sw_weight) + Gposs, Gnegs = encoded_return[0], encoded_return[1] + sw_weight_shape = sw_weight.shape + + # write noise + for i, Gpos in enumerate(Gposs): + if (self.write_noise > 0): + noise = torch.randn_like(Gposs[i]) * self.write_noise + 0.0 # 0 mean + Gposs[i] = Gposs[i] + noise + + + for i, Gneg in enumerate(Gnegs): + if (self.write_noise > 0): + noise = torch.randn_like(Gnegs[i]) * self.write_noise + 0.0 # 0 mean + Gnegs[i] = Gnegs[i] + noise + + return Gposs, Gnegs + def map_weights_to_array(self, sw_weight, pos_idxs=[], neg_idxs=[], additional_args={}): """ Map software weights onto the hardware array. @@ -210,6 +323,10 @@ def map_weights_to_array(self, sw_weight, pos_idxs=[], neg_idxs=[], additional_a - Adds Gaussian write noise if configured. - Defect map is reapplied to enforce stuck devices. """ + + if (not self.stateful): + return + encoded_return = self.weight_encoding_scheme(self, sw_weight, pos_idxs=pos_idxs, neg_idxs=neg_idxs, additional_args=additional_args) Gposs, Gnegs = encoded_return[0], encoded_return[1] sw_weight_shape = sw_weight.shape @@ -220,7 +337,7 @@ def map_weights_to_array(self, sw_weight, pos_idxs=[], neg_idxs=[], additional_a Gposs[i] = Gposs[i] + noise self._chip[pos_idx[0]:pos_idx[0]+sw_weight_shape[0], - pos_idx[1]:pos_idx[1]+sw_weight_shape[1]] = Gposs[i] + pos_idx[1]:pos_idx[1]+sw_weight_shape[1]] = Gposs[i] for i, neg_idx in enumerate(neg_idxs): if (self.write_noise > 0): @@ -228,7 +345,7 @@ def map_weights_to_array(self, sw_weight, pos_idxs=[], neg_idxs=[], additional_a Gnegs[i] = Gnegs[i] + noise self._chip[neg_idx[0]:neg_idx[0]+sw_weight_shape[0], - neg_idx[1]:neg_idx[1]+sw_weight_shape[1]] = Gnegs[i] + neg_idx[1]:neg_idx[1]+sw_weight_shape[1]] = Gnegs[i] # Add back defect map information in case the outer method attempted to do an illegal assignment self._chip[self.defect_map[0]] = self.defect_map[1] @@ -293,6 +410,10 @@ def plot_array(self, x_start=None, x_count=None, y_start=None, y_count=None, tit torch.Tensor The read subarray. """ + + if (not self.stateful): + return + import matplotlib.pyplot as plt fig = plt.figure() @@ -318,6 +439,7 @@ def plot_array(self, x_start=None, x_count=None, y_start=None, y_count=None, tit if show: plt.show() return read_chip +@register_accelerator("SimpleFixedPoint") class SimpleFixedPoint(GenericAccelerator): """ Simple fixed-point accelerator model. @@ -343,8 +465,24 @@ class SimpleFixedPoint(GenericAccelerator): """ - def __init__(self, adc_bits=5, dac_bits=5, g_min=50, g_max=100, v_read=0.3, read_noise=0, xb_size=(2500, 2500), write_noise=0, stuck_percentage=0.0, stuck_mode='real', xb_mapping_scheme=map_random, weight_encoding_scheme=encode_simple_binary, device='cpu'): - super().__init__(g_min, g_max, v_read, read_noise=read_noise, write_noise=write_noise, xb_size=xb_size, stuck_percentage=stuck_percentage, stuck_mode=stuck_mode, xb_mapping_scheme=xb_mapping_scheme, weight_encoding_scheme=weight_encoding_scheme, device=device) + def __init__(self, + adc_bits=5, + dac_bits=5, + g_min=50, + g_max=100, + v_read=0.3, + read_noise=0, + stateful=True, + xb_size=(2500, 2500), + write_noise=0, + stuck_percentage=0.0, + stuck_mode='real', + input_encoding_scheme='instant', + xb_mapping_scheme=map_random, + weight_encoding_scheme=encode_simple_binary, + device='cpu'): + # TODO: Input encoding modes and output encoding modes should go here + super().__init__(g_min, g_max, v_read, read_noise=read_noise, write_noise=write_noise, stateful=stateful, xb_size=xb_size, stuck_percentage=stuck_percentage, stuck_mode=stuck_mode, input_encoding_scheme=input_encoding_scheme, xb_mapping_scheme=xb_mapping_scheme, weight_encoding_scheme=weight_encoding_scheme, device=device) self.adc_bits = adc_bits self.dac_bits = dac_bits @@ -362,8 +500,37 @@ def DAC_quantize(self, vector): torch.Tensor Quantized voltage vector. """ - max_val = torch.max(vector) - return max_val * fixed_point_quantize(vector / max_val, wl=self.dac_bits, fl=self.dac_bits-1, symmetric=True) + + max_val = torch.max(torch.abs(vector)) + + if self.input_encoding_scheme == "instant": + + return max_val * fixed_point_quantize( + vector / max_val, + wl=self.dac_bits, + fl=self.dac_bits - 1, + symmetric=True + ) + + elif self.input_encoding_scheme == "linear": + + normalized = torch.clamp(vector / max_val, -1.0, 1.0) + + levels = 2 ** self.dac_bits - 1 + quantized = torch.round(normalized * levels) + + slices = [] + sign = torch.sign(quantized) + mag = torch.abs(quantized).long() + + for bit in range(self.dac_bits): + bit_slice = ((mag >> bit) & 1).float() + slices.append(sign * bit_slice * max_val) + + return slices + + else: + raise ValueError("Unsupported encoding scheme") def ADC_quantize(self, vector): """ @@ -379,10 +546,19 @@ def ADC_quantize(self, vector): torch.Tensor Quantized current vector. """ - max_val = torch.max(vector) - return max_val * fixed_point_quantize(vector / max_val, wl=self.adc_bits, fl=self.adc_bits-1, symmetric=True) + max_val = torch.max(torch.abs(vector)) + + return max_val * fixed_point_quantize( + vector / max_val, + wl=self.adc_bits, + fl=self.adc_bits - 1, + symmetric=True + ) + + # TODO: Support for ADC per-slice is not implemented +@register_accelerator("Daffodil") class Daffodil(GenericAccelerator): """ Experimental Daffodil accelerator model. @@ -412,8 +588,8 @@ class Daffodil(GenericAccelerator): """ - def __init__(self, g_min=50, g_max=100, v_read=0.3, read_noise=10, write_noise=10, stuck_percentage=0.0, stuck_mode='real', xb_mapping_scheme=map_random, device='cpu'): - super().__init__(g_min, g_max, v_read, read_noise, write_noise, stuck_percentage, stuck_mode, xb_mapping_scheme, device=device) + def __init__(self, g_min=50, g_max=100, v_read=0.3, read_noise=10, write_noise=10, stuck_percentage=0.0, stuck_mode='real', input_encoding_scheme='instant', xb_mapping_scheme=map_random, device='cpu'): + super().__init__(g_min, g_max, v_read, read_noise, write_noise, stuck_percentage, stuck_mode, input_encoding_scheme, xb_mapping_scheme, device=device) # Board level parameters, calibrated from hardware experiments # Can be overridenn based on further experimentation diff --git a/src/xbtorch/deployment/encoding.py b/src/xbtorch/deployment/weight_encoding.py similarity index 100% rename from src/xbtorch/deployment/encoding.py rename to src/xbtorch/deployment/weight_encoding.py diff --git a/src/xbtorch/patches/__init__.py b/src/xbtorch/patches/__init__.py index 2945665..1d2f773 100644 --- a/src/xbtorch/patches/__init__.py +++ b/src/xbtorch/patches/__init__.py @@ -1,4 +1,4 @@ """ Decorators for patching PyTorch models and optimizers for XBTorch """ -from .model import xbtorch_model \ No newline at end of file +from .model import xbtorch_model, replace_all_layers_stateless \ No newline at end of file diff --git a/src/xbtorch/patches/decorators.py b/src/xbtorch/patches/decorators.py index cf9a639..c62403a 100644 --- a/src/xbtorch/patches/decorators.py +++ b/src/xbtorch/patches/decorators.py @@ -69,59 +69,198 @@ def backward_hook(module, grad_input, grad_output): def xbtorch_forward(self, input): # TODO: Add support for non-linear layers if (self._xb_inference): - if (not hasattr(self, '_array_mappings')): - raise ValueError("Array mappings are not present, likely an issue during initialization.") + if (self.inference_accelerator.stateful): + if (not hasattr(self, '_array_mappings')): + raise ValueError("Array mappings are not present, likely an issue during initialization.") + + gnorm_scale = 1.0 + if (not self.inference_accelerator): raise ValueError('XB inference called without proper initialization of an accelerator profile.') + + # ternary mapping scheme + v_read = self.inference_accelerator.v_read + g_norm = self.inference_accelerator.g_max - self.inference_accelerator.g_min + + sw_weight = self.weight.data + + gamma = torch.unique(sw_weight)[-1] # WAGE quantization learns matrices [-gamma, 0, gamma], and so it's important to scale either G matrices or input voltage vector + + pos_idxs = self._array_mappings['Gpos'] + neg_idxs = self._array_mappings['Gneg'] + + # pass to the crossbar, perform VMM, averaging/summing over multiple instances + pos_outputs = [] + neg_outputs = [] + + if self.inference_accelerator.input_encoding_scheme == "instant": + # convert inputs to voltages, then quantize to DAC-based precision + input_voltages = input * v_read * gamma + input_voltages = self.inference_accelerator.DAC_quantize(input_voltages) + + # Todo: can be possibly batched and made faster by using torch.bmm + for pos_idx in pos_idxs: + gpos = self.inference_accelerator.read_chip(pos_idx[0], sw_weight.shape[0], pos_idx[1], sw_weight.shape[1]) + pos_outputs.append(input_voltages @ gpos.T) + + for neg_idx in neg_idxs: + gneg = self.inference_accelerator.read_chip(neg_idx[0], sw_weight.shape[0], neg_idx[1], sw_weight.shape[1]) + neg_outputs.append(input_voltages @ gneg.T) + + # Convert the list of tensors to a single tensor + pos_outputs = torch.stack(pos_outputs) + neg_outputs = torch.stack(neg_outputs) + + else: + # Linear temporal bit-sliced encoding + input_slices = self.inference_accelerator.DAC_quantize( + input * v_read * gamma + ) - gnorm_scale = 1.0 - if (not self.inference_accelerator): raise ValueError('XB inference called without proper initialization of an accelerator profile.') + pos_outputs_accum = [] + neg_outputs_accum = [] + + for bit_idx, input_slice in enumerate(input_slices): + bit_weight = 2 ** bit_idx + + pos_slice_outputs = [] + neg_slice_outputs = [] + + for pos_idx in pos_idxs: + gpos = self.inference_accelerator.read_chip( + pos_idx[0], + sw_weight.shape[0], + pos_idx[1], + sw_weight.shape[1] + ) + pos_slice_outputs.append(input_slice @ gpos.T) + + for neg_idx in neg_idxs: + gneg = self.inference_accelerator.read_chip( + neg_idx[0], + sw_weight.shape[0], + neg_idx[1], + sw_weight.shape[1] + ) + neg_slice_outputs.append(input_slice @ gneg.T) + + pos_slice_outputs = torch.stack(pos_slice_outputs) + neg_slice_outputs = torch.stack(neg_slice_outputs) + + pos_outputs_accum.append(bit_weight * pos_slice_outputs) + neg_outputs_accum.append(bit_weight * neg_slice_outputs) + + pos_outputs = torch.stack(pos_outputs_accum).sum(dim=0) + neg_outputs = torch.stack(neg_outputs_accum).sum(dim=0) + + # normalize temporal accumulation + scale = (2 ** self.inference_accelerator.dac_bits - 1) + pos_outputs = pos_outputs / scale + neg_outputs = neg_outputs / scale + + # for MAO, this has to be sum, for regular mapping, this will be average + if (self._array_mappings['output_polling_mode'] == 'avg'): + output = torch.mean(pos_outputs, dim=0) - torch.mean(neg_outputs, dim=0) + elif (self._array_mappings['output_polling_mode'] == 'sum'): + output = torch.sum(pos_outputs, dim=0) - torch.sum(neg_outputs, dim=0) + elif (self._array_mappings['output_polling_mode'] == 'reduced_avg'): + output = torch.sum(pos_outputs * self._array_mappings['maskpos'], dim=0) / self._array_mappings['alpha'] - torch.sum(neg_outputs * self._array_mappings['maskneg'], dim=0) / self._array_mappings['alpha'] + else: + raise ValueError("output_polling_mode not implemented") + # readback currents from ADC by simulated quantization again + output = self.inference_accelerator.ADC_quantize(output) # equivalent to optimizing the TIA potentiometer resistance. ADC quantization + output = output / (gnorm_scale * g_norm * v_read) + + if (self.bias is not None): output += self.bias.data + return output + else: + # stateless forward, weights were never encoded and mapped to a crossbar. Here, we'll do everything on-the-fly. - # ternary mapping scheme - v_read = self.inference_accelerator.v_read - g_norm = self.inference_accelerator.g_max - self.inference_accelerator.g_min + gnorm_scale = 1.0 + if (not self.inference_accelerator): raise ValueError('XB inference called without proper initialization of an accelerator profile.') - sw_weight = self.weight.data + # ternary mapping scheme + v_read = self.inference_accelerator.v_read + g_norm = self.inference_accelerator.g_max - self.inference_accelerator.g_min - gamma = torch.unique(sw_weight)[-1] # WAGE quantization learns matrices [-gamma, 0, gamma], and so it's important to scale either G matrices or input voltage vector + sw_weight = self.weight.data - # convert inputs to voltages, then quantize to DAC-based precision - input_voltages = input * v_read * gamma - input_voltages = self.inference_accelerator.DAC_quantize(input_voltages) + gamma = torch.unique(sw_weight)[-1] # WAGE quantization learns matrices [-gamma, 0, gamma], and so it's important to scale either G matrices or input voltage vector - pos_idxs = self._array_mappings['Gpos'] - neg_idxs = self._array_mappings['Gneg'] - - # pass to the crossbar, perform VMM, averaging/summing over multiple instances - pos_outputs = [] - neg_outputs = [] - # Todo: can be possibly batched and made faster by using torch.bmm - for pos_idx in pos_idxs: - gpos = self.inference_accelerator.read_chip(pos_idx[0], sw_weight.shape[0], pos_idx[1], sw_weight.shape[1]) - pos_outputs.append(input_voltages @ gpos.T) + # stateless operation + # first, let's convert weights to conductances + # TODO: raise error if redundnancy based scheme required with stateless operation, not supported atm + # would require splitting initialize layer mappings for stateless and stateful + + Gposs, Gnegs = self.inference_accelerator.map_weights_to_array_stateless(sw_weight) + + # pass to the crossbar, perform VMM, averaging/summing over multiple instances + pos_outputs = [] + neg_outputs = [] + + if self.inference_accelerator.input_encoding_scheme == "instant": + # convert inputs to voltages, then quantize to DAC-based precision + input_voltages = input * v_read * gamma + input_voltages = self.inference_accelerator.DAC_quantize(input_voltages) + + # Todo: can be possibly batched and made faster by using torch.bmm + for Gpos in Gposs: + gpos = self.inference_accelerator.read_chip_stateless(Gpos) + pos_outputs.append(input_voltages @ gpos.T) + + for Gneg in Gnegs: + gneg = self.inference_accelerator.read_chip_stateless(Gneg) + neg_outputs.append(input_voltages @ gneg.T) + + # Convert the list of tensors to a single tensor + pos_outputs = torch.stack(pos_outputs) + neg_outputs = torch.stack(neg_outputs) + + else: + # Linear temporal bit-sliced encoding + input_slices = self.inference_accelerator.DAC_quantize( + input * v_read * gamma + ) + + pos_outputs_accum = [] + neg_outputs_accum = [] - for neg_idx in neg_idxs: - gneg = self.inference_accelerator.read_chip(neg_idx[0], sw_weight.shape[0], neg_idx[1], sw_weight.shape[1]) - neg_outputs.append(input_voltages @ gneg.T) + for bit_idx, input_slice in enumerate(input_slices): + bit_weight = 2 ** bit_idx - # Convert the list of tensors to a single tensor - pos_outputs = torch.stack(pos_outputs) - neg_outputs = torch.stack(neg_outputs) + pos_slice_outputs = [] + neg_slice_outputs = [] + + for Gpos in Gposs: + gpos = self.inference_accelerator.read_chip_stateless(Gpos) + pos_slice_outputs.append(input_slice @ gpos.T) + + for Gneg in Gnegs: + gneg = self.inference_accelerator.read_chip_stateless(Gneg) + neg_slice_outputs.append(input_slice @ gneg.T) + + pos_slice_outputs = torch.stack(pos_slice_outputs) + neg_slice_outputs = torch.stack(neg_slice_outputs) + + pos_outputs_accum.append(bit_weight * pos_slice_outputs) + neg_outputs_accum.append(bit_weight * neg_slice_outputs) + + pos_outputs = torch.stack(pos_outputs_accum).sum(dim=0) + neg_outputs = torch.stack(neg_outputs_accum).sum(dim=0) + + # normalize temporal accumulation + scale = (2 ** self.inference_accelerator.dac_bits - 1) + pos_outputs = pos_outputs / scale + neg_outputs = neg_outputs / scale - # for MAO, this has to be sum, for regular mapping, this will be average - if (self._array_mappings['output_polling_mode'] == 'avg'): output = torch.mean(pos_outputs, dim=0) - torch.mean(neg_outputs, dim=0) - elif (self._array_mappings['output_polling_mode'] == 'sum'): - output = torch.sum(pos_outputs, dim=0) - torch.sum(neg_outputs, dim=0) - elif (self._array_mappings['output_polling_mode'] == 'reduced_avg'): - output = torch.sum(pos_outputs * self._array_mappings['maskpos'], dim=0) / self._array_mappings['alpha'] - torch.sum(neg_outputs * self._array_mappings['maskneg'], dim=0) / self._array_mappings['alpha'] - else: - raise ValueError("output_polling_mode not implemented") - # readback currents from ADC by simulated quantization again - output = self.inference_accelerator.ADC_quantize(output) # equivalent to optimizing the TIA potentiometer resistance. ADC quantization - output = output / (gnorm_scale * g_norm * v_read) + # TODO: output polling modes are not implemented for now - if (self.bias is not None): output += self.bias.data - return output + # readback currents from ADC by simulated quantization again + output = self.inference_accelerator.ADC_quantize(output) # equivalent to optimizing the TIA potentiometer resistance. ADC quantization + output = output / (gnorm_scale * g_norm * v_read) + + if (self.bias is not None): output += self.bias.data + return output else: if (hasattr(self, 'weight')): self.weight.input = input diff --git a/src/xbtorch/patches/model.py b/src/xbtorch/patches/model.py index 36cd5b7..2beb688 100644 --- a/src/xbtorch/patches/model.py +++ b/src/xbtorch/patches/model.py @@ -4,15 +4,9 @@ """ from xbtorch import get_xbtorch_param -import xbtorch.quant.wage_init as wage_init +from .utils import replace_all_layers_stateful, replace_all_layers_stateless, toggle_xb_eval_all_layers_stateless -import torch.nn as nn -import xbtorch.nn as xbnn -import xbtorch - -import torch - -def xbtorch_model(original_model): +def xbtorch_model(original_model, replace_all=False, exclude=None): """ Patch a PyTorch model for XBTorch compatibility. @@ -71,8 +65,6 @@ def xbtorch_model(original_model): - Inference accelerator mappings assume that crossbar dimensions and encoding/mapping schemes are defined during initialization. """ - # TODO: .model shouldn't be required, should have an option to specify layers - # TODO: This should be used if a model was already created i.e. on an instance # Copies state dictionary as well if (not get_xbtorch_param('initialized')): raise RuntimeError('XBTorch needs to be initialized, please refer to API for instructions.') @@ -80,90 +72,32 @@ def xbtorch_model(original_model): xb_inference_accelerator = get_xbtorch_param('inference_accelerator') original_model.xb_forward = False # declare this to be false, xb_eval() has to be explicitly called - # detect the device from the original model - try: - device = next(original_model.parameters()).device - except StopIteration: - device = torch.device("cpu") # fallback if model has no parameters - - if (wage_quantize): - wage_params = get_xbtorch_param('wage_params') - quantizer_act_error = wage_params['quantizer_act_error'] - wl_activation = wage_params['wl_activation'] - wl_error = wage_params['wl_error'] - wl_weight = wage_params['wl_weight'] - - if (not hasattr(original_model, 'model')): raise RuntimeError('Unable to find module list for patching, see network training example for correct patching workflow.') - - new_model = [] - if (wage_quantize): - new_model.append(quantizer_act_error(wl_activation, -1)) - - for module in original_model.model: - # TODO: A cleaner way to implement this could be to specify regex patterns for layers to be patched, or just specify them as a list - # This should simplify model definitions, as well as patched re-creations here - if (type(module) in xbtorch.layer_types): - if (type(module)) == nn.Linear: - args = (module.in_features, module.out_features, module.bias is not None) - xbnn_layer = xbnn.Linear(*args) - elif (type(module)) == nn.Conv2d: - args = (module.in_channels, module.out_channels, module.kernel_size, module.stride, module.padding, module.dilation, module.groups, module.bias is not None, module.padding_mode) - xbnn_layer = xbnn.Conv2d(*args) - - elif (type(module)) == nn.RNN: - args = (module.input_size, module.hidden_size, module.num_layers, module.nonlinearity, module.bias, module.batch_first, module.dropout, module.bidirectional) - xbnn_layer = xbnn.RNN(*args) - - elif (type(module)) == nn.LSTM: - args = (module.input_size, module.hidden_size, module.num_layers, module.bias, module.batch_first, module.dropout, module.bidirectional, module.proj_size) - xbnn_layer = xbnn.LSTM(*args) - else: - raise ValueError(f"An xbtorch supported layer, {type(module)}, is missing an implementation.") - - # move to device before loading weights - xbnn_layer = xbnn_layer.to(device) - xbnn_layer.load_state_dict(module.state_dict()) - xbnn_layer._array_mappings = {} # we add this to make initialization easier later, since pre-trained weights would be loaded after patching - new_model.append(xbnn_layer) - - # activations - elif (type(module) in xbtorch.activation_types): - new_model.append(module) - if (wage_quantize): new_model.append(quantizer_act_error(wl_activation, wl_error)) - - elif (type(module) in xbtorch.misc_types): - new_model.append(module) - - # else copy unpatched module, but notify user - else: - print(f'XBPatching for module {module} is not defined, using as is.') - new_model.append(module) - # exit() - - if (wage_quantize and new_model[-1] not in xbtorch.activation_types): - # add act/error quantizer if the last module is an activation - new_model.append(quantizer_act_error(-1, wl_error)) - - original_model.model = nn.Sequential(*new_model) - - if (wage_quantize): - # wage parameters - original_model.weight_scale = {} - original_model.weight_acc = {} - for name, param in original_model.named_parameters(): - if ("weight" in name): wage_init.wage_init_(param, wl_weight, factor=1.0) - param.weight_acc = param.data - - # print('Patched XBTorch Model', original_model.model) + # How to replace layers? stateless or stateful. + if (replace_all): + # stateless + replace_all_layers_stateless(model=original_model, + exclude=[] if exclude is None else exclude) + else: + # stateful + replace_all_layers_stateful(model=original_model, wage_quantize=wage_quantize) if (xb_inference_accelerator): # if initialization included an inference accelerator def toggle(enable=True): original_model.xb_forward = enable - for module in original_model.model: - module._xb_inference = enable + if not replace_all: + # stateless + for module in original_model.model: + module._xb_inference = enable + else: + toggle_xb_eval_all_layers_stateless(original_model, enable=enable) def initialize_array_mappings(output_polling_mode='avg', existing_mappings=[], additional_args={}): + + if replace_all or not xb_inference_accelerator.stateful: + raise ValueError("Can not map to array in stateless operation.") + # return + # existing_mappings can be used as a reference to avoid conflicting mappings across unique models on the xb (primary use case: committee machines) # reset/initialize mappings of this layer on the simulated crossbar original_model._array_mappings_all = existing_mappings @@ -177,12 +111,12 @@ def initialize_array_mappings(output_polling_mode='avg', existing_mappings=[], a # get indices where the conductance matrices will be mapped on the simulated xbar # xb_mapping_schemes = ['random', 'layer_ensemble'] etc. pos_idxs = xb_inference_accelerator.xb_mapping_scheme(accelerator=xb_inference_accelerator, - layer_shape=sw_weight.shape, - current_mappings=original_model._array_mappings_all) + layer_shape=sw_weight.shape, + current_mappings=original_model._array_mappings_all) neg_idxs = xb_inference_accelerator.xb_mapping_scheme(accelerator=xb_inference_accelerator, - layer_shape=sw_weight.shape, - current_mappings=original_model._array_mappings_all) + layer_shape=sw_weight.shape, + current_mappings=original_model._array_mappings_all) # map the sw_weight matrix to the simulated array as device conductances at indices extracted above # internally handles conversion of the sw_weight matrix to conductance matrices diff --git a/src/xbtorch/patches/utils.py b/src/xbtorch/patches/utils.py new file mode 100644 index 0000000..e328c45 --- /dev/null +++ b/src/xbtorch/patches/utils.py @@ -0,0 +1,127 @@ +import xbtorch.quant.wage_init as wage_init +import torch.nn as nn +import xbtorch.nn as xbnn +import xbtorch +import transformers + +from xbtorch import get_xbtorch_param + +def replace_all_layers_stateless(model: transformers.models, + exclude: list=[]) -> None: + """ + Recursively replace all modules within a model with XBTorch modules for stateless operation. + """ + + for name, module in model.named_children(): + + # this is not a nested exclusion + if exclude and name in exclude: + continue + + # TODO: could be a simple map between nn vs. xbnn modules, removing the elifs + if isinstance(module, nn.Linear): + args = (module.in_features, module.out_features, module.bias is not None) + xbnn_layer = xbnn.Linear(*args).to(next(module.parameters()).device) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._xb_inference = False # add attribute for xb evaluation for proper toggling later + setattr(model, name, xbnn_layer) + elif (type(module)) == nn.Conv2d: + args = (module.in_channels, module.out_channels, module.kernel_size, module.stride, module.padding, module.dilation, module.groups, module.bias is not None, module.padding_mode) + xbnn_layer = xbnn.Conv2d(*args) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._xb_inference = False # add attribute for xb evaluation for proper toggling later + setattr(model, name, xbnn_layer) + elif (type(module)) == nn.RNN: + args = (module.input_size, module.hidden_size, module.num_layers, module.nonlinearity, module.bias, module.batch_first, module.dropout, module.bidirectional) + xbnn_layer = xbnn.RNN(*args) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._xb_inference = False # add attribute for xb evaluation for proper toggling later + setattr(model, name, xbnn_layer) + elif (type(module)) == nn.LSTM: + args = (module.input_size, module.hidden_size, module.num_layers, module.bias, module.batch_first, module.dropout, module.bidirectional, module.proj_size) + xbnn_layer = xbnn.LSTM(*args) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._xb_inference = False # add attribute for xb evaluation for proper toggling later + setattr(model, name, xbnn_layer) + else: + replace_all_layers_stateless(model=module, exclude=exclude) + +def replace_all_layers_stateful(model, wage_quantize): + """ + Recursively replace all modules within a model with XBTorch modules for stateful operation. + Supports wage quantization. + """ + new_model = [] + if (wage_quantize): + + wage_params = get_xbtorch_param('wage_params') + quantizer_act_error = wage_params['quantizer_act_error'] + wl_activation = wage_params['wl_activation'] + wl_error = wage_params['wl_error'] + wl_weight = wage_params['wl_weight'] + + new_model.append(quantizer_act_error(wl_activation, -1)) + + for module in model.model: + if (type(module) in xbtorch.layer_types): + if (type(module)) == nn.Linear: + args = (module.in_features, module.out_features, module.bias is not None) + xbnn_layer = xbnn.Linear(*args) + elif (type(module)) == nn.Conv2d: + args = (module.in_channels, module.out_channels, module.kernel_size, module.stride, module.padding, module.dilation, module.groups, module.bias is not None, module.padding_mode) + xbnn_layer = xbnn.Conv2d(*args) + + elif (type(module)) == nn.RNN: + args = (module.input_size, module.hidden_size, module.num_layers, module.nonlinearity, module.bias, module.batch_first, module.dropout, module.bidirectional) + xbnn_layer = xbnn.RNN(*args) + + elif (type(module)) == nn.LSTM: + args = (module.input_size, module.hidden_size, module.num_layers, module.bias, module.batch_first, module.dropout, module.bidirectional, module.proj_size) + xbnn_layer = xbnn.LSTM(*args) + else: + raise ValueError(f"An xbtorch supported layer, {type(module)}, is missing an implementation.") + + # move to device before loading weights + xbnn_layer = xbnn_layer.to(next(module.parameters()).device) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._array_mappings = {} # we add this to make initialization easier later, since pre-trained weights would be loaded after patching + new_model.append(xbnn_layer) + + # activations + elif (type(module) in xbtorch.activation_types): + new_model.append(module) + if (wage_quantize): new_model.append(quantizer_act_error(wl_activation, wl_error)) + + elif (type(module) in xbtorch.misc_types): + new_model.append(module) + + # else copy unpatched module, but notify user + else: + print(f'XBPatching for module {module} is not defined, using as is.') + new_model.append(module) + + if (wage_quantize and new_model[-1] not in xbtorch.activation_types): + # add act/error quantizer if the last module is an activation + new_model.append(quantizer_act_error(-1, wl_error)) + + if (hasattr(model, 'model')): + model.model = nn.Sequential(*new_model) + + if (wage_quantize): + # wage parameters + model.weight_scale = {} + model.weight_acc = {} + for name, param in model.named_parameters(): + if ("weight" in name): wage_init.wage_init_(param, wl_weight, factor=1.0) + param.weight_acc = param.data + +def toggle_xb_eval_all_layers_stateless(model, enable): + """ + Recursively enable or disable linear modules within a model with some customized layer module. + """ + for _, module in model.named_children(): + if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d) or isinstance(module, nn.RNN) or isinstance(module, nn.LSTM): + module._xb_inference = enable # add attribute for xb evaluation for proper toggling later + # add elifs for other supported layers + else: + toggle_xb_eval_all_layers_stateless(module, enable) diff --git a/tests/conftest.py b/tests/conftest.py index 9010e39..6284f6c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,22 @@ def forward(self, x): x = x.view(-1, self.input_size) # Flatten the image x = self.model(x) return x + +class SimpleMLPNoModel(nn.Module): + def __init__(self, input_size, hidden_size, output_size): + self.input_size = input_size + super(SimpleMLPNoModel, self).__init__() + self.fc1 = nn.Linear(input_size, hidden_size, bias=False) + self.fc2 = nn.Linear(hidden_size, output_size, bias=False) + self.a1 = nn.ReLU() + def forward(self, x): + x = x.view(-1, self.input_size) # Flatten the image + x = self.fc1(x) + x = self.a1(x) + x = self.fc2(x) + return x + # ----------------------- # General Utilities # ----------------------- @@ -87,6 +102,15 @@ def mlp_model(device): model = SimpleMLP(input_size, hidden_size, output_size).to(device) return model +@pytest.fixture +def mlp_model_regular(device): + """Return a fresh 2-layer MLP patched for XBTorch.""" + input_size = 28*28 + hidden_size = 150 + output_size = 10 + model = SimpleMLPNoModel(input_size, hidden_size, output_size).to(device) + return model + # ----------------------- # XBTorch Accelerator Fixtures # ----------------------- diff --git a/tests/test_core.py b/tests/test_core.py index 5a1982c..93e1644 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -19,6 +19,9 @@ def test_device_initialization(simple_fixedpoint_accelerator): def test_model_patching(mlp_model, simple_fixedpoint_accelerator): """Test that xbtorch_model patches models correctly.""" + + # Stateful + # Model without HWA enabled model = mlp_model xbtorch.initialize() # baseline initialization diff --git a/tests/test_hwa_inference.py b/tests/test_hwa_inference.py index fc40f7a..40fb02a 100644 --- a/tests/test_hwa_inference.py +++ b/tests/test_hwa_inference.py @@ -1,4 +1,7 @@ import torch +from xbtorch.deployment import SimpleFixedPoint +import xbtorch +from xbtorch.patches import xbtorch_model def test_xb_eval_toggle(model_and_acc): model, _, _ = model_and_acc @@ -21,4 +24,23 @@ def test_initialize_array_mappings(model_and_acc): def test_plot_array_returns_tensor(model_and_acc): _, acc, _ = model_and_acc arr = acc.plot_array() - assert isinstance(arr, torch.Tensor) \ No newline at end of file + assert isinstance(arr, torch.Tensor) + +def test_stateless_accelerator(mlp_model_regular): + device = "cpu" + + acc = SimpleFixedPoint( + g_min=100, + g_max=200, + device=device, + stateful=False + ) + + xbtorch.initialize(pytorch_device=device, inference_accelerator=acc) + + model = mlp_model_regular.to(device) + model = xbtorch_model(model, replace_all=True) + + model.xb_eval(enable=False) + + return model, acc, device \ No newline at end of file