diff --git a/examples/11_hwa_lm_eval.ipynb b/examples/11_hwa_lm_eval.ipynb new file mode 100644 index 0000000..901c762 --- /dev/null +++ b/examples/11_hwa_lm_eval.ipynb @@ -0,0 +1,470 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "547ae66e-5da6-43bb-8bb5-7ebdee8697ac", + "metadata": {}, + "source": [ + "# XBTorch::Example 09: Hardware-Aware Language Model Evaluation\n", + "\n", + "## Introduction\n", + "\n", + "In this example, we demonstrate how to use **XBTorch** to evaluate large language models (LLMs)\n", + "under simulated hardware noise and quantization effects.\n", + "This notebook uses the [`lm_eval`](https://github.com/EleutherAI/lm-evaluation-harness)\n", + "framework [1] for standardized evaluation, while wrapping the model with XBTorch's inference accelerator.\n", + "\n", + "Contrary to Example 07 on hardware-aware inference, here we will use the stateless mode of operation for the inference accelerator. This is because maintaining a stateful representation of a crossbar that is large enough to fit all parameters of an LLM would be cost-prohibitive.\n", + "\n", + "This enables researchers to **quantify degradation in performance** due to things like ADC/DAC bit precision limits\n", + "and analog crossbar variability.\n", + "\n", + "---\n", + "**Key Concepts**\n", + "- Integrating XBTorch into transformer model evaluation\n", + "- Simulating low-precision analog inference with `SimpleFixedPoint`\n", + "- Measuring task accuracy using `lm_eval`\n", + "\n", + "**We’ll demonstrate this using a Hugging Face model on the PIQA benchmark.**\n", + "---\n", + "\n", + "## Getting Started\n", + "\n", + "Let's begin by importing the required dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "4e143fb6-baec-4ad0-bc47-b71d390ed508", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/mnt/osama.yousuf1/xbtorch/env/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "W1010 23:09:48.035000 2851489 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. \n", + "W1010 23:09:48.035000 2851489 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures.\n" + ] + } + ], + "source": [ + "# General imports\n", + "import torch\n", + "import numpy as np\n", + "from transformers import AutoTokenizer, AutoModelForCausalLM\n", + "\n", + "# LM Evaluation Harness imports\n", + "from lm_eval import evaluator\n", + "from lm_eval.models.huggingface import HFLM\n", + "\n", + "# XBTorch imports\n", + "import xbtorch\n", + "from xbtorch.patches import xbtorch_model\n", + "from xbtorch.deployment import SimpleFixedPoint" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "200c8554-103a-45c5-91dc-944231f52be1", + "metadata": {}, + "outputs": [], + "source": [ + "def noisy_lm_evaluate(\n", + " model_path,\n", + " eval_tasks,\n", + " acc=None,\n", + " eval_limit=None,\n", + " num_fewshot=0,\n", + " exclude=None,\n", + " batch_size=32,\n", + " **config_kwargs\n", + "):\n", + " \"\"\"\n", + " Evaluate a Hugging Face causal LM under simulated noisy inference conditions\n", + " using XBTorch's SimpleFixedPoint accelerator and lm-evaluation-harness.\n", + "\n", + " Parameters\n", + " ----------\n", + " model_path : str\n", + " Path or Hugging Face Hub ID of the pretrained model.\n", + " eval_tasks : list[str]\n", + " Evaluation tasks (e.g., [\"piqa\", \"hellaswag\"]).\n", + " eval_limit : int, optional\n", + " Number of examples to limit evaluation to.\n", + " num_fewshot : int, optional\n", + " Number of few-shot examples per evaluation (default: 0).\n", + " exclude : list[str], optional\n", + " Layer names to exclude from XBTorch patching (default: [\"lm_head\"]).\n", + " batch_size : int, optional\n", + " Batch size used during evaluation.\n", + " config_kwargs : dict\n", + " Extra keyword arguments forwarded to model loading.\n", + "\n", + " Returns\n", + " -------\n", + " dict\n", + " A dictionary containing task results (accuracy, perplexity, etc.)\n", + " \"\"\"\n", + " if exclude is None:\n", + " exclude = []\n", + "\n", + " device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n", + "\n", + " # --- Step 1: Initialize inference accelerator ---\n", + " if (acc is None):\n", + " acc = SimpleFixedPoint(\n", + " g_min=100,\n", + " g_max=200,\n", + " adc_bits=8,\n", + " dac_bits=8,\n", + " device=device,\n", + " stateful=False # This is critical! LLMs can be too big for the stateful mode of operation.\n", + " )\n", + "\n", + " xbtorch.initialize(\n", + " inference_accelerator=acc,\n", + " pytorch_device=device\n", + " )\n", + "\n", + " # --- Step 2: Load pretrained model ---\n", + " default_config_kwargs = {\n", + " \"device_map\": device,\n", + " \"trust_remote_code\": True,\n", + " }\n", + " config_kwargs = {**default_config_kwargs, **config_kwargs}\n", + "\n", + " model = AutoModelForCausalLM.from_pretrained(model_path, **config_kwargs)\n", + " model = xbtorch_model(model, replace_all=True, exclude=exclude)\n", + " model.xb_eval(enable=True)\n", + "\n", + " tokenizer = AutoTokenizer.from_pretrained(model_path)\n", + "\n", + " # --- Step 3: Prepare evaluation harness ---\n", + " patched_lm_eval_model = HFLM(\n", + " pretrained=model,\n", + " tokenizer=tokenizer,\n", + " batch_size=batch_size\n", + " )\n", + "\n", + " # --- Step 4: Run evaluation ---\n", + " patched_results = evaluator.simple_evaluate(\n", + " model=patched_lm_eval_model,\n", + " tasks=eval_tasks,\n", + " num_fewshot=num_fewshot,\n", + " limit=eval_limit\n", + " )\n", + "\n", + " return patched_results[\"results\"]" + ] + }, + { + "cell_type": "markdown", + "id": "542a5e73-6be4-4d07-a7e0-0121699b694e", + "metadata": {}, + "source": [ + "## Hardware-Aware Evaluation Example\n", + "\n", + "Let’s now apply this function to a transformer model on the **PIQA** task. We will use [SpectraSuite's TriLM models](https://huggingface.co/SpectraSuite) [2] for this purpose.\n", + "You can substitute any Hugging Face model and benchmark supported by `lm_eval`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "342c7092-7bba-470a-9ec6-c36ee65aa28e", + "metadata": {}, + "outputs": [], + "source": [ + "eval_tasks = [\"piqa\"] # Task(s) to evaluate\n", + "num_fewshot = 0 # Zero-shot evaluation\n", + "eval_limit = 200#None # Limit evaluation samples for speed\n", + "batch_size = 40\n", + "\n", + "models = [\n", + " \"SpectraSuite/TriLM_830M_Unpacked\", \n", + " \"SpectraSuite/TriLM_3.9B_Unpacked\"\n", + " ]\n", + "\n", + "# Optionally, you can define a sweep for ADC/DAC precision bits\n", + "layer_sweep_params = {\n", + " \"adc_dac_bits\": [4, 8, 16]\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "f5866a73-83cd-4b7a-90b7-6bbcd17cdddc", + "metadata": {}, + "source": [ + "## Performing the Evaluation\n", + "\n", + "We can now loop over models (and optionally over ADC/DAC precision values)\n", + "to observe how quantization affects performance." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "445e2679-9002-45bd-b6b0-f1db068ee249", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluating model: SpectraSuite/TriLM_830M_Unpacked\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1158.99it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:33<00:00, 11.92it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'piqa': {'alias': 'piqa', 'acc,none': 0.635, 'acc_stderr,none': 0.03412767927155776, 'acc_norm,none': 0.675, 'acc_norm_stderr,none': 0.033202212797844806}}\n", + "Evaluating model: SpectraSuite/TriLM_3.9B_Unpacked\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████| 2/2 [00:10<00:00, 5.37s/it]\n", + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1153.23it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:45<00:00, 8.71it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'piqa': {'alias': 'piqa', 'acc,none': 0.67, 'acc_stderr,none': 0.03333249580187342, 'acc_norm,none': 0.695, 'acc_norm_stderr,none': 0.03263741725420572}}\n" + ] + } + ], + "source": [ + "results_summary = {}\n", + "\n", + "for model_path in models:\n", + " print(f\"Evaluating model: {model_path}\")\n", + " result = noisy_lm_evaluate(\n", + " model_path=model_path,\n", + " eval_limit=eval_limit,\n", + " batch_size=batch_size,\n", + " num_fewshot=num_fewshot,\n", + " eval_tasks=eval_tasks,\n", + " exclude=[\"lm_head\"]\n", + " )\n", + "\n", + " results_summary[model_path] = result\n", + " print(result)" + ] + }, + { + "cell_type": "markdown", + "id": "bde5edfd-75e2-4c51-a299-23c78daaa885", + "metadata": {}, + "source": [ + "## (Optional) Parameter Sweep\n", + "\n", + "To simulate progressively more aggressive quantization,\n", + "we can vary the ADC/DAC bit precision of the inference accelerator.\n", + "\n", + "Note: this section can be time-consuming for large models." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f8500d83-4432-49db-8ecf-4bf0b35418ec", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Testing with ADC/DAC precision = 4 bits\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1165.72it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:07<00:00, 51.70it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Testing with ADC/DAC precision = 8 bits\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1186.42it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:06<00:00, 62.97it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Testing with ADC/DAC precision = 16 bits\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.\n", + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration\n", + "Overwriting default num_fewshot of piqa from None to 0\n", + "100%|███████████████████████████████████████████████████████████████████████████████| 200/200 [00:00<00:00, 1193.11it/s]\n", + "Running loglikelihood requests: 100%|█████████████████████████████████████████████████| 400/400 [00:11<00:00, 34.24it/s]\n" + ] + } + ], + "source": [ + "sweep_results = {}\n", + "\n", + "for bits in layer_sweep_params[\"adc_dac_bits\"]:\n", + " print(f\"Testing with ADC/DAC precision = {bits} bits\")\n", + "\n", + " acc = SimpleFixedPoint(\n", + " g_min=100,\n", + " g_max=200,\n", + " adc_bits=bits,\n", + " dac_bits=bits,\n", + " device=\"cuda\" if torch.cuda.is_available() else \"cpu\",\n", + " stateful=False\n", + " )\n", + "\n", + " xbtorch.initialize(inference_accelerator=acc)\n", + "\n", + " result = noisy_lm_evaluate(\n", + " model_path=models[0], # using the 830M model for this experiment\n", + " acc=acc,\n", + " eval_limit=eval_limit,\n", + " batch_size=batch_size,\n", + " num_fewshot=num_fewshot,\n", + " eval_tasks=eval_tasks,\n", + " exclude=[\"lm_head\"]\n", + " )\n", + "\n", + " sweep_results[bits] = result" + ] + }, + { + "cell_type": "markdown", + "id": "05722f0c-8349-4f5b-8d0a-230126bc8a29", + "metadata": {}, + "source": [ + "## Visualization\n", + "\n", + "Let's visualize the effect of ADC/DAC bit precision on task accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "848f07b0-3fca-4119-989b-975d70b30f97", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm4AAAGJCAYAAAAzAb+0AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAajxJREFUeJzt3XlYVOXbB/DvDDvIvoNs4oK4AIKSuyYuaS6pueWaqbkvZWa9rpmWlZqamqVZuWtuWZpKmvpTMcF9QVQElUWRVRAYZp73D2RynEFBB4aB7+e6uGrOPOec+zwOcHPOue8jEUIIEBEREVGFJ9V1AERERERUMkzciIiIiPQEEzciIiIiPcHEjYiIiEhPMHEjIiIi0hNM3IiIiIj0BBM3IiIiIj3BxI2IiIhITzBxIyIiItITTNyIqiCJRILZs2eXer3bt29DIpFg3bp1Wo+J6Fne3t4YOnSorsOodNq0aYM2bdqUap1169ZBIpHg9u3bZRITlRwTN3quom/WM2fO6DqUSqdobiUSCY4fP672vhACHh4ekEgkePPNN3UQoXb8+eefkEgkcHNzg0Kh0HU49AISiQTjxo3T+B5/Hryc2bNnK7/XJRIJzM3N4e/vj//7v/9DZmamrsMjPcPEjUjHTE1NsXHjRrXl//zzD+7evQsTExMdRKU9GzZsgLe3NxITE/H333/rOhwinVm5ciV+/fVXLFq0CH5+fvj888/RqVMnlPcjww8cOIADBw6Uap1Bgwbh8ePH8PLyKqOoqKSYuBHpWOfOnbFt2zYUFBSoLN+4cSOCg4Ph4uKio8heXXZ2Nnbv3o0pU6YgKCgIGzZs0HVIxcrOztZ1CFWOPsy5NmPs3bs3Bg4ciPfffx87duxAz549cfLkSZw6darYdXJycrS2/yLGxsYwNjYu1ToGBgYwNTWFRCLRejxUOkzc6JXl5+dj5syZCA4OhrW1NSwsLNCyZUscPnxYZVzR/VFff/01Vq9eDV9fX5iYmKBx48b4999/1ba7bds2+Pv7w9TUFPXr18fOnTsxdOhQeHt7K8ccOXIEEokER44c0bivp+/FunDhAoYOHYoaNWrA1NQULi4uePfdd/Hw4UO1fR85cgQhISEwNTWFr68vvv/+e+XljmetX78ewcHBMDMzg52dHfr164c7d+6UeP769++Phw8f4uDBg8pl+fn52L59OwYMGKBxnezsbHzwwQfw8PCAiYkJ6tSpg6+//lrtL/e8vDxMnjwZjo6OsLS0RLdu3XD37l2N27x37x7effddODs7w8TEBPXq1cPatWtLfBya7Ny5E48fP8bbb7+Nfv36YceOHcjNzVUbl5ubi9mzZ6N27dowNTWFq6srevbsiZs3byrHKBQKfPvtt2jQoAFMTU3h6OiITp06KS/bPe/+u2fv6Sv6t7xy5QoGDBgAW1tbtGjRAkDpPif37t3D8OHD4ebmBhMTE/j4+GD06NHIz8/HrVu3IJFIsHjxYrX1Tpw4AYlEgk2bNmmct+TkZBgaGmLOnDlq70VHR0MikWD58uUAAJlMhjlz5qBWrVowNTWFvb09WrRoofJ5Kkslna/nzbkQAvPmzUP16tVhbm6Otm3b4vLlyyrrp6enw8DAAEuXLlUuS0lJgVQqhb29vcpnf/To0Sp/8Bw7dgxvv/02PD09YWJiAg8PD0yePBmPHz9W2cfQoUNRrVo13Lx5E507d4alpSXeeecdAIWfvyVLlqBevXowNTWFs7MzRo0ahbS0tJeeu9dffx0AEBsbC6Dw3rP69esjMjISrVq1grm5OT755BMAhd/Ls2bNQs2aNZXH8NFHHyEvL09tu+vXr0eTJk1gbm4OW1tbtGrVSuUMm6Z73JYtW4Z69eop1wkJCVG5ElDcPW4rVqxAvXr1YGJiAjc3N4wdOxbp6ekqY4qO68qVK2jbti3Mzc3h7u6OhQsXvuzUVWmGug6A9F9mZiZ+/PFH9O/fHyNGjEBWVhbWrFmDjh074vTp0wgMDFQZv3HjRmRlZWHUqFGQSCRYuHAhevbsiVu3bsHIyAgA8Mcff6Bv375o0KABFixYgLS0NAwfPhzu7u4vHefBgwdx69YtDBs2DC4uLrh8+TJWr16Ny5cv49SpU8qk7OzZs+jUqRNcXV0xZ84cyOVyzJ07F46Ojmrb/PzzzzFjxgz06dMH7733Hh48eIBly5ahVatWOHv2LGxsbF4Yl7e3N5o2bYpNmzbhjTfeAADs27cPGRkZ6Nevn8ovKqDwl1y3bt1w+PBhDB8+HIGBgfjrr78wdepU3Lt3TyVReO+997B+/XoMGDAAzZo1w99//40uXbqoxZCcnIzXXntNeX+To6Mj9u3bh+HDhyMzMxOTJk0qxUz/Z8OGDWjbti1cXFzQr18/fPzxx/j999/x9ttvK8fI5XK8+eabCA8PR79+/TBx4kRkZWXh4MGDuHTpEnx9fQEAw4cPx7p16/DGG2/gvffeQ0FBAY4dO4ZTp04hJCTkpeJ7++23UatWLcyfP1/5i7+kn5OEhAQ0adIE6enpGDlyJPz8/HDv3j1s374dOTk5qFGjBpo3b44NGzZg8uTJavNiaWmJ7t27a4zL2dkZrVu3xtatWzFr1iyV97Zs2QIDAwPlHM6ePRsLFizAe++9hyZNmiAzMxNnzpxBVFQU2rdv/1Lzkpubi5SUFLXljx49UltW0vkqomnOZ86ciXnz5qFz587o3LkzoqKi0KFDB+Tn5yvXs7GxQf369XH06FFMmDABAHD8+HFIJBKkpqbiypUrqFevHoDCRK1ly5bKdbdt24acnByMHj0a9vb2OH36NJYtW4a7d+9i27ZtKvEVFBSgY8eOaNGiBb7++muYm5sDAEaNGoV169Zh2LBhmDBhAmJjY7F8+XKcPXsW//vf/5Q/u0qj6A8Te3t75bKHDx/ijTfeQL9+/TBw4EA4OztDoVCgW7duOH78OEaOHIm6devi4sWLWLx4Ma5fv45du3Yp158zZw5mz56NZs2aYe7cuTA2NkZERAT+/vtvdOjQQWMcP/zwAyZMmIDevXtj4sSJyM3NxYULFxAREVHsH49A4Wdvzpw5CAsLw+jRoxEdHY2VK1fi33//VZuTtLQ0dOrUCT179kSfPn2wfft2TJs2DQ0aNFD+3KMSEkTP8dNPPwkA4t9//y12TEFBgcjLy1NZlpaWJpydncW7776rXBYbGysACHt7e5Gamqpcvnv3bgFA/P7778plDRo0ENWrVxdZWVnKZUeOHBEAhJeXl3LZ4cOHBQBx+PBhlf0X7eunn35SLsvJyVGLfdOmTQKAOHr0qHJZ165dhbm5ubh3755yWUxMjDA0NBRPf8vcvn1bGBgYiM8//1xlmxcvXhSGhoZqy5/19NwuX75cWFpaKmN8++23Rdu2bYUQQnh5eYkuXboo19u1a5cAIObNm6eyvd69ewuJRCJu3LghhBDi3LlzAoAYM2aMyrgBAwYIAGLWrFnKZcOHDxeurq4iJSVFZWy/fv2EtbW1Mi5N81qc5ORkYWhoKH744QflsmbNmonu3burjFu7dq0AIBYtWqS2DYVCIYQQ4u+//xYAxIQJE4od87zYnj3eWbNmCQCif//+amNL+jkZPHiwkEqlGr83imL6/vvvBQBx9epV5Xv5+fnCwcFBDBkyRG29pxWte/HiRZXl/v7+4vXXX1e+DggIUPl8vCoAL/x6+phLOl/Fzfn9+/eFsbGx6NKli3LehBDik08+EQBU5mns2LHC2dlZ+XrKlCmiVatWwsnJSaxcuVIIIcTDhw+FRCIR33777XNjXLBggZBIJCIuLk65bMiQIQKA+Pjjj1XGHjt2TAAQGzZsUFm+f/9+jcufVXTs0dHR4sGDByI2NlZ8//33wsTERDg7O4vs7GwhhBCtW7cWAMSqVatU1v/111+FVCoVx44dU1m+atUqAUD873//E0IU/pySSqXirbfeEnK5XGXs03PbunVr0bp1a+Xr7t27i3r16j33GIp+XsXGxgoh/vt369Chg8q+li9fLgCItWvXquwPgPjll1+Uy/Ly8oSLi4vo1avXc/dL6niplF6ZgYGB8n4JhUKB1NRUFBQUICQkBFFRUWrj+/btC1tbW+Xror+Mb926BaDwTMbFixcxePBgVKtWTTmudevWaNCgwUvHaWZmpvz/ojMKr732GgAo45TL5Th06BB69OgBNzc35fiaNWuq/VW4Y8cOKBQK9OnTBykpKcovFxcX1KpVS+1S8fP06dMHjx8/xt69e5GVlYW9e/cW+5fun3/+CQMDA+VZhyIffPABhBDYt2+fchwAtXHPnj0TQuC3335D165dIYRQOZaOHTsiIyND47/ji2zevBlSqRS9evVSLuvfvz/27duncnnpt99+g4ODA8aPH6+2jaKzNb/99hskEona2aenx7yM999/X21ZST4nCoUCu3btQteuXTWe7SuKqU+fPjA1NVW5t++vv/5CSkoKBg4c+NzYevbsCUNDQ2zZskW57NKlS7hy5Qr69u2rXGZjY4PLly8jJiamJIdcIt27d8fBgwfVvqZOnao2tiTz9bRn5/zQoUPIz8/H+PHjVf4tNZ3lbdmyJZKTkxEdHQ2g8Mxaq1at0LJlSxw7dgxA4Vk4IYTKGbenY8zOzkZKSgqaNWsGIQTOnj2rtp/Ro0ervN62bRusra3Rvn17le+P4OBgVKtWrcTf63Xq1IGjoyN8fHwwatQo1KxZE3/88YfyrB4AmJiYYNiwYWr7r1u3Lvz8/FT2X3SptWj/u3btgkKhwMyZMyGVqv56f973iY2NDe7evavxlpXiFP27TZo0SWVfI0aMgJWVFf744w+V8dWqVVP5zBsbG6NJkybKn/tUcrxUSlrx888/45tvvsG1a9cgk8mUy318fNTGenp6qrwuSuKKfpnHxcUBKEyWnlWzZs2XSiIAIDU1FXPmzMHmzZtx//59lfcyMjIAAPfv38fjx4+L3ffTYmJiIIRArVq1NO6vNJdOHB0dERYWho0bNyInJwdyuRy9e/fWODYuLg5ubm6wtLRUWV63bl3l+0X/lUqlykuNRerUqaPy+sGDB0hPT8fq1auxevVqjft8dr5Koug+m4cPHyrvdwoKCkJ+fj62bduGkSNHAii8XFSnTh0YGhb/4+jmzZtwc3ODnZ1dqeN4Hk2fz5J8Th48eIDMzEzUr1//udu3sbFB165dsXHjRnz22WcACi+Turu7K3/pFsfBwQHt2rXD1q1bletu2bIFhoaG6Nmzp3Lc3Llz0b17d9SuXRv169dHp06dMGjQIDRs2PDFE1CM6tWrIywsTG25pvsjSzJfT3t2zos+r89+Hzk6Oqr8gQf890fesWPHUL16dZw9exbz5s2Do6Mjvv76a+V7VlZWCAgIUK4XHx+PmTNnYs+ePWr3pD0bo6GhIapXr66yLCYmBhkZGXByclI7HqDk3x+//fYbrKysYGRkhOrVq6t9bwKAu7u7WuFATEwMrl69qvF2jaf3f/PmTUilUvj7+5coniLTpk3DoUOH0KRJE9SsWRMdOnTAgAED0Lx582LXKfp3e/bnibGxMWrUqKF8v0j16tXVkkdbW1tcuHChVLESEzfSgvXr12Po0KHo0aMHpk6dCicnJxgYGGDBggUqN5cXMTAw0Lgd8RIl8cX9FSmXy9WW9enTBydOnMDUqVMRGBiIatWqQaFQoFOnTi/VX0yhUEAikWDfvn0aj+nps4UlMWDAAIwYMQJJSUl44403SnR/nDYUHfvAgQMxZMgQjWNKmwTExMQo/3rXlNhu2LBBmbhpS2k+C0WePhNTRNufk8GDB2Pbtm04ceIEGjRogD179mDMmDFqZ0Q06devH4YNG4Zz584hMDAQW7duRbt27eDg4KAc06pVK9y8eRO7d+/GgQMH8OOPP2Lx4sVYtWoV3nvvvVLHW1qlnS9Nc15Sbm5u8PHxwdGjR+Ht7Q0hBJo2bQpHR0dMnDgRcXFxOHbsGJo1a6acX7lcjvbt2yM1NRXTpk2Dn58fLCwscO/ePQwdOlQtRhMTE7V/G4VCAScnp2KrootLqJ7VqlUrlX87TTTNj0KhQIMGDbBo0SKN63h4eJRo/8WpW7cuoqOjsXfvXuzfvx+//fYbVqxYgZkzZ2oskHkZ2vy5X9UxcaNXtn37dtSoUQM7duxQ+eWp6bJWSRT1Cbpx44bae88uK/qL/Nkqpmf/2ktLS0N4eDjmzJmDmTNnKpc/e3nJyckJpqamJdq3r68vhBDw8fFB7dq1X3BUL/bWW29h1KhROHXqlMrlsWd5eXnh0KFDyMrKUjnrdu3aNeX7Rf9VKBTKM1pFii4zFSmqOJXL5RrPsryMDRs2wMjICL/++qvaD+zjx49j6dKliI+Ph6enJ3x9fREREQGZTFbsWUpfX1/89ddfSE1NLfasW0k/C89T0s+Jo6MjrKyscOnSpRdus1OnTnB0dMSGDRsQGhqKnJwcDBo0qETx9OjRA6NGjVJ+Hq5fv47p06erjbOzs8OwYcMwbNgwPHr0CK1atcLs2bPLPHEr6Xw9T9HnNSYmBjVq1FAuf/DggcaKzZYtW+Lo0aPw8fFBYGAgLC0tERAQAGtra+zfvx9RUVEqycbFixdx/fp1/Pzzzxg8eLByeWmqbn19fXHo0CE0b978lRLPl+Xr64vz58+jXbt2z73k6evrC4VCgStXrqgVhb2IhYUF+vbti759+yI/Px89e/bE559/junTp8PU1FRtfNG/W3R0tMq/W35+PmJjY7X2s4TU8R43emVFv5if/sspIiICJ0+efKntubm5oX79+vjll19Uqtj++ecfXLx4UWWsl5cXDAwMcPToUZXlK1aseGGMALBkyRK1cWFhYdi1axcSEhKUy2/cuKG8d6xIz549YWBggDlz5qhtVwihsX3E81SrVg0rV67E7Nmz0bVr12LHde7cGXK5XNkOosjixYshkUiU9+IV/ffZqlRNx9yrVy/89ttvGhORBw8elOo4gMLErWXLlujbty969+6t8lV0n1RRK4xevXohJSVF7XiA//69evXqBSGExr/+i8ZYWVnBwcHhhZ+F5ynp50QqlaJHjx74/fffNT5F4On1DQ0N0b9/f2zduhXr1q1DgwYNSnwG08bGBh07dsTWrVuxefNmGBsbo0ePHipjnv2cVatWDTVr1lRpE5GRkYFr165pvHT5Kko6X88TFhYGIyMjLFu2TGU7xW2jZcuWuH37NrZs2aK8dCqVStGsWTMsWrQIMplM5f42TTEKIfDtt9+WOMY+ffpALpcrL1k/raCgQO2PBW3r06cP7t27hx9++EHtvcePHyt7zfXo0QNSqRRz585VO5P4vDNbz36GjI2N4e/vDyGEyq0vTwsLC4OxsTGWLl2qsu01a9YgIyNDY/U6aQfPuFGJrF27Fvv371dbPnHiRLz55pvYsWMH3nrrLXTp0gWxsbFYtWoV/P39NbYPKIn58+eje/fuaN68OYYNG4a0tDQsX74c9evXV9mmtbU13n77bSxbtgwSiQS+vr7Yu3ev2j0nVlZWaNWqFRYuXAiZTAZ3d3ccOHBA2T/pabNnz8aBAwfQvHlzjB49Wpkk1a9fH+fOnVOO8/X1xbx58zB9+nTcvn0bPXr0gKWlJWJjY7Fz506MHDkSH374YamOu7hLlU/r2rUr2rZti08//RS3b99GQEAADhw4gN27d2PSpEnK+2YCAwPRv39/rFixAhkZGWjWrBnCw8M1nk384osvcPjwYYSGhmLEiBHw9/dHamoqoqKicOjQIaSmppb4GCIiInDjxo1iH5vk7u6ORo0aYcOGDZg2bRoGDx6MX375BVOmTMHp06fRsmVLZGdn49ChQxgzZgy6d++Otm3bYtCgQVi6dCliYmKUl+GOHTuGtm3bKvf13nvv4YsvvsB7772HkJAQHD16FNevXy9x7KX5nMyfPx8HDhxA69atlS0aEhMTsW3bNhw/flzlUvfgwYOxdOlSHD58GF9++WWJ4wEKi3kGDhyIFStWoGPHjmqX0P39/dGmTRsEBwfDzs4OZ86cwfbt21Xmf+fOnRg2bBh++uknrT77szTzVRxHR0d8+OGHWLBgAd5880107twZZ8+exb59+zReVixKyqKjozF//nzl8latWmHfvn3K3pBF/Pz84Ovriw8//BD37t2DlZUVfvvtt1L1X2vdujVGjRqFBQsW4Ny5c+jQoQOMjIwQExODbdu24dtvvy32nlRtGDRoELZu3Yr3338fhw8fRvPmzSGXy3Ht2jVs3boVf/31F0JCQlCzZk18+umn+Oyzz9CyZUv07NkTJiYm+Pfff+Hm5oYFCxZo3H6HDh3g4uKC5s2bw9nZGVevXsXy5cvRpUsXtXtpizg6OmL69OmYM2cOOnXqhG7duiE6OhorVqxA48aNX1h8Q6+gvMpXST8VlYAX93Xnzh2hUCjE/PnzhZeXlzAxMRFBQUFi7969YsiQISqtO4raNXz11Vdq+8Ez7RqEEGLz5s3Cz89PmJiYiPr164s9e/aIXr16CT8/P5VxDx48EL169RLm5ubC1tZWjBo1Sly6dEmtNcTdu3fFW2+9JWxsbIS1tbV4++23RUJCgsZ9h4eHi6CgIGFsbCx8fX3Fjz/+KD744ANhamqqFvtvv/0mWrRoISwsLISFhYXw8/MTY8eOFdHR0SWa2+e1WhFCvR2IEEJkZWWJyZMnCzc3N2FkZCRq1aolvvrqK5WSfyGEePz4sZgwYYKwt7cXFhYWomvXruLOnTsajzk5OVmMHTtWeHh4CCMjI+Hi4iLatWsnVq9erRxTknYg48ePFwDEzZs3ix0ze/ZsAUCcP39eCFHYruHTTz8VPj4+yn337t1bZRsFBQXiq6++En5+fsLY2Fg4OjqKN954Q0RGRirH5OTkiOHDhwtra2thaWkp+vTpI+7fv19sO5AHDx6oxVaaz0lcXJwYPHiwcHR0FCYmJqJGjRpi7Nixau1xhBCiXr16QiqVirt37xY7L5pkZmYKMzMzAUCsX79e7f158+aJJk2aCBsbG2FmZib8/PzE559/LvLz85Vjij5rJWnjAkCMHTtW43uaPrMlna/nzblcLhdz5swRrq6uwszMTLRp00ZcunRJeHl5aWyb4uTkJACI5ORk5bLjx48LAKJly5Zq469cuSLCwsJEtWrVhIODgxgxYoQ4f/682pwMGTJEWFhYFDs3q1evFsHBwcLMzExYWlqKBg0aiI8++kgkJCQUu86Ljv1prVu3LrYtR35+vvjyyy9FvXr1hImJibC1tRXBwcFizpw5IiMjQ2Xs2rVrRVBQkHJc69atxcGDB1X283Q7kO+//160atVK2NvbCxMTE+Hr6yumTp2qst1n24EUWb58ufDz8xNGRkbC2dlZjB49WqSlpZXouJ79HUElIxGCdwaS/ggMDISjo2O5dYV/Wo8ePbTedoGqjqCgINjZ2SE8PFzXoRCRHuM9blQhyWQytWd3HjlyBOfPn1d7VEtZePZRODExMfjzzz/LZd9U+Zw5cwbnzp1TuTmeiOhl8IwbVUi3b99GWFgYBg4cCDc3N1y7dg2rVq2CtbU1Ll26pPKImLLg6uqqfP5iXFwcVq5ciby8PJw9e7bYvm1Ez7p06RIiIyPxzTffICUlBbdu3dJYoUdEVFIsTqAKydbWFsHBwfjxxx/x4MEDWFhYoEuXLvjiiy/KPGkDCls4bNq0CUlJSTAxMUHTpk0xf/58Jm1UKtu3b8fcuXNRp04dbNq0iUkbEb2yCnHG7bvvvsNXX32FpKQkBAQEYNmyZWjSpInGsW3atME///yjtrxz587KR2wIITBr1iz88MMPSE9PR/PmzbFy5Ur+0iUiIiK9pvN73LZs2YIpU6Zg1qxZiIqKQkBAADp27FjsI0R27NiBxMRE5delS5dgYGCAt99+Wzlm4cKFWLp0KVatWoWIiAhYWFigY8eOyM3NLa/DIiIiItI6nZ9xCw0NRePGjZXNNxUKBTw8PDB+/Hh8/PHHL1x/yZIlmDlzJhITE2FhYQEhBNzc3PDBBx8oe2hlZGTA2dkZ69atQ79+/cr0eIiIiIjKik7vccvPz0dkZKTKI1ykUinCwsJK3HV/zZo16NevHywsLAAAsbGxSEpKUnnchrW1NUJDQ3Hy5EmNiVteXp5Kl3GFQoHU1FTY29s/9/EiRERERNoghEBWVhbc3Nye+yxjnSZuKSkpkMvlcHZ2Vlnu7OysfO7i85w+fRqXLl3CmjVrlMuSkpKU23h2m0XvPWvBggVae5AuERER0cu6c+cOqlevXuz7el1VumbNGjRo0KDYQoaSmj59OqZMmaJ8nZGRAU9PT8TGxhb7uI9XJZPJcPjwYbRt27bYB2vTi3EetYPzqB2cR+3gPGoP51I7ymMes7Ky4OPj88K8Q6eJm4ODAwwMDJCcnKyyPDk5GS4uLs9dNzs7G5s3b8bcuXNVlhetl5ycDFdXV5VtBgYGatyWiYkJTExM1Jbb2dnBysqqJIdSajKZDObm5rC3t+c30yvgPGoH51E7OI/awXnUHs6ldpTHPBZt90W3aOm0qtTY2BjBwcEqj4BRKBQIDw9H06ZNn7vutm3bkJeXp/YgWx8fH7i4uKhsMzMzExERES/cJhEREVFFpvNLpVOmTMGQIUMQEhKCJk2aYMmSJcjOzsawYcMAAIMHD4a7uzsWLFigst6aNWvQo0cPtWasEokEkyZNwrx581CrVi34+PhgxowZcHNzQ48ePcrrsIiIiIi0TueJW9++ffHgwQPMnDkTSUlJCAwMxP79+5XFBfHx8WrVFdHR0Th+/DgOHDigcZsfffQRsrOzMXLkSKSnp6NFixbYv38/u5YTERGRXtN54gYA48aNw7hx4zS+d+TIEbVlderUwfPaz0kkEsydO1ft/jciIiIifabzJycQERERUckwcSMiIiLSE0zciIiIiIohVwhExKYiMkWCiNhUyBU6fVJoxbjHjYiIiKii2X8pEXN+v4LEjFwABvgl5gxcrU0xq6s/OtV3feH6ZYFn3IiIiIiesf9SIkavj3qStP0nKSMXo9dHYf+lRJ3ExcSNiIiI6ClyhcCc369A00XRomVzfr+ik8umTNyIiIiIAAghcDctB4sPRqudaVMZByAxIxenY1PLL7gneI8bERERVUkyuQJXEjIRGZem/ErKLD5he9b9rJKP1RYmbkRERFQlpOfkIyo+DWduFyZp5++mI1emUBljKJXAy94cNx9kv3B7Tpbl/0QmJm5ERERU6QghcCslu/BM2u00RMan4cb9R2rjbMyNEOxpi0ZetgjxskXD6jYwNpSixZd/IykjV+N9bhIALtamaOJjV+bH8SwmbkRERKT3cmVyXLib8eSSZyoi49KQliNTG1fD0QIhXrYI9rJFsJcdajhYQCqVqI2b1dUfo9dHQQKoJG+Sp9430LBeWWPiRkRERHrnflZu4Zm0uDSciUvD5YQMyOSq58dMDKUIqG6DYG9b5Vk1OwvjEm2/U31XrBzY6Kk+boVcdNzHjYkbERERVWhyhcD15CyciUtDVFwazsSl4k7qY7VxTpYmCPG2RSPPwjNq9dysYWz48g00OtV3RXt/F5y8cR8HjkWgQ8tQNK3ppJMzbUWYuBEREVGF8iivAOfi03HmySXPc/HpyMorUBkjlQB1XKyeuuxpi+q2ZpBItJtUGUglCPWxw8OrAqE+djpN2gAmbkRERKRDhb3THqtUe15LysSzvW2rmRgiyNMGjTxtEeJti0APG1iaGukmaB1i4kZERETlRiZX4LKyd1rhGbXkzDy1cR52Zgj2tEWwtx2CPW1Rx8VS52e7KgImbkRERFRm0rILe6cVFRFc0NA7zchAgnpu1gh+0pKjkZctnK3Kv0eaPmDiRkRERFohhMDNB9mIiitK1FI1NrK1NTdCsFdR7zQ7NKxuDVMjAx1ErH+YuBEREdFLyZXJcf5OOiLjC5vcRsVr7p3m62iBEC+7wiICb1vUcLDQehFBVcHEjYiIiErkfmYuzsQ91TvtXgYKFBp6p3nYKKs9G3nawraEvdPoxZi4ERERkRq5QiA6KUtZQHAmLg1304rvnRb85Iyav6vVK/VOo+dj4kZERETIypXh3J10nHlyyfNsfDoeaeid5udiVVhE8KTRbVn0TqPiMXEjIiKqYop6pxUVEETGpSP6Ob3Tgp8UEQR4WFfJ3mkVCRM3IiKiSi6/QIFLiWlPeqcVft3P0tw7LcTL7km1py1qO7N3WkXDxI2IiKiSScvOR2RcGv6NfYhDlwzw0b9/I69Ac++0px8Z5cTeaRUeEzciIiI9VtQ77ekiglsqvdMkABTK3mlFRQTsnaafmLgRERHpkcf5cly4m44zcWmFjW7j05CuoXdaTadqaORhDYP0eAzt0gq1Xa1ZRFAJMHEjIiKqwJIzc5UPX4+M19w7zdRIioDqNk/achRWe9qYG0Mmk+HPP+NQw5ENbysLJm5EREQVhFwhcC0pU1lAcOZ2Gu6lq/dOc7YyUSki8HezgpEBe6dVBUzciIiIdCQzV4Zz8f9d9jwbn4bsfLnKGKkEqOtqpSwgCPayhbsNe6dVVUzciIiIyoEQAndSHyMyPlV56TM6OQvimd5pliaGCPKyRbBnYZPbAA8bVDPhr2sqxE8CERFRGcgvUOBSQgainlzyjIxPwwMNvdM87cwR4mVbeNnT2xa1nNg7jYrHxI2IiEgLUp/0Tiv8SsX5uxnI19A7rb77f73TGnnZwsmSvdOo5Ji4ERERlZJCIXAr5dF/1Z5xabiVkq02zs7CGI2eXPIM9rJFA3f2TqNXw8SNiIjoBR7ny3H+broySYsqpndaLadqKkUEPg5sw0HaxcSNiIjoGUkZucoHsEfFpeFyQqbG3mmBHv89gD3I0wY25sY6ipiqCiZuRERUpRXIFbiWlIWo+DTlpU9NvdNcrEwR7P1ftWddV/ZOo/LHxI2IiKqUzFwZzsanK4sIzsWnF9s77b9qTzu4WZvysifpHBM3IiKqtIQQiE/Neara8/m904qqPQM9bGDB3mlUAfFTSURElUZegRyX7mUW9k6LS0VkXDpSHqn3TvOyN0ewp23hpU8vW9R2soSUvdNIDzBxIyIivfXwUZ7y4euRt9Nw4Z567zRjAynqu1shxNsOjTxt0cjLhr3TSG8xcSMiIr2gUAjcfPAIZ5667BmroXeavYWx8uHrwV62qM/eaVSJMHEjIqIKKSe/AFfiMxEZl/qkd1o6Mh6r906r7VzUO80OwV628LY3ZxEBVVpM3IiIqEJIzHiMyLg0nL71EIcvGmBKxGHIn+mdZmZkoOydFuxti0YetrA2N9JRxETlj4kbERGVu6LeaYVNbtMQpdY7TQJAwNXaVPkUghAvO/i5WrJ3GlVpTNyIiKjMZTyW4Wx82pNqzzScu5OOnGd6pxlIJajraokgDxtIH8ZiePe28HSw1FHERBUTEzciItKqot5pZ27/V+15/b6G3mmmhoUPYH9yRi3gSe80mUyGP/+8BVdrVn4SPYuJGxERvZKi3mlFRQTF9U7ztjd/Uu1ZWERQy6kae6cRlRITNyIiKpWUJ73Top605Ciud1qD6tbK+9MaedrC0dJERxETVR5M3IiIqFgKhcCNB4+UD1+PjEvF7Yc5auPsLYz/KyLwtkU9N/ZOIyoLTNyIiEgpJ78A5+6kI/LJ/WlRcWnIzC1QG1fYO83uSbWnLbzYO42oXDBxIyKqwhLSH6s8gP1KYmaxvdNCvG3RyIu904h0iYkbEVEVUSBX4GpiFiLjUpW90xIyctXGuVmbPvXIKDvUdbWEIXunEVUITNyIiCqpjMcyRBX1Trtd2DvtsUy9d5q/q5Xy/rRgL1u42ZjpKGIiehEmbkRElYAQAnEPc556AHsqYu4/UuudZmVqiEZetgj2LHxkVED1wt5pRKQf+N1KRKSHcmVyXE7IUFZ7RsWnIeVRvto4b3tzBHvZIcS78GxaTUf2TiPSZ0zciIj0wIOsPETF/1dEcPFuBvLl6r3TGj7dO83LFg7V2DuNqDLReeL23Xff4auvvkJSUhICAgKwbNkyNGnSpNjx6enp+PTTT7Fjxw6kpqbCy8sLS5YsQefOnQEAs2fPxpw5c1TWqVOnDq5du1amx0FEpC0KhUDM/UdPHsCeiqi4NI290xyqGT91b5od6rtbwcSQvdOIKjOdJm5btmzBlClTsGrVKoSGhmLJkiXo2LEjoqOj4eTkpDY+Pz8f7du3h5OTE7Zv3w53d3fExcXBxsZGZVy9evVw6NAh5WtDQ53np0RExcrOK8D5O+lPErXCy55Zz/ROk0iA2k6WCPYuvD8txNsWnnbsnUZU1eg0o1m0aBFGjBiBYcOGAQBWrVqFP/74A2vXrsXHH3+sNn7t2rVITU3FiRMnYGRU2EPI29tbbZyhoSFcXFzKNHYiopd170nvtKgnZ9SuJmap9U4zN37SO+3JJc8gT1tYm7F3GlFVp7PELT8/H5GRkZg+fbpymVQqRVhYGE6ePKlxnT179qBp06YYO3Ysdu/eDUdHRwwYMADTpk2DgcF/lwdiYmLg5uYGU1NTNG3aFAsWLICnp2exseTl5SEv778HImdmZgIAZDIZZDLZqx6qRkXbLavtVxWcR+3gPGqHpnmUyRWITnpU+BSC+HRExacjKVP9Aexu1qYI8rRBsKcNGnnaoI5zNbXeaVXl34efR+3hXGpHecxjSbctEeLZYvHykZCQAHd3d5w4cQJNmzZVLv/oo4/wzz//ICIiQm0dPz8/3L59G++88w7GjBmDGzduYMyYMZgwYQJmzZoFANi3bx8ePXqEOnXqIDExEXPmzMG9e/dw6dIlWFpaaoxF031xALBx40aYm5tr6YiJqCrIKQBisyS4nSXBrSwg/pEE+QrVy5lSCLhbAD6WAjUsBXwsBWxYQ0BUpeXk5GDAgAHIyMiAlZVVseP0KnGrXbs2cnNzERsbqzzDtmjRInz11VdITEzUuJ/09HR4eXlh0aJFGD58uMYxms64eXh4ICUl5bmT9ypkMhkOHjyI9u3bKy/7UulxHrWD8/hyhBCIS81BZFw6zt5Jx5m4NNx8oF5EYGVqiCBPGzTysEGwlw0auFvB3Jj33haHn0ft4VxqR3nMY2ZmJhwcHF6YuOnsJ4eDgwMMDAyQnJyssjw5ObnY+9NcXV1hZGSkclm0bt26SEpKQn5+PoyNjdXWsbGxQe3atXHjxo1iYzExMYGJifqfu0ZGRmX+QS+PfVQFnEft4Dw+X65Mjkv3MpRNbqPi0vAwW3PvtBDv/x7A7sveaS+Fn0ft4VxqR1nOY0m3q7PEzdjYGMHBwQgPD0ePHj0AAAqFAuHh4Rg3bpzGdZo3b46NGzdCoVBAKi289+P69etwdXXVmLQBwKNHj3Dz5k0MGjSoTI6DiCqvB1l5yqcQRMal4dK9TPXeaYZSNHS3RrC3LQLdrZB6/Qz6dG/BX5JEVCZ0eq5+ypQpGDJkCEJCQtCkSRMsWbIE2dnZyirTwYMHw93dHQsWLAAAjB49GsuXL8fEiRMxfvx4xMTEYP78+ZgwYYJymx9++CG6du0KLy8vJCQkYNasWTAwMED//v11coxEpB8UCoHr97MKE7XbaYiMT0Ocxt5pJgj2skGIlx0aedmq9E6TyWT4M7a8IyeiqkSniVvfvn3x4MEDzJw5E0lJSQgMDMT+/fvh7OwMAIiPj1eeWQMADw8P/PXXX5g8eTIaNmwId3d3TJw4EdOmTVOOuXv3Lvr374+HDx/C0dERLVq0wKlTp+Do6Fjux0dEFVd2XgHOPdU77WwxvdPqOFui0ZNLnsFe7J1GRLql87tjx40bV+yl0SNHjqgta9q0KU6dOlXs9jZv3qyt0IiokhBCICEjF2dupz7pnZaGq4mZeKZ1GsyNDZ605LBFsLcdAj1s2DuNiCoUnSduRETaJpMrcDUxs/AB7PGFlz6TMnPVxrnbmD31yChb+LlYqvVOIyKqSJi4EZHeS8/Jx9n4dJx5UkRw/k4GHsvkKmMMpBLUc7N6Uulph0ZeNnC1NtNRxEREL4eJGxHpFSEEYlOyC5/p+eSy5437j9TGWZsZqZxNa1jdmr3TiEjv8acYEVVouTI5Lt7LKLzs+eQB7KkaeqfVcLBQJmkh3rao4cDeaURU+TBxI6IK5X5WbuGZtCf3p126lwGZXLWKwNhQioDq1gj2Kmxy28jTBvbV+MwoIqr8mLgRkc7IFQLXk5/0TnvyFZ+quXdayJMzaY28bFHfzRrGhiwiIKKqh4kbEZWbR3kFOBdf1DstFefi05GVp7l3WtElz2BPO3jYmbF3GhERmLgRURkRQuBe+mPlmbQzt9NwLUm9d5qFsQGCPG2VTW4DPW1gZcreaUREmjBxIyKtkMkVuJKQ+VS1ZyqSM/PUxrnbmBWeSXtSSFDHmb3TiIhKiokbURUnVwhExKYiMkUC+9hUNK3pBIMSVGOm5+QjKj5NWe15/m46cmWqD2A3VPZOs1Mmai7WpmV1KERElR4TN6IqbP+lRMz5/QoSM3IBGOCXmDNwtTbFrK7+6FTfVTlOCIFbKdkqD2AvSe+0gOo2MDM2KMcjIiKq3Ji4EVVR+y8lYvT6KDxzyxmSMnIxen0UprSvDUMDKSKfPI0gLUemto0ajhYI9rRVXvpk7zQiorLFxI2oCpIrBOb8fkUtaQOgXPbNwesqy00MpQiobqMsImjkZQs7C+Myj5WIiP7DxI2oCjodm/rk8ujzNfG2RYd6Lgj2skU99k4jItI5Jm5EVdD9rBcnbQDwzmte6B7oXsbREBFRSfHPZ6IqyMmyZJWdJR1HRETlg4kbURXUxMcO1mbFN7mVAHC1NkUTH7vyC4qIiF6IiRtRFXT+bjqyn3nUVJGimtBZXf1L1M+NiIjKDxM3oirmfmYu3v81EgUKgUAPG7hYqV4OdbE2xcqBjVT6uBERUcXA4gSiKiSvQI7310fiflYeajlVw/r3QmFmZICTN+7jwLEIdGgZWuInJxARUflj4kZUhczecwVR8emwMjXED4NDUM2k8EdAqI8dHl4VCPWxY9JGRFSB8VIpURWxISIOm07HQyIBlvYPgreDha5DIiKiUmLiRlQFnLmditl7LgMApnasgzZ1nHQcERERvQwmbkSVXFJGLkZviIJMLtClgStGt/bVdUhERPSSmLgRVWJFxQgPsvLg52KJhb0bQiLhPWxERPqKiRtRJSWEwIxdl3DuTjqszYzw/aBgWJiwHomISJ8xcSOqpNafisPWM3chlQDL+gfBy57FCERE+o6JG1EldDo2FXN+vwIAmNbJD61qO+o4IiIi0gYmbkSVTGLGY4zZUPhkhK4BbhjZqoauQyIiIi1h4kZUieTK5Hj/10ikPMqHn4slvuzVgMUIRESVCBM3okpCCIFPd17C+bsZsDE3wg+DQ2BuzGIEIqLKhIkbUSXx84nb+C2qsBhhef9G8LAz13VIRESkZUzciCqBU7ce4rM/rgIAPulcFy1qOeg4IiIiKgtM3Ij03L30xxi7IQpyhUCPQDcMb+Gj65CIiKiMMHEj0mO5MjlG/XoGD7PzUc/NCgt68skIRESVGRM3Ij0lhMD0HRdx6V4m7CyM8f2gYJgZG+g6LCIiKkNM3Ij01Nr/3cbOs/dgIJVg+YAgVLdlMQIRUWXHxI1ID524kYL5fxYWI3zauS6a+bIYgYioKmDiRqRn7qTmYOzGwmKEno3cMay5t65DIiKicsLEjUiPPM6XY9SvkUjLkaGBuzXmv8UnIxARVSVM3Ij0hBACH++4gCuJmbB/UoxgasRiBCKiqoSJG5Ge+PFYLHafS4ChVILv3mkENxszXYdERETljIkbkR44HpOCBfsKixFmvOmP12rY6zgiIiLSBSZuRBXcndQcjNsUBYUAegdXx+CmXroOiYiIdISJG1EFlpNfgBG/nEF6jgwB1a0xr0d9FiMQEVVhTNyIKighBD7afgHXkrLgUM0Yq1iMQERU5TFxI6qgvj96C3svJMJQKsGKd4Lhas1iBCKiqo6JG1EF9M/1B1i4/xoAYFa3emjiY6fjiIiIqCJg4kZUwcQ9zMb4jYXFCH1DPDAw1FPXIRERUQXBxI2oAsnOK8DIXyKRmVuAIE8bzO1Rj8UIRESkVOrEzdvbG3PnzkV8fHxZxENUZQkhMHX7eUQnZ8HR0gSrBgbDxJDFCERE9J9SJ26TJk3Cjh07UKNGDbRv3x6bN29GXl5eWcRGVKWsOHITf15MgpGBBKsGNoKzlamuQyIiogrmpRK3c+fO4fTp06hbty7Gjx8PV1dXjBs3DlFRUWURI1Gldzj6Pr4+EA0AmNOtPoK9WIxARETqXvoet0aNGmHp0qVISEjArFmz8OOPP6Jx48YIDAzE2rVrIYTQZpxElVZsSjYmbDoLIYD+TTwxgMUIRERUDMOXXVEmk2Hnzp346aefcPDgQbz22msYPnw47t69i08++QSHDh3Cxo0btRkrUaXzKK8AI385g6zcAgR72WJ2N39dh0RERBVYqRO3qKgo/PTTT9i0aROkUikGDx6MxYsXw8/PTznmrbfeQuPGjbUaKFFlo1AIfLD1HGLuP4KzlQlWvtOIxQhERPRcpU7cGjdujPbt22PlypXo0aMHjIyM1Mb4+PigX79+WgmQqLL67vAN/HU5GcYGUqwcGAwnFiMQEdELlDpxu3XrFry8vJ47xsLCAj/99NNLB0VU2YVfTcaiQ9cBAJ/1qIdGnrY6joiIiPRBqYsT7t+/j4iICLXlEREROHPmTKkD+O677+Dt7Q1TU1OEhobi9OnTzx2fnp6OsWPHwtXVFSYmJqhduzb+/PPPV9omUXm6+eARJm0+ByGAga95om9jFiMQEVHJlDpxGzt2LO7cuaO2/N69exg7dmyptrVlyxZMmTIFs2bNQlRUFAICAtCxY0fcv39f4/j8/Hy0b98et2/fxvbt2xEdHY0ffvgB7u7uL71NovKUlSsrLEbIK0Bjb1vMfLOerkMiIiI9UurE7cqVK2jUqJHa8qCgIFy5cqVU21q0aBFGjBiBYcOGwd/fH6tWrYK5uTnWrl2rcfzatWuRmpqKXbt2oXnz5vD29kbr1q0REBDw0tskKi8KhcDkLedx80E2XKxMseKdYBgb8qlzRERUcqW+x83ExATJycmoUaOGyvLExEQYGpZ8c/n5+YiMjMT06dOVy6RSKcLCwnDy5EmN6+zZswdNmzbF2LFjsXv3bjg6OmLAgAGYNm0aDAwMXmqbAJCXl6fy9IfMzEwAhS1PZDJZiY+pNIq2W1bbryr0aR6X/X0Th64mw9hQiu/6B8DGVFph4taneazIOI/awXnUHs6ldpTHPJZ026VO3Dp06IDp06dj9+7dsLa2BlB439knn3yC9u3bl3g7KSkpkMvlcHZ2Vlnu7OyMa9euaVzn1q1b+Pvvv/HOO+/gzz//xI0bNzBmzBjIZDLMmjXrpbYJAAsWLMCcOXPUlh84cADm5uYlPqaXcfDgwTLdflVR0efxYqoEP0YXtvro7SXD3Qv/w90LOg5Kg4o+j/qC86gdnEft4VxqR1nOY05OTonGlTpx+/rrr9GqVSt4eXkhKCgIAHDu3Dk4Ozvj119/Le3mSkWhUMDJyQmrV6+GgYEBgoODce/ePXz11VeYNWvWS293+vTpmDJlivJ1ZmYmPDw80KFDB1hZWWkjdDUymQwHDx5E+/btNbZUoZLRh3m8+SAbn3x/CoAcg0I9MPPNuroOSY0+zKM+4DxqB+dReziX2lEe81h0te9FSp24ubu748KFC9iwYQPOnz8PMzMzDBs2DP379y/VwTg4OMDAwADJyckqy5OTk+Hi4qJxHVdXVxgZGcHA4L8mpXXr1kVSUhLy8/NfaptA4eVfExMTteVGRkZl/kEvj31UBRV1HjNzZRiz8Ryy8+Ro4mOHmd3qw8ig4t7XVlHnUd9wHrWD86g9nEvtKMt5LOl2X+qRVxYWFhg5cuTLrKpkbGyM4OBghIeHo0ePHgAKz6iFh4dj3LhxGtdp3rw5Nm7cCIVCAam08Jff9evX4erqCmNjYwAo9TaJyopCITB58zncSsmGm7UpVrzTqEInbUREVPG99LNKr1y5gvj4eOTn56ss79atW4m3MWXKFAwZMgQhISFo0qQJlixZguzsbAwbNgwAMHjwYLi7u2PBggUAgNGjR2P58uWYOHEixo8fj5iYGMyfPx8TJkwo8TaJysuSQ9cRfu0+TAyl+H5QCByqqZ/VJSIiKo2XenLCW2+9hYsXL0IikUAIAQCQSCQAALlcXuJt9e3bFw8ePMDMmTORlJSEwMBA7N+/X1lcEB8frzyzBgAeHh7466+/MHnyZDRs2BDu7u6YOHEipk2bVuJtEpWH/ZcSsfTvGwCABT0boEF1ax1HRERElUGpE7eJEyfCx8cH4eHh8PHxwenTp/Hw4UN88MEH+Prrr0sdwLhx44q9jHnkyBG1ZU2bNsWpU6deeptEZS0mOQsfbD0PAHi3uQ96Nqqu44iIiKiyKHXidvLkSfz9999wcHCAVCqFVCpFixYtsGDBAkyYMAFnz54tiziJ9ELGYxlG/HIG2flyNK1hj086++k6JCIiqkRKfae0XC6HpaUlgMLK0ISEBACAl5cXoqOjtRsdkR6RKwQmbj6L2w9z4G5jhuUDgmDIYgQiItKiUp9xq1+/Ps6fPw8fHx+EhoZi4cKFMDY2xurVq9WepkBUlSw6GI0j0Q9gaiTF94OCYc9iBCIi0rJSJ27/93//h+zsbADA3Llz8eabb6Jly5awt7fHli1btB4gkT7482Iivjt8EwDwZa+GqO/OYgQiItK+UiduHTt2VP5/zZo1ce3aNaSmpsLW1lZZWUpUlUQnZeHDbYXFCCNa+qB7oLuOIyIiosqqVDfgyGQyGBoa4tKlSyrL7ezsmLRRlZSek48Rv5xBTr4czWvaY1onFiMQEVHZKVXiZmRkBE9Pz1L1aiOqrOQKgfGbziI+NQfVbc2wvH8jFiMQEVGZKvVvmU8//RSffPIJUlNTyyIeIr3x1V/ROBaTAlMjKVYPCoGthbGuQyIiokqu1Pe4LV++HDdu3ICbmxu8vLxgYWGh8n5UVJTWgiOqqH4/n4BV/xQWIyzsHQB/NysdR0RERFVBqRO3ooe3E1VVVxIy8dH2CwCAUa1roFuAm44jIiKiqqLUidusWbPKIg4ivZCWnY+Rv57BY5kcLWs54KOOLEYgIqLywzupiUqoQK7A+E1ncTftMTztzLGsfxAMpKymJiKi8lPqM25SqfS5rT9YcUqV1cK/onH8RgrMjQ2wenAwbMxZjEBEROWr1Inbzp07VV7LZDKcPXsWP//8M+bMmaO1wIgqkt3n7mH10VsAgK96B8DPhcUIRERU/kqduHXv3l1tWe/evVGvXj1s2bIFw4cP10pgRBXFpXsZmPZbYTHCmDa+6NLQVccRERFRVaW1e9xee+01hIeHa2tzRBVCanY+Rv0aiVyZAm3qOOKDDnV0HRIREVVhWkncHj9+jKVLl8Ldnc9opMqjQK7AuI1RuJf+GN725vi2L4sRiIhIt0p9qfTZh8kLIZCVlQVzc3OsX79eq8ER6dKCfddw4uZDWBgbYPXgEFibG+k6JCIiquJKnbgtXrxYJXGTSqVwdHREaGgobG1ttRocka7siLqLNcdjAQDf9AlAbWdLHUdERET0Eonb0KFDyyAMoorj4t0MTN9xEQAw/vWa6FSfxQhERFQxlPoet59++gnbtm1TW75t2zb8/PPPWgmKSFdSHuVh1K9nkFegwOt+TpgcVlvXIRERESmVOnFbsGABHBwc1JY7OTlh/vz5WgmKSBdkcgXGbohCQkYuajhYYHHfQEhZjEBERBVIqRO3+Ph4+Pj4qC338vJCfHy8VoIi0oXP/7iKiNhUVDMxxOrBwbA2YzECERFVLKVO3JycnHDhwgW15efPn4e9vb1WgiIqb9vO3MG6E7cBAIv6BKCmE4sRiIio4il14ta/f39MmDABhw8fhlwuh1wux99//42JEyeiX79+ZREjUZk6fycdn+66BACY2K4WOtRz0XFEREREmpW6qvSzzz7D7du30a5dOxgaFq6uUCgwePBg3uNGeudBVh5G/RqJ/AIFwuo6Y2K7WroOiYiIqFilTtyMjY2xZcsWzJs3D+fOnYOZmRkaNGgALy+vsoiPqMzkFygwZkMkkjJz4etogcV9A1iMQEREFVqpE7citWrVQq1aPDtB+uuzvVfw7+00WJoYYvXgEFiashiBiIgqtlLf49arVy98+eWXassXLlyIt99+WytBEZW1rf/ewa+n4iCRAEv6BcLXsZquQyIiInqhUiduR48eRefOndWWv/HGGzh69KhWgiIqS2fj0/B/T4oRJofVRru6zjqOiIiIqGRKnbg9evQIxsbGasuNjIyQmZmplaCIysr9rFy8vz4S+XIFOvg7Y1zbmroOiYiIqMRKnbg1aNAAW7ZsUVu+efNm+Pv7ayUoorKQX6DA6PVRSM7MQy2naljEJyMQEZGeKXVxwowZM9CzZ0/cvHkTr7/+OgAgPDwcGzduxPbt27UeIJG2zP79MiLj0mBpWliMUM3kpWtziIiIdKLUv7m6du2KXbt2Yf78+di+fTvMzMwQEBCAv//+G3Z2dmURI9Er23Q6Hhsj4iGRAEv7BcHHwULXIREREZXaS51y6NKlC7p06QIAyMzMxKZNm/Dhhx8iMjIScrlcqwESvarIuFTM3F1YjPBhhzpo6+ek44iIiIheTqnvcSty9OhRDBkyBG5ubvjmm2/w+uuv49SpU9qMjeiVJWfm4v31UZDJBd6o74IxbXx1HRIREdFLK9UZt6SkJKxbtw5r1qxBZmYm+vTpg7y8POzatYuFCVTh5BXI8f76SDzIykMdZ0t8/XYAJBIWIxARkf4q8Rm3rl27ok6dOrhw4QKWLFmChIQELFu2rCxjI3ppQgjM2n0ZZ+PTYWVqiNWDg2HBYgQiItJzJf5Ntm/fPkyYMAGjR4/mo66owtsQEY/N/96BVAIsG9AIXvYsRiAiIv1X4jNux48fR1ZWFoKDgxEaGorly5cjJSWlLGMjein/3k7F7D2XAQBTO/qhdW1HHUdERESkHSVO3F577TX88MMPSExMxKhRo7B582a4ublBoVDg4MGDyMrKKss4iUokMeMxRq+PQoFCoEtDV7zfuoauQyIiItKaUleVWlhY4N1338Xx48dx8eJFfPDBB/jiiy/g5OSEbt26lUWMRCWSK5Pj/fVRSHmUBz8XS3zVuyGLEYiIqFJ56XYgAFCnTh0sXLgQd+/exaZNm7QVE1GpCSEwY9clnL+TDmszI6weFAJzYxYjEBFR5fJKiVsRAwMD9OjRA3v27NHG5ohKbcPpO9gWeRdSCbB8QBA87c11HRIREZHW8ZQE6b0bmcDKiGgAwMdv+KFlLRYjEBFR5aSVM25EupKYkYufog1QoBDoFuCGES1ZjEBERJUXEzfSW7kyOcZsPIdHBRLUdbHEl71YjEBERJUbEzfSS0IIfLLzIi4lZMLCUGDFgECYGRvoOiwiIqIyxcSN9NK6E7exI+oeDKQSDKmtQHVbM12HREREVOaYuJHeOXEzBfP+uAoAmNaxNupYCx1HREREVD6YuJFeuZuWg3Ebz0KuEHgryB1Dm3rqOiQiIqJyw8SN9MbjfDlG/RqJ1Ox81He3woKeDViMQEREVQoTN9ILQghM33EBlxMyYWdhjO8HhcDUiMUIRERUtTBxI72w5ngsdp1LgIFUgu8GNIK7DYsRiIio6mHiRhXe/26kYP6fhcUI/9elLpr62us4IiIiIt1g4kYV2p3UHIzbGAWFAHo1qo6hzbx1HRIREZHOMHGjCutxvhwjf41EWo4MDatb4/O36rMYgYiIqrQKkbh999138Pb2hqmpKUJDQ3H69Olix65btw4SiUTly9TUVGXM0KFD1cZ06tSprA+DtEgIgY9+u4CriZlwqGaMVQODWYxARERVnqGuA9iyZQumTJmCVatWITQ0FEuWLEHHjh0RHR0NJycnjetYWVkhOjpa+VrTWZhOnTrhp59+Ur42MTHRfvBUZn44dgu/n0+AoVSCFe8Ew43FCERERLo/47Zo0SKMGDECw4YNg7+/P1atWgVzc3OsXbu22HUkEglcXFyUX87OzmpjTExMVMbY2tqW5WGQFh2LeYAv9l0DAMzs6o8mPnY6joiIiKhi0OkZt/z8fERGRmL69OnKZVKpFGFhYTh58mSx6z169AheXl5QKBRo1KgR5s+fj3r16qmMOXLkCJycnGBra4vXX38d8+bNg7295mrEvLw85OXlKV9nZmYCAGQyGWQy2ascYrGKtltW29dX8U8VI/Ru5I5+wW7PnSPOo3ZwHrWD86gdnEft4VxqR3nMY0m3LRFC6OxBjwkJCXB3d8eJEyfQtGlT5fKPPvoI//zzDyIiItTWOXnyJGJiYtCwYUNkZGTg66+/xtGjR3H58mVUr14dALB582aYm5vDx8cHN2/exCeffIJq1arh5MmTMDBQv09q9uzZmDNnjtryjRs3wtzcXItHTM+TJwcWXzJAYo4EXtUExteTw0jn54SJiIjKXk5ODgYMGICMjAxYWVkVO07vErdnyWQy1K1bF/3798dnn32mccytW7fg6+uLQ4cOoV27dmrvazrj5uHhgZSUlOdO3quQyWQ4ePAg2rdvDyMjozLZhz4RQmDilgvYdzkZDtWMsXP0a3CxMn3hepxH7eA8agfnUTs4j9rDudSO8pjHzMxMODg4vDBx0+mlUgcHBxgYGCA5OVlleXJyMlxcXEq0DSMjIwQFBeHGjRvFjqlRowYcHBxw48YNjYmbiYmJxuIFIyOjMv+gl8c+9MGKIzew73IyjAwkWDUwGB72lqVan/OoHZxH7eA8agfnUXs4l9pRlvNY0u3q9EKUsbExgoODER4erlymUCgQHh6ucgbueeRyOS5evAhXV9dix9y9excPHz587hjSnSPR9/HVX4VVwrO61kOIN4sRiIiINNH5HURTpkzBDz/8gJ9//hlXr17F6NGjkZ2djWHDhgEABg8erFK8MHfuXBw4cAC3bt1CVFQUBg4ciLi4OLz33nsACgsXpk6dilOnTuH27dsIDw9H9+7dUbNmTXTs2FEnx0jFu52SjQmbzkIIoH8TD7wT6qnrkIiIiCosnfdx69u3Lx48eICZM2ciKSkJgYGB2L9/v7LFR3x8PKTS//LLtLQ0jBgxAklJSbC1tUVwcDBOnDgBf39/AICBgQEuXLiAn3/+Genp6XBzc0OHDh3w2WefsZdbBfMorwAjfz2DzNwCNPK0wexu9fhkBCIioufQeeIGAOPGjcO4ceM0vnfkyBGV14sXL8bixYuL3ZaZmRn++usvbYZHZUAIgQ+3nsf15EdwsjTByoHBMDHkkxGIiIieR+eXSqlq+u7wDey/nAQjAwlWDgyGcwkqSImIiKo6Jm5U7v6+loxvDl4HAMztXh/BXnyqBRERUUkwcaNydevBI0zcfA5CAO+EeqJ/ExYjEBERlRQTNyo3WbkyjPw1Elm5BQjxssWsrvVevBIREREpMXGjcqFQCHyw9Txu3H8EZysTrBjYCMaG/PgRERGVBn9zUrlY9vcNHLiSDGMDKVYNDIaTJYsRiIiISouJG5W5Q1eSsfhQYTHCvLfqI8iTxQhEREQvg4kblakb9x9h8pZzAIDBTb3QJ8RDtwERERHpMSZuVGYyc2UY+esZZOUVoIm3HWa86a/rkIiIiPQaEzcqEwqFwJQt53DrQTZcrU3x3TuNYGTAjxsREdGr4G9SKhNLwmNw6Op9GBtK8f2gYDha8jmxREREr4qJG2ndX5eTsDQ8BgCw4K0GaFjdRrcBERERVRJM3EirYpKzMOVJMcLQZt7oFVxdtwERERFVIkzcSGsyHhc+GSE7X45QHzt82qWurkMiIiKqVJi4kVbIFQKTNp9FbEo23G3MsILFCERERFrH36ykFYsPXsfh6AcweVKMYF+NxQhERETaxsSNXtm+i4lYfvgGAOCLXg1Q391axxERERFVTkzc6JVEJ2Xhg23nAQDDW/jgrSAWIxAREZUVJm700jJyCp+MkJMvRzNfe0x/w0/XIREREVVqTNzopcgVAhM2n0Xcwxy425hh+YBGMGQxAhERUZnib1p6KV8fiMY/1x/A1EiK1YODYWdhrOuQiIiIKj0mblRqey8kYOWRmwCAL3s1RD03FiMQERGVByZuVCpXEzMxddsFAMDIVjXQPdBdxxERERFVHUzcqMTSc/Ix8tczeCyTo2UtB3zUsY6uQyIiIqpSmLhRiRTIFRi/6SzupD6Gh50ZlvUPYjECERFROeNvXiqRr/6KxrGYFJgZGWD1oBDYmLMYgYiIqLwxcaMX2nM+Ad8fvQUA+OrthqjraqXjiIiIiKomJm70XJcTMvDR9sInI7zf2hdvNnTTcURERERVFxM3KlZqdj5G/RqJXJkCrWo7YiqLEYiIiHSKiRtpVCBXYNzGKNxNewwve3Ms6xcEA6lE12ERERFVaUzcSKMv9l3DiZsPYW5cWIxgbW6k65CIiIiqPCZupGbX2Xv48XgsAOCbtwNQx8VSxxERERERwMSNnnHpXgam/Vb4ZISxbX3xRgNXHUdERERERZi4kdLDR3kY9Wsk8goUaFvHEVPasxiBiIioImHiRgAAmVyBsRujcC/9MXwcLLCExQhEREQVDhM3AgDM//MqTt1KhYWxAVYPCoa1GYsRiIiIKhomboTfIu/ip//dBgAs6huIWs4sRiAiIqqImLhVcRfupmP6zosAgAmv10THei46joiIiIiKw8StCnuQVViMkF+gQDs/J0wKq63rkIiIiOg5mLhVUUXFCIkZuajhaIHF/QIhZTECERFRhcbErYqat/cKTsemopqJIVYPCoGVKYsRiIiIKjomblXQ1jN38PPJOADA4r6BqOlUTccRERERUUkwcatizt1Jx//tvAQAmBRWC+39nXUcEREREZUUE7cq5H5WLt7/NRL5cgXa+ztjwuu1dB0SERERlQITtyoiv0CBMeujkJSZC19HCyzqE8BiBCIiIj3DxK2KmLv3Ms7EpcHSxBA/DA6BJYsRiIiI9A4Ttypg8+l4rD8VD4kE+LZ/IGo4shiBiIhIHzFxq+Si4tMwc/dlAMAH7WvjdT8WIxAREekrJm6V2P3M/4oROtVzwdi2NXUdEhEREb0CJm6VVF6BHO+vj8T9rDzUcqqGr/sEQCJhMQIREZE+Y+JWSc3ecwVR8emwMi0sRqhmYqjrkIiIiOgVMXGrhDZExGHT6cJihKX9g+DtYKHrkIiIiEgLmLhVMmdup2L2nsJihKkd66BNHScdR0RERETawsStEknKyMX766Mgkwt0aeCK0a19dR0SERERaRETt0qiqBgh5VEe/FwssbB3QxYjEBERVTJM3CoBIQRm7LqEc3fSYW1mhO8HBcOCxQhERESVDhO3SmD9qThsPXMXUgmwrH8QvOxZjEBERFQZVYjE7bvvvoO3tzdMTU0RGhqK06dPFzt23bp1kEgkKl+mpqYqY4QQmDlzJlxdXWFmZoawsDDExMSU9WHoxOnYVMz5/QoAYFonP7Sq7ajjiIiIiKis6Dxx27JlC6ZMmYJZs2YhKioKAQEB6NixI+7fv1/sOlZWVkhMTFR+xcXFqby/cOFCLF26FKtWrUJERAQsLCzQsWNH5ObmlvXhlKuE9McYsyESBQqBrgFuGNmqhq5DIiIiojKk88Rt0aJFGDFiBIYNGwZ/f3+sWrUK5ubmWLt2bbHrSCQSuLi4KL+cnf97/qYQAkuWLMH//d//oXv37mjYsCF++eUXJCQkYNeuXeVwROUjVybH6PWRSHmUDz8XS3zZqwGLEYiIiCo5nd7Bnp+fj8jISEyfPl25TCqVIiwsDCdPnix2vUePHsHLywsKhQKNGjXC/PnzUa9ePQBAbGwskpKSEBYWphxvbW2N0NBQnDx5Ev369VPbXl5eHvLy8pSvMzMzAQAymQwymeyVj1OTou2+zPaFEJi+8zLO382AjZkRVgwIgJFElFmsFdmrzCP9h/OoHZxH7eA8ag/nUjvKYx5Lum2dJm4pKSmQy+UqZ8wAwNnZGdeuXdO4Tp06dbB27Vo0bNgQGRkZ+Prrr9GsWTNcvnwZ1atXR1JSknIbz26z6L1nLViwAHPmzFFbfuDAAZibm7/MoZXYwYMHS73O0UQJdt42gAQCA7xzcfHkEVwsg9j0ycvMI6njPGoH51E7OI/aw7nUjrKcx5ycnBKN07ueEU2bNkXTpk2Vr5s1a4a6devi+++/x2efffZS25w+fTqmTJmifJ2ZmQkPDw906NABVlZWrxyzJjKZDAcPHkT79u1hZGRU4vUiYlOxKyISgMDHnerg3ebeZRKfvnjZeSRVnEft4DxqB+dReziX2lEe81h0te9FdJq4OTg4wMDAAMnJySrLk5OT4eLiUqJtGBkZISgoCDdu3AAA5XrJyclwdXVV2WZgYKDGbZiYmMDExETjtsv6g16afdxLf4wJWy5ArhDoEeiGka1r8r62J8rj36oq4DxqB+dROziP2sO51I6ynMeSblenxQnGxsYIDg5GeHi4cplCoUB4eLjKWbXnkcvluHjxojJJ8/HxgYuLi8o2MzMzERERUeJtVkS5MjlG/XoGqdn5qOdmhQU9+WQEIiKiqkbnl0qnTJmCIUOGICQkBE2aNMGSJUuQnZ2NYcOGAQAGDx4Md3d3LFiwAAAwd+5cvPbaa6hZsybS09Px1VdfIS4uDu+99x6AworTSZMmYd68eahVqxZ8fHwwY8YMuLm5oUePHro6zFcihMD0HRdx6V4m7CyM8f2gYJgZG+g6LCIiIipnOk/c+vbtiwcPHmDmzJlISkpCYGAg9u/frywuiI+Ph1T634nBtLQ0jBgxAklJSbC1tUVwcDBOnDgBf39/5ZiPPvoI2dnZGDlyJNLT09GiRQvs379frVGvvlj7v9vYefYeDKQSLB8QhOq2ZVswQURERBWTzhM3ABg3bhzGjRun8b0jR46ovF68eDEWL1783O1JJBLMnTsXc+fO1VaIOnPiRgrm/3kVAPBp57po5uug44iIiIhIV3TegJeKdyc1B2M3RkGuEOjZyB3DqngFKRERUVXHxK2Cepwvx6hfI5GWI0MDd2vMf4tPRiAiIqrqmLhVQEIITPvtAq4kZsL+STGCqRGLEYiIiKo6Jm4V0I/HYrHnfAIMpRJ8904juNmY6TokIiIiqgCYuFUwx2NSsGBfYTHCjDf98VoNex1HRERERBUFE7cK5E5qDsZtioJCAL2Dq2NwUy9dh0REREQVCBO3CiInvwAjfjmD9BwZAqpbY16P+ixGICIiIhVM3CoAIQSmbr+Aa0lZcKhmjFUsRiAiIiINmLhVAN8fvYU/LiTCUCrByoHBcLVmMQIRERGpqxBPTqhq5AqBiNhURKZIcO9YLL46EAMAmNWtHhp72+k4OiIiIqqomLiVs/2XEjHn9ytIzMgFYADEFCZtzX3tMTDUU7fBERERUYXGS6XlaP+lRIxeH/UkaVN14uZD/HU5SQdRERERkb5g4lZO5AqBOb9fgXjOmDm/X4Fc8bwRREREVJUxcSsnp2NTNZ5pKyIAJGbk4nRsavkFRURERHqFiVs5uZ9VfNL2MuOIiIio6mHiVk6cLE21Oo6IiIiqHiZu5aSJjx1crU1R3LMQJABcrU3RxIftQIiIiEgzJm7lxEAqwayu/gCglrwVvZ7V1R8GUj7mioiIiDRj4laOOtV3xcqBjeBirXo51MXaFCsHNkKn+q46ioyIiIj0ARvwlrNO9V3R3t8FJ2/cx4FjEejQMhRNazrxTBsRERG9EBM3HTCQShDqY4eHVwVCfeyYtBEREVGJ8FIpERERkZ5g4kZERESkJ5i4EREREekJJm5EREREeoKJGxEREZGeYOJGREREpCfYDkQDIQQAIDMzs8z2IZPJkJOTg8zMTBgZGZXZfio7zqN2cB61g/OoHZxH7eFcakd5zGNRzlGUgxSHiZsGWVlZAAAPDw8dR0JERERVSVZWFqytrYt9XyJelNpVQQqFAgkJCbC0tIREUjbNcTMzM+Hh4YE7d+7AysqqTPZRFXAetYPzqB2cR+3gPGoP51I7ymMehRDIysqCm5sbpNLi72TjGTcNpFIpqlevXi77srKy4jeTFnAetYPzqB2cR+3gPGoP51I7ynoen3emrQiLE4iIiIj0BBM3IiIiIj3BxE1HTExMMGvWLJiYmOg6FL3GedQOzqN2cB61g/OoPZxL7ahI88jiBCIiIiI9wTNuRERERHqCiRsRERGRnmDiRkRERKQnmLgRERER6Qkmbjr0xRdfQCKRYNKkSboORe/cu3cPAwcOhL29PczMzNCgQQOcOXNG12HpHblcjhkzZsDHxwdmZmbw9fXFZ5999sJn5VV1R48eRdeuXeHm5gaJRIJdu3apvC+EwMyZM+Hq6gozMzOEhYUhJiZGN8FWYM+bR5lMhmnTpqFBgwawsLCAm5sbBg8ejISEBN0FXEG96PP4tPfffx8SiQRLliwpt/j0SUnm8urVq+jWrRusra1hYWGBxo0bIz4+vtxiZOKmI//++y++//57NGzYUNeh6J20tDQ0b94cRkZG2LdvH65cuYJvvvkGtra2ug5N73z55ZdYuXIlli9fjqtXr+LLL7/EwoULsWzZMl2HVqFlZ2cjICAA3333ncb3Fy5ciKVLl2LVqlWIiIiAhYUFOnbsiNzc3HKOtGJ73jzm5OQgKioKM2bMQFRUFHbs2IHo6Gh069ZNB5FWbC/6PBbZuXMnTp06BTc3t3KKTP+8aC5v3ryJFi1awM/PD0eOHMGFCxcwY8YMmJqall+QgspdVlaWqFWrljh48KBo3bq1mDhxoq5D0ivTpk0TLVq00HUYlUKXLl3Eu+++q7KsZ8+e4p133tFRRPoHgNi5c6fytUKhEC4uLuKrr75SLktPTxcmJiZi06ZNOohQPzw7j5qcPn1aABBxcXHlE5QeKm4e7969K9zd3cWlS5eEl5eXWLx4cbnHpm80zWXfvn3FwIEDdRPQEzzjpgNjx45Fly5dEBYWputQ9NKePXsQEhKCt99+G05OTggKCsIPP/yg67D0UrNmzRAeHo7r168DAM6fP4/jx4/jjTfe0HFk+is2NhZJSUkq39/W1tYIDQ3FyZMndRiZ/svIyIBEIoGNjY2uQ9ErCoUCgwYNwtSpU1GvXj1dh6O3FAoF/vjjD9SuXRsdO3aEk5MTQkNDn3tpuiwwcStnmzdvRlRUFBYsWKDrUPTWrVu3sHLlStSqVQt//fUXRo8ejQkTJuDnn3/WdWh65+OPP0a/fv3g5+cHIyMjBAUFYdKkSXjnnXd0HZreSkpKAgA4OzurLHd2dla+R6WXm5uLadOmoX///nxYeil9+eWXMDQ0xIQJE3Qdil67f/8+Hj16hC+++AKdOnXCgQMH8NZbb6Fnz574559/yi0Ow3LbE+HOnTuYOHEiDh48WL7XwysZhUKBkJAQzJ8/HwAQFBSES5cuYdWqVRgyZIiOo9MvW7duxYYNG7Bx40bUq1cP586dw6RJk+Dm5sa5pApDJpOhT58+EEJg5cqVug5Hr0RGRuLbb79FVFQUJBKJrsPRawqFAgDQvXt3TJ48GQAQGBiIEydOYNWqVWjdunW5xMEzbuUoMjIS9+/fR6NGjWBoaAhDQ0P8888/WLp0KQwNDSGXy3Udol5wdXWFv7+/yrK6deuWa1VPZTF16lTlWbcGDRpg0KBBmDx5Ms8IvwIXFxcAQHJyssry5ORk5XtUckVJW1xcHA4ePMizbaV07Ngx3L9/H56ensrfO3Fxcfjggw/g7e2t6/D0ioODAwwNDXX++4dn3MpRu3btcPHiRZVlw4YNg5+fH6ZNmwYDAwMdRaZfmjdvjujoaJVl169fh5eXl44i0l85OTmQSlX/fjMwMFD+ZUml5+PjAxcXF4SHhyMwMBAAkJmZiYiICIwePVq3wemZoqQtJiYGhw8fhr29va5D0juDBg1Su5+6Y8eOGDRoEIYNG6ajqPSTsbExGjdurPPfP0zcypGlpSXq16+vsszCwgL29vZqy6l4kydPRrNmzTB//nz06dMHp0+fxurVq7F69Wpdh6Z3unbtis8//xyenp6oV68ezp49i0WLFuHdd9/VdWgV2qNHj3Djxg3l69jYWJw7dw52dnbw9PTEpEmTMG/ePNSqVQs+Pj6YMWMG3Nzc0KNHD90FXQE9bx5dXV3Ru3dvREVFYe/evZDL5cp7BO3s7GBsbKyrsCucF30en014jYyM4OLigjp16pR3qBXei+Zy6tSp6Nu3L1q1aoW2bdti//79+P3333HkyJHyC1KnNa3EdiAv6ffffxf169cXJiYmws/PT6xevVrXIemlzMxMMXHiROHp6SlMTU1FjRo1xKeffiry8vJ0HVqFdvjwYQFA7WvIkCFCiMKWIDNmzBDOzs7CxMREtGvXTkRHR+s26AroefMYGxur8T0A4vDhw7oOvUJ50efxWWwHUrySzOWaNWtEzZo1hampqQgICBC7du0q1xglQrBFOhEREZE+YHECERERkZ5g4kZERESkJ5i4EREREekJJm5EREREeoKJGxEREZGeYOJGREREpCeYuBERERHpCSZuRERERHqCiRsRURkYOnSo3jzi6vbt25BIJDh37pxWxxKR9jFxI6ISOXnyJAwMDNClSxe194p+mRd9WVpaol69ehg7dixiYmLUxufn52PhwoUICAiAubk5HBwc0Lx5c/z000+QyWQqY4cNG4b/+7//AwCVfVhYWKBWrVoYOnQoIiMjNcZ89+5dGBsbF/ssYCEEVq9ejdDQUFSrVg02NjYICQnBkiVLkJOTo3GdZ4/V2NgYNWvWxLx58/D0g2i+/fZbrFu3Tvm6TZs2mDRpksZtPq1NmzbKbZuamsLf3x8rVqx44XqvwsPDA4mJiSV6ZnJpxhKR9jFxI6ISWbNmDcaPH4+jR48iISFB45hDhw4hMTER58+fx/z583H16lUEBAQgPDxcOSY/Px8dO3bEF198gZEjR+LEiRM4ffo0xo4di2XLluHy5cvKsXK5HHv37kW3bt2Uy3766SckJibi8uXL+O677/Do0SOEhobil19+UYtn3bp16NOnDzIzMxEREaH2/qBBgzBp0iR0794dhw8fxrlz5zBjxgzs3r0bBw4ceO58FB1rTEwM5syZg88//xxr165Vvm9tbQ0bG5vnbqM4I0aMQGJiIq5cuYI+ffpg7Nix2LRpk8ax+fn5L7WPpxkYGMDFxQWGhoZaHUtEZaBcn4xKRHopKytLVKtWTVy7dk307dtXfP755yrvFz0Q/OzZsyrL5XK5aNOmjfDy8hIFBQVCCCG+/PJLIZVKRVRUlNp+8vPzxaNHj5Svjx49KlxdXYVCoRBCCAFA7Ny5U229wYMHC0tLS5GamqpcplAoRI0aNcT+/fvFtGnTxIgRI1TW2bJliwCg8QHRCoVCpKena5yL4o61Xbt2YsyYMcrXQ4YMEd27d1f+P555aHVsbKzG7bdu3VpMnDhRZVmtWrVEv379lO+PHTtWTJw4Udjb24s2bdoIIYS4ePGi6NSpk7CwsBBOTk5i4MCB4sGDB8ptyOVy8eWXXwpfX19hbGwsPDw8xLx58zQeU2pqqhgwYIBwcHAQpqamombNmmLt2rXFHv+RI0dE48aNhbGxsXBxcRHTpk0TMplM5ZjGjx8vpk6dKmxtbYWzs7OYNWuWxuMnoufjGTcieqGtW7fCz88PderUwcCBA7F27VqVy4LFkUqlmDhxIuLi4pSXMzds2ICwsDAEBQWpjTcyMoKFhYXy9Z49e9C1a1dIJJLn7mfy5MnIysrCwYMHlcsOHz6MnJwchIWFYeDAgdi8eTOys7OV72/YsAF16tRB9+7d1bYnkUhgbW39wuMrcubMGURGRiI0NFTj+99++y2aNm2qPJOWmJgIDw+PEm/fzMxM5czazz//DGNjY/zvf//DqlWrkJ6ejtdffx1BQUE4c+YM9u/fj+TkZPTp00e5zvTp0/HFF19gxowZuHLlCjZu3AhnZ2eN+ysas2/fPly9ehUrV66Eg4ODxrH37t1D586d0bhxY5w/fx4rV67EmjVrMG/ePJVxP//8MywsLBAREYGFCxdi7ty5Kv9eRFQyPNdNRC+0Zs0aDBw4EADQqVMnZGRk4J9//kGbNm1euK6fnx+AwnvDmjRpgpiYmBKtBwC7d+/G4sWLS7WPp2Pu168fDAwMUL9+fdSoUQPbtm3D0KFDAQAxMTGoU6dOieLQpFmzZpBKpcjPz4dMJsPIkSMxePBgjWOtra1hbGwMc3NzuLi4lHgfcrkcmzZtwoULFzBy5Ejl8lq1amHhwoXK1/PmzUNQUBDmz5+vXLZ27Vp4eHjg+vXrcHV1xbfffovly5djyJAhAABfX1+0aNFC437j4+MRFBSEkJAQAIC3t3exMa5YsQIeHh5Yvnw5JBIJ/Pz8kJCQgGnTpmHmzJmQSgvPDzRs2BCzZs1Sxr98+XKEh4ejffv2JZ4PIuI9bkT0AtHR0Th9+jT69+8PADA0NETfvn2xZs2aEq1fdGau6KxZSc7UAcDVq1eRkJCAdu3alXof6enp2LFjhzLZBICBAweqxFzSOIqzZcsWnDt3DufPn8fWrVuxe/dufPzxx6+0zSIrVqxAtWrVYGZmhhEjRmDy5MkYPXq08v3g4GCV8efPn8fhw4dRrVo15VdRMnvz5k1cvXoVeXl5JZpLABg9ejQ2b96MwMBAfPTRRzhx4kSxY69evYqmTZuqnBVt3rw5Hj16hLt37yqXNWzYUGU9V1dX3L9/v0TxENF/eMaNiJ5rzZo1KCgogJubm3KZEAImJiZYvnz5Cy8pXr16FQDg4+MDAKhduzauXbv2wv3u2bMH7du3h6mp6QvHPruPjRs3Ijc3V+XSpRACCoUC169fR+3atUscR3E8PDxQs2ZNAEDdunVx8+ZNzJgxA7Nnzy5RzM/zzjvv4NNPP4WZmRlcXV2VZ62KPH05GQAePXqErl274ssvv1TblqurK27dulWq/b/xxhuIi4vDn3/+iYMHD6Jdu3YYO3Ysvv7669IfzBNGRkYqryUSCRQKxUtvj6iq4hk3IipWQUEBfvnlF3zzzTc4d+6c8uv8+fNwc3MrttKxiEKhwNKlS+Hj46O8p23AgAE4dOgQzp49qzZeJpMp70PbvXu3xvvPNFmyZAmsrKwQFhYGoDDZ/OCDD9RibtmypbLyc8CAAbh+/Tp2796ttj0hBDIyMkq07yIGBgYoKCgotsrT2NgYcrm8RNuytrZGzZo14e7urpa0adKoUSNcvnwZ3t7eqFmzpspXUdsUMzMzlereF3F0dMSQIUOwfv16LFmyBKtXr9Y4rm7dujh58qTKGcz//e9/sLS0RPXq1Uu8PyIqGSZuRFSsvXv3Ii0tDcOHD0f9+vVVvnr16qV2ufThw4dISkrCrVu3sGfPHoSFheH06dNYs2YNDAwMAACTJk1C8+bN0a5dO3z33Xc4f/48bt26ha1bt+K1115DTEwM7t+/jzNnzuDNN99Uiyk9PR1JSUmIi4vDwYMH0bt3b2zcuBErV66EjY0Nzp07h6ioKLz33ntqMffv3x8///wzCgoK0KdPH/Tt2xf9+/fH/PnzcebMGcTFxWHv3r0ICwvD4cOHnzs3Rcd69+5d7Nu3D99++y3atm0LKysrjeO9vb0RERGB27dvIyUlRatnm8aOHYvU1FT0798f//77L27evIm//voLw4YNg1wuh6mpKaZNm4aPPvoIv/zyC27evIlTp04Ve7l75syZ2L17N27cuIHLly9j7969qFu3rsaxY8aMwZ07dzB+/Hhcu3YNu3fvxqxZszBlypQSJZ1EVEo6q2clogrvzTffFJ07d9b4XkREhAAgzp8/r2wRUfRlbm4u6tatK8aMGSNiYmLU1s3NzRULFiwQDRo0EKampsLOzk40b95crFu3TshkMvHjjz+K5s2bq6339D5MTU2Fr6+vGDJkiIiMjFSOGTdunPD399cYc2JiopBKpWL37t1CiMIWGStXrhSNGzcW5ubmwsrKSgQHB4tvv/1W5OTkaNzGs8dqYGAgqlevLkaMGCHu37+vHPd0OxAhhIiOjhavvfaaMDMzK3U7kJK8f/36dfHWW28JGxsbYWZmJvz8/MSkSZOUrVTkcrmYN2+e8PLyEkZGRsLT01PMnz9f5ZiKWnx89tlnom7dusLMzEzY2dmJ7t27i1u3bmkcK0TJ2oE8G3P37t3FkCFDij1OItJMIsQr3qFLRKRl3bp1Q4sWLfDRRx/pOhQiogqF57GJqMJp0aKFsoqViIj+wzNuRERERHqCZ9yIiIiI9AQTNyIiIiI9wcSNiIiISE8wcSMiIiLSE0zciIiIiPQEEzciIiIiPcHEjYiIiEhPMHEjIiIi0hNM3IiIiIj0xP8DdI5gXI/7YAwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "bit_values = list(sweep_results.keys())\n", + "accuracies = [\n", + " list(sweep_results[b].values())[0][\"acc,none\"] for b in bit_values\n", + "]\n", + "\n", + "plt.figure(figsize=(7, 4))\n", + "plt.plot(bit_values, accuracies, marker='o')\n", + "plt.xlabel(\"ADC/DAC Bit Precision\")\n", + "plt.ylabel(\"Accuracy\")\n", + "plt.title(\"Language Model Accuracy vs. Hardware Precision\")\n", + "plt.grid(True)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "c9c0b5a4-6f0c-474b-acfd-41083ca9f024", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "We demonstrated how XBTorch can be integrated into the LM Evaluation Harness\n", + "to simulate low-precision inference in large language models.\n", + "\n", + "**Key Takeaways:**\n", + "- The `SimpleFixedPoint` accelerator enables simulation of analog crossbars and quantization. For large models, the stateful mode of operation can be used.\n", + "- Language model evaluation can be done seamlessly using `lm_eval`.\n", + "- Reducing ADC/DAC precision degrades model performance, providing a direct measure of hardware sensitivity.\n", + "\n", + "---\n", + "**References**\n", + "1. Gao et al., *\"The Language Model Evaluation Harness*, EleutherAI, 2021. \n", + "2. Kaushal et al., *\"Spectra: Surprising effectiveness of pretraining ternary language models at scale\"*, 2024." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 5af2652..e4315e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,8 @@ dependencies = [ "ninja>=1.11", "qtorch>=0.3", "smt>=2.9.4", + "transfomers>=4.57.0", + "lm_eval>=0.4.9" ] # List additional groups of dependencies here (e.g. development diff --git a/src/xbtorch/deployment/__init__.py b/src/xbtorch/deployment/__init__.py index cc919ed..5d787c6 100644 --- a/src/xbtorch/deployment/__init__.py +++ b/src/xbtorch/deployment/__init__.py @@ -1,7 +1,7 @@ """ Deployment (mapping, encoding, etc.) of solutions to inference accelerators """ -from .base import Daffodil, SimpleFixedPoint +from .base import Daffodil, SimpleFixedPoint, register_accelerator from .mapping import map_random from .encoding import encode_simple_binary, encode_MAO, encode_LEA1, encode_LEA2 from .metrics import compute_error diff --git a/src/xbtorch/deployment/base.py b/src/xbtorch/deployment/base.py index 4311486..cc6d655 100644 --- a/src/xbtorch/deployment/base.py +++ b/src/xbtorch/deployment/base.py @@ -25,6 +25,16 @@ from xbtorch.deployment.mapping import map_random from xbtorch.deployment.encoding import encode_simple_binary, encode_LEA1, encode_LEA2 +ACCELERATOR_REGISTRY = {} + +def register_accelerator(name: str): + """Decorator to register a custom layer under a string name.""" + def decorator(cls): + ACCELERATOR_REGISTRY[name] = cls + return cls + return decorator + +@register_accelerator("Generic") class GenericAccelerator(metaclass=abc.ABCMeta): """ Abstract base class for hardware accelerator models in XBTorch. @@ -46,8 +56,11 @@ class GenericAccelerator(metaclass=abc.ABCMeta): Amplitude of uniform read noise applied during chip readout. write_noise : float Standard deviation of Gaussian noise applied during weight writes. + stateful: bool, optional + In stateful mode, a physical representation of the entire crossbar is maintained, and weights are mapped to these limited devices. + In stateless mode, weights are mapped and VMM is performed on the fly. This is more memory-efficient. Essentially behaves like an infinite size stateful crossbar. xb_size : tuple of int, optional - Dimensions of the crossbar array (columns, rows). Default: (2500, 2500). + Dimensions of the crossbar array (columns, rows). Default: (2500, 2500). Utilized only when stateful is True. stuck_percentage : float, optional Fraction of devices randomly stuck at high or low values. Default: 0.0. stuck_mode : {"ideal", "real"}, optional @@ -80,7 +93,19 @@ class GenericAccelerator(metaclass=abc.ABCMeta): read noise. """ - def __init__(self, g_min, g_max, v_read, read_noise, write_noise, xb_size=(2500, 2500), stuck_percentage=0.0, stuck_mode='real', weight_encoding_scheme=encode_simple_binary, xb_mapping_scheme=map_random, device="cpu"): + def __init__(self, + g_min, + g_max, + v_read, + read_noise, + write_noise, + stateful=True, + xb_size=(2500, 2500), + stuck_percentage=0.0, + stuck_mode='real', + weight_encoding_scheme=encode_simple_binary, + xb_mapping_scheme=map_random, + device="cpu"): self.read_noise = read_noise self.write_noise = write_noise self.g_min = g_min @@ -89,8 +114,12 @@ def __init__(self, g_min, g_max, v_read, read_noise, write_noise, xb_size=(2500, self.weight_encoding_scheme = weight_encoding_scheme self.xb_mapping_scheme = xb_mapping_scheme self.stuck_percentage = stuck_percentage + self.stateful = stateful + + if (self.stuck_percentage > 0 and not self.stateful): + raise ValueError("Stuck devices can not be simulated without a stateful representation of a crossbar. See examples for usage.") - self.columns, self.rows = xb_size + self.columns, self.rows = xb_size if stateful else (-1, -1) # self.stuck_low = 0 # self.stuck_high = self.g_max * 2 self.stuck_mode = stuck_mode @@ -104,26 +133,28 @@ def __init__(self, g_min, g_max, v_read, read_noise, write_noise, xb_size=(2500, else: raise ValueError(f"Stuck mode {stuck_mode} not implemented") - # Create defect map - # TODO: Separate out defect maps + # TODO: Separate out defect maps; - self.name = f'cols_{self.columns}_row_{self.rows}_stuck_{self.stuck_percentage}' + self.name = f'stateful_{self.stateful}_cols_{self.columns}_row_{self.rows}_stuck_{self.stuck_percentage}' self.device = device - self.initialize_chip() + if (self.stateful): + self.initialize_chip() def initialize_chip(self): """ - Initialize the simulated chip state. + Initialize the simulated chip state, assuming stateful mode. + If stateless, defect maps will be patched on dynamically during operation. - Fills the array with uninitialized values (-1). - Generates a defect map based on the specified stuck percentage. """ - self._chip = torch.ones((self.columns, self.rows)).to(self.device) * -1 # uninitialized devices - self.defect_map = self.gen_defect_map(self.stuck_percentage) # defect map is a paired list of (defective indices, defective conductance states) - self._chip[self.defect_map[0]] = self.defect_map[1] + if (self.stateful): + self._chip = torch.ones((self.columns, self.rows)).to(self.device) * -1 # uninitialized devices + self.defect_map = self.gen_defect_map(self.stuck_percentage) # defect map is a paired list of (defective indices, defective conductance states) + self._chip[self.defect_map[0]] = self.defect_map[1] def get_xb_size(self): """ @@ -135,6 +166,9 @@ def read_chip(self, row, n_rows, col, n_cols, fast_mode=True): """ Read a subarray of the chip, optionally with read noise. + This method requires the object to be in a stateful mode. + If `self.stateful` is False, a RuntimeError is raised. + Parameters ---------- row : int @@ -152,12 +186,55 @@ def read_chip(self, row, n_rows, col, n_cols, fast_mode=True): ------- torch.Tensor Subarray with applied read noise (if configured). + + + Raises + ------ + RuntimeError + If `self.stateful` is False. + ValueError + If `fast_mode` is False (not implemented yet). + """ + + if not self.stateful: + raise RuntimeError("Cannot read chip when self.stateful is False.") + subarray = self._chip[row:row+n_rows, col:col+n_cols] noise = torch.empty_like(subarray).uniform_(-self.read_noise, self.read_noise) if (not fast_mode): raise ValueError("Not implemented") if (self.read_noise > 0): subarray = subarray + noise return subarray + + def read_chip_stateless(self, subarray): + """ + Read a subarray of the chip, optionally with read noise. + + This method requires the object to be in a stateful mode. + If `self.stateful` is False, a RuntimeError is raised. + + Parameters + ---------- + G + + Returns + ------- + torch.Tensor + Subarray with applied read noise (if configured). + + + Raises + ------ + RuntimeError + If `self.stateful` is False. + ValueError + If `fast_mode` is False (not implemented yet). + + """ + + noise = torch.empty_like(subarray).uniform_(-self.read_noise, self.read_noise) + if (self.read_noise > 0): subarray = subarray + noise + return subarray def gen_defect_map(self, stuck_percentage): """ @@ -175,6 +252,9 @@ def gen_defect_map(self, stuck_percentage): - indices are tensor indices of defective devices, - values are their fixed conductances (stuck_high or stuck_low). """ + if (not self.stateful): + return + num_elements = int(stuck_percentage * self._chip.numel()) defect_indices = np.unravel_index( np.random.choice(self._chip.shape[0] * self._chip.shape[1], num_elements, replace=False), (self._chip.shape[0], self._chip.shape[1]) @@ -184,6 +264,30 @@ def gen_defect_map(self, stuck_percentage): defect_values[defect_values == 1] = self.stuck_high return defect_indices, defect_values.to(self.device) + def map_weights_to_array_stateless(self, sw_weight): + + if (self.stateful): + return + + encoded_return = self.weight_encoding_scheme(self, sw_weight) + Gposs, Gnegs = encoded_return[0], encoded_return[1] + sw_weight_shape = sw_weight.shape + + # write noise + for i, Gpos in enumerate(Gposs): + if (self.write_noise > 0): + noise = torch.randn_like(Gposs[i]) * self.write_noise + 0.0 # 0 mean + Gposs[i] = Gposs[i] + noise + + + for i, Gneg in enumerate(Gnegs): + if (self.write_noise > 0): + noise = torch.randn_like(Gnegs[i]) * self.write_noise + 0.0 # 0 mean + Gnegs[i] = Gnegs[i] + noise + + # TODO: read noise + return Gposs, Gnegs + def map_weights_to_array(self, sw_weight, pos_idxs=[], neg_idxs=[], additional_args={}): """ Map software weights onto the hardware array. @@ -210,6 +314,10 @@ def map_weights_to_array(self, sw_weight, pos_idxs=[], neg_idxs=[], additional_a - Adds Gaussian write noise if configured. - Defect map is reapplied to enforce stuck devices. """ + + if (not self.stateful): + return + encoded_return = self.weight_encoding_scheme(self, sw_weight, pos_idxs=pos_idxs, neg_idxs=neg_idxs, additional_args=additional_args) Gposs, Gnegs = encoded_return[0], encoded_return[1] sw_weight_shape = sw_weight.shape @@ -220,7 +328,7 @@ def map_weights_to_array(self, sw_weight, pos_idxs=[], neg_idxs=[], additional_a Gposs[i] = Gposs[i] + noise self._chip[pos_idx[0]:pos_idx[0]+sw_weight_shape[0], - pos_idx[1]:pos_idx[1]+sw_weight_shape[1]] = Gposs[i] + pos_idx[1]:pos_idx[1]+sw_weight_shape[1]] = Gposs[i] for i, neg_idx in enumerate(neg_idxs): if (self.write_noise > 0): @@ -228,7 +336,7 @@ def map_weights_to_array(self, sw_weight, pos_idxs=[], neg_idxs=[], additional_a Gnegs[i] = Gnegs[i] + noise self._chip[neg_idx[0]:neg_idx[0]+sw_weight_shape[0], - neg_idx[1]:neg_idx[1]+sw_weight_shape[1]] = Gnegs[i] + neg_idx[1]:neg_idx[1]+sw_weight_shape[1]] = Gnegs[i] # Add back defect map information in case the outer method attempted to do an illegal assignment self._chip[self.defect_map[0]] = self.defect_map[1] @@ -293,6 +401,10 @@ def plot_array(self, x_start=None, x_count=None, y_start=None, y_count=None, tit torch.Tensor The read subarray. """ + + if (not self.stateful): + return + import matplotlib.pyplot as plt fig = plt.figure() @@ -318,6 +430,7 @@ def plot_array(self, x_start=None, x_count=None, y_start=None, y_count=None, tit if show: plt.show() return read_chip +@register_accelerator("SimpleFixedPoint") class SimpleFixedPoint(GenericAccelerator): """ Simple fixed-point accelerator model. @@ -343,8 +456,22 @@ class SimpleFixedPoint(GenericAccelerator): """ - def __init__(self, adc_bits=5, dac_bits=5, g_min=50, g_max=100, v_read=0.3, read_noise=0, xb_size=(2500, 2500), write_noise=0, stuck_percentage=0.0, stuck_mode='real', xb_mapping_scheme=map_random, weight_encoding_scheme=encode_simple_binary, device='cpu'): - super().__init__(g_min, g_max, v_read, read_noise=read_noise, write_noise=write_noise, xb_size=xb_size, stuck_percentage=stuck_percentage, stuck_mode=stuck_mode, xb_mapping_scheme=xb_mapping_scheme, weight_encoding_scheme=weight_encoding_scheme, device=device) + def __init__(self, + adc_bits=5, + dac_bits=5, + g_min=50, + g_max=100, + v_read=0.3, + read_noise=0, + stateful=True, + xb_size=(2500, 2500), + write_noise=0, + stuck_percentage=0.0, + stuck_mode='real', + xb_mapping_scheme=map_random, + weight_encoding_scheme=encode_simple_binary, + device='cpu'): + super().__init__(g_min, g_max, v_read, read_noise=read_noise, write_noise=write_noise, stateful=stateful, xb_size=xb_size, stuck_percentage=stuck_percentage, stuck_mode=stuck_mode, xb_mapping_scheme=xb_mapping_scheme, weight_encoding_scheme=weight_encoding_scheme, device=device) self.adc_bits = adc_bits self.dac_bits = dac_bits @@ -382,7 +509,7 @@ def ADC_quantize(self, vector): max_val = torch.max(vector) return max_val * fixed_point_quantize(vector / max_val, wl=self.adc_bits, fl=self.adc_bits-1, symmetric=True) - +@register_accelerator("Daffodil") class Daffodil(GenericAccelerator): """ Experimental Daffodil accelerator model. diff --git a/src/xbtorch/patches/__init__.py b/src/xbtorch/patches/__init__.py index 2945665..1d2f773 100644 --- a/src/xbtorch/patches/__init__.py +++ b/src/xbtorch/patches/__init__.py @@ -1,4 +1,4 @@ """ Decorators for patching PyTorch models and optimizers for XBTorch """ -from .model import xbtorch_model \ No newline at end of file +from .model import xbtorch_model, replace_all_layers_stateless \ No newline at end of file diff --git a/src/xbtorch/patches/decorators.py b/src/xbtorch/patches/decorators.py index cf9a639..fc0fdb2 100644 --- a/src/xbtorch/patches/decorators.py +++ b/src/xbtorch/patches/decorators.py @@ -69,59 +69,112 @@ def backward_hook(module, grad_input, grad_output): def xbtorch_forward(self, input): # TODO: Add support for non-linear layers if (self._xb_inference): - if (not hasattr(self, '_array_mappings')): - raise ValueError("Array mappings are not present, likely an issue during initialization.") + if (self.inference_accelerator.stateful): + if (not hasattr(self, '_array_mappings')): + raise ValueError("Array mappings are not present, likely an issue during initialization.") + + gnorm_scale = 1.0 + if (not self.inference_accelerator): raise ValueError('XB inference called without proper initialization of an accelerator profile.') + + # ternary mapping scheme + v_read = self.inference_accelerator.v_read + g_norm = self.inference_accelerator.g_max - self.inference_accelerator.g_min + + sw_weight = self.weight.data + + gamma = torch.unique(sw_weight)[-1] # WAGE quantization learns matrices [-gamma, 0, gamma], and so it's important to scale either G matrices or input voltage vector + + # convert inputs to voltages, then quantize to DAC-based precision + input_voltages = input * v_read * gamma + input_voltages = self.inference_accelerator.DAC_quantize(input_voltages) + + pos_idxs = self._array_mappings['Gpos'] + neg_idxs = self._array_mappings['Gneg'] + + # pass to the crossbar, perform VMM, averaging/summing over multiple instances + pos_outputs = [] + neg_outputs = [] + + # Todo: can be possibly batched and made faster by using torch.bmm + for pos_idx in pos_idxs: + gpos = self.inference_accelerator.read_chip(pos_idx[0], sw_weight.shape[0], pos_idx[1], sw_weight.shape[1]) + pos_outputs.append(input_voltages @ gpos.T) + + for neg_idx in neg_idxs: + gneg = self.inference_accelerator.read_chip(neg_idx[0], sw_weight.shape[0], neg_idx[1], sw_weight.shape[1]) + neg_outputs.append(input_voltages @ gneg.T) + + # Convert the list of tensors to a single tensor + pos_outputs = torch.stack(pos_outputs) + neg_outputs = torch.stack(neg_outputs) + + # for MAO, this has to be sum, for regular mapping, this will be average + if (self._array_mappings['output_polling_mode'] == 'avg'): + output = torch.mean(pos_outputs, dim=0) - torch.mean(neg_outputs, dim=0) + elif (self._array_mappings['output_polling_mode'] == 'sum'): + output = torch.sum(pos_outputs, dim=0) - torch.sum(neg_outputs, dim=0) + elif (self._array_mappings['output_polling_mode'] == 'reduced_avg'): + output = torch.sum(pos_outputs * self._array_mappings['maskpos'], dim=0) / self._array_mappings['alpha'] - torch.sum(neg_outputs * self._array_mappings['maskneg'], dim=0) / self._array_mappings['alpha'] + else: + raise ValueError("output_polling_mode not implemented") + # readback currents from ADC by simulated quantization again + output = self.inference_accelerator.ADC_quantize(output) # equivalent to optimizing the TIA potentiometer resistance. ADC quantization + output = output / (gnorm_scale * g_norm * v_read) + + if (self.bias is not None): output += self.bias.data + return output + else: + # stateless forward, weights were never encoded and mapped to a crossbar. Here, we'll do everything on-the-fly. - gnorm_scale = 1.0 - if (not self.inference_accelerator): raise ValueError('XB inference called without proper initialization of an accelerator profile.') + gnorm_scale = 1.0 + if (not self.inference_accelerator): raise ValueError('XB inference called without proper initialization of an accelerator profile.') - # ternary mapping scheme - v_read = self.inference_accelerator.v_read - g_norm = self.inference_accelerator.g_max - self.inference_accelerator.g_min + # ternary mapping scheme + v_read = self.inference_accelerator.v_read + g_norm = self.inference_accelerator.g_max - self.inference_accelerator.g_min - sw_weight = self.weight.data + sw_weight = self.weight.data - gamma = torch.unique(sw_weight)[-1] # WAGE quantization learns matrices [-gamma, 0, gamma], and so it's important to scale either G matrices or input voltage vector + gamma = torch.unique(sw_weight)[-1] # WAGE quantization learns matrices [-gamma, 0, gamma], and so it's important to scale either G matrices or input voltage vector - # convert inputs to voltages, then quantize to DAC-based precision - input_voltages = input * v_read * gamma - input_voltages = self.inference_accelerator.DAC_quantize(input_voltages) + # convert inputs to voltages, then quantize to DAC-based precision + input_voltages = input * v_read * gamma + input_voltages = self.inference_accelerator.DAC_quantize(input_voltages) - pos_idxs = self._array_mappings['Gpos'] - neg_idxs = self._array_mappings['Gneg'] - - # pass to the crossbar, perform VMM, averaging/summing over multiple instances - pos_outputs = [] - neg_outputs = [] + # stateless operation + # first, let's convert weights to conductances + # TODO: raise error if redundnancy based scheme required with stateless operation, not supported atm + # would require splitting initialize layer mappings for stateless and stateful + + + Gposs, Gnegs = self.inference_accelerator.map_weights_to_array_stateless(sw_weight) + + # pass to the crossbar, perform VMM, averaging/summing over multiple instances + pos_outputs = [] + neg_outputs = [] - # Todo: can be possibly batched and made faster by using torch.bmm - for pos_idx in pos_idxs: - gpos = self.inference_accelerator.read_chip(pos_idx[0], sw_weight.shape[0], pos_idx[1], sw_weight.shape[1]) - pos_outputs.append(input_voltages @ gpos.T) + # Todo: can be possibly batched and made faster by using torch.bmm + for Gpos in Gposs: + gpos = self.inference_accelerator.read_chip_stateless(Gpos) + pos_outputs.append(input_voltages @ gpos.T) - for neg_idx in neg_idxs: - gneg = self.inference_accelerator.read_chip(neg_idx[0], sw_weight.shape[0], neg_idx[1], sw_weight.shape[1]) - neg_outputs.append(input_voltages @ gneg.T) + for Gneg in Gnegs: + gneg = self.inference_accelerator.read_chip_stateless(Gneg) + neg_outputs.append(input_voltages @ gneg.T) - # Convert the list of tensors to a single tensor - pos_outputs = torch.stack(pos_outputs) - neg_outputs = torch.stack(neg_outputs) + # Convert the list of tensors to a single tensor + pos_outputs = torch.stack(pos_outputs) + neg_outputs = torch.stack(neg_outputs) - # for MAO, this has to be sum, for regular mapping, this will be average - if (self._array_mappings['output_polling_mode'] == 'avg'): output = torch.mean(pos_outputs, dim=0) - torch.mean(neg_outputs, dim=0) - elif (self._array_mappings['output_polling_mode'] == 'sum'): - output = torch.sum(pos_outputs, dim=0) - torch.sum(neg_outputs, dim=0) - elif (self._array_mappings['output_polling_mode'] == 'reduced_avg'): - output = torch.sum(pos_outputs * self._array_mappings['maskpos'], dim=0) / self._array_mappings['alpha'] - torch.sum(neg_outputs * self._array_mappings['maskneg'], dim=0) / self._array_mappings['alpha'] - else: - raise ValueError("output_polling_mode not implemented") - # readback currents from ADC by simulated quantization again - output = self.inference_accelerator.ADC_quantize(output) # equivalent to optimizing the TIA potentiometer resistance. ADC quantization - output = output / (gnorm_scale * g_norm * v_read) + # TODO: output polling modes are not implemented for now - if (self.bias is not None): output += self.bias.data - return output + # readback currents from ADC by simulated quantization again + output = self.inference_accelerator.ADC_quantize(output) # equivalent to optimizing the TIA potentiometer resistance. ADC quantization + output = output / (gnorm_scale * g_norm * v_read) + + if (self.bias is not None): output += self.bias.data + return output else: if (hasattr(self, 'weight')): self.weight.input = input diff --git a/src/xbtorch/patches/model.py b/src/xbtorch/patches/model.py index 36cd5b7..90855cf 100644 --- a/src/xbtorch/patches/model.py +++ b/src/xbtorch/patches/model.py @@ -4,15 +4,9 @@ """ from xbtorch import get_xbtorch_param -import xbtorch.quant.wage_init as wage_init +from .utils import replace_all_layers_stateful, replace_all_layers_stateless, toggle_xb_eval_all_layers_stateless -import torch.nn as nn -import xbtorch.nn as xbnn -import xbtorch - -import torch - -def xbtorch_model(original_model): +def xbtorch_model(original_model, replace_all=False, exclude=None): """ Patch a PyTorch model for XBTorch compatibility. @@ -71,8 +65,6 @@ def xbtorch_model(original_model): - Inference accelerator mappings assume that crossbar dimensions and encoding/mapping schemes are defined during initialization. """ - # TODO: .model shouldn't be required, should have an option to specify layers - # TODO: This should be used if a model was already created i.e. on an instance # Copies state dictionary as well if (not get_xbtorch_param('initialized')): raise RuntimeError('XBTorch needs to be initialized, please refer to API for instructions.') @@ -80,90 +72,32 @@ def xbtorch_model(original_model): xb_inference_accelerator = get_xbtorch_param('inference_accelerator') original_model.xb_forward = False # declare this to be false, xb_eval() has to be explicitly called - # detect the device from the original model - try: - device = next(original_model.parameters()).device - except StopIteration: - device = torch.device("cpu") # fallback if model has no parameters - - if (wage_quantize): - wage_params = get_xbtorch_param('wage_params') - quantizer_act_error = wage_params['quantizer_act_error'] - wl_activation = wage_params['wl_activation'] - wl_error = wage_params['wl_error'] - wl_weight = wage_params['wl_weight'] - - if (not hasattr(original_model, 'model')): raise RuntimeError('Unable to find module list for patching, see network training example for correct patching workflow.') - - new_model = [] - if (wage_quantize): - new_model.append(quantizer_act_error(wl_activation, -1)) - - for module in original_model.model: - # TODO: A cleaner way to implement this could be to specify regex patterns for layers to be patched, or just specify them as a list - # This should simplify model definitions, as well as patched re-creations here - if (type(module) in xbtorch.layer_types): - if (type(module)) == nn.Linear: - args = (module.in_features, module.out_features, module.bias is not None) - xbnn_layer = xbnn.Linear(*args) - elif (type(module)) == nn.Conv2d: - args = (module.in_channels, module.out_channels, module.kernel_size, module.stride, module.padding, module.dilation, module.groups, module.bias is not None, module.padding_mode) - xbnn_layer = xbnn.Conv2d(*args) - - elif (type(module)) == nn.RNN: - args = (module.input_size, module.hidden_size, module.num_layers, module.nonlinearity, module.bias, module.batch_first, module.dropout, module.bidirectional) - xbnn_layer = xbnn.RNN(*args) - - elif (type(module)) == nn.LSTM: - args = (module.input_size, module.hidden_size, module.num_layers, module.bias, module.batch_first, module.dropout, module.bidirectional, module.proj_size) - xbnn_layer = xbnn.LSTM(*args) - else: - raise ValueError(f"An xbtorch supported layer, {type(module)}, is missing an implementation.") - - # move to device before loading weights - xbnn_layer = xbnn_layer.to(device) - xbnn_layer.load_state_dict(module.state_dict()) - xbnn_layer._array_mappings = {} # we add this to make initialization easier later, since pre-trained weights would be loaded after patching - new_model.append(xbnn_layer) - - # activations - elif (type(module) in xbtorch.activation_types): - new_model.append(module) - if (wage_quantize): new_model.append(quantizer_act_error(wl_activation, wl_error)) - - elif (type(module) in xbtorch.misc_types): - new_model.append(module) - - # else copy unpatched module, but notify user - else: - print(f'XBPatching for module {module} is not defined, using as is.') - new_model.append(module) - # exit() - - if (wage_quantize and new_model[-1] not in xbtorch.activation_types): - # add act/error quantizer if the last module is an activation - new_model.append(quantizer_act_error(-1, wl_error)) - - original_model.model = nn.Sequential(*new_model) - - if (wage_quantize): - # wage parameters - original_model.weight_scale = {} - original_model.weight_acc = {} - for name, param in original_model.named_parameters(): - if ("weight" in name): wage_init.wage_init_(param, wl_weight, factor=1.0) - param.weight_acc = param.data - - # print('Patched XBTorch Model', original_model.model) + # How to replace layers? stateless or stateful. + if (replace_all): + # stateless + replace_all_layers_stateless(model=original_model, + exclude=[] if exclude is None else exclude) + else: + # stateful + replace_all_layers_stateful(model=original_model, wage_quantize=wage_quantize) if (xb_inference_accelerator): # if initialization included an inference accelerator def toggle(enable=True): original_model.xb_forward = enable - for module in original_model.model: - module._xb_inference = enable + if not replace_all: + # stateless + for module in original_model.model: + module._xb_inference = enable + else: + toggle_xb_eval_all_layers_stateless(original_model, enable=enable) def initialize_array_mappings(output_polling_mode='avg', existing_mappings=[], additional_args={}): + + if replace_all: + print("Can not map to array in stateless operation.") + return + # existing_mappings can be used as a reference to avoid conflicting mappings across unique models on the xb (primary use case: committee machines) # reset/initialize mappings of this layer on the simulated crossbar original_model._array_mappings_all = existing_mappings @@ -177,12 +111,12 @@ def initialize_array_mappings(output_polling_mode='avg', existing_mappings=[], a # get indices where the conductance matrices will be mapped on the simulated xbar # xb_mapping_schemes = ['random', 'layer_ensemble'] etc. pos_idxs = xb_inference_accelerator.xb_mapping_scheme(accelerator=xb_inference_accelerator, - layer_shape=sw_weight.shape, - current_mappings=original_model._array_mappings_all) + layer_shape=sw_weight.shape, + current_mappings=original_model._array_mappings_all) neg_idxs = xb_inference_accelerator.xb_mapping_scheme(accelerator=xb_inference_accelerator, - layer_shape=sw_weight.shape, - current_mappings=original_model._array_mappings_all) + layer_shape=sw_weight.shape, + current_mappings=original_model._array_mappings_all) # map the sw_weight matrix to the simulated array as device conductances at indices extracted above # internally handles conversion of the sw_weight matrix to conductance matrices diff --git a/src/xbtorch/patches/utils.py b/src/xbtorch/patches/utils.py new file mode 100644 index 0000000..e328c45 --- /dev/null +++ b/src/xbtorch/patches/utils.py @@ -0,0 +1,127 @@ +import xbtorch.quant.wage_init as wage_init +import torch.nn as nn +import xbtorch.nn as xbnn +import xbtorch +import transformers + +from xbtorch import get_xbtorch_param + +def replace_all_layers_stateless(model: transformers.models, + exclude: list=[]) -> None: + """ + Recursively replace all modules within a model with XBTorch modules for stateless operation. + """ + + for name, module in model.named_children(): + + # this is not a nested exclusion + if exclude and name in exclude: + continue + + # TODO: could be a simple map between nn vs. xbnn modules, removing the elifs + if isinstance(module, nn.Linear): + args = (module.in_features, module.out_features, module.bias is not None) + xbnn_layer = xbnn.Linear(*args).to(next(module.parameters()).device) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._xb_inference = False # add attribute for xb evaluation for proper toggling later + setattr(model, name, xbnn_layer) + elif (type(module)) == nn.Conv2d: + args = (module.in_channels, module.out_channels, module.kernel_size, module.stride, module.padding, module.dilation, module.groups, module.bias is not None, module.padding_mode) + xbnn_layer = xbnn.Conv2d(*args) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._xb_inference = False # add attribute for xb evaluation for proper toggling later + setattr(model, name, xbnn_layer) + elif (type(module)) == nn.RNN: + args = (module.input_size, module.hidden_size, module.num_layers, module.nonlinearity, module.bias, module.batch_first, module.dropout, module.bidirectional) + xbnn_layer = xbnn.RNN(*args) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._xb_inference = False # add attribute for xb evaluation for proper toggling later + setattr(model, name, xbnn_layer) + elif (type(module)) == nn.LSTM: + args = (module.input_size, module.hidden_size, module.num_layers, module.bias, module.batch_first, module.dropout, module.bidirectional, module.proj_size) + xbnn_layer = xbnn.LSTM(*args) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._xb_inference = False # add attribute for xb evaluation for proper toggling later + setattr(model, name, xbnn_layer) + else: + replace_all_layers_stateless(model=module, exclude=exclude) + +def replace_all_layers_stateful(model, wage_quantize): + """ + Recursively replace all modules within a model with XBTorch modules for stateful operation. + Supports wage quantization. + """ + new_model = [] + if (wage_quantize): + + wage_params = get_xbtorch_param('wage_params') + quantizer_act_error = wage_params['quantizer_act_error'] + wl_activation = wage_params['wl_activation'] + wl_error = wage_params['wl_error'] + wl_weight = wage_params['wl_weight'] + + new_model.append(quantizer_act_error(wl_activation, -1)) + + for module in model.model: + if (type(module) in xbtorch.layer_types): + if (type(module)) == nn.Linear: + args = (module.in_features, module.out_features, module.bias is not None) + xbnn_layer = xbnn.Linear(*args) + elif (type(module)) == nn.Conv2d: + args = (module.in_channels, module.out_channels, module.kernel_size, module.stride, module.padding, module.dilation, module.groups, module.bias is not None, module.padding_mode) + xbnn_layer = xbnn.Conv2d(*args) + + elif (type(module)) == nn.RNN: + args = (module.input_size, module.hidden_size, module.num_layers, module.nonlinearity, module.bias, module.batch_first, module.dropout, module.bidirectional) + xbnn_layer = xbnn.RNN(*args) + + elif (type(module)) == nn.LSTM: + args = (module.input_size, module.hidden_size, module.num_layers, module.bias, module.batch_first, module.dropout, module.bidirectional, module.proj_size) + xbnn_layer = xbnn.LSTM(*args) + else: + raise ValueError(f"An xbtorch supported layer, {type(module)}, is missing an implementation.") + + # move to device before loading weights + xbnn_layer = xbnn_layer.to(next(module.parameters()).device) + xbnn_layer.load_state_dict(module.state_dict()) + xbnn_layer._array_mappings = {} # we add this to make initialization easier later, since pre-trained weights would be loaded after patching + new_model.append(xbnn_layer) + + # activations + elif (type(module) in xbtorch.activation_types): + new_model.append(module) + if (wage_quantize): new_model.append(quantizer_act_error(wl_activation, wl_error)) + + elif (type(module) in xbtorch.misc_types): + new_model.append(module) + + # else copy unpatched module, but notify user + else: + print(f'XBPatching for module {module} is not defined, using as is.') + new_model.append(module) + + if (wage_quantize and new_model[-1] not in xbtorch.activation_types): + # add act/error quantizer if the last module is an activation + new_model.append(quantizer_act_error(-1, wl_error)) + + if (hasattr(model, 'model')): + model.model = nn.Sequential(*new_model) + + if (wage_quantize): + # wage parameters + model.weight_scale = {} + model.weight_acc = {} + for name, param in model.named_parameters(): + if ("weight" in name): wage_init.wage_init_(param, wl_weight, factor=1.0) + param.weight_acc = param.data + +def toggle_xb_eval_all_layers_stateless(model, enable): + """ + Recursively enable or disable linear modules within a model with some customized layer module. + """ + for _, module in model.named_children(): + if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d) or isinstance(module, nn.RNN) or isinstance(module, nn.LSTM): + module._xb_inference = enable # add attribute for xb evaluation for proper toggling later + # add elifs for other supported layers + else: + toggle_xb_eval_all_layers_stateless(module, enable) diff --git a/tests/conftest.py b/tests/conftest.py index 9010e39..6284f6c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,22 @@ def forward(self, x): x = x.view(-1, self.input_size) # Flatten the image x = self.model(x) return x + +class SimpleMLPNoModel(nn.Module): + def __init__(self, input_size, hidden_size, output_size): + self.input_size = input_size + super(SimpleMLPNoModel, self).__init__() + self.fc1 = nn.Linear(input_size, hidden_size, bias=False) + self.fc2 = nn.Linear(hidden_size, output_size, bias=False) + self.a1 = nn.ReLU() + def forward(self, x): + x = x.view(-1, self.input_size) # Flatten the image + x = self.fc1(x) + x = self.a1(x) + x = self.fc2(x) + return x + # ----------------------- # General Utilities # ----------------------- @@ -87,6 +102,15 @@ def mlp_model(device): model = SimpleMLP(input_size, hidden_size, output_size).to(device) return model +@pytest.fixture +def mlp_model_regular(device): + """Return a fresh 2-layer MLP patched for XBTorch.""" + input_size = 28*28 + hidden_size = 150 + output_size = 10 + model = SimpleMLPNoModel(input_size, hidden_size, output_size).to(device) + return model + # ----------------------- # XBTorch Accelerator Fixtures # ----------------------- diff --git a/tests/test_core.py b/tests/test_core.py index 5a1982c..93e1644 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -19,6 +19,9 @@ def test_device_initialization(simple_fixedpoint_accelerator): def test_model_patching(mlp_model, simple_fixedpoint_accelerator): """Test that xbtorch_model patches models correctly.""" + + # Stateful + # Model without HWA enabled model = mlp_model xbtorch.initialize() # baseline initialization diff --git a/tests/test_hwa_inference.py b/tests/test_hwa_inference.py index fc40f7a..40fb02a 100644 --- a/tests/test_hwa_inference.py +++ b/tests/test_hwa_inference.py @@ -1,4 +1,7 @@ import torch +from xbtorch.deployment import SimpleFixedPoint +import xbtorch +from xbtorch.patches import xbtorch_model def test_xb_eval_toggle(model_and_acc): model, _, _ = model_and_acc @@ -21,4 +24,23 @@ def test_initialize_array_mappings(model_and_acc): def test_plot_array_returns_tensor(model_and_acc): _, acc, _ = model_and_acc arr = acc.plot_array() - assert isinstance(arr, torch.Tensor) \ No newline at end of file + assert isinstance(arr, torch.Tensor) + +def test_stateless_accelerator(mlp_model_regular): + device = "cpu" + + acc = SimpleFixedPoint( + g_min=100, + g_max=200, + device=device, + stateful=False + ) + + xbtorch.initialize(pytorch_device=device, inference_accelerator=acc) + + model = mlp_model_regular.to(device) + model = xbtorch_model(model, replace_all=True) + + model.xb_eval(enable=False) + + return model, acc, device \ No newline at end of file