From b4f51325c9f1f720391e1a70f7f86b5b93f0aae7 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang
Date: Thu, 28 May 2026 18:46:40 -0700
Subject: [PATCH] docs(cookbook): migrate cookbook examples from
BerriAI/litellm
Bring the current state of the cookbook/ directory (notebooks and
example projects) into the docs repo as the new home for these guides.
---
cookbook/Benchmarking_LLMs_by_use_case.ipynb | 753 ++++++++++++
...pic)_with_Streaming_liteLLM_Examples.ipynb | 406 +++++++
cookbook/Evaluating_LLMs.ipynb | 579 ++++++++++
.../LiteLLM_Azure_and_OpenAI_example.ipynb | 422 +++++++
cookbook/LiteLLM_Bedrock.ipynb | 310 +++++
cookbook/LiteLLM_CometAPI.ipynb | 474 ++++++++
cookbook/LiteLLM_Comparing_LLMs.ipynb | 441 ++++++++
cookbook/LiteLLM_Completion_Cost.ipynb | 241 ++++
cookbook/LiteLLM_HuggingFace.ipynb | 252 +++++
cookbook/LiteLLM_NovitaAI_Cookbook.ipynb | 97 ++
cookbook/LiteLLM_OpenRouter.ipynb | 179 +++
cookbook/LiteLLM_Petals.ipynb | 568 ++++++++++
cookbook/LiteLLM_PromptLayer.ipynb | 130 +++
cookbook/LiteLLM_User_Based_Rate_Limits.ipynb | 224 ++++
cookbook/LiteLLM_batch_completion.ipynb | 163 +++
...teLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb | 565 +++++++++
cookbook/Parallel_function_calling.ipynb | 478 ++++++++
cookbook/Proxy_Batch_Users.ipynb | 205 ++++
cookbook/TogetherAI_liteLLM.ipynb | 1006 +++++++++++++++++
..._Nemo_Guardrails_with_LiteLLM_Server.ipynb | 157 +++
cookbook/VLLM_Model_Testing.ipynb | 404 +++++++
.../claude_code_quickstart/guide.md | 295 +++++
cookbook/ai_coding_tool_guides/index.json | 134 +++
cookbook/anthropic_agent_sdk/README.md | 144 +++
.../anthropic_agent_sdk/agent_with_mcp.py | 144 +++
cookbook/anthropic_agent_sdk/common.py | 164 +++
.../anthropic_agent_sdk/config.example.yaml | 25 +
cookbook/anthropic_agent_sdk/main.py | 99 ++
cookbook/anthropic_agent_sdk/requirements.txt | 2 +
cookbook/benchmark/benchmark.py | 90 ++
.../auto_evals.py | 30 +
cookbook/benchmark/readme.md | 181 +++
cookbook/codellama-server/README.MD | 153 +++
.../codellama-server/imgs/code-output.png | Bin 0 -> 238123 bytes
.../imgs/promptlayer_logging.png | Bin 0 -> 300105 bytes
cookbook/codellama-server/main.py | 102 ++
cookbook/community-resources/get_hf_models.py | 89 ++
cookbook/community-resources/max_tokens.json | 93 ++
cookbook/gollem_go_agent_framework/README.md | 119 ++
.../gollem_go_agent_framework/basic/main.go | 41 +
cookbook/gollem_go_agent_framework/go.mod | 5 +
cookbook/gollem_go_agent_framework/go.sum | 2 +
.../proxy_config.yaml | 16 +
.../streaming/main.go | 56 +
.../gollem_go_agent_framework/tools/main.go | 64 ++
cookbook/google_adk_litellm_tutorial.ipynb | 412 +++++++
cookbook/liteLLM_A121_Jurrasic_example.ipynb | 251 ++++
cookbook/liteLLM_Baseten.ipynb | 146 +++
cookbook/liteLLM_Getting_Started.ipynb | 411 +++++++
cookbook/liteLLM_IBM_Watsonx.ipynb | 300 +++++
cookbook/liteLLM_Langchain_Demo.ipynb | 195 ++++
cookbook/liteLLM_Ollama.ipynb | 289 +++++
cookbook/liteLLM_Replicate_Demo.ipynb | 238 ++++
cookbook/liteLLM_Streaming_Demo.ipynb | 226 ++++
cookbook/liteLLM_VertextAI_Example.ipynb | 199 ++++
cookbook/liteLLM_clarifai_Demo.ipynb | 187 +++
cookbook/liteLLM_function_calling.ipynb | 331 ++++++
.../litellm-ollama-docker-image/Dockerfile | 25 +
.../requirements.txt | 1 +
cookbook/litellm-ollama-docker-image/start.sh | 2 +
cookbook/litellm-ollama-docker-image/test.py | 35 +
.../litellm_Test_Multiple_Providers.ipynb | 571 ++++++++++
cookbook/litellm_model_fallback.ipynb | 51 +
.../batch_api/bedrock/bedrock.py | 25 +
.../bedrock/bedrock_batch_completions.jsonl | 128 +++
.../braintrust_prompt_wrapper_README.md | 279 +++++
.../braintrust_prompt_wrapper_server.py | 274 +++++
.../litellm_proxy_server/cli_token_usage.py | 63 ++
.../dashboard_1/grafana_dashboard.json | 614 ++++++++++
.../grafana_dashboard/dashboard_1/readme.md | 6 +
.../dashboard_v2/grafana_dashboard.json | 827 ++++++++++++++
.../grafana_dashboard/readme.md | 14 +
.../mcp/mcp_with_litellm_proxy.py | 37 +
cookbook/litellm_proxy_server/readme.md | 177 +++
.../custom_secret_manager_config.yaml | 20 +
.../secret_manager/my_secret_manager.py | 80 ++
cookbook/litellm_router/error_log.txt | 1004 ++++++++++++++++
cookbook/litellm_router/load_test_proxy.py | 148 +++
cookbook/litellm_router/load_test_queuing.py | 164 +++
cookbook/litellm_router/load_test_router.py | 143 +++
cookbook/litellm_router/request_log.txt | 48 +
cookbook/litellm_router/response_log.txt | 0
.../test_questions/question1.txt | 43 +
.../test_questions/question2.txt | 65 ++
.../test_questions/question3.txt | 50 +
.../memory_usage/router_endpoint.py | 65 ++
.../memory_usage/router_memory_usage copy.py | 91 ++
.../memory_usage/router_memory_usage.py | 91 ++
.../memory_usage/send_request.py | 28 +
.../test_loadtest_openai_client.py | 73 ++
.../test_loadtest_router.py | 87 ++
.../test_loadtest_router_withs3_cache.py | 93 ++
cookbook/litellm_test_multiple_llm_demo.ipynb | 55 +
cookbook/livekit_agent_sdk/README.md | 114 ++
.../livekit_agent_sdk/config.example.yaml | 21 +
cookbook/livekit_agent_sdk/main.py | 121 ++
cookbook/livekit_agent_sdk/requirements.txt | 2 +
.../logging_observability/LiteLLM_Arize.ipynb | 172 +++
.../LiteLLM_Langfuse.ipynb | 197 ++++
.../LiteLLM_Lunary.ipynb | 348 ++++++
.../LiteLLM_Proxy_Langfuse.ipynb | 252 +++++
.../litellm_proxy_langfuse.png | Bin 0 -> 315558 bytes
.../RELEASE_NOTES_GENERATION_INSTRUCTIONS.md | 543 +++++++++
cookbook/misc/add_new_models.py | 71 ++
cookbook/misc/config.yaml | 73 ++
cookbook/misc/dev_release.txt | 11 +
cookbook/misc/migrate_proxy_config.py | 95 ++
cookbook/misc/openai_timeouts.py | 33 +
cookbook/misc/sagmaker_streaming.py | 55 +
cookbook/misc/test_responses_api.py | 49 +
cookbook/misc/update_json_caching.py | 54 +
...flow_langchain_tracing_litellm_proxy.ipynb | 311 +++++
.../mock_bedrock_guardrail_server.py | 540 +++++++++
.../mock_prompt_management_server/README.md | 293 +++++
.../mock_prompt_management_server.py | 390 +++++++
cookbook/nova_sonic_realtime.py | 291 +++++
cookbook/result.html | 22 +
cookbook/veo_video_generation.py | 321 ++++++
118 files changed, 23143 insertions(+)
create mode 100644 cookbook/Benchmarking_LLMs_by_use_case.ipynb
create mode 100644 cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb
create mode 100644 cookbook/Evaluating_LLMs.ipynb
create mode 100644 cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb
create mode 100644 cookbook/LiteLLM_Bedrock.ipynb
create mode 100644 cookbook/LiteLLM_CometAPI.ipynb
create mode 100644 cookbook/LiteLLM_Comparing_LLMs.ipynb
create mode 100644 cookbook/LiteLLM_Completion_Cost.ipynb
create mode 100644 cookbook/LiteLLM_HuggingFace.ipynb
create mode 100644 cookbook/LiteLLM_NovitaAI_Cookbook.ipynb
create mode 100644 cookbook/LiteLLM_OpenRouter.ipynb
create mode 100644 cookbook/LiteLLM_Petals.ipynb
create mode 100644 cookbook/LiteLLM_PromptLayer.ipynb
create mode 100644 cookbook/LiteLLM_User_Based_Rate_Limits.ipynb
create mode 100644 cookbook/LiteLLM_batch_completion.ipynb
create mode 100644 cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb
create mode 100644 cookbook/Parallel_function_calling.ipynb
create mode 100644 cookbook/Proxy_Batch_Users.ipynb
create mode 100644 cookbook/TogetherAI_liteLLM.ipynb
create mode 100644 cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb
create mode 100644 cookbook/VLLM_Model_Testing.ipynb
create mode 100644 cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
create mode 100644 cookbook/ai_coding_tool_guides/index.json
create mode 100644 cookbook/anthropic_agent_sdk/README.md
create mode 100644 cookbook/anthropic_agent_sdk/agent_with_mcp.py
create mode 100644 cookbook/anthropic_agent_sdk/common.py
create mode 100644 cookbook/anthropic_agent_sdk/config.example.yaml
create mode 100644 cookbook/anthropic_agent_sdk/main.py
create mode 100644 cookbook/anthropic_agent_sdk/requirements.txt
create mode 100644 cookbook/benchmark/benchmark.py
create mode 100644 cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py
create mode 100644 cookbook/benchmark/readme.md
create mode 100644 cookbook/codellama-server/README.MD
create mode 100644 cookbook/codellama-server/imgs/code-output.png
create mode 100644 cookbook/codellama-server/imgs/promptlayer_logging.png
create mode 100644 cookbook/codellama-server/main.py
create mode 100644 cookbook/community-resources/get_hf_models.py
create mode 100644 cookbook/community-resources/max_tokens.json
create mode 100644 cookbook/gollem_go_agent_framework/README.md
create mode 100644 cookbook/gollem_go_agent_framework/basic/main.go
create mode 100644 cookbook/gollem_go_agent_framework/go.mod
create mode 100644 cookbook/gollem_go_agent_framework/go.sum
create mode 100644 cookbook/gollem_go_agent_framework/proxy_config.yaml
create mode 100644 cookbook/gollem_go_agent_framework/streaming/main.go
create mode 100644 cookbook/gollem_go_agent_framework/tools/main.go
create mode 100644 cookbook/google_adk_litellm_tutorial.ipynb
create mode 100644 cookbook/liteLLM_A121_Jurrasic_example.ipynb
create mode 100644 cookbook/liteLLM_Baseten.ipynb
create mode 100644 cookbook/liteLLM_Getting_Started.ipynb
create mode 100644 cookbook/liteLLM_IBM_Watsonx.ipynb
create mode 100644 cookbook/liteLLM_Langchain_Demo.ipynb
create mode 100644 cookbook/liteLLM_Ollama.ipynb
create mode 100644 cookbook/liteLLM_Replicate_Demo.ipynb
create mode 100644 cookbook/liteLLM_Streaming_Demo.ipynb
create mode 100644 cookbook/liteLLM_VertextAI_Example.ipynb
create mode 100644 cookbook/liteLLM_clarifai_Demo.ipynb
create mode 100644 cookbook/liteLLM_function_calling.ipynb
create mode 100644 cookbook/litellm-ollama-docker-image/Dockerfile
create mode 100644 cookbook/litellm-ollama-docker-image/requirements.txt
create mode 100644 cookbook/litellm-ollama-docker-image/start.sh
create mode 100644 cookbook/litellm-ollama-docker-image/test.py
create mode 100644 cookbook/litellm_Test_Multiple_Providers.ipynb
create mode 100644 cookbook/litellm_model_fallback.ipynb
create mode 100644 cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py
create mode 100644 cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl
create mode 100644 cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md
create mode 100644 cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py
create mode 100644 cookbook/litellm_proxy_server/cli_token_usage.py
create mode 100644 cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/grafana_dashboard.json
create mode 100644 cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/readme.md
create mode 100644 cookbook/litellm_proxy_server/grafana_dashboard/dashboard_v2/grafana_dashboard.json
create mode 100644 cookbook/litellm_proxy_server/grafana_dashboard/readme.md
create mode 100644 cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py
create mode 100644 cookbook/litellm_proxy_server/readme.md
create mode 100644 cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml
create mode 100644 cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py
create mode 100644 cookbook/litellm_router/error_log.txt
create mode 100644 cookbook/litellm_router/load_test_proxy.py
create mode 100644 cookbook/litellm_router/load_test_queuing.py
create mode 100644 cookbook/litellm_router/load_test_router.py
create mode 100644 cookbook/litellm_router/request_log.txt
create mode 100644 cookbook/litellm_router/response_log.txt
create mode 100644 cookbook/litellm_router/test_questions/question1.txt
create mode 100644 cookbook/litellm_router/test_questions/question2.txt
create mode 100644 cookbook/litellm_router/test_questions/question3.txt
create mode 100644 cookbook/litellm_router_load_test/memory_usage/router_endpoint.py
create mode 100644 cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py
create mode 100644 cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py
create mode 100644 cookbook/litellm_router_load_test/memory_usage/send_request.py
create mode 100644 cookbook/litellm_router_load_test/test_loadtest_openai_client.py
create mode 100644 cookbook/litellm_router_load_test/test_loadtest_router.py
create mode 100644 cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py
create mode 100644 cookbook/litellm_test_multiple_llm_demo.ipynb
create mode 100644 cookbook/livekit_agent_sdk/README.md
create mode 100644 cookbook/livekit_agent_sdk/config.example.yaml
create mode 100644 cookbook/livekit_agent_sdk/main.py
create mode 100644 cookbook/livekit_agent_sdk/requirements.txt
create mode 100644 cookbook/logging_observability/LiteLLM_Arize.ipynb
create mode 100644 cookbook/logging_observability/LiteLLM_Langfuse.ipynb
create mode 100644 cookbook/logging_observability/LiteLLM_Lunary.ipynb
create mode 100644 cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb
create mode 100644 cookbook/logging_observability/litellm_proxy_langfuse.png
create mode 100644 cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md
create mode 100644 cookbook/misc/add_new_models.py
create mode 100644 cookbook/misc/config.yaml
create mode 100644 cookbook/misc/dev_release.txt
create mode 100644 cookbook/misc/migrate_proxy_config.py
create mode 100644 cookbook/misc/openai_timeouts.py
create mode 100644 cookbook/misc/sagmaker_streaming.py
create mode 100644 cookbook/misc/test_responses_api.py
create mode 100644 cookbook/misc/update_json_caching.py
create mode 100644 cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb
create mode 100644 cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py
create mode 100644 cookbook/mock_prompt_management_server/README.md
create mode 100644 cookbook/mock_prompt_management_server/mock_prompt_management_server.py
create mode 100644 cookbook/nova_sonic_realtime.py
create mode 100644 cookbook/result.html
create mode 100644 cookbook/veo_video_generation.py
diff --git a/cookbook/Benchmarking_LLMs_by_use_case.ipynb b/cookbook/Benchmarking_LLMs_by_use_case.ipynb
new file mode 100644
index 000000000..6ea6211bf
--- /dev/null
+++ b/cookbook/Benchmarking_LLMs_by_use_case.ipynb
@@ -0,0 +1,753 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "4Cq-_Y-TKf0r"
+ },
+ "source": [
+ "# LiteLLM - Benchmark Llama2, Claude1.2 and GPT3.5 for a use case\n",
+ "In this notebook for a given use case we run the same question and view:\n",
+ "* LLM Response\n",
+ "* Response Time\n",
+ "* Response Cost\n",
+ "\n",
+ "## Sample output for a question\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "O3ENsWYB27Mb"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Pk55Mjq_3DiR"
+ },
+ "source": [
+ "## Example Use Case 1 - Code Generator\n",
+ "### For this use case enter your system prompt and questions\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {
+ "id": "_1SZYJFB3HmQ"
+ },
+ "outputs": [],
+ "source": [
+ "# enter your system prompt if you have one\n",
+ "system_prompt = \"\"\"\n",
+ "You are a coding assistant helping users using litellm.\n",
+ "litellm is a light package to simplify calling OpenAI, Azure, Cohere, Anthropic, Huggingface API Endpoints\n",
+ "--\n",
+ "Sample Usage:\n",
+ "```\n",
+ "pip install litellm\n",
+ "from litellm import completion\n",
+ "## set ENV variables\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"openai key\"\n",
+ "os.environ[\"COHERE_API_KEY\"] = \"cohere key\"\n",
+ "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
+ "# openai call\n",
+ "response = completion(model=\"gpt-3.5-turbo\", messages=messages)\n",
+ "# cohere call\n",
+ "response = completion(\"command-nightly\", messages)\n",
+ "```\n",
+ "\n",
+ "\"\"\"\n",
+ "\n",
+ "\n",
+ "# qustions/logs you want to run the LLM on\n",
+ "questions = [\n",
+ " \"what is litellm?\",\n",
+ " \"why should I use LiteLLM\",\n",
+ " \"does litellm support Anthropic LLMs\",\n",
+ " \"write code to make a litellm completion call\",\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "AHH3cqeU3_ZT"
+ },
+ "source": [
+ "## Running questions\n",
+ "### Select from 100+ LLMs here: https://docs.litellm.ai/docs/providers"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "BpQD4A5339L3"
+ },
+ "outputs": [],
+ "source": [
+ "from litellm import completion, completion_cost\n",
+ "import os\n",
+ "import time\n",
+ "\n",
+ "# optional use litellm dashboard to view logs\n",
+ "# litellm.use_client = True\n",
+ "# litellm.token = \"ishaan_2@berri.ai\" # set your email\n",
+ "\n",
+ "\n",
+ "# set API keys\n",
+ "os.environ['TOGETHERAI_API_KEY'] = \"\"\n",
+ "os.environ['OPENAI_API_KEY'] = \"\"\n",
+ "os.environ['ANTHROPIC_API_KEY'] = \"\"\n",
+ "\n",
+ "\n",
+ "# select LLMs to benchmark\n",
+ "# using https://api.together.xyz/playground for llama2\n",
+ "# try any supported LLM here: https://docs.litellm.ai/docs/providers\n",
+ "\n",
+ "models = ['togethercomputer/llama-2-70b-chat', 'gpt-3.5-turbo', 'claude-instant-1.2']\n",
+ "data = []\n",
+ "\n",
+ "for question in questions: # group by question\n",
+ " for model in models:\n",
+ " print(f\"running question: {question} for model: {model}\")\n",
+ " start_time = time.time()\n",
+ " # show response, response time, cost for each question\n",
+ " response = completion(\n",
+ " model=model,\n",
+ " max_tokens=500,\n",
+ " messages = [\n",
+ " {\n",
+ " \"role\": \"system\", \"content\": system_prompt\n",
+ " },\n",
+ " {\n",
+ " \"role\": \"user\", \"content\": question\n",
+ " }\n",
+ " ],\n",
+ " )\n",
+ " end = time.time()\n",
+ " total_time = end-start_time # response time\n",
+ " # print(response)\n",
+ " cost = completion_cost(response) # cost for completion\n",
+ " raw_response = response['choices'][0]['message']['content'] # response string\n",
+ "\n",
+ "\n",
+ " # add log to pandas df\n",
+ " data.append(\n",
+ " {\n",
+ " 'Model': model,\n",
+ " 'Question': question,\n",
+ " 'Response': raw_response,\n",
+ " 'ResponseTime': total_time,\n",
+ " 'Cost': cost\n",
+ " })"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "apOSV3PBLa5Y"
+ },
+ "source": [
+ "## View Benchmarks for LLMs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 1000
+ },
+ "id": "CJqBlqUh_8Ws",
+ "outputId": "e02c3427-d8c6-4614-ff07-6aab64247ff6"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Question: does litellm support Anthropic LLMs\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " Model \n",
+ " Question \n",
+ " Response \n",
+ " ResponseTime \n",
+ " Cost \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " 6 \n",
+ " togethercomputer/llama-2-70b-chat \n",
+ " does litellm support Anthropic LLMs \n",
+ " Yes, litellm supports Anthropic LLMs.\\n\\nIn the example usage you provided, the `completion` function is called with the `model` parameter set to `\"gpt-3.5-turbo\"` for OpenAI and `\"command-nightly\"` for Cohere.\\n\\nTo use an Anthropic LLM with litellm, you would set the `model` parameter to the name of the Anthropic model you want to use, followed by the version number, if applicable. For example:\\n```\\nresponse = completion(model=\"anthropic-gpt-2\", messages=messages)\\n```\\nThis would call the Anthropic GPT-2 model to generate a completion for the given input messages.\\n\\nNote that you will need to set the `ANTHROPIC_API_KEY` environment variable to your Anthropic API key before making the call. You can do this by running the following command in your terminal:\\n```\\nos.environ[\"ANTHROPIC_API_KEY\"] = \"your-anthropic-api-key\"\\n```\\nReplace `\"your-anthropic-api-key\"` with your actual Anthropic API key.\\n\\nOnce you've set the environment variable, you can use the `completion` function with the `model` parameter set to an Anthropic model name to call the Anthropic API and generate a completion. \n",
+ " 21.513009 \n",
+ " 0.001347 \n",
+ " \n",
+ " \n",
+ " 7 \n",
+ " gpt-3.5-turbo \n",
+ " does litellm support Anthropic LLMs \n",
+ " No, currently litellm does not support Anthropic LLMs. It mainly focuses on simplifying the usage of OpenAI, Azure, Cohere, and Huggingface API endpoints. \n",
+ " 8.656510 \n",
+ " 0.000342 \n",
+ " \n",
+ " \n",
+ " 8 \n",
+ " claude-instant-1.2 \n",
+ " does litellm support Anthropic LLMs \n",
+ " Yes, litellm supports calling Anthropic LLMs through the completion function.\\n\\nTo use an Anthropic model with litellm:\\n\\n1. Set the ANTHROPIC_API_KEY environment variable with your Anthropic API key\\n\\n2. Pass the model name as the 'model' argument to completion(). Anthropic model names follow the format 'anthropic/<model_name>'\\n\\nFor example:\\n\\n```python \\nimport os\\nfrom litellm import completion\\n\\nos.environ[\"ANTHROPIC_API_KEY\"] = \"your_anthropic_api_key\"\\n\\nmessages = [{\"content\": \"Hello\", \"role\": \"user\"}]\\n\\nresponse = completion(model=\"anthropic/constitutional\", messages=messages)\\n```\\n\\nThis would call the Constitutional AI model from Anthropic.\\n\\nSo in summary, litellm provides a simple interface to call any Anthropic models as long as you specify the model name correctly and set the ANTHROPIC_API_KEY env variable. \n",
+ " 9.698195 \n",
+ " 0.001342 \n",
+ " \n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Question: what is litellm?\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " Model \n",
+ " Question \n",
+ " Response \n",
+ " ResponseTime \n",
+ " Cost \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 \n",
+ " togethercomputer/llama-2-70b-chat \n",
+ " what is litellm? \n",
+ " Litellm is a lightweight Python package that simplifies calling various AI API endpoints, including OpenAI, Azure, Cohere, Anthropic, and Hugging Face. It provides a convenient interface for making requests to these APIs, allowing developers to easily integrate them into their applications. With Litellm, developers can quickly and easily interact with multiple AI models and services, without having to handle the details of authentication, API calls, and response parsing. This makes it easier to build and deploy AI-powered applications, and can help developers save time and effort. \n",
+ " 13.479644 \n",
+ " 0.000870 \n",
+ " \n",
+ " \n",
+ " 1 \n",
+ " gpt-3.5-turbo \n",
+ " what is litellm? \n",
+ " litellm is a light package that provides a simplified interface for making API calls to various language models and APIs. It abstracts away the complexities of handling network requests, authentication, and response parsing, making it easier for developers to integrate powerful language models into their applications.\\n\\nWith litellm, you can quickly make API calls to models like OpenAI's GPT-3.5 Turbo, Azure's Text Analytics, Cohere's Command API, Anthropic's API, and Huggingface's models. It also supports additional functionality like conversational AI, summarization, translation, and more.\\n\\nBy using litellm, you can focus on your application logic without getting tangled in the details of API integration, allowing you to quickly build intelligent and conversational applications. \n",
+ " 8.324332 \n",
+ " 0.000566 \n",
+ " \n",
+ " \n",
+ " 2 \n",
+ " claude-instant-1.2 \n",
+ " what is litellm? \n",
+ " litellm is a Python library that simplifies calling various AI API endpoints like OpenAI, Azure, Cohere, Anthropic, and Huggingface. \\n\\nSome key things to know about litellm:\\n\\n- It provides a consistent interface for completing prompts and generating responses from different AI models through a single method called completion().\\n\\n- You specify the API (e.g. OpenAI, Cohere etc.) and model either by name or by setting environment variables before making the completion call.\\n\\n- This avoids having to use different SDKs or APIs for each provider and standardizes the call structure. \\n\\n- It handles things like setting headers, encoding inputs, parsing responses so the user doesn't have to deal with those details.\\n\\n- The goal is to make it easy to try different AI APIs and models without having to change code or learn different interfaces.\\n\\n- It's lightweight with no other dependencies required besides what's needed for each API (e.g. openai, azure SDKs etc.).\\n\\nSo in summary, litellm is a small library that provides a common way to interact with multiple conversational AI APIs through a single Python method, avoiding the need to directly use each provider's specific SDK. \n",
+ " 10.316488 \n",
+ " 0.001603 \n",
+ " \n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Question: why should I use LiteLLM\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " Model \n",
+ " Question \n",
+ " Response \n",
+ " ResponseTime \n",
+ " Cost \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " 3 \n",
+ " togethercomputer/llama-2-70b-chat \n",
+ " why should I use LiteLLM \n",
+ " \\nThere are several reasons why you might want to use LiteLLM:\\n\\n1. Simplified API calls: LiteLLM provides a simple and consistent API for calling various language models, making it easier to use multiple models and switch between them.\\n2. Environment variable configuration: LiteLLM allows you to set environment variables for API keys and model names, making it easier to manage and switch between different models and APIs.\\n3. Support for multiple models and APIs: LiteLLM supports a wide range of language models and APIs, including OpenAI, Azure, Cohere, Anthropic, and Hugging Face.\\n4. Easy integration with popular frameworks: LiteLLM can be easily integrated with popular frameworks such as PyTorch and TensorFlow, making it easy to use with your existing codebase.\\n5. Lightweight: LiteLLM is a lightweight package, making it easy to install and use, even on resource-constrained devices.\\n6. Flexible: LiteLLM allows you to define your own models and APIs, making it easy to use with custom models and APIs.\\n7. Extensive documentation: LiteLLM has extensive documentation, making it easy to get started and learn how to use the package.\\n8. Active community: LiteLLM has an active community of developers and users, making it easy to get help and feedback on your projects.\\n\\nOverall, LiteLLM can help you to simplify your workflow, improve your productivity, and make it easier to work with multiple language models and APIs. \n",
+ " 23.777885 \n",
+ " 0.001443 \n",
+ " \n",
+ " \n",
+ " 4 \n",
+ " gpt-3.5-turbo \n",
+ " why should I use LiteLLM \n",
+ " LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience. \n",
+ " 12.109881 \n",
+ " 0.000881 \n",
+ " \n",
+ " \n",
+ " 5 \n",
+ " claude-instant-1.2 \n",
+ " why should I use LiteLLM \n",
+ " Here are some key reasons why you may want to consider using LiteLLM:\\n\\n- Simplifies calling multiple large language models - LiteLLM provides a unified API to call models from different providers like OpenAI, Azure, HuggingFace, Anthropic etc. This avoids having to deal with different APIs from each provider.\\n\\n- Easy to get started - LiteLLM is very lightweight and simple to install with just one pip install. It has minimal configuration needed with just API keys. This lowers the barrier to start experimenting with LLMs.\\n\\n- Standard interface - The completion API provides a standard way to send messages and get responses back regardless of underlying model. This keeps the code interface consistent. \\n\\n- Model agnostic - Your code doesn't need to change to switch between different models. You can easily compare performance of models from different providers.\\n\\n- Open source - As it is open source, the library can be expanded to support more models and providers over time via community contributions.\\n\\n- Cost effective - LiteLLM helps avoid vendor lock-in. You can optimize costs by testing multiple models and choosing the best suitable and cost-effective option for your needs.\\n\\nSo in summary, L \n",
+ " 12.610259 \n",
+ " 0.001568 \n",
+ " \n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Question: write code to make a litellm completion call\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " Model \n",
+ " Question \n",
+ " Response \n",
+ " ResponseTime \n",
+ " Cost \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " 9 \n",
+ " togethercomputer/llama-2-70b-chat \n",
+ " write code to make a litellm completion call \n",
+ " \\nTo make a completion call using litellm, you can use the `completion` function from the litellm package. Here's an example of how to use it:\\n```\\nfrom litellm import completion\\n\\n# Set ENV variables\\nos.environ[\"OPENAI_API_KEY\"] = \"your_openai_api_key\"\\nos.environ[\"COHERE_API_KEY\"] = \"your_cohere_api_key\"\\n\\n# Define the messages to be completed\\nmessages = [\\n {\\n \"content\": \"Hello, how are you?\",\\n \"role\": \"user\"\\n }\\n]\\n\\n# Make a completion call using OpenAI\\nresponse = completion(model=\"gpt-3.5-turbo\", messages=messages)\\n\\n# Make a completion call using Cohere\\nresponse = completion(\"command-nightly\", messages)\\n```\\nIn this example, we first set the ENV variables for the OpenAI and Cohere API keys. Then, we define a list of messages to be completed, which in this case contains a single message with the content \"Hello, how are you?\" and the role \"user\".\\n\\nNext, we make two completion calls using the `completion` function from litellm. The first call uses the OpenAI model `gpt-3.5-turbo` and passes in the list of messages. The second call uses the Cohere model `command-nightly` and passes in the same list of messages.\\n\\nThe `completion` function returns a response object that contains the completed messages. You can then use the `response.messages` attribute to access the completed messages.\\n\\nHere's an example of how to access the completed messages:\\n```\\n# Print the completed messages\\nprint(response.messages)\\n```\\nThis will print the completed messages, which should look something like this:\\n```\\n[{'content': 'Hello, how are you?', 'role': 'user', 'completion': 'I am doing well, thanks for asking.'}]\\n```\\nNote that the `completion` attribute contains the completed message. You can use this attribute to retrieve the completed message and use it in your application. \n",
+ " 32.192453 \n",
+ " 0.001821 \n",
+ " \n",
+ " \n",
+ " 10 \n",
+ " gpt-3.5-turbo \n",
+ " write code to make a litellm completion call \n",
+ " Sure! Here's an example code to make a completion call using the litellm package:\\n\\n```python\\nfrom litellm import completion\\n\\n# Set the model and messages\\nmodel = \"gpt-3.5-turbo\"\\nmessages = [\\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\\n {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"},\\n {\"role\": \"assistant\", \"content\": \"The Los Angeles Dodgers won the World Series in 2020.\"},\\n {\"role\": \"user\", \"content\": \"Where was it played?\"}\\n]\\n\\n# Make the completion call\\nresponse = completion(model=model, messages=messages)\\n\\n# Print the assistant's reply\\nassistant_reply = response[\"choices\"][0][\"message\"][\"content\"]\\nprint(\"Assistant: \", assistant_reply)\\n```\\n\\nMake sure you have the litellm package installed (`pip install litellm`) and set the necessary environment variables for the API keys before running this code. \n",
+ " 9.377155 \n",
+ " 0.000686 \n",
+ " \n",
+ " \n",
+ " 11 \n",
+ " claude-instant-1.2 \n",
+ " write code to make a litellm completion call \n",
+ " Here is an example of making a completion call using litellm:\\n\\n```python\\nimport os\\nfrom litellm import completion\\n\\n# Set API keys as environment variables\\nos.environ[\"OPENAI_API_KEY\"] = \"your openai api key\" \\n\\n# Conversation context \\nmessages = [{\\n \"content\": \"Hello, how can I help you today?\",\\n \"role\": \"assistant\"\\n}]\\n\\n# Make completion call with GPT-3 model\\nresponse = completion(\\n model=\"gpt-3.5-turbo\", \\n messages=messages\\n)\\n\\nprint(response)\\n```\\n\\nTo break it down:\\n\\n- Import completion from litellm\\n- Set the OPENAI_API_KEY env var \\n- Define a messages list with the conversation context\\n- Call completion(), specifying the model (\"gpt-3.5-turbo\") and messages\\n- It will return the response from the API\\n- Print the response\\n\\nThis makes a simple completion call to OpenAI GPT-3 using litellm to handle the API details. You can also call other models like Cohere or Anthropic by specifying their name instead of the OpenAI \n",
+ " 9.839988 \n",
+ " 0.001578 \n",
+ " \n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from IPython.core.interactiveshell import InteractiveShell\n",
+ "InteractiveShell.ast_node_interactivity = \"all\"\n",
+ "from IPython.display import HTML\n",
+ "import pandas as pd\n",
+ "\n",
+ "df = pd.DataFrame(data)\n",
+ "grouped_by_question = df.groupby('Question')\n",
+ "\n",
+ "for question, group_data in grouped_by_question:\n",
+ " print(f\"Question: {question}\")\n",
+ " HTML(group_data.to_html())\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "bmtAbC1rGVAm"
+ },
+ "source": [
+ "## Use Case 2 - Rewrite user input concisely"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {
+ "id": "boiHO1PhGXSL"
+ },
+ "outputs": [],
+ "source": [
+ "# enter your system prompt if you have one\n",
+ "system_prompt = \"\"\"\n",
+ "For a given user input, rewrite the input to make be more concise.\n",
+ "\"\"\"\n",
+ "\n",
+ "# user input for re-writing questions\n",
+ "questions = [\n",
+ " \"LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience\",\n",
+ " \"Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!\",\n",
+ " \"Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.\"\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "fwNcC_obICUc"
+ },
+ "source": [
+ "## Run Questions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "KtBjZ1mUIBiJ"
+ },
+ "outputs": [],
+ "source": [
+ "from litellm import completion, completion_cost\n",
+ "import os\n",
+ "import time\n",
+ "\n",
+ "# optional use litellm dashboard to view logs\n",
+ "# litellm.use_client = True\n",
+ "# litellm.token = \"ishaan_2@berri.ai\" # set your email\n",
+ "\n",
+ "os.environ['TOGETHERAI_API_KEY'] = \"\"\n",
+ "os.environ['OPENAI_API_KEY'] = \"\"\n",
+ "os.environ['ANTHROPIC_API_KEY'] = \"\"\n",
+ "\n",
+ "models = ['togethercomputer/llama-2-70b-chat', 'gpt-3.5-turbo', 'claude-instant-1.2'] # enter llms to benchmark\n",
+ "data_2 = []\n",
+ "\n",
+ "for question in questions: # group by question\n",
+ " for model in models:\n",
+ " print(f\"running question: {question} for model: {model}\")\n",
+ " start_time = time.time()\n",
+ " # show response, response time, cost for each question\n",
+ " response = completion(\n",
+ " model=model,\n",
+ " max_tokens=500,\n",
+ " messages = [\n",
+ " {\n",
+ " \"role\": \"system\", \"content\": system_prompt\n",
+ " },\n",
+ " {\n",
+ " \"role\": \"user\", \"content\": \"User input:\" + question\n",
+ " }\n",
+ " ],\n",
+ " )\n",
+ " end = time.time()\n",
+ " total_time = end-start_time # response time\n",
+ " # print(response)\n",
+ " cost = completion_cost(response) # cost for completion\n",
+ " raw_response = response['choices'][0]['message']['content'] # response string\n",
+ " #print(raw_response, total_time, cost)\n",
+ "\n",
+ " # add to pandas df\n",
+ " data_2.append(\n",
+ " {\n",
+ " 'Model': model,\n",
+ " 'Question': question,\n",
+ " 'Response': raw_response,\n",
+ " 'ResponseTime': total_time,\n",
+ " 'Cost': cost\n",
+ " })\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "-PCYIzG5M0II"
+ },
+ "source": [
+ "## View Logs - Group by Question"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 1000
+ },
+ "id": "-3R5-2q8IiL2",
+ "outputId": "c4a0d9e5-bb21-4de0-fc4c-9f5e71d0f177"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Question: Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " Model \n",
+ " Question \n",
+ " Response \n",
+ " ResponseTime \n",
+ " Cost \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " 3 \n",
+ " togethercomputer/llama-2-70b-chat \n",
+ " Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need! \n",
+ " \\nHere's a more concise version of the user input:\\n\\n\"Hi everyone! I'm [your name] and I'm working on [your project/role involving LLMs]. I recently discovered LiteLLM and I'm excited to use it to [build an app/simplify my code/test different models etc]. Before LiteLLM, I struggled with [describe any issues you faced working with multiple LLMs]. I'm looking forward to using LiteLLM's unified API and automatic translation to achieve my goals. I'm eager to learn more about building impactful applications powered by LLMs and to be part of this community. Let me know if you have any questions or need further clarification.\"\\n\\nIn this revised version, we've kept the essential information and removed some of the extraneous language. We've also rephrased some of the sentences to make them more concise and easier to read. \n",
+ " 18.300620 \n",
+ " 0.001200 \n",
+ " \n",
+ " \n",
+ " 4 \n",
+ " gpt-3.5-turbo \n",
+ " Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need! \n",
+ " User input: Hi, I'm [your name] and I'm excited about using LiteLLM to simplify working with different LLM providers. Before finding LiteLLM, I faced challenges working with multiple LLMs. With LiteLLM's unified API and automatic translation, I believe it will help me achieve my goals of [state your goals]. I look forward to being part of this community and learning how to build impactful applications with LLMs. Let me know if you need any further clarification or details. \n",
+ " 7.385472 \n",
+ " 0.000525 \n",
+ " \n",
+ " \n",
+ " 5 \n",
+ " claude-instant-1.2 \n",
+ " Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need! \n",
+ " Here is a more concise rewrite of the user input:\\n\\nHi everyone, I'm [your name]. I'm currently [your project/role] and came across LiteLLM, which simplifies working with different LLMs through its unified API. I hope to [build an app/simplify code/test models] with LiteLLM since I previously struggled with [issues]. LiteLLM's automatic translation between providers will help me [goals] and build impactful LLM applications. Looking forward to learning more as part of this community. Let me know if you need any clarification on my plans to use LiteLLM. \n",
+ " 8.628217 \n",
+ " 0.001022 \n",
+ " \n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Question: LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\n",
+ "\n",
+ "1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\n",
+ "\n",
+ "2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\n",
+ "\n",
+ "3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\n",
+ "\n",
+ "4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\n",
+ "\n",
+ "5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\n",
+ "\n",
+ "6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\n",
+ "\n",
+ "Overall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " Model \n",
+ " Question \n",
+ " Response \n",
+ " ResponseTime \n",
+ " Cost \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 \n",
+ " togethercomputer/llama-2-70b-chat \n",
+ " LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience \n",
+ " Here's a more concise version of the user input:\\n\\nLiteLLM is a lightweight Python package that simplifies API calls to various language models. It abstracts away complexity, integrates seamlessly, supports multiple models, and handles authentication. It's ideal for rapid prototyping and has community support. It saves time and effort while providing flexibility and convenience. \n",
+ " 11.294250 \n",
+ " 0.001251 \n",
+ " \n",
+ " \n",
+ " 1 \n",
+ " gpt-3.5-turbo \n",
+ " LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience \n",
+ " LiteLLM is a lightweight Python package that simplifies API calls to various language models. Here's why you should use it:\\n1. Simplified API Calls: Works with multiple models (OpenAI, Azure, Cohere, Anthropic, Huggingface).\\n2. Easy Integration: Import and start using it quickly in your codebase.\\n3. Flexibility: Supports GPT-3, GPT-Neo, chatGPT, etc. easily switch between models.\\n4. Convenience: Handles authentication and connection details, just set environment variables.\\n5. Quick Prototyping: Great for rapid prototyping and building interactive applications.\\n6. Community Support: Actively maintained and supported by a developer community. \n",
+ " 9.778315 \n",
+ " 0.000795 \n",
+ " \n",
+ " \n",
+ " 2 \n",
+ " claude-instant-1.2 \n",
+ " LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience \n",
+ " Here is a concise rewrite of the user input:\\n\\nLiteLLM is a lightweight Python package that simplifies accessing various language models. It provides a unified interface for models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more. Key benefits include simplified API calls, easy integration, flexibility to use different models, automated handling of authentication, and support for quick prototyping. The actively maintained package saves time by abstracting away complexity while offering convenience and a collaborative community. \n",
+ " 7.697520 \n",
+ " 0.001098 \n",
+ " \n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Question: Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " Model \n",
+ " Question \n",
+ " Response \n",
+ " ResponseTime \n",
+ " Cost \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " 6 \n",
+ " togethercomputer/llama-2-70b-chat \n",
+ " Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack. \n",
+ " \\nRewritten input: Traceloop is a platform for monitoring and debugging LLM outputs. It allows users to track performance, rollout changes confidently, and debug issues in production. It uses OpenTelemetry for full visibility into LLM requests, vector DB usage, and other infrastructure. \n",
+ " 9.060444 \n",
+ " 0.000525 \n",
+ " \n",
+ " \n",
+ " 7 \n",
+ " gpt-3.5-turbo \n",
+ " Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack. \n",
+ " Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It helps track performance, rollout changes, and debug issues in production. It is based on OpenTelemetry, providing visibility to LLM requests, vector DB usage, and other infrastructure in your stack. \n",
+ " 7.304661 \n",
+ " 0.000283 \n",
+ " \n",
+ " \n",
+ " 8 \n",
+ " claude-instant-1.2 \n",
+ " Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack. \n",
+ " Here is a more concise rewrite of the user input:\\n\\nTraceloop monitors and debugs LLM quality. It tracks LLM performance, enables confident changes, and debugs production issues. Based on OpenTelemetry, Traceloop provides full visibility into LLM requests, vector DB usage, and other stack infrastructure. \n",
+ " 7.976158 \n",
+ " 0.000538 \n",
+ " \n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from IPython.core.interactiveshell import InteractiveShell\n",
+ "InteractiveShell.ast_node_interactivity = \"all\"\n",
+ "from IPython.display import HTML\n",
+ "import pandas as pd\n",
+ "\n",
+ "df = pd.DataFrame(data_2)\n",
+ "grouped_by_question = df.groupby('Question')\n",
+ "\n",
+ "for question, group_data in grouped_by_question:\n",
+ " print(f\"Question: {question}\")\n",
+ " HTML(group_data.to_html())\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb b/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb
new file mode 100644
index 000000000..338785ea5
--- /dev/null
+++ b/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb
@@ -0,0 +1,406 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "ZwuaylskLxFu",
+ "outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Collecting litellm==0.1.363\n",
+ " Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n",
+ "Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n",
+ "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n",
+ "Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n",
+ "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n",
+ "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n",
+ "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n",
+ "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n",
+ "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n",
+ "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n",
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n",
+ "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n",
+ "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n",
+ "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n",
+ "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n",
+ "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n",
+ "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n",
+ "Installing collected packages: litellm\n",
+ " Attempting uninstall: litellm\n",
+ " Found existing installation: litellm 0.1.362\n",
+ " Uninstalling litellm-0.1.362:\n",
+ " Successfully uninstalled litellm-0.1.362\n",
+ "Successfully installed litellm-0.1.363\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install litellm==\"0.1.363\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "W216G__XL19Q"
+ },
+ "outputs": [],
+ "source": [
+ "# @title Import litellm & Set env variables\n",
+ "import litellm\n",
+ "import os\n",
+ "\n",
+ "os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "ff1lKwUMMLJj",
+ "outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "\n",
+ " Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n",
+ "\n",
+ "\n",
+ " Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n"
+ ]
+ }
+ ],
+ "source": [
+ "# @title Request Claude Instant-1 and Claude-2\n",
+ "messages = [\n",
+ " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
+ " {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n",
+ " ]\n",
+ "\n",
+ "result = litellm.completion('claude-instant-1', messages)\n",
+ "print(\"\\n\\n Result from claude-instant-1\", result)\n",
+ "result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n",
+ "print(\"\\n\\n Result from claude-2\", result)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "06hWKnNQMrV-",
+ "outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " Here\n",
+ "'s\n",
+ " a\n",
+ " quick\n",
+ " overview\n",
+ " of\n",
+ " how\n",
+ " a\n",
+ " court\n",
+ " case\n",
+ " can\n",
+ " reach\n",
+ " the\n",
+ " U\n",
+ ".\n",
+ "S\n",
+ ".\n",
+ " Supreme\n",
+ " Court\n",
+ ":\n",
+ "\n",
+ "\n",
+ "-\n",
+ " The\n",
+ " case\n",
+ " must\n",
+ " first\n",
+ " be\n",
+ " heard\n",
+ " in\n",
+ " a\n",
+ " lower\n",
+ " trial\n",
+ " court\n",
+ " (\n",
+ "either\n",
+ " a\n",
+ " state\n",
+ " court\n",
+ " or\n",
+ " federal\n",
+ " district\n",
+ " court\n",
+ ").\n",
+ " The\n",
+ " trial\n",
+ " court\n",
+ " makes\n",
+ " initial\n",
+ " r\n",
+ "ulings\n",
+ " and\n",
+ " produces\n",
+ " a\n",
+ " record\n",
+ " of\n",
+ " the\n",
+ " case\n",
+ ".\n",
+ "\n",
+ "\n",
+ "-\n",
+ " The\n",
+ " losing\n",
+ " party\n",
+ " can\n",
+ " appeal\n",
+ " the\n",
+ " decision\n",
+ " to\n",
+ " an\n",
+ " appeals\n",
+ " court\n",
+ " (\n",
+ "a\n",
+ " state\n",
+ " appeals\n",
+ " court\n",
+ " for\n",
+ " state\n",
+ " cases\n",
+ ",\n",
+ " or\n",
+ " a\n",
+ " federal\n",
+ " circuit\n",
+ " court\n",
+ " for\n",
+ " federal\n",
+ " cases\n",
+ ").\n",
+ " The\n",
+ " appeals\n",
+ " court\n",
+ " reviews\n",
+ " the\n",
+ " trial\n",
+ " court\n",
+ "'s\n",
+ " r\n",
+ "ulings\n",
+ " and\n",
+ " can\n",
+ " affirm\n",
+ ",\n",
+ " reverse\n",
+ ",\n",
+ " or\n",
+ " modify\n",
+ " the\n",
+ " decision\n",
+ ".\n",
+ "\n",
+ "\n",
+ "-\n",
+ " If\n",
+ " a\n",
+ " party\n",
+ " is\n",
+ " still\n",
+ " unsat\n",
+ "isf\n",
+ "ied\n",
+ " after\n",
+ " the\n",
+ " appeals\n",
+ " court\n",
+ " rules\n",
+ ",\n",
+ " they\n",
+ " can\n",
+ " petition\n",
+ " the\n",
+ " Supreme\n",
+ " Court\n",
+ " to\n",
+ " hear\n",
+ " the\n",
+ " case\n",
+ " through\n",
+ " a\n",
+ " writ\n",
+ " of\n",
+ " cert\n",
+ "ior\n",
+ "ari\n",
+ ".\n",
+ " \n",
+ "\n",
+ "\n",
+ "-\n",
+ " The\n",
+ " Supreme\n",
+ " Court\n",
+ " gets\n",
+ " thousands\n",
+ " of\n",
+ " cert\n",
+ " petitions\n",
+ " every\n",
+ " year\n",
+ " but\n",
+ " usually\n",
+ " only\n",
+ " agrees\n",
+ " to\n",
+ " hear\n",
+ " about\n",
+ " 100\n",
+ "-\n",
+ "150\n",
+ " of\n",
+ " cases\n",
+ " that\n",
+ " have\n",
+ " significant\n",
+ " national\n",
+ " importance\n",
+ " or\n",
+ " where\n",
+ " lower\n",
+ " courts\n",
+ " disagree\n",
+ " on\n",
+ " federal\n",
+ " law\n",
+ ".\n",
+ " \n",
+ "\n",
+ "\n",
+ "-\n",
+ " If\n",
+ " 4\n",
+ " out\n",
+ " of\n",
+ " the\n",
+ " 9\n",
+ " Just\n",
+ "ices\n",
+ " vote\n",
+ " to\n",
+ " grant\n",
+ " cert\n",
+ " (\n",
+ "agree\n",
+ " to\n",
+ " hear\n",
+ " the\n",
+ " case\n",
+ "),\n",
+ " it\n",
+ " goes\n",
+ " on\n",
+ " the\n",
+ " Supreme\n",
+ " Court\n",
+ "'s\n",
+ " do\n",
+ "cket\n",
+ " for\n",
+ " arguments\n",
+ ".\n",
+ "\n",
+ "\n",
+ "-\n",
+ " The\n",
+ " Supreme\n",
+ " Court\n",
+ " then\n",
+ " hears\n",
+ " oral\n",
+ " arguments\n",
+ ",\n",
+ " considers\n",
+ " written\n",
+ " brief\n",
+ "s\n",
+ ",\n",
+ " examines\n",
+ " the\n",
+ " lower\n",
+ " court\n",
+ " records\n",
+ ",\n",
+ " and\n",
+ " issues\n",
+ " a\n",
+ " final\n",
+ " ruling\n",
+ " on\n",
+ " the\n",
+ " case\n",
+ ",\n",
+ " which\n",
+ " serves\n",
+ " as\n",
+ " binding\n",
+ " precedent\n"
+ ]
+ }
+ ],
+ "source": [
+ "# @title Streaming Example: Request Claude-2\n",
+ "messages = [\n",
+ " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
+ " {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n",
+ " ]\n",
+ "\n",
+ "result = litellm.completion('claude-2', messages, stream=True)\n",
+ "for part in result:\n",
+ " print(part.choices[0].delta.content or \"\")\n",
+ "\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/Evaluating_LLMs.ipynb b/cookbook/Evaluating_LLMs.ipynb
new file mode 100644
index 000000000..e27e8934f
--- /dev/null
+++ b/cookbook/Evaluating_LLMs.ipynb
@@ -0,0 +1,579 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Ys9n20Es2IzT"
+ },
+ "source": [
+ "# Evaluate Multiple LLM Providers with LiteLLM\n",
+ "\n",
+ "\n",
+ "\n",
+ "* Quality Testing\n",
+ "* Load Testing\n",
+ "* Duration Testing\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ZXOXl23PIIP6"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm python-dotenv"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "LINuBzXDItq2"
+ },
+ "outputs": [],
+ "source": [
+ "from litellm import load_test_model, testing_batch_completion"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "EkxMhsWdJdu4"
+ },
+ "outputs": [],
+ "source": [
+ "import os \n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n",
+ "os.environ[\"ANTHROPIC_API_KEY\"] = \"...\"\n",
+ "os.environ[\"REPLICATE_API_KEY\"] = \"...\""
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "mv5XdnqeW5I_"
+ },
+ "source": [
+ "# Quality Test endpoint\n",
+ "\n",
+ "## Test the same prompt across multiple LLM providers\n",
+ "\n",
+ "In this example, let's ask some questions about Paul Graham"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "id": "XpzrR5m4W_Us"
+ },
+ "outputs": [],
+ "source": [
+ "models = [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-16k\", \"gpt-4\", \"claude-instant-1\", {\"model\": \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"custom_llm_provider\": \"replicate\"}]\n",
+ "context = \"\"\"Paul Graham (/ɔrƦm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n",
+ "prompts = [\"Who is Paul Graham?\", \"What is Paul Graham known for?\" , \"Is paul graham a writer?\" , \"Where does Paul Graham live?\", \"What has Paul Graham done?\"]\n",
+ "messages = [[{\"role\": \"user\", \"content\": context + \"\\n\" + prompt}] for prompt in prompts] # pass in a list of messages we want to test\n",
+ "result = testing_batch_completion(models=models, messages=messages)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "9nzeLySnvIIW"
+ },
+ "source": [
+ "## Visualize the data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 403
+ },
+ "id": "X-2n7hdAuVAY",
+ "outputId": "69cc0de1-68e3-4c12-a8ea-314880010d94"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " Model Name \n",
+ " claude-instant-1 \n",
+ " gpt-3.5-turbo-0613 \n",
+ " gpt-3.5-turbo-16k-0613 \n",
+ " gpt-4-0613 \n",
+ " replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781 \n",
+ " \n",
+ " \n",
+ " Prompt \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \\nIs paul graham a writer? \n",
+ " Yes, Paul Graham is considered a writer in ad... \n",
+ " Yes, Paul Graham is a writer. He has written s... \n",
+ " Yes, Paul Graham is a writer. He has authored ... \n",
+ " Yes, Paul Graham is a writer. He is an essayis... \n",
+ " Yes, Paul Graham is an author. According to t... \n",
+ " \n",
+ " \n",
+ " \\nWhat has Paul Graham done? \n",
+ " Paul Graham has made significant contribution... \n",
+ " Paul Graham has achieved several notable accom... \n",
+ " Paul Graham has made significant contributions... \n",
+ " Paul Graham is known for his work on the progr... \n",
+ " Paul Graham has had a diverse career in compu... \n",
+ " \n",
+ " \n",
+ " \\nWhat is Paul Graham known for? \n",
+ " Paul Graham is known for several things:\\n\\n-... \n",
+ " Paul Graham is known for his work on the progr... \n",
+ " Paul Graham is known for his work on the progr... \n",
+ " Paul Graham is known for his work on the progr... \n",
+ " Paul Graham is known for many things, includi... \n",
+ " \n",
+ " \n",
+ " \\nWhere does Paul Graham live? \n",
+ " Based on the information provided:\\n\\n- Paul ... \n",
+ " According to the given information, Paul Graha... \n",
+ " Paul Graham currently lives in England, where ... \n",
+ " The text does not provide a current place of r... \n",
+ " Based on the information provided, Paul Graha... \n",
+ " \n",
+ " \n",
+ " \\nWho is Paul Graham? \n",
+ " Paul Graham is an influential computer scient... \n",
+ " Paul Graham is an English computer scientist, ... \n",
+ " Paul Graham is an English computer scientist, ... \n",
+ " Paul Graham is an English computer scientist, ... \n",
+ " Paul Graham is an English computer scientist,... \n",
+ " \n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "
\n",
+ "\n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ "
\n",
+ "\n",
+ "\n",
+ "\n",
+ " \n",
+ "\n",
+ "\n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ "
\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "Model Name claude-instant-1 \\\n",
+ "Prompt \n",
+ "\\nIs paul graham a writer? Yes, Paul Graham is considered a writer in ad... \n",
+ "\\nWhat has Paul Graham done? Paul Graham has made significant contribution... \n",
+ "\\nWhat is Paul Graham known for? Paul Graham is known for several things:\\n\\n-... \n",
+ "\\nWhere does Paul Graham live? Based on the information provided:\\n\\n- Paul ... \n",
+ "\\nWho is Paul Graham? Paul Graham is an influential computer scient... \n",
+ "\n",
+ "Model Name gpt-3.5-turbo-0613 \\\n",
+ "Prompt \n",
+ "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has written s... \n",
+ "\\nWhat has Paul Graham done? Paul Graham has achieved several notable accom... \n",
+ "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n",
+ "\\nWhere does Paul Graham live? According to the given information, Paul Graha... \n",
+ "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n",
+ "\n",
+ "Model Name gpt-3.5-turbo-16k-0613 \\\n",
+ "Prompt \n",
+ "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has authored ... \n",
+ "\\nWhat has Paul Graham done? Paul Graham has made significant contributions... \n",
+ "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n",
+ "\\nWhere does Paul Graham live? Paul Graham currently lives in England, where ... \n",
+ "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n",
+ "\n",
+ "Model Name gpt-4-0613 \\\n",
+ "Prompt \n",
+ "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He is an essayis... \n",
+ "\\nWhat has Paul Graham done? Paul Graham is known for his work on the progr... \n",
+ "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n",
+ "\\nWhere does Paul Graham live? The text does not provide a current place of r... \n",
+ "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n",
+ "\n",
+ "Model Name replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781 \n",
+ "Prompt \n",
+ "\\nIs paul graham a writer? Yes, Paul Graham is an author. According to t... \n",
+ "\\nWhat has Paul Graham done? Paul Graham has had a diverse career in compu... \n",
+ "\\nWhat is Paul Graham known for? Paul Graham is known for many things, includi... \n",
+ "\\nWhere does Paul Graham live? Based on the information provided, Paul Graha... \n",
+ "\\nWho is Paul Graham? Paul Graham is an English computer scientist,... "
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "# Create an empty list to store the row data\n",
+ "table_data = []\n",
+ "\n",
+ "# Iterate through the list and extract the required data\n",
+ "for item in result:\n",
+ " prompt = item['prompt'][0]['content'].replace(context, \"\") # clean the prompt for easy comparison\n",
+ " model = item['response']['model']\n",
+ " response = item['response']['choices'][0]['message']['content']\n",
+ " table_data.append([prompt, model, response])\n",
+ "\n",
+ "# Create a DataFrame from the table data\n",
+ "df = pd.DataFrame(table_data, columns=['Prompt', 'Model Name', 'Response'])\n",
+ "\n",
+ "# Pivot the DataFrame to get the desired table format\n",
+ "table = df.pivot(index='Prompt', columns='Model Name', values='Response')\n",
+ "table"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "zOxUM40PINDC"
+ },
+ "source": [
+ "# Load Test endpoint\n",
+ "\n",
+ "Run 100+ simultaneous queries across multiple providers to see when they fail + impact on latency"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ZkQf_wbcIRQ9"
+ },
+ "outputs": [],
+ "source": [
+ "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n",
+ "context = \"\"\"Paul Graham (/ɔrƦm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n",
+ "prompt = \"Where does Paul Graham live?\"\n",
+ "final_prompt = context + prompt\n",
+ "result = load_test_model(models=models, prompt=final_prompt, num_calls=5)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "8vSNBFC06aXY"
+ },
+ "source": [
+ "## Visualize the data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 552
+ },
+ "id": "SZfiKjLV3-n8",
+ "outputId": "00f7f589-b3da-43ed-e982-f9420f074b8d"
+ },
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAIXCAYAAACy1HXAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABn5UlEQVR4nO3dd1QT2d8G8Cf0ojQBEUFRsSv2FXvvvSx2saNi7733ihXELotd7KuIir33sjZUsIuKVGmS+/7hy/yM6K7RYEZ4PufkaO5Mkm/IJHly594ZhRBCgIiIiEiGdLRdABEREdG3MKgQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBCR7Dk5OaFLly7aLkNtc+fORd68eaGrq4uSJUtquxyNO3bsGBQKBbZv367tUtSmUCgwadIktW8XGhoKhUKBdevWabwm+joGFVKxfPlyKBQKlC9fXtulyI6TkxMUCoV0MTU1xR9//IENGzZou7TfTuoX3PdcfleHDh3CiBEjUKlSJaxduxYzZszQdkmys27dOul1PnXqVJrlQgg4OjpCoVCgcePGWqiQ5EBP2wWQvPj7+8PJyQkXLlxASEgInJ2dtV2SrJQsWRJDhw4FALx8+RKrVq2Cu7s7EhMT0bNnTy1X9/soXLgw/Pz8VNpGjx6NLFmyYOzYsWnWv3fvHnR0fq/fVUePHoWOjg5Wr14NAwMDbZcja0ZGRti4cSMqV66s0n78+HE8e/YMhoaGWqqM5IBBhSSPHz/GmTNnEBAQAA8PD/j7+2PixIm/tAalUomkpCQYGRn90sf9Xjlz5kTHjh2l6126dEHevHmxcOFCBhU1ZM+eXeXvCACzZs2CtbV1mnYAv+UXVXh4OIyNjTUWUoQQSEhIgLGxsUbuT04aNmyIbdu2YfHixdDT+9/X0saNG1GmTBm8fftWi9WRtv1eP1EoXfn7+8PS0hKNGjVC69at4e/vLy1LTk6GlZUVunbtmuZ20dHRMDIywrBhw6S2xMRETJw4Ec7OzjA0NISjoyNGjBiBxMREldsqFAr069cP/v7+KFq0KAwNDXHw4EEAwLx581CxYkVky5YNxsbGKFOmzFf3hcfHx2PAgAGwtrZG1qxZ0bRpUzx//vyr+6CfP3+Obt26IXv27DA0NETRokWxZs2aH/6b2djYoFChQnj48KFKu1KphJeXF4oWLQojIyNkz54dHh4eeP/+vcp6ly5dQr169WBtbQ1jY2PkyZMH3bp1k5an7g+fN28eFi5ciNy5c8PY2BjVqlXDrVu30tRz9OhRVKlSBaamprCwsECzZs1w584dlXUmTZoEhUKBkJAQdOnSBRYWFjA3N0fXrl3x4cMHlXWDgoJQuXJlWFhYIEuWLChYsCDGjBmjss73vtY/48sxKqm7DE6dOoUBAwbAxsYGFhYW8PDwQFJSEiIjI9G5c2dYWlrC0tISI0aMwJcnitfUa/Q1CoUCa9euRVxcnLRrI3VMw8ePHzF16lTky5cPhoaGcHJywpgxY9L8vZycnNC4cWMEBgaibNmyMDY2xooVK/71cc+fP4/69evD3NwcJiYmqFatGk6fPq2yTlhYGPr27YuCBQvC2NgY2bJlw59//onQ0NA09xcZGYnBgwfDyckJhoaGcHBwQOfOndMEB6VSienTp8PBwQFGRkaoVasWQkJC/rXWz7Vr1w7v3r1DUFCQ1JaUlITt27ejffv2X71NXFwchg4dCkdHRxgaGqJgwYKYN29emtc5MTERgwcPho2NjfT58OzZs6/ep6Y/H0hDBNH/K1SokOjevbsQQogTJ04IAOLChQvS8m7dugkLCwuRmJiocrv169cLAOLixYtCCCFSUlJE3bp1hYmJiRg0aJBYsWKF6Nevn9DT0xPNmjVTuS0AUbhwYWFjYyMmT54sli1bJq5evSqEEMLBwUH07dtXLF26VCxYsED88ccfAoDYt2+fyn24ubkJAKJTp05i2bJlws3NTZQoUUIAEBMnTpTWe/XqlXBwcBCOjo5iypQpwtvbWzRt2lQAEAsXLvzPv0/u3LlFo0aNVNqSk5OFnZ2dyJ49u0p7jx49hJ6enujZs6fw8fERI0eOFKampqJcuXIiKSlJCCHE69evhaWlpShQoICYO3euWLlypRg7dqwoXLiwdD+PHz8WAETx4sWFk5OTmD17tpg8ebKwsrISNjY24tWrV9K6QUFBQk9PTxQoUEDMmTNHTJ48WVhbWwtLS0vx+PFjab2JEycKAKJUqVKiZcuWYvny5aJHjx4CgBgxYoS03q1bt4SBgYEoW7asWLRokfDx8RHDhg0TVatWldZR57X+L0WLFhXVqlX75t/e3d1dur527VoBQJQsWVLUr19fLFu2THTq1El6DpUrVxbt27cXy5cvF40bNxYAxPr169PlNfoaPz8/UaVKFWFoaCj8/PyEn5+fePjwoRBCCHd3dwFAtG7dWixbtkx07txZABDNmzdP85ydnZ2FpaWlGDVqlPDx8RHBwcHffMwjR44IAwMDUaFCBTF//nyxcOFC4eLiIgwMDMT58+el9bZt2yZKlCghJkyYIHx9fcWYMWOEpaWlyJ07t4iLi5PWi4mJEcWKFRO6urqiZ8+ewtvbW0ydOlWUK1dOeo8GBwdL21KZMmXEwoULxaRJk4SJiYn4448//vVv9PnrePHiRVGxYkXRqVMnadmuXbuEjo6OeP78eZr3nlKpFDVr1hQKhUL06NFDLF26VDRp0kQAEIMGDVJ5jI4dOwoAon379mLp0qWiZcuWwsXF5Yc/H1Lfk2vXrv3P50eawaBCQgghLl26JACIoKAgIcSnDwIHBwcxcOBAaZ3AwEABQOzdu1fltg0bNhR58+aVrvv5+QkdHR1x8uRJlfV8fHwEAHH69GmpDYDQ0dERt2/fTlPThw8fVK4nJSWJYsWKiZo1a0ptly9f/uqHU5cuXdJ8EHXv3l3kyJFDvH37VmXdtm3bCnNz8zSP96XcuXOLunXrijdv3og3b96ImzdvSl+Onp6e0nonT54UAIS/v7/K7Q8ePKjSvnPnTpWA9zWpH4rGxsbi2bNnUvv58+cFADF48GCprWTJksLW1la8e/dOart+/brQ0dERnTt3ltpSg0q3bt1UHqtFixYiW7Zs0vWFCxcKAOLNmzffrE+d1/q//EhQqVevnlAqlVJ7hQoVhEKhEL1795baPn78KBwcHFTuW5Ov0be4u7sLU1NTlbZr164JAKJHjx4q7cOGDRMAxNGjR1WeMwBx8ODB/3wspVIp8ufPn+bv8eHDB5EnTx5Rp04dlbYvnT17VgAQGzZskNomTJggAIiAgICvPp4Q/wsqhQsXVvkBs2jRIgFA3Lx581/r/jyoLF26VGTNmlWq788//xQ1atSQ/hafB5Vdu3YJAGLatGkq99e6dWuhUChESEiIEOJ/f+++ffuqrNe+ffsf/nxgUPn1uOuHAHza7ZM9e3bUqFEDwKeu6zZt2mDz5s1ISUkBANSsWRPW1tbYsmWLdLv3798jKCgIbdq0kdq2bduGwoULo1ChQnj79q10qVmzJgAgODhY5bGrVauGIkWKpKnp833x79+/R1RUFKpUqYIrV65I7am7ifr27aty2/79+6tcF0Jgx44daNKkCYQQKnXVq1cPUVFRKvf7LYcOHYKNjQ1sbGxQvHhx+Pn5oWvXrpg7d67K8zc3N0edOnVUHqdMmTLIkiWL9PwtLCwAAPv27UNycvK/Pm7z5s2RM2dO6foff/yB8uXL4++//wbwaWDvtWvX0KVLF1hZWUnrubi4oE6dOtJ6n+vdu7fK9SpVquDdu3eIjo5WqW/37t1QKpVfrUvd11rTunfvrjIzqHz58hBCoHv37lKbrq4uypYti0ePHqnUrenX6Hukvg5DhgxRaU8doL1//36V9jx58qBevXr/eb/Xrl3DgwcP0L59e7x79056PnFxcahVqxZOnDghvYafv6+Sk5Px7t07ODs7w8LCQuU9sGPHDpQoUQItWrRI83hfzsbq2rWrylicKlWqAIDK3/y/uLm5IT4+Hvv27UNMTAz27dv3zd0+f//9N3R1dTFgwACV9qFDh0IIgQMHDkjrAUiz3qBBg1Sua+rzgdJHhgkqJ06cQJMmTWBvbw+FQoFdu3al+2M+f/4cHTt2lMZQFC9eHJcuXUr3x9W0lJQUbN68GTVq1MDjx48REhKCkJAQlC9fHq9fv8aRI0cAAHp6emjVqhV2794t7U8PCAhAcnKySlB58OABbt++LX2hp14KFCgA4NMgw8/lyZPnq3Xt27cPrq6uMDIygpWVFWxsbODt7Y2oqChpnbCwMOjo6KS5jy9nK7158waRkZHw9fVNU1fquJsv6/qa8uXLIygoCAcPHsS8efNgYWGB9+/fq3xIP3jwAFFRUbC1tU3zWLGxsdLjVKtWDa1atcLkyZNhbW2NZs2aYe3atV8d25E/f/40bQUKFJDGFYSFhQEAChYsmGa9woULS19an8uVK5fKdUtLSwCQxmi0adMGlSpVQo8ePZA9e3a0bdsWW7duVQkt6r7WmvblczA3NwcAODo6pmn/fOxJerxG3yN1e/1y+7Szs4OFhYX0Oqb61nvjSw8ePAAAuLu7p3k+q1atQmJiovS+iY+Px4QJE6SxHdbW1rCxsUFkZKTKe+vhw4coVqzYdz3+f21L38PGxga1a9fGxo0bERAQgJSUFLRu3fqr64aFhcHe3h5Zs2ZVaS9cuLC0PPVfHR0d5MuXT2W9L98nmvp8oPSRYWb9xMXFoUSJEujWrRtatmyZ7o/3/v17VKpUCTVq1MCBAwdgY2ODBw8eSG/Q38nRo0fx8uVLbN68GZs3b06z3N/fH3Xr1gUAtG3bFitWrMCBAwfQvHlzbN26FYUKFUKJEiWk9ZVKJYoXL44FCxZ89fG+/BL52iyGkydPomnTpqhatSqWL1+OHDlyQF9fH2vXrsXGjRvVfo6pX64dO3aEu7v7V9dxcXH5z/uxtrZG7dq1AQD16tVDoUKF0LhxYyxatEj6laxUKmFra6syGPlzNjY2ACAdKOvcuXPYu3cvAgMD0a1bN8yfPx/nzp1DlixZ1H6e6tDV1f1qu/j/wYjGxsY4ceIEgoODsX//fhw8eBBbtmxBzZo1cejQIejq6qr9Wmvat57D19rFZ4Mstf0afe/xYb53hk/q9j137txvHlgutdb+/ftj7dq1GDRoECpUqABzc3MoFAq0bdv2mz1n/+W/tqXv1b59e/Ts2ROvXr1CgwYNpB6t9KapzwdKHxkmqDRo0AANGjT45vLExESMHTsWmzZtQmRkJIoVK4bZs2ejevXqP/R4s2fPhqOjI9auXSu1fe+vH7nx9/eHra0tli1blmZZQEAAdu7cCR8fHxgbG6Nq1arIkSMHtmzZgsqVK+Po0aNpjnuRL18+XL9+HbVq1frhA3bt2LEDRkZGCAwMVJma+vnfGwBy584NpVKJx48fq/Q6fDnjIHXEf0pKihQ0NKFRo0aoVq0aZsyYAQ8PD5iamiJfvnw4fPgwKlWq9F1fNK6urnB1dcX06dOxceNGdOjQAZs3b0aPHj2kdVJ/MX/u/v37cHJyAvDp7wB8Ot7Il+7evQtra2uYmpqq/fx0dHRQq1Yt1KpVCwsWLMCMGTMwduxYBAcHo3bt2hp5rbUhPV6j75G6vT548ED69Q8Ar1+/RmRkpPQ6qiu1x8DMzOw/t+/t27fD3d0d8+fPl9oSEhIQGRmZ5j6/NrMsPbVo0QIeHh44d+6cyi7mL+XOnRuHDx9GTEyMSq/K3bt3peWp/yqVSjx8+FClF+XL90l6fT6QZmSYXT//pV+/fjh79iw2b96MGzdu4M8//0T9+vW/+gXwPfbs2YOyZcvizz//hK2tLUqVKoWVK1dquOr0Fx8fj4CAADRu3BitW7dOc+nXrx9iYmKwZ88eAJ++uFq3bo29e/fCz88PHz9+VNntA3za1/z8+fOv/j3i4+PT7IL4Gl1dXSgUCml8DPBpqu6Xu/RS998vX75cpX3JkiVp7q9Vq1bYsWPHVz9837x58581fcvIkSPx7t076fm6ubkhJSUFU6dOTbPux48fpS+E9+/fp/nFmfpr+MtdC7t27cLz58+l6xcuXMD58+elcJ4jRw6ULFkS69evV/nCuXXrFg4dOoSGDRuq/bwiIiLStH1ZnyZea21Ij9foe6S+Dl5eXirtqT1SjRo1Uvs+AaBMmTLIly8f5s2bh9jY2DTLP9++dXV10zynJUuWqLzXAKBVq1a4fv06du7cmeb+1O0p+V5ZsmSBt7c3Jk2ahCZNmnxzvYYNGyIlJQVLly5VaV+4cCEUCoX0vkj9d/HixSrrffn3T8/PB/p5GaZH5d88efIEa9euxZMnT2Bvbw8AGDZsGA4ePPjDh7Z+9OgRvL29MWTIEIwZMwYXL17EgAEDYGBg8M2uQznas2cPYmJi0LRp068ud3V1hY2NDfz9/aVA0qZNGyxZsgQTJ05E8eLFVX4ZAkCnTp2wdetW9O7dG8HBwahUqRJSUlJw9+5dbN26VTouxL9p1KgRFixYgPr166N9+/YIDw/HsmXL4OzsjBs3bkjrlSlTBq1atYKXlxfevXsHV1dXHD9+HPfv3weg2sU+a9YsBAcHo3z58ujZsyeKFCmCiIgIXLlyBYcPH/7qF/P3aNCgAYoVK4YFCxbA09MT1apVg4eHB2bOnIlr166hbt260NfXx4MHD7Bt2zYsWrQIrVu3xvr167F8+XK0aNEC+fLlQ0xMDFauXAkzM7M0wcLZ2RmVK1dGnz59kJiYCC8vL2TLlg0jRoyQ1pk7dy4aNGiAChUqoHv37oiPj8eSJUtgbm7+Q+c0mTJlCk6cOIFGjRohd+7cCA8Px/Lly+Hg4CAdQVQTr7U2pMdr9D1KlCgBd3d3+Pr6IjIyEtWqVcOFCxewfv16NG/eXBrMri4dHR2sWrUKDRo0QNGiRdG1a1fkzJkTz58/R3BwMMzMzLB3714AQOPGjeHn5wdzc3MUKVIEZ8+exeHDh5EtWzaV+xw+fDi2b9+OP//8E926dUOZMmUQERGBPXv2wMfHR2V3ryZ9z+dnkyZNUKNGDYwdOxahoaEoUaIEDh06hN27d2PQoEFSD1PJkiXRrl07LF++HFFRUahYsSKOHDny1WO8pNfnA2mAVuYapTMAYufOndL1ffv2CQDC1NRU5aKnpyfc3NyEEELcuXNHAPjXy8iRI6X71NfXFxUqVFB53P79+wtXV9df8hw1pUmTJsLIyEjl+Alf6tKli9DX15em7SmVSuHo6PjV6YGpkpKSxOzZs0XRokWFoaGhsLS0FGXKlBGTJ08WUVFR0nr4Ymrv51avXi3y588vDA0NRaFChcTatWulqbWfi4uLE56ensLKykpkyZJFNG/eXNy7d08AELNmzVJZ9/Xr18LT01M4OjoKfX19YWdnJ2rVqiV8fX3/82/1teOopFq3bl2aKYu+vr6iTJkywtjYWGTNmlUUL15cjBgxQrx48UIIIcSVK1dEu3btRK5cuYShoaGwtbUVjRs3FpcuXZLuI3Uq5Ny5c8X8+fOFo6OjMDQ0FFWqVBHXr19PU8fhw4dFpUqVhLGxsTAzMxNNmjQR//zzj8o6qX/DL6cdp04VTT3mypEjR0SzZs2Evb29MDAwEPb29qJdu3bi/v37Krf73tf6v/zI9OQvpw1/67l9baqwEJp5jb7lW4+ZnJwsJk+eLPLkySP09fWFo6OjGD16tEhISEjznL+1vX3L1atXRcuWLUW2bNmEoaGhyJ07t3BzcxNHjhyR1nn//r3o2rWrsLa2FlmyZBH16tUTd+/eTfM3FkKId+/eiX79+omcOXMKAwMD4eDgINzd3aXPgtTpydu2bVO53fdO4f3W6/ilr/0tYmJixODBg4W9vb3Q19cX+fPnF3PnzlWZni2EEPHx8WLAgAEiW7ZswtTUVDRp0kQ8ffo0zfRkIb7v84HTk389hRDp1IenRQqFAjt37kTz5s0BAFu2bEGHDh1w+/btNIO+smTJAjs7OyQlJf3nVLps2bJJg+xy586NOnXqYNWqVdJyb29vTJs2TaWLnrTj2rVrKFWqFP766y906NBB2+X8sNDQUOTJkwdz585VOfIvEVFmkSl2/ZQqVQopKSkIDw+X5vd/ycDAAIUKFfru+6xUqVKaAVn379//4cFw9OPi4+PTDIj08vKCjo4OqlatqqWqiIhIEzJMUImNjVXZ7/j48WNcu3YNVlZWKFCgADp06IDOnTtj/vz5KFWqFN68eYMjR47AxcXlhwawDR48GBUrVsSMGTPg5uaGCxcuwNfXF76+vpp8WvQd5syZg8uXL6NGjRrQ09PDgQMHcODAAfTq1Svdp8cSEVE60/a+J01J3Vf65SV1n2tSUpKYMGGCcHJyEvr6+iJHjhyiRYsW4saNGz/8mHv37hXFihWTxlB8zzgH0rxDhw6JSpUqCUtLS6Gvry/y5csnJk2aJJKTk7Vd2k/7fIwKEVFmlCHHqBAREVHGkGmOo0JERES/HwYVIiIikq3fejCtUqnEixcvkDVr1t/q8N1ERESZmRACMTExsLe3h47Ov/eZ/NZB5cWLF5zVQURE9Jt6+vQpHBwc/nWd3zqopJ6M6unTpzAzM9NyNURERPQ9oqOj4ejoqHJSyW/5rYNK6u4eMzMzBhUiIqLfzPcM2+BgWiIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki09bRdARETy5TRqv7ZLIC0LndVIq4/PHhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki3ZBJVZs2ZBoVBg0KBB2i6FiIiIZEIWQeXixYtYsWIFXFxctF0KERERyYjWg0psbCw6dOiAlStXwtLSUtvlEBERkYxoPah4enqiUaNGqF279n+um5iYiOjoaJULERERZVx62nzwzZs348qVK7h48eJ3rT9z5kxMnjw5nasiIiIiudBaj8rTp08xcOBA+Pv7w8jI6LtuM3r0aERFRUmXp0+fpnOVREREpE1a61G5fPkywsPDUbp0aaktJSUFJ06cwNKlS5GYmAhdXV2V2xgaGsLQ0PBXl0pERERaorWgUqtWLdy8eVOlrWvXrihUqBBGjhyZJqQQERFR5qO1oJI1a1YUK1ZMpc3U1BTZsmVL005ERESZk9Zn/RARERF9i1Zn/Xzp2LFj2i6BiIiIZIQ9KkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWz8UVB4+fIhx48ahXbt2CA8PBwAcOHAAt2/f1mhxRERElLmpHVSOHz+O4sWL4/z58wgICEBsbCwA4Pr165g4caLGCyQiIqLMS+2gMmrUKEybNg1BQUEwMDCQ2mvWrIlz585ptDgiIiLK3NQOKjdv3kSLFi3StNva2uLt27caKYqIiIgI+IGgYmFhgZcvX6Zpv3r1KnLmzKmRooiIiIiAHwgqbdu2xciRI/Hq1SsoFAoolUqcPn0aw4YNQ+fOndOjRiIiIsqk1A4qM2bMQKFCheDo6IjY2FgUKVIEVatWRcWKFTFu3Lj0qJGIiIgyKT11b2BgYICVK1di/PjxuHXrFmJjY1GqVCnkz58/PeojIiKiTEztoJIqV65cyJUrlyZrISIiIlKhdlARQmD79u0IDg5GeHg4lEqlyvKAgACNFUdERESZm9pBZdCgQVixYgVq1KiB7NmzQ6FQpEddREREROoHFT8/PwQEBKBhw4bpUQ8RERGRRO1ZP+bm5sibN2961EJERESkQu2gMmnSJEyePBnx8fHpUQ8RERGRRO1dP25ubti0aRNsbW3h5OQEfX19leVXrlzRWHFERESUuakdVNzd3XH58mV07NiRg2mJiIgoXakdVPbv34/AwEBUrlw5PeohIiIikqg9RsXR0RFmZmbpUQsRERGRCrWDyvz58zFixAiEhoamQzlERERE/6P2rp+OHTviw4cPyJcvH0xMTNIMpo2IiNBYcUSZndOo/dougbQsdFYjbZdApFVqBxUvL690KIOIiIgorR+a9UNERET0K3xXUImOjpYG0EZHR//ruhxoS0RERJryXUHF0tISL1++hK2tLSwsLL567BQhBBQKBVJSUjReJBEREWVO3xVUjh49CisrKwBAcHBwuhZERERElOq7gkq1atWQN29eXLx4EdWqVUvvmoiIiIgAqHEcldDQUO7WISIiol9K7QO+aZK3tzdcXFxgZmYGMzMzVKhQAQcOHNBmSURERCQjak1PDgwMhLm5+b+u07Rp0+++PwcHB8yaNQv58+eHEALr169Hs2bNcPXqVRQtWlSd0oiIiCgDUiuo/NcxVNSd9dOkSROV69OnT4e3tzfOnTvHoEJERETqBZVXr17B1tY2XQpJSUnBtm3bEBcXhwoVKnx1ncTERCQmJkrX/+uYLkRERPR7++4xKl87doom3Lx5E1myZIGhoSF69+6NnTt3okiRIl9dd+bMmTA3N5cujo6O6VITERERycN3BxUhRLoUULBgQVy7dg3nz59Hnz594O7ujn/++eer644ePRpRUVHS5enTp+lSExEREcnDd+/6cXd3h7GxscYLMDAwgLOzMwCgTJkyuHjxIhYtWoQVK1akWdfQ0BCGhoYar4GIiIjk6buDytq1a9OzDolSqVQZh0JERESZl9pnT9ak0aNHo0GDBsiVKxdiYmKwceNGHDt2DIGBgdosi4iIiGRCq0ElPDwcnTt3xsuXL2Fubg4XFxcEBgaiTp062iyLiIiIZEKrQWX16tXafHgiIiKSuR8+hH5ISAgCAwMRHx8PIP1mBREREVHmpXZQeffuHWrXro0CBQqgYcOGePnyJQCge/fuGDp0qMYLJCIiosxL7aAyePBg6Onp4cmTJzAxMZHa27Rpg4MHD2q0OCIiIsrc1B6jcujQIQQGBsLBwUGlPX/+/AgLC9NYYURERERq96jExcWp9KSkioiI4MHYiIiISKPUDipVqlTBhg0bpOsKhQJKpRJz5sxBjRo1NFocERERZW5q7/qZM2cOatWqhUuXLiEpKQkjRozA7du3ERERgdOnT6dHjURERJRJqd2jUqxYMdy/fx+VK1dGs2bNEBcXh5YtW+Lq1avIly9fetRIREREmdQPHfDN3NwcY8eO1XQtRERERCrU7lE5ePAgTp06JV1ftmwZSpYsifbt2+P9+/caLY6IiIgyN7WDyvDhwxEdHQ0AuHnzJoYMGYKGDRvi8ePHGDJkiMYLJCIiosxL7V0/jx8/RpEiRQAAO3bsQJMmTTBjxgxcuXIFDRs21HiBRERElHmp3aNiYGCADx8+AAAOHz6MunXrAgCsrKyknhYiIiIiTVC7R6Vy5coYMmQIKlWqhAsXLmDLli0AgPv376c5Wi0RERHRz1C7R2Xp0qXQ09PD9u3b4e3tjZw5cwIADhw4gPr162u8QCIiIsq81O5RyZUrF/bt25emfeHChRopiIiIiCjVDx1HRalUIiQkBOHh4VAqlSrLqlatqpHCiIiIiNQOKufOnUP79u0RFhYGIYTKMoVCgZSUFI0VR0RERJmb2kGld+/eKFu2LPbv348cOXJAoVCkR11ERERE6geVBw8eYPv27XB2dk6PeoiIiIgkas/6KV++PEJCQtKjFiIiIiIVaveo9O/fH0OHDsWrV69QvHhx6Ovrqyx3cXHRWHFERESUuakdVFq1agUA6Natm9SmUCgghOBgWiIiItKoHzrXDxEREdGvoHZQyZ07d3rUQURERJTGDx3w7eHDh/Dy8sKdO3cAAEWKFMHAgQORL18+jRZHREREmZvaQSUwMBBNmzZFyZIlUalSJQDA6dOnUbRoUezduxd16tTReJHa4jRqv7ZLIC0LndVI2yUQEWVqageVUaNGYfDgwZg1a1aa9pEjR2aooEJERETapfZxVO7cuYPu3bunae/WrRv++ecfjRRFREREBPxAULGxscG1a9fStF+7dg22traaqImIiIgIwA/s+unZsyd69eqFR48eoWLFigA+jVGZPXs2hgwZovECiYiIKPNSO6iMHz8eWbNmxfz58zF69GgAgL29PSZNmoQBAwZovEAiIiLKvNQOKgqFAoMHD8bgwYMRExMDAMiaNavGCyMiIiL6oeOoAEB4eDju3bsHAChUqBBsbGw0VhQRERER8AODaWNiYtCpUyfY29ujWrVqqFatGuzt7dGxY0dERUWlR41ERESUSakdVHr06IHz589j//79iIyMRGRkJPbt24dLly7Bw8MjPWokIiKiTErtXT/79u1DYGAgKleuLLXVq1cPK1euRP369TVaHBEREWVuaveoZMuWDebm5mnazc3NYWlpqZGiiIiIiIAfCCrjxo3DkCFD8OrVK6nt1atXGD58OMaPH6/R4oiIiChzU3vXj7e3N0JCQpArVy7kypULAPDkyRMYGhrizZs3WLFihbTulStXNFcpERERZTpqB5XmzZunQxlEREREaakdVCZOnJgedRARERGlofYYladPn+LZs2fS9QsXLmDQoEHw9fXVaGFEREREageV9u3bIzg4GMCnQbS1a9fGhQsXMHbsWEyZMkXjBRIREVHmpXZQuXXrFv744w8AwNatW1G8eHGcOXMG/v7+WLdunabrIyIiokxM7aCSnJwMQ0NDAMDhw4fRtGlTAJ/O9/Py5UvNVkdERESZmtpBpWjRovDx8cHJkycRFBQkHY32xYsXyJYtm8YLJCIiosxL7aAye/ZsrFixAtWrV0e7du1QokQJAMCePXukXUJEREREmqD29OTq1avj7du3iI6OVjlkfq9evWBiYqLR4oiIiChzU7tHBQCEELh8+TJWrFiBmJgYAICBgQGDChEREWmU2j0qYWFhqF+/Pp48eYLExETUqVMHWbNmxezZs5GYmAgfH5/0qJOIiIgyIbV7VAYOHIiyZcvi/fv3MDY2ltpbtGiBI0eOaLQ4IiIiytzU7lE5efIkzpw5AwMDA5V2JycnPH/+XGOFEREREando6JUKpGSkpKm/dmzZ8iaNatGiiIiIiICfiCo1K1bF15eXtJ1hUKB2NhYTJw4EQ0bNtRkbURERJTJqb3rZ/78+ahXrx6KFCmChIQEtG/fHg8ePIC1tTU2bdqUHjUSERFRJqV2UHFwcMD169exZcsWXL9+HbGxsejevTs6dOigMriWiIiI6GepHVQAQE9PDx06dECHDh2ktpcvX2L48OFYunSpxoojIiKizE2toHL79m0EBwfDwMAAbm5usLCwwNu3bzF9+nT4+Pggb9686VUnERERZULfPZh2z549KFWqFAYMGIDevXujbNmyCA4ORuHChXHnzh3s3LkTt2/fTs9aiYiIKJP57qAybdo0eHp6Ijo6GgsWLMCjR48wYMAA/P333zh48KB0FmUiIiIiTfnuoHLv3j14enoiS5Ys6N+/P3R0dLBw4UKUK1cuPesjIiKiTOy7g0pMTAzMzMwAALq6ujA2NuaYFCIiIkpXag2mDQwMhLm5OYBPR6g9cuQIbt26pbJO06ZNNVcdERERZWpqBRV3d3eV6x4eHirXFQrFVw+vT0RERPQjvjuoKJXK9KyDiIiIKA21z/VDRERE9KtoNajMnDkT5cqVQ9asWWFra4vmzZvj3r172iyJiIiIZESrQeX48ePw9PTEuXPnEBQUhOTkZNStWxdxcXHaLIuIiIhk4ofO9aMpBw8eVLm+bt062Nra4vLly6hataqWqiIiIiK50GpQ+VJUVBQAwMrK6qvLExMTkZiYKF2Pjo7+JXURERGRdvzQrp/IyEisWrUKo0ePRkREBADgypUreP78+Q8XolQqMWjQIFSqVAnFihX76jozZ86Eubm5dHF0dPzhxyMiIiL5Uzuo3LhxAwUKFMDs2bMxb948REZGAgACAgIwevToHy7E09MTt27dwubNm7+5zujRoxEVFSVdnj59+sOPR0RERPKndlAZMmQIunTpggcPHsDIyEhqb9iwIU6cOPFDRfTr1w/79u1DcHAwHBwcvrmeoaEhzMzMVC5ERESUcak9RuXixYtYsWJFmvacOXPi1atXat2XEAL9+/fHzp07cezYMeTJk0fdcoiIiCgDUzuoGBoafnUQ6/3792FjY6PWfXl6emLjxo3YvXs3smbNKgUdc3NzGBsbq1saERERZTBq7/pp2rQppkyZguTkZACfzu/z5MkTjBw5Eq1atVLrvry9vREVFYXq1asjR44c0mXLli3qlkVEREQZkNpBZf78+YiNjYWtrS3i4+NRrVo1ODs7I2vWrJg+fbpa9yWE+OqlS5cu6pZFREREGZDau37Mzc0RFBSEU6dO4caNG4iNjUXp0qVRu3bt9KiPiIiIMrEfPuBb5cqVUblyZU3WQkRERKRC7aCyePHir7YrFAoYGRnB2dkZVatWha6u7k8XR0RERJmb2kFl4cKFePPmDT58+ABLS0sAwPv372FiYoIsWbIgPDwcefPmRXBwMI8cS0RERD9F7cG0M2bMQLly5fDgwQO8e/cO7969w/3791G+fHksWrQIT548gZ2dHQYPHpwe9RIREVEmonaPyrhx47Bjxw7ky5dPanN2dsa8efPQqlUrPHr0CHPmzFF7qjIRERHRl9TuUXn58iU+fvyYpv3jx4/SAdvs7e0RExPz89URERFRpqZ2UKlRowY8PDxw9epVqe3q1avo06cPatasCQC4efMmD4dPREREP03toLJ69WpYWVmhTJkyMDQ0hKGhIcqWLQsrKyusXr0aAJAlSxbMnz9f48USERFR5qL2GBU7OzsEBQXh7t27uH//PgCgYMGCKFiwoLROjRo1NFchERERZVo/fMC3QoUKoVChQpqshYiIiEjFDwWVZ8+eYc+ePXjy5AmSkpJUli1YsEAjhRERERGpHVSOHDmCpk2bIm/evLh79y6KFSuG0NBQCCFQunTp9KiRiIiIMim1B9OOHj0aw4YNw82bN2FkZIQdO3bg6dOnqFatGv7888/0qJGIiIgyKbWDyp07d9C5c2cAgJ6eHuLj45ElSxZMmTIFs2fP1niBRERElHmpHVRMTU2lcSk5cuTAw4cPpWVv377VXGVERESU6ak9RsXV1RWnTp1C4cKF0bBhQwwdOhQ3b95EQEAAXF1d06NGIiIiyqTUDioLFixAbGwsAGDy5MmIjY3Fli1bkD9/fs74ISIiIo1SK6ikpKTg2bNncHFxAfBpN5CPj0+6FEZERESk1hgVXV1d1K1bF+/fv0+veoiIiIgkag+mLVasGB49epQetRARERGpUDuoTJs2DcOGDcO+ffvw8uVLREdHq1yIiIiINEXtwbQNGzYEADRt2hQKhUJqF0JAoVAgJSVFc9URERFRpqZ2UAkODk6POoiIiIjSUDuoVKtWLT3qICIiIkpD7TEqAHDy5El07NgRFStWxPPnzwEAfn5+OHXqlEaLIyIiosxN7aCyY8cO1KtXD8bGxrhy5QoSExMBAFFRUZgxY4bGCyQiIqLM64dm/fj4+GDlypXQ19eX2itVqoQrV65otDgiIiLK3NQOKvfu3UPVqlXTtJubmyMyMlITNREREREB+IGgYmdnh5CQkDTtp06dQt68eTVSFBERERHwA0GlZ8+eGDhwIM6fPw+FQoEXL17A398fw4YNQ58+fdKjRiIiIsqk1J6ePGrUKCiVStSqVQsfPnxA1apVYWhoiGHDhqF///7pUSMRERFlUmoHFYVCgbFjx2L48OEICQlBbGwsihQpgixZsqRHfURERJSJqb3r56+//sKHDx9gYGCAIkWK4I8//mBIISIionShdlAZPHgwbG1t0b59e/z99988tw8RERGlG7WDysuXL7F582YoFAq4ubkhR44c8PT0xJkzZ9KjPiIiIsrE1A4qenp6aNy4Mfz9/REeHo6FCxciNDQUNWrUQL58+dKjRiIiIsqk1B5M+zkTExPUq1cP79+/R1hYGO7cuaOpuoiIiIh+7KSEHz58gL+/Pxo2bIicOXPCy8sLLVq0wO3btzVdHxEREWViaveotG3bFvv27YOJiQnc3Nwwfvx4VKhQIT1qIyIiokxO7aCiq6uLrVu3ol69etDV1VVZduvWLRQrVkxjxREREVHmpnZQ8ff3V7keExODTZs2YdWqVbh8+TKnKxMREZHG/NAYFQA4ceIE3N3dkSNHDsybNw81a9bEuXPnNFkbERERZXJq9ai8evUK69atw+rVqxEdHQ03NzckJiZi165dKFKkSHrVSERERJnUd/eoNGnSBAULFsSNGzfg5eWFFy9eYMmSJelZGxEREWVy392jcuDAAQwYMAB9+vRB/vz507MmIiIiIgBq9KicOnUKMTExKFOmDMqXL4+lS5fi7du36VkbERERZXLfHVRcXV2xcuVKvHz5Eh4eHti8eTPs7e2hVCoRFBSEmJiY9KyTiIiIMiG1Z/2YmpqiW7duOHXqFG7evImhQ4di1qxZsLW1RdOmTdOjRiIiIsqkfnh6MgAULFgQc+bMwbNnz7Bp0yZN1UREREQE4CeDSipdXV00b94ce/bs0cTdEREREQHQUFAhIiIiSg8MKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbWg0qJ06cQJMmTWBvbw+FQoFdu3ZpsxwiIiKSGa0Glbi4OJQoUQLLli3TZhlEREQkU3rafPAGDRqgQYMG2iyBiIiIZEyrQUVdiYmJSExMlK5HR0drsRoiIiJKb7/VYNqZM2fC3Nxcujg6Omq7JCIiIkpHv1VQGT16NKKioqTL06dPtV0SERERpaPfatePoaEhDA0NtV0GERER/SK/VY8KERERZS5a7VGJjY1FSEiIdP3x48e4du0arKyskCtXLi1WRkRERHKg1aBy6dIl1KhRQ7o+ZMgQAIC7uzvWrVunpaqIiIhILrQaVKpXrw4hhDZLICIiIhnjGBUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLVkElWXLlsHJyQlGRkYoX748Lly4oO2SiIiISAa0HlS2bNmCIUOGYOLEibhy5QpKlCiBevXqITw8XNulERERkZZpPagsWLAAPXv2RNeuXVGkSBH4+PjAxMQEa9as0XZpREREpGVaDSpJSUm4fPkyateuLbXp6Oigdu3aOHv2rBYrIyIiIjnQ0+aDv337FikpKciePbtKe/bs2XH37t006ycmJiIxMVG6HhUVBQCIjo5Ol/qUiR/S5X7p95Fe29b34jZI3AZJ29JjG0y9TyHEf66r1aCirpkzZ2Ly5Mlp2h0dHbVQDWUG5l7aroAyO26DpG3puQ3GxMTA3Nz8X9fRalCxtraGrq4uXr9+rdL++vVr2NnZpVl/9OjRGDJkiHRdqVQiIiIC2bJlg0KhSPd6M5Po6Gg4Ojri6dOnMDMz03Y5lAlxGyRt4zaYfoQQiImJgb29/X+uq9WgYmBggDJlyuDIkSNo3rw5gE/h48iRI+jXr1+a9Q0NDWFoaKjSZmFh8QsqzbzMzMz4BiWt4jZI2sZtMH38V09KKq3v+hkyZAjc3d1RtmxZ/PHHH/Dy8kJcXBy6du2q7dKIiIhIy7QeVNq0aYM3b95gwoQJePXqFUqWLImDBw+mGWBLREREmY/WgwoA9OvX76u7ekh7DA0NMXHixDS72oh+FW6DpG3cBuVBIb5nbhARERGRFmj9yLRERERE38KgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCvyWlUqntEoiI6BdgUKHfko7Op0337du3AACeCYJ+tS/DMrdB0oYvt8OM+COOQYV+W4sWLULz5s3x8OFDKBQKbZdDmYyOjg6ioqIQGBgIANwGSSt0dHQQGRmJuXPn4v3799KPuIwk4z0jyrC+/MWqr68PY2NjGBgYaKkiysyUSiXmz58PDw8P7Nu3T9vlUCZ26NAhLFiwAEuXLtV2KemCZ0+m3050dDTMzMwAAFFRUTA3N9dyRZRZKJVKlV+sd+7cwerVqzF79mzo6upqsTLKTFJSUlS2t+TkZGzZsgXt2rXLkNshgwr9VgYPHoyUlBSMHj0aOXLk0HY5lAlFRkYiMjISjo6OKl8KX355EP2ML0Pxl969e4fTp0+jYsWKsLa2ltoz4nbIXT8ka1/maAcHB2zYsCHDvRHp9yCEwKhRo1C+fHmEhoaqLOM2ST/j5cuXePHiBd68eQPg09iTf+tH2Lp1K5o3b47jx4+rtGfE7ZA9KiQbqb8EhBBQKBTf/EXx/v17WFpaaqFCymj+61fr19YJCwvDuHHjsG7dugz5pUC/3tq1a7Fs2TI8ffoU+fLlQ+XKlTFnzhyVdb7WU+Ll5YV+/fpBT0/vV5b7yzGokFakhhHg0xtQCAE9PT08f/4cO3fuRNeuXWFqagrg0+4eS0tLTJgwIc1tiX7U5wHk6NGjePLkCZydnZE3b17Y29urrBMVFQWlUpkmIGfEbnb6tfbt2wc3NzcsX74cJiYmePToEebMmYOKFSti/fr1yJYtm/SZ9/btW4SEhMDV1VXlPj5+/Jihwwp3/dAvkZqHo6OjER8fD4VCgUOHDiEkJAS6urrQ09NDWFgYSpUqhRcvXkghJS4uDvr6+li4cCEiIiIYUkgjhBBSSBk1ahS6dOmCefPmoVevXhg2bBguXrwI4FP3e2JiIiZMmIDSpUvj3bt3KvfDkEI/6+LFi2jUqBG6dOkCNzc3jBgxAoGBgbhx4wY6dOgA4NPU9+TkZPj5+aFixYo4deqUyn1k5JACMKjQL/Tq1SsUL14cx48fx8aNG1G/fn38888/AD7tzilatChatGiB6dOnS7cxNTXFiBEj8ODBA1hZWTGkkEakbkfz5s3DX3/9hU2bNuHWrVto2bIl9u7di3HjxuHs2bMAAAMDA5QqVQq1atWChYWFFqumjOjx48d4+fKlSlu5cuWwZ88eXL58GT179gTw6XAMjRs3xvTp09P0qGR4gugX6tq1qzAzMxM6Ojpi5cqVUntSUpLYsmWLSElJkdqUSqU2SqRM4vXr16Jly5ZizZo1Qggh9uzZI8zMzETv3r1FqVKlRK1atcS5c+eEEKrb4sePH7VSL2VMgYGBInv27GLz5s1SW+r25u/vL5ydncXFixfT3C45OfmX1aht7FGhXyL1sM6enp6IiYmBgYEB7OzskJCQAODTrwU3NzeVQYvsPaH0ZGtrixEjRqB+/fq4evUqPD09MW3aNHh7e6NVq1Y4d+4cPD09cfnyZZVtkbt7SJMKFy6M6tWrw8/PD0eOHAHwv8++kiVLIjw8XDpVyOcy+u6ezzGo0C+RGkAcHR1x6tQpuLu7o23btti9ezfi4+PTrJ8Rz1dB2vOt7alUqVLIkSMHDhw4ABcXF/Tq1QsAYGVlBVdXVzRp0gSlSpX6laVSJuPo6IjevXsjMjISCxcuxJ49e6RlOXLkQJ48ebRYnTxknkhGWiH+f/Dry5cvkZycjFy5csHW1hYVK1ZEQkICunfvjnXr1qFx48YwMjKCj48PateuDWdnZ22XThmE+Gzg7KpVqxAeHg4DAwMMGzZMOv1CYmIinj9/jtDQUBQsWBCHDh1C06ZN0b9//3+dKk/0M1JnjVWvXh3Lly/HmDFjMHLkSAQGBsLFxQVbt26FQqFAnTp1tF2qVnF6MqW7gIAATJo0Ca9fv0ajRo3QokULNGnSBADQtWtX7Ny5E0OHDsXr16/h7e2NmzdvokiRIlqumjKaiRMnwsvLC+XKlcOFCxdQvnx5+Pn5wc7ODnv37sW0adPw/v176OvrQwiBGzduQE9PjzPNKF2kblcBAQFYvnw5Dh06hLt37yI4OBhLly6Fo6MjLCws4O/vD319/Uw9FZ5BhdLV7du3Ua9ePQwePBgmJibYtGkTDA0N4e7ujo4dOwIABg4ciCtXriAxMRG+vr4oWbKkdoumDOHzXpCPHz/C3d0d/fv3R6lSpRAaGopGjRrBzs4OO3fuhI2NDfbv34+QkBDExsZi5MiR0NPTy9RfDqQZqYFEfHHsKF1dXQQEBKBz585YsGCBtNsR+LS96ujoqGy/mWlMypcYVCjd3L17F9u2bUN8fDxmzJgBALh58yYmTJiA6OhodO3aVQorr169gqmpKbJmzarNkimD+Dyk3LlzB9HR0VixYgUmTJgAJycnAJ+mhdapUwfZs2fHrl27YGNjo3IfDCn0sz7fDt++fQuFQoFs2bIB+PSZV7p0aUyYMAG9e/eWbvNlDx579BhUKB0IIfD+/Xs0btwY//zzD5o0aQI/Pz9p+Y0bNzBhwgTEx8ejbdu26Nq1qxarpYxs+PDhUtf569evERAQgAYNGkgf/I8fP0aDBg0ghMDp06dVTu5G9DM+DxhTp07Frl27EB0dDWtra0yfPh01a9bE8+fPkTNnTi1XKn8cHUYap1AoYGVlhZkzZ6Jo0aK4cuUKgoKCpOUuLi6YOnUqkpOTpTcvkSZ8Prtn3759OHjwIBYvXozly5cjT548GDt2LK5fvy4dKTlPnjzYt28fSpYsyfNHkUalhpQpU6Zg0aJF0vR3a2trdOjQAevXr0/Ti0dfxx4V0ohvdU8eP34cY8aMgZ2dHTw9PVGzZk1p2e3bt2Fubg4HB4dfWSplAgEBAThz5gyyZcuG0aNHAwBiY2NRunRpmJmZYdWqVShRokSabZa7e0iT3r17h7p168LT0xPdunWT2nv16oW9e/ciODgYhQoV4u6d/8AeFfppqW+yM2fOYMGCBRg/fjxOnz6N5ORkVKtWDVOmTMGrV6+wdOlSHDt2TLpd0aJFGVJI4+Lj4zF+/HgsWLAAt2/fltqzZMmCK1euICYmBh4eHtL5fD7HkEKa9PHjR7x9+1bqrUs9wKWvry/s7e2xcOFCADy45X9hUKGf8vkUuwYNGuD06dPYs2cPxowZg+nTpyMpKQm1atXClClT8O7dO0ydOhUnT57UdtmUgRkbG+PkyZOoXbs2Ll++jD179iAlJQXA/8LK3bt3sWLFCi1XShnJ13ZOZM+eHXZ2dlizZg0AwMjICElJSQAAZ2dnBpTvxKBCPyW1J2XAgAFYsGABduzYgW3btuHy5cvYsmULxo0bJ4WVUaNGQV9fn0daJI35fEyKEEL6srCyssLGjRthaWmJuXPnIjAwUFpmamqKV69ewdfXVys1U8ajVCql0PHixQuEh4fjw4cPAIBJkybh7t270sye1IMMPnv2jCe5/E4co0I/JPWNqVAosHz5cly7dg2+vr54/PgxateujcqVK8PMzAzbtm2Dh4cHxowZA0NDQ3z48AEmJibaLp8ygM+nfi5ZsgTXr1/Ho0ePMGjQIJQuXRoODg548+YNmjVrBl1dXYwZMwb16tVTOcIsx6TQz/D394erqyvy5csHABg9ejQCAwMRFhaG2rVro2nTpujQoQNWrlyJqVOnIlu2bChWrBgePnyIyMhI6aCC9O8YVOi7pH4pfB40rl27hpIlSyI6OhpPnz6Fs7Mz6tevjzx58mDNmjWIioqSjjDbpUsXTJ8+nYPG6Kd9uQ2NHj0aq1evRq9evfDs2TOcPXsWzZo1Q69eveDs7Iw3b96gZcuWePPmDdatWwdXV1ctVk8ZxYEDB9C4cWOMHDkSgwYNwoEDBzBixAh4eXnh3bt3uHLlCgIDAzF+/Hj07t0bN2/ehJeXF3R0dGBpaYkZM2bwoILfK13PzUwZyqNHj0S7du3EP//8I7Zu3SoUCoW4cOGCdErymzdvikKFConz588LIYR4+PChaNy4sRgzZox48uSJNkunDCYlJUUIIYSfn5/IkyePuHz5shBCiJMnTwqFQiHy588vBg4cKB49eiSEEOLly5eiV69e4uPHj1qrmTKepUuXCgcHBzF16lTRr18/sXLlSmnZ06dPxZQpU4STk5M4ePDgV2+fnJz8q0r9rbHPib5bQkICTp48iS5duuDatWtYu3YtypUrJ+0GEkLg48ePOHv2LIoWLYoNGzYAAIYNG8ZjVNBP69SpE2xsbLBgwQLo6OggOTkZBgYG6N27N0qXLo1du3aha9euWLVqFV69eoVp06ZBR0cHPXv2ROHChaXBs/wFSz8rKSkJBgYG8PT0hImJCUaPHo2YmBhMmzZNWsfBwQGdO3fGoUOHcOnSJdSrVy/NyS252+c7aTsp0e8h9Resj4+P0NHRESVKlBBXr15VWScqKkp06dJF5MuXTzg5OQkbGxvply7Rz4iKihKTJ08WVlZWYtKkSVL78+fPxevXr8XLly9F2bJlxfz586X17e3tRY4cOcSiRYuEEELq+SPSlJkzZ4rw8HDh7+8vTExMRMOGDcX9+/dV1mnTpo1o2bKllirMGDjrh/6TEAI6OjoQQsDe3h7z58/Hx48fMW7cOJw6dUpaz8zMDPPmzcPy5csxceJEnD9/HqVLl9Zi5ZQRxMTEwMzMDH369MG4cePg5eWFiRMnAgDs7e1ha2uLly9f4v3799L4k+fPn6Nu3bqYMGECPD09AfBYFfTzxGdDOtevX4+pU6fiwYMHaN++PRYuXIgrV67Ax8cH9+7dAwBER0fj8ePHyJUrl7ZKzhDY70T/Svz/wMWjR4/i+PHjGDRoEJo0aYLatWvDzc0Ns2bNwpgxY1CxYkUAn046WLduXS1XTRnFiBEjsGLFCjx8+BA2Njbo2LEjhBCYOnUqAGDy5MkAPoUZXV1dnD59GkIIzJo1CyYmJtKUUO7uIU1IDbtHjhzB1atX4evrK3329erVC8nJyZg8eTIOHjyI0qVLIy4uDklJSZgzZ442y/79abM7h+Qttat8+/btwtzcXIwePVpcvHhRWn7jxg1RpEgR0bhxY/HXX3+JSZMmCYVCIZ4+fcpudtKI69evi6pVq4qCBQuKN2/eCCGECA8PF/PnzxcWFhZiwoQJ0rr9+vUT+fLlEw4ODsLV1VUkJSUJIbjLhzTr2LFjonjx4iJbtmxi165dQgghEhMTpeWrV68WWbJkEaVLlxYbNmyQBnBz4OyP4/Rk+lcXLlxA/fr1MXv2bPTs2VNqj46OhpmZGe7cuYOePXsiPj4eUVFR2Lp1K3f3kEacPXsWb968QZEiRdCmTRvExsZKZzh+8+YN/Pz8MHXqVOlkb8CnKfMKhQLFixeHjo4OPn78yAGL9FPEF9PhY2NjMXfuXPj6+qJ8+fLYtGkTjI2NkZycDH19fQDAggULcObMGWzbtg0KhYI9ej+JQYX+1dKlS7Fz504cOXIEUVFROHr0KP766y/cuXMHw4YNQ7du3RAeHo6oqCiYm5vD1tZW2yVTBtG5c2e8ePEChw8fRmhoKFq3bo2YmJg0YWXatGno168fpkyZonJ7fjmQJi1btgwODg5o1qwZ4uPjMW/ePOzcuRPVq1fHjBkzYGRkpBJWUgPOl0GH1MfBtPSv7OzscPnyZcycOROtW7fG2rVrYWRkhEaNGqFHjx64f/8+bG1tkT9/foYU0qhly5bh2bNnWLp0KZycnLBp0yaYm5ujUqVKePv2LWxsbNCpUydMmDAB06ZNw+rVq1Vuz5BCmvLmzRscPXoUffv2xcGDB2FsbIwhQ4agcePGOHPmDMaOHYuEhATo6+vj48ePAMCQokHsUSFJ6psqNjYWWbJkAQC8fv0aS5YswdatW1GzZk106dIFf/zxB16/fo2mTZti3bp1KFq0qJYrp4wmtTdk8eLFuHr1KhYsWABLS0vcvXsXnTt3RlRUlNSz8urVKxw/fhytWrXibh7SiC+PdwIA169fx+LFi3H48GH4+PigQYMGiIuLw5w5c3D48GEULlwYy5cvl87lQ5rDHhWSKBQK7N+/H+3atUP16tWxbt066OnpYdq0aTh//jx8fHzg6uoKHR0dLFmyBHFxcexFoXSR2htSvXp1nDhxAvv37wcAFCxYEH5+frC0tETVqlXx+vVr2NnZoU2bNtDT05N+zRL9jNSQ8urVK6mtRIkSGDhwIGrUqIHevXvj4MGDMDU1xYgRI/DHH39AR0dH2u1DGqalQbwkQ6dPnxZGRkZi+PDhon79+sLFxUV4eHiIkJAQaZ3g4GDRq1cvYWVlleaAb0Q/KvWAgl/j4+MjChQoIO7duye13bt3Tzg5OYm2bdv+ivIok/h8O9y8ebPImzevykxHIYS4du2aaNasmciVK5c4duyYEEKI+Ph4aXbZv23L9GPYo0IAgLCwMAQFBWH69OmYM2cODhw4gF69euHGjRuYOXMmHj16hLi4OJw9exbh4eE4fvw4SpYsqe2yKQP4vJv9woULOHPmDI4fPy4tb9q0KcqXL4/g4GCprUCBAjhx4gT++uuvX14vZUyJiYnSdpiUlIR8+fKhUKFC8PT0xOXLl6X1SpQogebNm+Pp06eoW7cuzpw5AyMjI2lMype7jOjn8S+aCS1duhR///23dP3evXto06YN1qxZAyMjI6nd09MTHTp0wO3btzFnzhxERkZi+PDhWL9+PYoVK6aN0imD+fyDfcyYMejSpQu6desGd3d3tGnTBtHR0ciRI4e0/z85OVm6raOjI3R1dZGSkqKt8imDOHDgAPz8/AAAPXv2RM2aNVG2bFkMHToUdnZ28PDwwKVLl6T1c+XKhbZt22L+/PkoX7681M6Bs+lE21069Gs9fvxYtG/fXjx48EClfdSoUcLW1la0bNlSOrBWKm9vb1GwYEExYMAAHrSI0sW8efNEtmzZxPnz50VKSoqYMWOGUCgU4tSpU9I6lSpVEh4eHlqskjKqdu3aCScnJ1GvXj1hbW0trl+/Li07evSoaN68uShWrJg4cOCAePz4sWjevLkYOnSotA7Pyp2+GFQyobi4OCGEEOfOnRPbt2+X2idMmCCKFy8uxo0bJ16/fq1ym5UrV4rHjx//yjIpk1AqlcLd3V34+voKIYTYsWOHsLCwED4+PkIIIWJiYoQQQhw4cEA0bdpU3LhxQ2u1UsZVsmRJoVAoVE56merkyZOiU6dOQqFQiAIFCggXFxfpRxuPfJz+OJcvEzI2NkZkZCRmzpyJ58+fQ1dXF82bN8fkyZORnJyM/fv3QwiBgQMHwsbGBgDQo0cPLVdNGVVCQgLOnz+P6tWr49ixY3B3d8fcuXPh4eGBjx8/Ys6cOahQoQJcXV0xZcoUXLhwAcWLF9d22ZRBJCUlISEhAc7OzsiVKxe2bNmCnDlzom3bttJhGipXrozy5cujZ8+eSE5ORrVq1aCrq8sjH/8iHKOSCSkUClhYWGDo0KHIkycPvLy8EBAQAACYMWMG6tevj6CgIMyYMQNv377VcrWUkdy4cQPPnj0DAAwePBjHjx+HsbEx2rdvj7/++gsNGzbEwoULpZMJvn//HpcuXcK9e/dgaWkJPz8/5M6dW5tPgTIYAwMDmJmZYdu2bdi9ezfKlSuHOXPmYPPmzYiJiZHWS0hIQJUqVVCzZk1pbBRDyq/BoJIJiU+7/FClShUMHjwYlpaWWLx4sUpYcXV1xdWrV1VOa070o4QQuH//PmrUqIE1a9agd+/eWLRoESwtLQEArq6uCAsLQ/ny5VGhQgUAwIsXL9ClSxdERkaiX79+AIB8+fKhdu3aWnselPEIIaBUKqXr69evR8WKFbFw4UJs2LABT548Qc2aNfHnn39K6wM88vGvxCPTZkKpR/2MioqCiYkJbty4genTp+P9+/cYOHAgmjdvDuDTYaNTd/0QacLKlSsxYsQIJCQkYPfu3ahbt650ROQtW7ZgypQpEEJAT08PxsbGUCqVOHPmDPT19XnuHvppERERsLKyUmlL3f62bduGoKAg+Pr6AgB69eqFY8eOISUlBVZWVjh9+jSPOqsl7FHJZD5+/AhdXV2EhoaievXqOHToEMqUKYNhw4bBxsYGkydPxr59+wCAIYU0JvUXq6OjIwwNDWFmZoZz584hNDRUmtLZpk0bbNiwAVOmTIGbmxtGjhyJc+fOSedPYUihn7Fo0SKUK1dOZXcOACmkdOnSBSVKlJDafX19sWLFCixZsgTnzp2DgYEBj3ysLdoZw0u/wrdGo4eEhIjs2bOLHj16qEyrO3bsmOjUqZMIDQ39VSVSBvflNpiUlCTi4+OFt7e3yJkzpxgzZsx/bm+c+kk/a8WKFcLQ0FBs3LgxzbInT56I4sWLi6VLl0ptX9vmuB1qD3f9ZFDi/7szz549izt37iAkJASdO3dGjhw5sH79ely6dAnr169Pc4bPhIQElYO+Ef2oz484GxERgZiYGJWBsF5eXpg3bx66d++Orl27wsnJCU2aNMHYsWPh6uqqrbIpg1m5ciX69+8PPz8//Pnnn4iMjERcXBwSEhJga2uLrFmz4sGDB8ifP7+2S6VvYFDJwHbs2IFevXpJJ2978+YN2rRpg5EjRyJr1qzaLo8ysM9DypQpU3Do0CHcunULbm5uaNGiBRo0aADgU1jx8vJCsWLF8O7dOzx58gShoaE8uRtpxKNHj+Ds7Aw3Nzds3rwZt27dQt++ffHmzRuEhYWhRo0a6NOnDxo3bqztUulfcG5VBnXr1i0MHjwY8+fPR5cuXRAdHQ0LCwsYGxszpFC6Sw0pEyZMgK+vL+bOnQsnJyf07t0bDx48QGRkJNq1a4dBgwbB2toa169fR0JCAk6ePCmdBZlTP+ln2djYYPbs2ZgwYQKGDRuGQ4cOoUqVKmjWrBmio6Oxfft2jBs3DtbW1uzFkzNt7ncizTh69Kh4+PBhmrYKFSoIIYS4c+eOyJ07t+jRo4e0/OHDh9znSunq6NGjomjRouLEiRNCCCHOnDkjDAwMRJEiRUT58uXFtm3bpHU/PzUDT9NAmpSQkCDmzZsndHR0RLdu3URSUpK07NKlS6JgwYJi2bJlWqyQ/gtn/fzGhBC4evUqGjRoAG9vb4SFhUnLnj9/DiEEYmNjUb9+fdStWxcrVqwAAAQFBcHb2xvv37/XVumUAYkv9iLnzJkTffr0QZUqVXDo0CE0btwYvr6+CAoKwsOHD7F48WKsXr0aAFR6T9iTQppkaGiI3r17Y8eOHejRowf09fWlbbVMmTIwMjLC06dPtVwl/RsGld+YQqFAqVKlMH/+fGzduhXe3t549OgRAKBRo0Z4/fo1zMzM0KhRI/j6+krd8YGBgbhx4wane5LGKJVKaUD2o0ePEBcXh/z586Ndu3ZISEjAokWLMGDAAHTq1An29vYoWrQoQkJCcOfOHS1XTpmBqakpGjRoIB1MMHVbDQ8Ph7GxMYoWLarN8ug/8KfLbyx1P76npycAYO7cudDV1UWPHj2QJ08ejB8/HjNmzMDHjx/x4cMHhISEYNOmTVi1ahVOnTolHRWU6Gd8PnB2woQJOHv2LIYPH44aNWrAysoKcXFxePnyJUxMTKCjo4PExEQ4OTlhxIgRqF+/vparp4xIfDaTMZWhoaH0/5SUFLx9+xY9e/aEQqFAu3btfnWJpAYGld9Yao/IoUOHoKOjg+TkZHh5eSEhIQEjR46Em5sb4uPjMWPGDGzfvh3Zs2eHgYEBgoODUaxYMS1XTxnF5yFlxYoV8PX1RalSpaSZO4mJibCyssKpU6ekAbPv3r3DmjVroKOjoxJ0iH5EWFgYIiIikC1bNtjZ2f3rEWSTk5Ph5+eHTZs2ISIiAufOnZPO3cNeZnni9OTfXGBgoHQiN1NTUzx48ACLFy9G3759MXLkSNjY2CAmJgbHjx+Hk5MTbG1tYWtrq+2y6Tf3Zbi4f/8+mjdvjtmzZ6NJkyZp1rt48SLGjRuH2NhYWFlZISAgAPr6+gwp9NM2bNiA+fPnIzw8HNbW1ujfv7/UU5Lqy+0sKCgIt2/fRr9+/TjL7DfAoPIbUyqV6NChAxQKBTZu3Ci1L1myBCNGjICnpyf69u2LvHnzarFKymhatmyJMWPGoGzZslLbtWvXUL9+fRw/fhwFCxb86kEEExISIISAkZERFAoFvxzop23YsAGenp7S4fFnzJiBR48e4fTp09K2lRpSIiMjcejQIbi5uancB3tS5I8/ZX5jqb8QUrvYk5KSAAD9+/eHh4cH1q5di8WLF6vMBiL6Webm5nBxcVFpMzIywvv373Hr1i2pLfX8PmfPnsWOHTugo6MDY2NjKBQKKJVKhhT6KZcuXcLUqVOxdOlSdOvWDcWLF8fgwYPh7OyMM2fO4Pbt24iOjpZ2i69fvx59+/bFX3/9pXI/DCnyx6DyG3rx4oX0/4IFC2Lv3r0IDw+HgYEBkpOTAQAODg4wMTFBcHAwjI2NtVUqZSDPnz8HAKxduxYGBgZYvHgxDh06hKSkJDg7O6NNmzaYO3cuDh8+DIVCAR0dHaSkpGD69OkIDg5WGTfA3T30sxITEzFo0CA0atRIaps0aRKOHDmCdu3aoXPnzmjbti0iIiKgr6+Phg0bYtiwYRw4+xvirp/fzPXr19GvXz+0b98effr0QVJSEmrWrIm3b9/i2LFjsLOzAwCMHDkSRYsWRePGjdOc1pxIXT179gQAjB49WtqV6OLigrdv32Lz5s2oWrUqTp48iYULF+LmzZvo0KEDDAwMcOTIEbx58wZXrlxhDwpplFKpxJs3b5A9e3YAQOfOnXH48GHs2bMHjo6OOH78OKZNm4aRI0eiffv2KmNWuLvn98KfNb8ZExMTWFhYYPv27Vi3bh0MDAywYsUK2NjYoHDhwmjevDnq1q2LRYsWoWzZsgwppBEuLi44ePAgvL29ERISAgC4ceMGChYsiA4dOuDEiROoUqUKpkyZgs6dO8PPzw9Hjx5Frly5cPnyZWnAIpGm6OjoSCEFAIYNG4bz58+jbNmyyJ49Oxo0aICIiAi8fv06zVRlhpTfC3tUfkMhISEYM2YMXr16hZ49e6JTp05ISUnBvHnzEBYWBiEE+vfvjyJFimi7VMpA1qxZgwkTJqBt27bo2bMnChYsCACoWrUqHj9+DH9/f1StWhUA8OHDB5iYmEi35cBZ+tWePXuGjh07YtiwYTzp4G+OQeU3cOXKFbx8+VJlX2xISAjGjRuH0NBQ9O/fHx06dNBihZSRfT61c/Xq1ZgwYQLatWuXJqyEhYVhw4YNqFChgsp4lK8dfItIHZ9vQ6n/T/33zZs3sLGxUVk/Li4O7dq1Q1RUFI4ePcoelN8cg4rMxcTEoFGjRtDV1cWIESPQoEEDaVloaCjq168PExMT9OjRA3379tVipZTRfOsYJytXrsTkyZPRpk0b9OrVSworNWvWxOnTp3Hu3DmUKlXqV5dLGdTXtsPUtoCAAGzatAmLFi2Cvb094uPjsXv3bvj5+eH58+e4ePEi9PX1OSblN8cxKjKVmh+zZs2KOXPmQE9PD0uXLsX+/fuldZycnFCjRg28evUKR44cQWRkpJaqpYzm8y+HM2fOIDg4GNevXwfwaWDt+PHjsXnzZvj6+uLevXsAgKNHj6JHjx5ppi4T/ahTp05JJwwcMmQIZs2aBeDT+JQtW7agc+fOqF27Nuzt7QF8OqHl48ePkTdvXly6dAn6+vr4+PEjQ8pvjj0qMpPanZn6CyD1C+P8+fMYNWoUTE1N0adPH2k30NChQ5E3b160bNkSOXLk0HL1lBF83s0+ZMgQbNmyBbGxsXBwcECuXLlw4MABAMCKFSswbdo0tG3bFu7u7iqnZeAvWPoZQghERUXB1tYWDRo0gLW1NQICAnDy5EkUK1YMkZGRcHV1haenJ/r37y/d5vPPToDbYUbBoCIjqW+04OBg7NmzBxEREahcuTL+/PNPWFhY4Ny5cxg/fjwSExORN29emJiYYMuWLbh+/TocHBy0XT5lAJ+HlEOHDmHQoEHw9fWFhYUF/vnnH0ycOBGmpqa4dOkSgE9jVjw8PODl5YV+/fpps3TKgMLDw5E3b16kpKRgx44daNiwobTsa2NTvjaWhX5/3PUjIwqFAjt37kSTJk3w4cMHfPjwAX5+fujTpw8iIiLg6uqKefPmoVq1aggJCcGjR49w9OhRhhTSmNQP9j179mDz5s2oXbs2KleujGLFiqF169bYsGEDYmNj0adPHwBA9+7dsXv3buk6kaYkJibi1atXMDExga6uLtasWSNNjQcAa2tr6f+pR0H+PJgwpGQc7FGRkUuXLqFt27YYNWoUevTogbCwMJQuXRrGxsYoWbIkNmzYACsrK+ncKV9OASXShIiICDRu3BjXr19HjRo1sG/fPpXlY8aMwenTp/H333/D1NRUamc3O/2sbw3gDg0NhYuLC2rUqIEFCxYgX758WqiOtIU9Kloyc+ZMjB07VvolAHw6RLmrqyt69OiB0NBQ1KpVC82bN8e4ceNw8eJF9O3bFxERETAyMgIAhhTSiM+3QQCwsrLC+vXrUadOHVy9ehVr165VWZ4/f368e/cO8fHxKu0MKfQzPg8px44dw8aNG3H9+nU8f/4cTk5OOH36NIKDgzFixAhpAHeLFi2wZMkSbZZNvwB7VLRkyZIlGDhwIGbMmIERI0ZIb9A7d+6gYMGCaNasmfSFoVQqUbJkSYSEhKBRo0bYsmULz5VCGvH5l8PDhw+hUChgYmICOzs7PH78GJ6enoiLi8Off/4JDw8PvH79Gu7u7jAyMsK+ffvYvU4aN2zYMKxfvx56enrIkiUL7OzssHDhQpQtWxY3b95EjRo14OTkhKSkJHz8+BHXr1+XTsxKGZSgX06pVAohhFi5cqXQ0dERU6dOFcnJydLyp0+fisKFC4t9+/YJIYSIiIgQ7dq1E0uWLBHPnj3TSs2U8aRuh0IIMXHiRFG8eHFRqFAhkSNHDuHr6yuEECIkJEQ0bNhQGBkZiYIFC4oWLVqIevXqifj4eCGEECkpKVqpnTKOz7fDoKAgUaJECXHy5EkREREhdu/eLVq0aCGcnZ3FlStXhBBCPHjwQEyZMkVMnz5d+tz8/POTMh4GlV9MqVRKb0ylUin++usvoaOjI6ZNmyZ96IeHh4uSJUsKDw8PERoaKsaMGSPKlSsnXr9+rc3SKYOaMmWKsLGxEYGBgSI2Nla0aNFCWFhYiNu3bwshhHj06JFo1KiRKFmypFi4cKF0u4SEBC1VTBnR+vXrRb9+/USvXr1U2i9evCjq168v3N3dRWxsrBBCNdwwpGR83H+gBQqFAocPH8bQoUNRpkwZ6Rwqs2bNghAClpaW6NChA44fPw5XV1ds2LABPj4+sLW11XbplAF8PiZFqVTiwoULWLhwIerWrYugoCAcO3YMM2bMQJEiRZCcnIw8efJg/vz5yJ49O/bv34+AgAAAgKGhobaeAmUA4otRB7t27cKyZctw7do1JCYmSu1ly5ZFlSpVcOrUKaSkpABQndHDc0hlAtpOSpnRjh07hLGxsZg6daq4ePGiEEIIX19faTeQEEIkJiaK27dvi6CgIPH06VNtlksZ1IQJE8SsWbNEzpw5xb1790RwcLDIkiWL8Pb2FkII8eHDBzF27FgRGhoqhBDi/v37onHjxqJs2bIiICBAm6XTb+7zHhF/f3+xYcMGIYQQ/fr1ExYWFmLZsmUiKipKWicwMFAUKlRI2hYpc2FQ+cXu3bsn8uTJI5YvX55m2YoVK6TdQESa9vl4ks2bNwtHR0dx69Yt0bFjR1GvXj1hYmIiVq9eLa3z/PlzUaVKFbFhwwbptnfu3BGtW7cWYWFhv7x+yhg+3w5v3bolSpUqJUqUKCF2794thBDC3d1d5M+fX0yfPl2EhISIkJAQUatWLVGtWjWVgEOZB/vMfrEnT55AX19f5QiLqTMvevXqBVNTU3Tq1AmGhoYYNmyYFiuljCZ1ds/x48dx7NgxDB06FEWLFpUOJFirVi1069YNwKeTYfbo0QO6urpo3749dHR0oFQqUahQIWzcuJGzLOiHpW6Hw4cPx+PHj2FsbIy7d+9i8ODB+PjxI9atW4du3bph3LhxWLJkCSpVqoQsWbJgy5YtUCgU3zzWCmVcDCq/WGxsrMrxJ5RKpbS/9dixYyhTpgy2bNmict4UIk159eoVunfvjvDwcIwZMwYA0Lt3bzx8+BBHjx5FqVKlkD9/fjx58gQJCQm4ePEidHV1VQ7mxjEB9LPWrVuHVatW4ciRI8iTJw8SExPh7u6OmTNnQkdHB2vWrIGJiQm2bt2K+vXro23btjA0NERSUhIMDAy0XT79Yoylv1iJEiXw9u1b+Pr6Avj06yI1qOzevRsbN25Ey5YtUbhwYW2WSRmUnZ0dAgICkD17duzduxeXL1+Grq4u5s6diylTpqBmzZqws7NDmzZtvnn2WR47hX5WSEgIihUrhpIlS8Lc3Bx2dnZYs2YNdHV1MXjwYOzcuRNLly5F7dq1sWDBAuzZswcxMTEMKZkUfxr9Ynny5MHSpUvRu3dvJCcno3PnztDV1cW6deuwbt06nD17lkf4pHTl4uKCHTt2wN3dHT4+Pujfvz9cXFzQtGlTNG3aVGXdlJQU9qCQxoj/P1GgoaEhEhISkJSUBCMjIyQnJyNnzpyYOXMmGjduDC8vLxgbG2Pjxo1o3749hg0bBj09Pbi5uWn7KZAW8Mi0WqBUKrFjxw54eHjA1NQURkZG0NXVxaZNm1CqVCltl0eZxNWrV9GjRw+UKVMGAwcORNGiRbVdEmUSN2/eRKlSpTB+/HhMnDhRag8MDMTKlSvx/v17pKSk4NixYwCArl27Yvz48cibN6+WKiZtYlDRohcvXiAsLAwKhQJ58uRB9uzZtV0SZTJXr16Fh4cHcufOjTlz5iBPnjzaLokyiXXr1qFXr14YNGgQ2rRpA0tLSwwYMAAVK1ZEixYtULRoUezfvx8NGjTQdqmkZQwqRJnchQsX4OPjg1WrVnE2Bf1SO3bsQN++fWFgYAAhBGxtbXHmzBm8fv0aderUwfbt2+Hi4qLtMknLGFSISBo7wKmf9Ks9f/4cT58+RXJyMipVqgQdHR2MHj0au3btQnBwMOzs7LRdImkZgwoRAfhfWCHSltu3b2P27Nn4+++/cfjwYZQsWVLbJZEMcDg/EQHgtGPSro8fPyIpKQm2trY4fvw4B3eThD0qREQkG8nJyTzyMalgUCEiIiLZ4qg5IiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSL6rRw7dgwKhQKRkZHffRsnJyd4eXmlW01ElH4YVIhIo7p06QKFQoHevXunWebp6QmFQoEuXbr8+sKI6LfEoEJEGufo6IjNmzcjPj5eaktISMDGjRuRK1cuLVZGRL8bBhUi0rjSpUvD0dERAQEBUltAQABy5cqFUqVKSW2JiYkYMGAAbG1tYWRkhMqVK+PixYsq9/X333+jQIECMDY2Ro0aNRAaGprm8U6dOoUqVarA2NgYjo6OGDBgAOLi4tLt+RHRr8OgQkTpolu3bli7dq10fc2aNejatavKOiNGjMCOHTuwfv16XLlyBc7OzqhXrx4iIiIAAE+fPkXLli3RpEkTXLt2DT169MCoUaNU7uPhw4eoX78+WrVqhRs3bmDLli04deoU+vXrl/5PkojSHYMKEaWLjh074tSpUwgLC0NYWBhOnz6Njh07Ssvj4uLg7e2NuXPnokGDBihSpAhWrlwJY2NjrF69GgDg7e2NfPnyYf78+ShYsCA6dOiQZnzLzJkz0aFDBwwaNAj58+dHxYoVsXjxYmzYsAEJCQm/8ikTUTrgSQmJKF3Y2NigUaNGWLduHYQQaNSoEaytraXlDx8+RHJyMipVqiS16evr448//sCdO3cAAHfu3EH58uVV7rdChQoq169fv44bN27A399fahNCQKlU4vHjxyhcuHB6PD0i+kUYVIgo3XTr1k3aBbNs2bJ0eYzY2Fh4eHhgwIABaZZx4C7R749BhYjSTf369ZGUlASFQoF69eqpLMuXLx8MDAxw+vRp5M6dG8CnM+devHgRgwYNAgAULlwYe/bsUbnduXPnVK6XLl0a//zzD5ydndPviRCR1nCMChGlG11dXdy5cwf//PMPdHV1VZaZmpqiT58+GD58OA4ePIh//vkHPXv2xIcPH9C9e3cAQO/evfHgwQMMHz4c9+7dw8aNG7Fu3TqV+xk5ciTOnDmDfv364dq1a3jw4AF2797NwbREGQSDChGlKzMzM5iZmX112axZs9CqVSt06tQJpUuXRkhICAIDA2FpaQng066bHTt2YNeuXShRogR8fHwwY8YMlftwcXHB8ePHcf/+fVSpUgWlSpXChAkTYG9vn+7PjYjSn0IIIbRdBBEREdHXsEeFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhk6/8AHoK08GWUizwAAAAASUVORK5CYII=",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "## calculate avg response time\n",
+ "unique_models = set(result[\"response\"]['model'] for result in result[\"results\"])\n",
+ "model_dict = {model: {\"response_time\": []} for model in unique_models}\n",
+ "for completion_result in result[\"results\"]:\n",
+ " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n",
+ "\n",
+ "avg_response_time = {}\n",
+ "for model, data in model_dict.items():\n",
+ " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n",
+ "\n",
+ "models = list(avg_response_time.keys())\n",
+ "response_times = list(avg_response_time.values())\n",
+ "\n",
+ "plt.bar(models, response_times)\n",
+ "plt.xlabel('Model', fontsize=10)\n",
+ "plt.ylabel('Average Response Time')\n",
+ "plt.title('Average Response Times for each Model')\n",
+ "\n",
+ "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "inSDIE3_IRds"
+ },
+ "source": [
+ "# Duration Test endpoint\n",
+ "\n",
+ "Run load testing for 2 mins. Hitting endpoints with 100+ queries every 15 seconds."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {
+ "id": "ePIqDx2EIURH"
+ },
+ "outputs": [],
+ "source": [
+ "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n",
+ "context = \"\"\"Paul Graham (/ɔrƦm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n",
+ "prompt = \"Where does Paul Graham live?\"\n",
+ "final_prompt = context + prompt\n",
+ "result = load_test_model(models=models, prompt=final_prompt, num_calls=100, interval=15, duration=120)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 552
+ },
+ "id": "k6rJoELM6t1K",
+ "outputId": "f4968b59-3bca-4f78-a88b-149ad55e3cf7"
+ },
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAIXCAYAAABghH+YAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABwdUlEQVR4nO3dd1QU198G8GfpoNKUooKCYuwIaiL2GrGLJnYFOxrsNZbYFTsYG2JDjV2xRKOIir33EhsWLBGwUaXJ3vcPX+bnCiYsLi6Oz+ecPbp37ux+lx3YZ+/cmVEIIQSIiIiIZEJH2wUQERERaRLDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNEcmSg4MDunfvru0y1DZnzhyUKFECurq6cHFx0XY5GnfkyBEoFAps27ZN26WoTaFQYNKkSWqv9+jRIygUCgQFBWm8Jsoaww19tiVLlkChUKBatWraLiXPcXBwgEKhkG758uXDDz/8gLVr12q7tK9Oxodidm5fqwMHDmDUqFGoWbMmVq9ejRkzZmi7pDwnKChIep9PnDiRabkQAvb29lAoFGjRooUWKqS8QE/bBdDXb/369XBwcMC5c+cQHh4OJycnbZeUp7i4uGD48OEAgOfPn2PFihXw8vJCSkoK+vTpo+Xqvh5ly5bFunXrVNrGjBmD/PnzY9y4cZn637lzBzo6X9f3t8OHD0NHRwcrV66EgYGBtsvJ04yMjLBhwwbUqlVLpf3o0aN4+vQpDA0NtVQZ5QUMN/RZHj58iFOnTiE4OBje3t5Yv349Jk6c+EVrUCqVSE1NhZGR0Rd93uwqWrQounbtKt3v3r07SpQoAT8/P4YbNdjY2Kj8HAFg5syZKFSoUKZ2AF/lh1t0dDSMjY01FmyEEEhOToaxsbFGHi8vadasGbZu3Yrff/8denr/+yjbsGEDqlSpgpcvX2qxOtK2r+trDeU569evh4WFBZo3b46ff/4Z69evl5alpaXB0tISPXr0yLReXFwcjIyMMGLECKktJSUFEydOhJOTEwwNDWFvb49Ro0YhJSVFZV2FQoEBAwZg/fr1KF++PAwNDbF//34AwNy5c1GjRg0ULFgQxsbGqFKlSpb79pOSkjBo0CAUKlQIBQoUQKtWrfDs2bMs96k/e/YMPXv2hI2NDQwNDVG+fHmsWrUqxz8zKysrlClTBvfv31dpVyqV8Pf3R/ny5WFkZAQbGxt4e3vjzZs3Kv0uXLgAd3d3FCpUCMbGxnB0dETPnj2l5Rn79+fOnQs/Pz8UL14cxsbGqFu3Lm7cuJGpnsOHD6N27drIly8fzM3N0bp1a9y6dUulz6RJk6BQKBAeHo7u3bvD3NwcZmZm6NGjB96+favSNzQ0FLVq1YK5uTny58+P0qVLY+zYsSp9svtef46P59xk7M44ceIEBg0aBCsrK5ibm8Pb2xupqamIiYmBp6cnLCwsYGFhgVGjRkEIofKYmnqPsqJQKLB69WokJiZKu10y5mi8e/cOU6dORcmSJWFoaAgHBweMHTs208/LwcEBLVq0QEhICKpWrQpjY2MsW7bsX5/37NmzaNKkCczMzGBiYoK6devi5MmTKn0iIiLwyy+/oHTp0jA2NkbBggXRrl07PHr0KNPjxcTEYOjQoXBwcIChoSHs7Ozg6emZKWwolUpMnz4ddnZ2MDIyQsOGDREeHv6vtX6oU6dOePXqFUJDQ6W21NRUbNu2DZ07d85yncTERAwfPhz29vYwNDRE6dKlMXfu3Ezvc0pKCoYOHQorKyvp78PTp0+zfExN/30gDRFEn6FMmTKiV69eQgghjh07JgCIc+fOSct79uwpzM3NRUpKisp6a9asEQDE+fPnhRBCpKeni8aNGwsTExMxZMgQsWzZMjFgwAChp6cnWrdurbIuAFG2bFlhZWUlJk+eLBYvXiwuX74shBDCzs5O/PLLL2LRokVi/vz54ocffhAAxJ49e1Qeo3379gKA6Natm1i8eLFo3769qFSpkgAgJk6cKPWLjIwUdnZ2wt7eXkyZMkUsXbpUtGrVSgAQfn5+//nzKV68uGjevLlKW1pamrC1tRU2NjYq7b179xZ6enqiT58+IiAgQIwePVrky5dPfP/99yI1NVUIIURUVJSwsLAQ3333nZgzZ45Yvny5GDdunChbtqz0OA8fPhQARMWKFYWDg4OYNWuWmDx5srC0tBRWVlYiMjJS6hsaGir09PTEd999J2bPni0mT54sChUqJCwsLMTDhw+lfhMnThQAhKurq2jbtq1YsmSJ6N27twAgRo0aJfW7ceOGMDAwEFWrVhULFiwQAQEBYsSIEaJOnTpSH3Xe6/9Svnx5Ubdu3U/+7L28vKT7q1evFgCEi4uLaNKkiVi8eLHo1q2b9Bpq1aolOnfuLJYsWSJatGghAIg1a9bkynuUlXXr1onatWsLQ0NDsW7dOrFu3Tpx//59IYQQXl5eAoD4+eefxeLFi4Wnp6cAIDw8PDK9ZicnJ2FhYSF+/fVXERAQIMLCwj75nIcOHRIGBgaievXqYt68ecLPz084OzsLAwMDcfbsWanf1q1bRaVKlcSECRNEYGCgGDt2rLCwsBDFixcXiYmJUr/4+HhRoUIFoaurK/r06SOWLl0qpk6dKr7//nvpdzQsLEzalqpUqSL8/PzEpEmThImJifjhhx/+9Wf04ft4/vx5UaNGDdGtWzdp2c6dO4WOjo549uxZpt89pVIpGjRoIBQKhejdu7dYtGiRaNmypQAghgwZovIcXbt2FQBE586dxaJFi0Tbtm2Fs7Nzjv8+ZPxOrl69+j9fH2kGww3l2IULFwQAERoaKoR4/8fDzs5ODB48WOoTEhIiAIg///xTZd1mzZqJEiVKSPfXrVsndHR0xPHjx1X6BQQECADi5MmTUhsAoaOjI27evJmpprdv36rcT01NFRUqVBANGjSQ2i5evJjlH7Tu3btn+uPVq1cvUbhwYfHy5UuVvh07dhRmZmaZnu9jxYsXF40bNxYvXrwQL168ENevX5c+UH18fKR+x48fFwDE+vXrVdbfv3+/SvuOHTtUQmFWMv6QGhsbi6dPn0rtZ8+eFQDE0KFDpTYXFxdhbW0tXr16JbVdvXpV6OjoCE9PT6ktI9z07NlT5bnatGkjChYsKN338/MTAMSLFy8+WZ867/V/yUm4cXd3F0qlUmqvXr26UCgUol+/flLbu3fvhJ2dncpja/I9+hQvLy+RL18+lbYrV64IAKJ3794q7SNGjBAAxOHDh1VeMwCxf//+/3wupVIpSpUqlenn8fbtW+Ho6Ch+/PFHlbaPnT59WgAQa9euldomTJggAIjg4OAsn0+I/4WbsmXLqnzpWbBggQAgrl+//q91fxhuFi1aJAoUKCDV165dO1G/fn3pZ/FhuNm5c6cAIKZNm6byeD///LNQKBQiPDxcCPG/n/cvv/yi0q9z5845/vvAcPPlcbcU5dj69ethY2OD+vXrA3g/rN6hQwds2rQJ6enpAIAGDRqgUKFC2Lx5s7TemzdvEBoaig4dOkhtW7duRdmyZVGmTBm8fPlSujVo0AAAEBYWpvLcdevWRbly5TLV9OHcgjdv3iA2Nha1a9fGpUuXpPaMXVi//PKLyroDBw5UuS+EwPbt29GyZUsIIVTqcnd3R2xsrMrjfsqBAwdgZWUFKysrVKxYEevWrUOPHj0wZ84clddvZmaGH3/8UeV5qlSpgvz580uv39zcHACwZ88epKWl/evzenh4oGjRotL9H374AdWqVcNff/0F4P3k5itXrqB79+6wtLSU+jk7O+PHH3+U+n2oX79+Kvdr166NV69eIS4uTqW+Xbt2QalUZlmXuu+1pvXq1UvliKpq1apBCIFevXpJbbq6uqhatSoePHigUrem36PsyHgfhg0bptKeMUl97969Ku2Ojo5wd3f/z8e9cuUK7t27h86dO+PVq1fS60lMTETDhg1x7Ngx6T388PcqLS0Nr169gpOTE8zNzVV+B7Zv345KlSqhTZs2mZ7v46PYevTooTK3qHbt2gCg8jP/L+3bt0dSUhL27NmD+Ph47Nmz55O7pP766y/o6upi0KBBKu3Dhw+HEAL79u2T+gHI1G/IkCEq9zX194Fyxzcdbo4dO4aWLVuiSJEiUCgU2LlzZ64/57Nnz9C1a1dpTkjFihVx4cKFXH9eTUtPT8emTZtQv359PHz4EOHh4QgPD0e1atUQFRWFQ4cOAQD09PTw008/YdeuXdL8gODgYKSlpamEm3v37uHmzZtSCMi4fffddwDeT7T8kKOjY5Z17dmzB25ubjAyMoKlpSWsrKywdOlSxMbGSn0iIiKgo6OT6TE+PsrrxYsXiImJQWBgYKa6MuYRfVxXVqpVq4bQ0FDs378fc+fOhbm5Od68eaPyh/3evXuIjY2FtbV1pudKSEiQnqdu3br46aefMHnyZBQqVAitW7fG6tWrs5yrUqpUqUxt3333nTRPIiIiAgBQunTpTP3Kli0rfdB9qFixYir3LSwsAECac9KhQwfUrFkTvXv3ho2NDTp27IgtW7aoBB1132tN+/g1mJmZAQDs7e0ztX84lyY33qPsyNheP94+bW1tYW5uLr2PGT71u/Gxe/fuAQC8vLwyvZ4VK1YgJSVF+r1JSkrChAkTpLkqhQoVgpWVFWJiYlR+t+7fv48KFSpk6/n/a1vKDisrKzRq1AgbNmxAcHAw0tPT8fPPP2fZNyIiAkWKFEGBAgVU2suWLSstz/hXR0cHJUuWVOn38e+Jpv4+UO74po+WSkxMRKVKldCzZ0+0bds215/vzZs3qFmzJurXr499+/bBysoK9+7dk36pvyaHDx/G8+fPsWnTJmzatCnT8vXr16Nx48YAgI4dO2LZsmXYt28fPDw8sGXLFpQpUwaVKlWS+iuVSlSsWBHz58/P8vk+/uDJ6uiP48ePo1WrVqhTpw6WLFmCwoULQ19fH6tXr8aGDRvUfo0ZH8hdu3aFl5dXln2cnZ3/83EKFSqERo0aAQDc3d1RpkwZtGjRAgsWLJC+jSuVSlhbW6tMyP6QlZUVAEgnPztz5gz+/PNPhISEoGfPnpg3bx7OnDmD/Pnzq/061aGrq5tlu/j/CZnGxsY4duwYwsLCsHfvXuzfvx+bN29GgwYNcODAAejq6qr9Xmvap15DVu3ig4mm2n6Psnv+nuweGZWxfc+ZM+eTJwvMqHXgwIFYvXo1hgwZgurVq8PMzAwKhQIdO3b85Ajdf/mvbSm7OnfujD59+iAyMhJNmzaVRs5ym6b+PlDu+KbDTdOmTdG0adNPLk9JScG4ceOwceNGxMTEoEKFCpg1axbq1auXo+ebNWsW7O3tsXr1aqktu9+y8pr169fD2toaixcvzrQsODgYO3bsQEBAAIyNjVGnTh0ULlwYmzdvRq1atXD48OFM5yUpWbIkrl69ioYNG+b4JGzbt2+HkZERQkJCVA4D/vDnDQDFixeHUqnEw4cPVUY3Pj5SI+NIifT0dCmcaELz5s1Rt25dzJgxA97e3siXLx9KliyJgwcPombNmtn6cHJzc4ObmxumT5+ODRs2oEuXLti0aRN69+4t9cn4Zv6hu3fvwsHBAcD7nwPw/nwwH7t9+zYKFSqEfPnyqf36dHR00LBhQzRs2BDz58/HjBkzMG7cOISFhaFRo0Yaea+1ITfeo+zI2F7v3bsnjTIAQFRUFGJiYqT3UV0ZIxOmpqb/uX1v27YNXl5emDdvntSWnJyMmJiYTI+Z1RF5ualNmzbw9vbGmTNnVHZ/f6x48eI4ePAg4uPjVUZvbt++LS3P+FepVOL+/fsqozUf/57k1t8H0oxverfUfxkwYABOnz6NTZs24dq1a2jXrh2aNGmS5YdGduzevRtVq1ZFu3btYG1tDVdXVyxfvlzDVee+pKQkBAcHo0WLFvj5558z3QYMGID4+Hjs3r0bwPsPu59//hl//vkn1q1bh3fv3qnskgLe7zt/9uxZlj+PpKSkTLtHsqKrqwuFQiHN9wHeHxb98e7GjPkIS5YsUWlfuHBhpsf76aefsH379iz/YL948eI/a/qU0aNH49WrV9Lrbd++PdLT0zF16tRMfd+9eyd9iLx58ybTN9uMb90f7/bYuXMnnj17Jt0/d+4czp49KwX6woULw8XFBWvWrFH5kLpx4wYOHDiAZs2aqf26Xr9+nant4/o08V5rQ268R9mR8T74+/urtGeMfDVv3lztxwSAKlWqoGTJkpg7dy4SEhIyLf9w+9bV1c30mhYuXKjyuwYAP/30E65evYodO3Zkejx1R2SyK3/+/Fi6dCkmTZqEli1bfrJfs2bNkJ6ejkWLFqm0+/n5QaFQSL8XGf/+/vvvKv0+/vnn5t8H+nzf9MjNv3n8+DFWr16Nx48fo0iRIgCAESNGYP/+/Tk+LfqDBw+wdOlSDBs2DGPHjsX58+cxaNAgGBgYfHJYMy/avXs34uPj0apVqyyXu7m5wcrKCuvXr5dCTIcOHbBw4UJMnDgRFStWVPkGCgDdunXDli1b0K9fP4SFhaFmzZpIT0/H7du3sWXLFum8Hf+mefPmmD9/Ppo0aYLOnTsjOjoaixcvhpOTE65duyb1q1KlCn766Sf4+/vj1atXcHNzw9GjR3H37l0AqsP/M2fORFhYGKpVq4Y+ffqgXLlyeP36NS5duoSDBw9m+WGeHU2bNkWFChUwf/58+Pj4oG7duvD29oavry+uXLmCxo0bQ19fH/fu3cPWrVuxYMEC/Pzzz1izZg2WLFmCNm3aoGTJkoiPj8fy5cthamqaKYw4OTmhVq1a6N+/P1JSUuDv74+CBQti1KhRUp85c+agadOmqF69Onr16oWkpCQsXLgQZmZmObqGzpQpU3Ds2DE0b94cxYsXR3R0NJYsWQI7OzvpTLKaeK+1ITfeo+yoVKkSvLy8EBgYiJiYGNStWxfnzp3DmjVr4OHhIU3oV5eOjg5WrFiBpk2bonz58ujRoweKFi2KZ8+eISwsDKampvjzzz8BAC1atMC6detgZmaGcuXK4fTp0zh48CAKFiyo8pgjR47Etm3b0K5dO/Ts2RNVqlTB69evsXv3bgQEBKjsitak7Pz9bNmyJerXr49x48bh0aNHqFSpEg4cOIBdu3ZhyJAh0kiWi4sLOnXqhCVLliA2NhY1atTAoUOHsjwHT279fSAN0MoxWnkQALFjxw7p/p49ewQAkS9fPpWbnp6eaN++vRBCiFu3bgkA/3obPXq09Jj6+vqievXqKs87cOBA4ebm9kVeo6a0bNlSGBkZqZzf4mPdu3cX+vr60iGSSqVS2NvbZ3koZobU1FQxa9YsUb58eWFoaCgsLCxElSpVxOTJk0VsbKzUDx8dRv2hlStXilKlSglDQ0NRpkwZsXr1aukw5g8lJiYKHx8fYWlpKfLnzy88PDzEnTt3BAAxc+ZMlb5RUVHCx8dH2NvbC319fWFraysaNmwoAgMD//NnldV5bjIEBQVlOjw0MDBQVKlSRRgbG4sCBQqIihUrilGjRol//vlHCCHEpUuXRKdOnUSxYsWEoaGhsLa2Fi1atBAXLlyQHiPjsNM5c+aIefPmCXt7e2FoaChq164trl69mqmOgwcPipo1awpjY2NhamoqWrZsKf7++2+VPhk/w48P8c44LDfjnDiHDh0SrVu3FkWKFBEGBgaiSJEiolOnTuLu3bsq62X3vf4vOTkU/ONDtD/12rI6LFsIzbxHn/Kp50xLSxOTJ08Wjo6OQl9fX9jb24sxY8aI5OTkTK/5U9vbp1y+fFm0bdtWFCxYUBgaGorixYuL9u3bi0OHDkl93rx5I3r06CEKFSok8ufPL9zd3cXt27cz/YyFEOLVq1diwIABomjRosLAwEDY2dkJLy8v6W9BxqHgW7duVVkvu4dLf+p9/FhWP4v4+HgxdOhQUaRIEaGvry9KlSol5syZo3IovBBCJCUliUGDBomCBQuKfPnyiZYtW4onT55kOhRciOz9feCh4F+eQohcGiv8yigUCuzYsQMeHh4AgM2bN6NLly64efNmpolv+fPnh62tLVJTU//zsMWCBQtKEw2LFy+OH3/8EStWrJCWL126FNOmTVPZfUDaceXKFbi6uuKPP/5Aly5dtF1Ojj169AiOjo6YM2eOyhmgiYi+Fdwt9Qmurq5IT09HdHS0dP6FjxkYGKBMmTLZfsyaNWtmmpR29+7dHE8IpJxLSkrKNCnU398fOjo6qFOnjpaqIiIiTfimw01CQoLKftSHDx/iypUrsLS0xHfffYcuXbrA09MT8+bNg6urK168eIFDhw7B2dk5R5P4hg4diho1amDGjBlo3749zp07h8DAQAQGBmryZVE2zJ49GxcvXkT9+vWhp6eHffv2Yd++fejbt2+uH4pMRES5TNv7xbQpY9/vx7eMfcipqaliwoQJwsHBQejr64vChQuLNm3aiGvXruX4Of/8809RoUIFaU5IduZtkOYdOHBA1KxZU1hYWAh9fX1RsmRJMWnSJJGWlqbt0j7bh3NuiIi+RZxzQ0RERLLC89wQERGRrDDcEBERkax8cxOKlUol/vnnHxQoUOCrOvU7ERHRt0wIgfj4eBQpUgQ6Ov8+NvPNhZt//vmHR8MQERF9pZ48eQI7O7t/7fPNhZuMC6Y9efIEpqamWq6GiIiIsiMuLg729vYqFz79lG8u3GTsijI1NWW4ISIi+spkZ0oJJxQTERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs6Gm7ACLSLIdf92q7BNKyRzOba/X5uQ2StrdBjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrGg13CxduhTOzs4wNTWFqakpqlevjn379v3rOlu3bkWZMmVgZGSEihUr4q+//vpC1RIREdHXQKvhxs7ODjNnzsTFixdx4cIFNGjQAK1bt8bNmzez7H/q1Cl06tQJvXr1wuXLl+Hh4QEPDw/cuHHjC1dOREREeZVCCCG0XcSHLC0tMWfOHPTq1SvTsg4dOiAxMRF79uyR2tzc3ODi4oKAgIBsPX5cXBzMzMwQGxsLU1NTjdVNlFfwooWk7YsWchuk3NgG1fn8zjNzbtLT07Fp0yYkJiaievXqWfY5ffo0GjVqpNLm7u6O06dPf/JxU1JSEBcXp3IjIiIi+dJ6uLl+/Try588PQ0ND9OvXDzt27EC5cuWy7BsZGQkbGxuVNhsbG0RGRn7y8X19fWFmZibd7O3tNVo/ERER5S1aDzelS5fGlStXcPbsWfTv3x9eXl74+++/Nfb4Y8aMQWxsrHR78uSJxh6biIiI8h49bRdgYGAAJycnAECVKlVw/vx5LFiwAMuWLcvU19bWFlFRUSptUVFRsLW1/eTjGxoawtDQULNFExERUZ6l9ZGbjymVSqSkpGS5rHr16jh06JBKW2ho6Cfn6BAREdG3R6sjN2PGjEHTpk1RrFgxxMfHY8OGDThy5AhCQkIAAJ6enihatCh8fX0BAIMHD0bdunUxb948NG/eHJs2bcKFCxcQGBiozZdBREREeYhWw010dDQ8PT3x/PlzmJmZwdnZGSEhIfjxxx8BAI8fP4aOzv8Gl2rUqIENGzZg/PjxGDt2LEqVKoWdO3eiQoUK2noJRERElMdoNdysXLnyX5cfOXIkU1u7du3Qrl27XKqIiIiIvnZ5bs4NERER0edguCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWdFquPH19cX333+PAgUKwNraGh4eHrhz586/rhMUFASFQqFyMzIy+kIVExERUV6n1XBz9OhR+Pj44MyZMwgNDUVaWhoaN26MxMTEf13P1NQUz58/l24RERFfqGIiIiLK6/S0+eT79+9XuR8UFARra2tcvHgRderU+eR6CoUCtra2uV0eERERfYXy1Jyb2NhYAIClpeW/9ktISEDx4sVhb2+P1q1b4+bNm5/sm5KSgri4OJUbERERyVeeCTdKpRJDhgxBzZo1UaFChU/2K126NFatWoVdu3bhjz/+gFKpRI0aNfD06dMs+/v6+sLMzEy62dvb59ZLICIiojwgz4QbHx8f3LhxA5s2bfrXftWrV4enpydcXFxQt25dBAcHw8rKCsuWLcuy/5gxYxAbGyvdnjx5khvlExERUR6h1Tk3GQYMGIA9e/bg2LFjsLOzU2tdfX19uLq6Ijw8PMvlhoaGMDQ01ESZRERE9BXQ6siNEAIDBgzAjh07cPjwYTg6Oqr9GOnp6bh+/ToKFy6cCxUSERHR10arIzc+Pj7YsGEDdu3ahQIFCiAyMhIAYGZmBmNjYwCAp6cnihYtCl9fXwDAlClT4ObmBicnJ8TExGDOnDmIiIhA7969tfY6iIiIKO/QarhZunQpAKBevXoq7atXr0b37t0BAI8fP4aOzv8GmN68eYM+ffogMjISFhYWqFKlCk6dOoVy5cp9qbKJiIgoD9NquBFC/GefI0eOqNz38/ODn59fLlVEREREX7s8MaFYThx+3avtEkjLHs1sru0SiIi+aXnmUHAiIiIiTWC4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZyVG4uX//PsaPH49OnTohOjoaALBv3z7cvHlTo8URERERqUvtcHP06FFUrFgRZ8+eRXBwMBISEgAAV69excSJEzVeIBEREZE61A43v/76K6ZNm4bQ0FAYGBhI7Q0aNMCZM2c0WhwRERGRutQON9evX0ebNm0ytVtbW+Ply5caKYqIiIgop9QON+bm5nj+/Hmm9suXL6No0aIaKYqIiIgop9QONx07dsTo0aMRGRkJhUIBpVKJkydPYsSIEfD09MyNGomIiIiyTe1wM2PGDJQpUwb29vZISEhAuXLlUKdOHdSoUQPjx4/PjRqJiIiIsk3tq4IbGBhg+fLl+O2333Djxg0kJCTA1dUVpUqVyo36iIiIiNSidrjJUKxYMRQrVkyTtRARERF9NrXDjRAC27ZtQ1hYGKKjo6FUKlWWBwcHa6w4IiIiInWpHW6GDBmCZcuWoX79+rCxsYFCociNuoiIiIhyRO1ws27dOgQHB6NZs2a5UQ8RERHRZ1H7aCkzMzOUKFEiN2ohIiIi+mxqh5tJkyZh8uTJSEpKyo16iIiIiD6L2rul2rdvj40bN8La2hoODg7Q19dXWX7p0iWNFUdERESkLrXDjZeXFy5evIiuXbtyQjERERHlOWqHm7179yIkJAS1atXKjXqIiIiIPovac27s7e1hamqaG7UQERERfTa1w828efMwatQoPHr0KBfKISIiIvo8au+W6tq1K96+fYuSJUvCxMQk04Ti169fa6w4IiIiInWpHW78/f1zoQwiIiIizcjR0VJEREREeVW2wk1cXJw0iTguLu5f+3KyMREREWlTtsKNhYUFnj9/Dmtra5ibm2d5bhshBBQKBdLT0zVeJBEREVF2ZSvcHD58GJaWlgCAsLCwXC2IiIiI6HNkK9zUrVsXJUqUwPnz51G3bt3cromIiIgox7J9nptHjx5xlxMRERHleWqfxI+IiIgoL1PrUPCQkBCYmZn9a59WrVp9VkFEREREn0OtcPNf57jh0VJERESkbWrtloqMjIRSqfzkjcGGiIiItC3b4Sarc9sQERER5TXZDjdCiNysg4iIiEgjsh1uvLy8YGxsnJu1EBEREX22bE8oXr16dW7WQURERKQRPM8NERERyQrDDREREckKww0RERHJSo7DTXh4OEJCQpCUlAQgZ0dT+fr64vvvv0eBAgVgbW0NDw8P3Llz5z/X27p1K8qUKQMjIyNUrFgRf/31l9rPTURERPKkdrh59eoVGjVqhO+++w7NmjXD8+fPAQC9evXC8OHD1Xqso0ePwsfHB2fOnEFoaCjS0tLQuHFjJCYmfnKdU6dOoVOnTujVqxcuX74MDw8PeHh44MaNG+q+FCIiIpIhtcPN0KFDoaenh8ePH8PExERq79ChA/bv36/WY+3fvx/du3dH+fLlUalSJQQFBeHx48e4ePHiJ9dZsGABmjRpgpEjR6Js2bKYOnUqKleujEWLFqn7UoiIiEiG1Lq2FAAcOHAAISEhsLOzU2kvVaoUIiIiPquY2NhYAIClpeUn+5w+fRrDhg1TaXN3d8fOnTuz7J+SkoKUlBTpflxc3GfVSERERHmb2iM3iYmJKiM2GV6/fg1DQ8McF6JUKjFkyBDUrFkTFSpU+GS/yMhI2NjYqLTZ2NggMjIyy/6+vr4wMzOTbvb29jmukYiIiPI+tcNN7dq1sXbtWum+QqGAUqnE7NmzUb9+/RwX4uPjgxs3bmDTpk05foysjBkzBrGxsdLtyZMnGn18IiIiylvU3i01e/ZsNGzYEBcuXEBqaipGjRqFmzdv4vXr1zh58mSOihgwYAD27NmDY8eOZdrd9TFbW1tERUWptEVFRcHW1jbL/oaGhp81okRERERfF7VHbipUqIC7d++iVq1aaN26NRITE9G2bVtcvnwZJUuWVOuxhBAYMGAAduzYgcOHD8PR0fE/16levToOHTqk0hYaGorq1aur9dxEREQkT2qP3ACAmZkZxo0b99lP7uPjgw0bNmDXrl0oUKCANG/GzMxMukinp6cnihYtCl9fXwDA4MGDUbduXcybNw/NmzfHpk2bcOHCBQQGBn52PURERPT1U3vkZv/+/Thx4oR0f/HixXBxcUHnzp3x5s0btR5r6dKliI2NRb169VC4cGHptnnzZqnP48ePpXPpAECNGjWwYcMGBAYGolKlSti2bRt27tz5r5OQiYiI6Nuh9sjNyJEjMWvWLADA9evXMWzYMAwfPhxhYWEYNmyYWlcPz85ZjY8cOZKprV27dmjXrl22n4eIiIi+HWqHm4cPH6JcuXIAgO3bt6Nly5aYMWMGLl26hGbNmmm8QCIiIiJ1qL1bysDAAG/fvgUAHDx4EI0bNwbw/sR7PEEeERERaZvaIze1atXCsGHDULNmTZw7d06aH3P37t3/PIybiIiIKLepPXKzaNEi6OnpYdu2bVi6dCmKFi0KANi3bx+aNGmi8QKJiIiI1KH2yE2xYsWwZ8+eTO1+fn4aKYiIiIjoc+ToPDdKpRLh4eGIjo6GUqlUWVanTh2NFEZERESUE2qHmzNnzqBz586IiIjIdCi3QqFAenq6xoojIiIiUpfa4aZfv36oWrUq9u7di8KFC0OhUORGXUREREQ5ona4uXfvHrZt2wYnJ6fcqIeIiIjos6h9tFS1atUQHh6eG7UQERERfTa1R24GDhyI4cOHIzIyEhUrVoS+vr7KcmdnZ40VR0RERKQutcPNTz/9BADo2bOn1KZQKCCE4IRiIiIi0rocXVuKiIiIKK9SO9wUL148N+ogIiIi0ogcncTv/v378Pf3x61btwAA5cqVw+DBg1GyZEmNFkdERESkLrWPlgoJCUG5cuVw7tw5ODs7w9nZGWfPnkX58uURGhqaGzUSERERZZvaIze//vorhg4dipkzZ2ZqHz16NH788UeNFUdERESkLrVHbm7duoVevXplau/Zsyf+/vtvjRRFRERElFNqhxsrKytcuXIlU/uVK1dgbW2tiZqIiIiIckzt3VJ9+vRB37598eDBA9SoUQMAcPLkScyaNQvDhg3TeIFERERE6lA73Pz2228oUKAA5s2bhzFjxgAAihQpgkmTJmHQoEEaL5CIiIhIHWqHG4VCgaFDh2Lo0KGIj48HABQoUEDjhRERERHlRI7OcwMA0dHRuHPnDgCgTJkysLKy0lhRRERERDml9oTi+Ph4dOvWDUWKFEHdunVRt25dFClSBF27dkVsbGxu1EhERESUbWqHm969e+Ps2bPYu3cvYmJiEBMTgz179uDChQvw9vbOjRqJiIiIsk3t3VJ79uxBSEgIatWqJbW5u7tj+fLlaNKkiUaLIyIiIlKX2iM3BQsWhJmZWaZ2MzMzWFhYaKQoIiIiopxSO9yMHz8ew4YNQ2RkpNQWGRmJkSNH4rffftNocURERETqUnu31NKlSxEeHo5ixYqhWLFiAIDHjx/D0NAQL168wLJly6S+ly5d0lylRERERNmgdrjx8PDIhTKIiIiINEPtcDNx4sTcqIOIiIhII9Sec/PkyRM8ffpUun/u3DkMGTIEgYGBGi2MiIiIKCfUDjedO3dGWFgYgPcTiRs1aoRz585h3LhxmDJlisYLJCIiIlKH2uHmxo0b+OGHHwAAW7ZsQcWKFXHq1CmsX78eQUFBmq6PiIiISC1qh5u0tDQYGhoCAA4ePIhWrVoBeH99qefPn2u2OiIiIiI1qR1uypcvj4CAABw/fhyhoaHSWYn/+ecfFCxYUOMFEhEREalD7XAza9YsLFu2DPXq1UOnTp1QqVIlAMDu3bul3VVERERE2qL2oeD16tXDy5cvERcXp3K5hb59+8LExESjxRERERGpS+2RGwAQQuDixYtYtmwZ4uPjAQAGBgYMN0RERKR1ao/cREREoEmTJnj8+DFSUlLw448/okCBApg1axZSUlIQEBCQG3USERERZYvaIzeDBw9G1apV8ebNGxgbG0vtbdq0waFDhzRaHBEREZG61B65OX78OE6dOgUDAwOVdgcHBzx79kxjhRERERHlhNojN0qlEunp6Znanz59igIFCmikKCIiIqKcUjvcNG7cGP7+/tJ9hUKBhIQETJw4Ec2aNdNkbURERERqU3u31Lx58+Du7o5y5cohOTkZnTt3xr1791CoUCFs3LgxN2okIiIiyja1R27s7Oxw9epVjBs3DkOHDoWrqytmzpyJy5cvw9raWq3HOnbsGFq2bIkiRYpAoVBg586d/9r/yJEjUCgUmW6RkZHqvgwiIiKSKbVHbgBAT08PXbp0QZcuXaS258+fY+TIkVi0aFG2HycxMRGVKlVCz5490bZt22yvd+fOHZiamkr31Q1VREREJF9qhZubN28iLCwMBgYGaN++PczNzfHy5UtMnz4dAQEBKFGihFpP3rRpUzRt2lStdYD3Ycbc3Fzt9YiIiEj+sr1bavfu3XB1dcWgQYPQr18/VK1aFWFhYShbtixu3bqFHTt24ObNm7lZq8TFxQWFCxfGjz/+iJMnT/5r35SUFMTFxanciIiISL6yHW6mTZsGHx8fxMXFYf78+Xjw4AEGDRqEv/76C/v375euDp6bChcujICAAGzfvh3bt2+Hvb096tWrh0uXLn1yHV9fX5iZmUk3e3v7XK+TiIiItCfb4ebOnTvw8fFB/vz5MXDgQOjo6MDPzw/ff/99btanonTp0vD29kaVKlVQo0YNrFq1CjVq1ICfn98n1xkzZgxiY2Ol25MnT75YvURERPTlZXvOTXx8vDSJV1dXF8bGxmrPsckNP/zwA06cOPHJ5YaGhjA0NPyCFREREZE2qTWhOCQkBGZmZgDen6n40KFDuHHjhkqfVq1aaa66bLhy5QoKFy78RZ+TiIiI8i61wo2Xl5fKfW9vb5X7CoUiy0szfEpCQgLCw8Ol+w8fPsSVK1dgaWmJYsWKYcyYMXj27BnWrl0LAPD394ejoyPKly+P5ORkrFixAocPH8aBAwfUeRlEREQkY9kON0qlUuNPfuHCBdSvX1+6P2zYMADvQ1RQUBCeP3+Ox48fS8tTU1MxfPhwPHv2DCYmJnB2dsbBgwdVHoOIiIi+bTk6iZ+m1KtXD0KITy4PCgpSuT9q1CiMGjUql6siIiKir5nal18gIiIiyssYboiIiEhWGG6IiIhIVhhuiIiISFZyFG5iYmKwYsUKjBkzBq9fvwYAXLp0Cc+ePdNocURERETqUvtoqWvXrqFRo0YwMzPDo0eP0KdPH1haWiI4OBiPHz+WzklDREREpA1qj9wMGzYM3bt3x71792BkZCS1N2vWDMeOHdNocURERETqUjvcnD9/PtOZiQGgaNGiiIyM1EhRRERERDmldrgxNDREXFxcpva7d+/CyspKI0URERER5ZTa4aZVq1aYMmUK0tLSALy/ntTjx48xevRo/PTTTxovkIiIiEgdaoebefPmISEhAdbW1khKSkLdunXh5OSEAgUKYPr06blRIxEREVG2qX20lJmZGUJDQ3HixAlcu3YNCQkJqFy5Mho1apQb9RERERGpJccXzqxVqxZq1aqlyVqIiIiIPpva4eb333/Psl2hUMDIyAhOTk6oU6cOdHV1P7s4IiIiInWpHW78/Pzw4sULvH37FhYWFgCAN2/ewMTEBPnz50d0dDRKlCiBsLAw2Nvba7xgIiIion+j9oTiGTNm4Pvvv8e9e/fw6tUrvHr1Cnfv3kW1atWwYMECPH78GLa2thg6dGhu1EtERET0r9QeuRk/fjy2b9+OkiVLSm1OTk6YO3cufvrpJzx48ACzZ8/mYeFERESkFWqP3Dx//hzv3r3L1P7u3TvpDMVFihRBfHz851dHREREpCa1w039+vXh7e2Ny5cvS22XL19G//790aBBAwDA9evX4ejoqLkqiYiIiLJJ7XCzcuVKWFpaokqVKjA0NIShoSGqVq0KS0tLrFy5EgCQP39+zJs3T+PFEhEREf0Xtefc2NraIjQ0FLdv38bdu3cBAKVLl0bp0qWlPvXr19dchURERERqyPFJ/MqUKYMyZcposhYiIiKiz5ajcPP06VPs3r0bjx8/Rmpqqsqy+fPna6QwIiIiopxQO9wcOnQIrVq1QokSJXD79m1UqFABjx49ghAClStXzo0aiYiIiLJN7QnFY8aMwYgRI3D9+nUYGRlh+/btePLkCerWrYt27drlRo1ERERE2aZ2uLl16xY8PT0BAHp6ekhKSkL+/PkxZcoUzJo1S+MFEhEREalD7XCTL18+aZ5N4cKFcf/+fWnZy5cvNVcZERERUQ6oPefGzc0NJ06cQNmyZdGsWTMMHz4c169fR3BwMNzc3HKjRiIiIqJsUzvczJ8/HwkJCQCAyZMnIyEhAZs3b0apUqV4pBQRERFpnVrhJj09HU+fPoWzszOA97uoAgICcqUwIiIiopxQa86Nrq4uGjdujDdv3uRWPURERESfRe0JxRUqVMCDBw9yoxYiIiKiz6Z2uJk2bRpGjBiBPXv24Pnz54iLi1O5EREREWmT2hOKmzVrBgBo1aoVFAqF1C6EgEKhQHp6uuaqIyIiIlKT2uEmLCwsN+ogIiIi0gi1w03dunVzow4iIiIijVB7zg0AHD9+HF27dkWNGjXw7NkzAMC6detw4sQJjRZHREREpC61w8327dvh7u4OY2NjXLp0CSkpKQCA2NhYzJgxQ+MFEhEREakjR0dLBQQEYPny5dDX15faa9asiUuXLmm0OCIiIiJ1qR1u7ty5gzp16mRqNzMzQ0xMjCZqIiIiIsoxtcONra0twsPDM7WfOHECJUqU0EhRRERERDmldrjp06cPBg8ejLNnz0KhUOCff/7B+vXrMWLECPTv3z83aiQiIiLKNrUPBf/111+hVCrRsGFDvH37FnXq1IGhoSFGjBiBgQMH5kaNRERERNmmdrhRKBQYN24cRo4cifDwcCQkJKBcuXLInz9/btRHREREpBa1d0v98ccfePv2LQwMDFCuXDn88MMPDDZERESUZ6gdboYOHQpra2t07twZf/3112ddS+rYsWNo2bIlihQpAoVCgZ07d/7nOkeOHEHlypVhaGgIJycnBAUF5fj5iYiISH7UDjfPnz/Hpk2boFAo0L59exQuXBg+Pj44deqU2k+emJiISpUqYfHixdnq//DhQzRv3hz169fHlStXMGTIEPTu3RshISFqPzcRERHJk9pzbvT09NCiRQu0aNECb9++xY4dO7BhwwbUr18fdnZ2uH//frYfq2nTpmjatGm2+wcEBMDR0RHz5s0DAJQtWxYnTpyAn58f3N3d1X0pREREJENqh5sPmZiYwN3dHW/evEFERARu3bqlqbqydPr0aTRq1Eilzd3dHUOGDPnkOikpKdIlIgAgLi4ut8ojIiKiPCBHF858+/Yt1q9fj2bNmqFo0aLw9/dHmzZtcPPmTU3XpyIyMhI2NjYqbTY2NoiLi0NSUlKW6/j6+sLMzEy62dvb52qNREREpF1qh5uOHTvC2toaQ4cORYkSJXDkyBGEh4dj6tSpKFOmTG7U+FnGjBmD2NhY6fbkyRNtl0RERES5SO3dUrq6utiyZQvc3d2hq6ursuzGjRuoUKGCxor7mK2tLaKiolTaoqKiYGpqCmNj4yzXMTQ0hKGhYa7VRERERHmL2uFm/fr1Kvfj4+OxceNGrFixAhcvXvysQ8P/S/Xq1fHXX3+ptIWGhqJ69eq59pxERET0dcnRnBvg/TlqvLy8ULhwYcydOxcNGjTAmTNn1HqMhIQEXLlyBVeuXAHw/lDvK1eu4PHjxwDe71Ly9PSU+vfr1w8PHjzAqFGjcPv2bSxZsgRbtmzB0KFDc/oyiIiISGbUGrmJjIxEUFAQVq5cibi4OLRv3x4pKSnYuXMnypUrp/aTX7hwAfXr15fuDxs2DADg5eWFoKAgPH/+XAo6AODo6Ii9e/di6NChWLBgAezs7LBixQoeBk5ERESSbIebli1b4tixY2jevDn8/f3RpEkT6OrqIiAgIMdPXq9ePQghPrk8q7MP16tXD5cvX87xcxIREZG8ZTvc7Nu3D4MGDUL//v1RqlSp3KyJiIiIKMeyPefmxIkTiI+PR5UqVVCtWjUsWrQIL1++zM3aiIiIiNSW7XDj5uaG5cuX4/nz5/D29samTZtQpEgRKJVKhIaGIj4+PjfrJCIiIsoWtY+WypcvH3r27IkTJ07g+vXrGD58OGbOnAlra2u0atUqN2okIiIiyrYcHwoOAKVLl8bs2bPx9OlTbNy4UVM1EREREeXYZ4WbDLq6uvDw8MDu3bs18XBEREREOaaRcENERESUVzDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs5Ilws3jxYjg4OMDIyAjVqlXDuXPnPtk3KCgICoVC5WZkZPQFqyUiIqK8TOvhZvPmzRg2bBgmTpyIS5cuoVKlSnB3d0d0dPQn1zE1NcXz58+lW0RExBesmIiIiPIyrYeb+fPno0+fPujRowfKlSuHgIAAmJiYYNWqVZ9cR6FQwNbWVrrZ2Nh8wYqJiIgoL9NquElNTcXFixfRqFEjqU1HRweNGjXC6dOnP7leQkICihcvDnt7e7Ru3Ro3b978ZN+UlBTExcWp3IiIiEi+tBpuXr58ifT09EwjLzY2NoiMjMxyndKlS2PVqlXYtWsX/vjjDyiVStSoUQNPnz7Nsr+vry/MzMykm729vcZfBxEREeUdWt8tpa7q1avD09MTLi4uqFu3LoKDg2FlZYVly5Zl2X/MmDGIjY2Vbk+ePPnCFRMREdGXpKfNJy9UqBB0dXURFRWl0h4VFQVbW9tsPYa+vj5cXV0RHh6e5XJDQ0MYGhp+dq1ERET0ddDqyI2BgQGqVKmCQ4cOSW1KpRKHDh1C9erVs/UY6enpuH79OgoXLpxbZRIREdFXRKsjNwAwbNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwZcoUuLm5wcnJCTExMZgzZw4iIiLQu3dvbb4MIiIiyiO0Hm46dOiAFy9eYMKECYiMjISLiwv2798vTTJ+/PgxdHT+N8D05s0b9OnTB5GRkbCwsECVKlVw6tQplCtXTlsvgYiIiPIQrYcbABgwYAAGDBiQ5bIjR46o3Pfz84Ofn98XqIqIiIi+Rl/d0VJERERE/4bhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkJU+Em8WLF8PBwQFGRkaoVq0azp0796/9t27dijJlysDIyAgVK1bEX3/99YUqJSIiorxO6+Fm8+bNGDZsGCZOnIhLly6hUqVKcHd3R3R0dJb9T506hU6dOqFXr164fPkyPDw84OHhgRs3bnzhyomIiCgv0nq4mT9/Pvr06YMePXqgXLlyCAgIgImJCVatWpVl/wULFqBJkyYYOXIkypYti6lTp6Jy5cpYtGjRF66ciIiI8iKthpvU1FRcvHgRjRo1ktp0dHTQqFEjnD59Ost1Tp8+rdIfANzd3T/Zn4iIiL4tetp88pcvXyI9PR02NjYq7TY2Nrh9+3aW60RGRmbZPzIyMsv+KSkpSElJke7HxsYCAOLi4j6n9E9SprzNlcelr0dubVvZxW2QuA2StuXGNpjxmEKI/+yr1XDzJfj6+mLy5MmZ2u3t7bVQDX0LzPy1XQF967gNkrbl5jYYHx8PMzOzf+2j1XBTqFAh6OrqIioqSqU9KioKtra2Wa5ja2urVv8xY8Zg2LBh0n2lUonXr1+jYMGCUCgUn/kK6ENxcXGwt7fHkydPYGpqqu1y6BvEbZC0jdtg7hFCID4+HkWKFPnPvloNNwYGBqhSpQoOHToEDw8PAO/Dx6FDhzBgwIAs16levToOHTqEIUOGSG2hoaGoXr16lv0NDQ1haGio0mZubq6J8ukTTE1N+UtNWsVtkLSN22Du+K8Rmwxa3y01bNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwePBg1K1bF/PmzUPz5s2xadMmXLhwAYGBgdp8GURERJRHaD3cdOjQAS9evMCECRMQGRkJFxcX7N+/X5o0/PjxY+jo/O+grho1amDDhg0YP348xo4di1KlSmHnzp2oUKGCtl4CERER5SEKkZ1px0TZkJKSAl9fX4wZMybTrkCiL4HbIGkbt8G8geGGiIiIZEXrZygmIiIi0iSGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYb+mYolUptl0BERF8Aww19MzIuwPry5UsAAK88Ql/axwGb2yBpw8fboRy/+DHc0DdlwYIF8PDwwP3796FQKLRdDn1jdHR0EBsbi5CQEADgNkhaoaOjg5iYGMyZMwdv3ryRvvjJifxeEdEHPv5mrK+vD2NjYxgYGGipIvqWKZVKzJs3D97e3tizZ4+2y6Fv2IEDBzB//nwsWrRI26XkCl4VnL4JcXFxMDU1BQDExsbCzMxMyxXRt0KpVKp8M7516xZWrlyJWbNmQVdXV4uV0bckPT1dZXtLS0vD5s2b0alTJ1luhww3JHtDhw5Feno6xowZg8KFC2u7HPoGxcTEICYmBvb29iofJB9/4BB9jo+D9MdevXqFkydPokaNGihUqJDULsftkLulSHY+zut2dnZYu3at7H556esghMCvv/6KatWq4dGjRyrLuE3S53j+/Dn++ecfvHjxAsD7uTT/Nl6xZcsWeHh44OjRoyrtctwOOXJDX7WMbxxCCCgUik9+c3nz5g0sLCy0UCHJzX99O86qT0REBMaPH4+goCBZfpDQl7d69WosXrwYT548QcmSJVGrVi3Mnj1bpU9WIzL+/v4YMGAA9PT0vmS5XxzDDX01MgIM8P6XVggBPT09PHv2DDt27ECPHj2QL18+AO93RVlYWGDChAmZ1iXKqQ9Dy+HDh/H48WM4OTmhRIkSKFKkiEqf2NhYKJXKTKFajrsA6Mvas2cP2rdvjyVLlsDExAQPHjzA7NmzUaNGDaxZswYFCxaU/ua9fPkS4eHhcHNzU3mMd+/eyTrgcLcU5VkZuTsuLg5JSUlQKBQ4cOAAwsPDoaurCz09PURERMDV1RX//POPFGwSExOhr68PPz8/vH79msGGNEIIIQWbX3/9Fd27d8fcuXPRt29fjBgxAufPnwfwftdASkoKJkyYgMqVK+PVq1cqj8NgQ5/r/PnzaN68Obp374727dtj1KhRCAkJwbVr19ClSxcA708zkJaWhnXr1qFGjRo4ceKEymPIOdgADDeUx0VGRqJixYo4evQoNmzYgCZNmuDvv/8G8H5XU/ny5dGmTRtMnz5dWidfvnwYNWoU7t27B0tLSwYb0oiM7Wju3Ln4448/sHHjRty4cQNt27bFn3/+ifHjx+P06dMAAAMDA7i6uqJhw4YwNzfXYtUkRw8fPsTz589V2r7//nvs3r0bFy9eRJ8+fQC8P/VFixYtMH369EwjN7IniPK4Hj16CFNTU6GjoyOWL18utaemporNmzeL9PR0qU2pVGqjRPpGREVFibZt24pVq1YJIYTYvXu3MDU1Ff369ROurq6iYcOG4syZM0II1W3x3bt3WqmX5CkkJETY2NiITZs2SW0Z29v69euFk5OTOH/+fKb10tLSvliN2saRG8qzMk4J7uPjg/j4eBgYGMDW1hbJyckA3n8rad++vcrETY7SUG6ytrbGqFGj0KRJE1y+fBk+Pj6YNm0ali5dip9++glnzpyBj48PLl68qLItclcUaVLZsmVRr149rFu3DocOHQLwv799Li4uiI6Oli4z8yG574r6EMMN5VkZocXe3h4nTpyAl5cXOnbsiF27diEpKSlTfzleH4W051Pbk6urKwoXLox9+/bB2dkZffv2BQBYWlrCzc0NLVu2hKur65cslb4x9vb26NevH2JiYuDn54fdu3dLywoXLgxHR0ctVpc3fDsxjr4a4v8nAD9//hxpaWkoVqwYrK2tUaNGDSQnJ6NXr14ICgpCixYtYGRkhICAADRq1AhOTk7aLp1kQnwweXjFihWIjo6GgYEBRowYIV26IyUlBc+ePcOjR49QunRpHDhwAK1atcLAgQP/9bQERJ8j42i7evXqYcmSJRg7dixGjx6NkJAQODs7Y8uWLVAoFPjxxx+1XapW8VBwypOCg4MxadIkREVFoXnz5mjTpg1atmwJAOjRowd27NiB4cOHIyoqCkuXLsX169dRrlw5LVdNcjNx4kT4+/vj+++/x7lz51CtWjWsW7cOtra2+PPPPzFt2jS8efMG+vr6EELg2rVr0NPT4xF6lCsytqvg4GAsWbIEBw4cwO3btxEWFoZFixbB3t4e5ubmWL9+PfT19b/p0w4w3FCec/PmTbi7u2Po0KEwMTHBxo0bYWhoCC8vL3Tt2hUAMHjwYFy6dAkpKSkIDAyEi4uLdosmWfhwtOXdu3fw8vLCwIED4erqikePHqF58+awtbXFjh07YGVlhb179yI8PBwJCQkYPXo09PT0vukPFNKMjBAjPjq3l66uLoKDg+Hp6Yn58+dLu0SB99urjo6Oyvb7Lc2x+RjDDeUpt2/fxtatW5GUlIQZM2YAAK5fv44JEyYgLi4OPXr0kAJOZGQk8uXLhwIFCmizZJKJD4PNrVu3EBcXh2XLlmHChAlwcHAA8P4Q3B9//BE2NjbYuXMnrKysVB6DwYY+14fb4cuXL6FQKFCwYEEA7//mVa5cGRMmTEC/fv2kdT4eKeTIIcMN5RFCCLx58wYtWrTA33//jZYtW2LdunXS8mvXrmHChAlISkpCx44d0aNHDy1WS3I2cuRIaVg/KioKwcHBaNq0qfRh8fDhQzRt2hRCCJw8eVLlAoREn+PDUDJ16lTs3LkTcXFxKFSoEKZPn44GDRrg2bNnKFq0qJYrzfs4243yBIVCAUtLS/j6+qJ8+fK4dOkSQkNDpeXOzs6YOnUq0tLSpF94Ik348KioPXv2YP/+/fj999+xZMkSODo6Yty4cbh69ap0xmxHR0fs2bMHLi4uvF4ZaVRGsJkyZQoWLFggnWqgUKFC6NKlC9asWZNptJCyxpEb0ppPDZ0ePXoUY8eOha2tLXx8fNCgQQNp2c2bN2FmZgY7O7svWSp9A4KDg3Hq1CkULFgQY8aMAQAkJCSgcuXKMDU1xYoVK1CpUqVM2yx3RZEmvXr1Co0bN4aPjw969uwptfft2xd//vknwsLCUKZMGe56+g8cuSGtyPjFPHXqFObPn4/ffvsNJ0+eRFpaGurWrYspU6YgMjISixYtwpEjR6T1ypcvz2BDGpeUlITffvsN8+fPx82bN6X2/Pnz49KlS4iPj4e3t7d0/agPMdiQJr179w4vX76URgUzTloaGBiIIkWKwM/PDwBPWPpfGG7oi/vwcMamTZvi5MmT2L17N8aOHYvp06cjNTUVDRs2xJQpU/Dq1StMnToVx48f13bZJGPGxsY4fvw4GjVqhIsXL2L37t1IT08H8L+Ac/v2bSxbtkzLlZKcZLXjxMbGBra2tli1ahUAwMjICKmpqQAAJycnhppsYrihLy5jxGbQoEGYP38+tm/fjq1bt+LixYvYvHkzxo8fLwWcX3/9Ffr6+jzjJmnMh3NshBDSB4ylpSU2bNgACwsLzJkzByEhIdKyfPnyITIyEoGBgVqpmeRHqVRKQeWff/5BdHQ03r59CwCYNGkSbt++LR0RlXHiyKdPn/JCrNnEOTf0xWT8MisUCixZsgRXrlxBYGAgHj58iEaNGqFWrVowNTXF1q1b4e3tjbFjx8LQ0BBv376FiYmJtssnGfjwMNuFCxfi6tWrePDgAYYMGYLKlSvDzs4OL168QOvWraGrq4uxY8fC3d1d5UzDnGNDn2P9+vVwc3NDyZIlAQBjxoxBSEgIIiIi0KhRI7Rq1QpdunTB8uXLMXXqVBQsWBAVKlTA/fv3ERMTI50okv4dww3lmowPkg/DyZUrV+Di4oK4uDg8efIETk5OaNKkCRwdHbFq1SrExsZKZxru3r07pk+fzolz9Nk+3obGjBmDlStXom/fvnj69ClOnz6N1q1bo2/fvnBycsKLFy/Qtm1bvHjxAkFBQXBzc9Ni9SQX+/btQ4sWLTB69GgMGTIE+/btw6hRo+Dv749Xr17h0qVLCAkJwW+//YZ+/frh+vXr8Pf3h46ODiwsLDBjxgyeKDK7cvWa4/TNe/DggejUqZP4+++/xZYtW4RCoRDnzp0TSqVSCCHE9evXRZkyZcTZs2eFEELcv39ftGjRQowdO1Y8fvxYm6WTzKSnpwshhFi3bp1wdHQUFy9eFEIIcfz4caFQKESpUqXE4MGDxYMHD4QQQjx//lz07dtXvHv3Tms1k/wsWrRI2NnZialTp4oBAwaI5cuXS8uePHkipkyZIhwcHMT+/fuzXD8tLe1LlfpV49gW5ark5GQcP34c3bt3x5UrV7B69Wp8//330i4qIQTevXuH06dPo3z58li7di0AYMSIETyHCH22bt26wcrKCvPnz4eOjg7S0tJgYGCAfv36oXLlyti5cyd69OiBFStWIDIyEtOmTYOOjg769OmDsmXLShOI+U2ZPldqaioMDAzg4+MDExMTjBkzBvHx8Zg2bZrUx87ODp6enjhw4AAuXLgAd3f3TBdg5S6pbNJ2uiL5yvimHBAQIHR0dESlSpXE5cuXVfrExsaK7t27i5IlSwoHBwdhZWUlfaMm+hyxsbFi8uTJwtLSUkyaNElqf/bsmYiKihLPnz8XVatWFfPmzZP6FylSRBQuXFgsWLBACCGkEUYiTfH19RXR0dFi/fr1wsTERDRr1kzcvXtXpU+HDh1E27ZttVShPPBoKcoVQgjo6OhACIEiRYpg3rx5ePfuHcaPH48TJ05I/UxNTTF37lwsWbIEEydOxNmzZ1G5cmUtVk5yEB8fD1NTU/Tv3x/jx4+Hv78/Jk6cCAAoUqQIrK2t8fz5c7x580aaT/Ps2TM0btwYEyZMgI+PDwCeS4Q+n/hgWuuaNWswdepU3Lt3D507d4afnx8uXbqEgIAA3LlzBwAQFxeHhw8folixYtoqWRY4vkUaJ/5/8ubhw4dx9OhRDBkyBC1btkSjRo3Qvn17zJw5E2PHjkWNGjUAvL8wZuPGjbVcNcnFqFGjsGzZMty/fx9WVlbo2rUrhBCYOnUqAGDy5MkA3gcgXV1dnDx5EkIIzJw5EyYmJtLht9wVRZqQEZAPHTqEy5cvIzAwUPrb17dvX6SlpWHy5MnYv38/KleujMTERKSmpmL27NnaLPvrp81hI5KfjGH8bdu2CTMzMzFmzBhx/vx5afm1a9dEuXLlRIsWLcQff/whJk2aJBQKhXjy5Al3AZBGXL16VdSpU0eULl1avHjxQgghRHR0tJg3b54wNzcXEyZMkPoOGDBAlCxZUtjZ2Qk3NzeRmpoqhODuKNKsI0eOiIoVK4qCBQuKnTt3CiGESElJkZavXLlS5M+fX1SuXFmsXbtWmsTOycM5x0PBSePOnTuHJk2aYNasWejTp4/UHhcXB1NTU9y6dQt9+vRBUlISYmNjsWXLFu6KIo04ffo0Xrx4gXLlyqFDhw5ISEiQrtz94sULrFu3DlOnTpUuSAi8Pz2BQqFAxYoVoaOjg3fv3nHSJn0W8dGpBxISEjBnzhwEBgaiWrVq2LhxI4yNjZGWlgZ9fX0AwPz583Hq1Cls3boVCoWCI4efieGGNG7RokXYsWMHDh06hNjYWBw+fBh//PEHbt26hREjRqBnz56Ijo5GbGwszMzMYG1tre2SSSY8PT3xzz//4ODBg3j06BF+/vlnxMfHZwo406ZNw4ABAzBlyhSV9fmBQpq0ePFi2NnZoXXr1khKSsLcuXOxY8cO1KtXDzNmzICRkZFKwMkIRR+HI1IfJxSTxtna2uLixYvw9fXFzz//jNWrV8PIyAjNmzdH7969cffuXVhbW6NUqVIMNqRRixcvxtOnT7Fo0SI4ODhg48aNMDMzQ82aNfHy5UtYWVmhW7dumDBhAqZNm4aVK1eqrM9gQ5ry4sULHD58GL/88gv2798PY2NjDBs2DC1atMCpU6cwbtw4JCcnQ19fH+/evQMABhsN4sgNfZaMX8SEhATkz58fABAVFYWFCxdiy5YtaNCgAbp3744ffvgBUVFRaNWqFYKCglC+fHktV05ykzHq8vvvv+Py5cuYP38+LCwscPv2bXh6eiI2NlYawYmMjMTRo0fx008/cRcUacTH56MBgKtXr+L333/HwYMHERAQgKZNmyIxMRGzZ8/GwYMHUbZsWSxZskS6dhRpDkdu6LMoFArs3bsXnTp1Qr169RAUFAQ9PT1MmzYNZ8+eRUBAANzc3KCjo4OFCxciMTGRozWUKzJGXerVq4djx45h7969AIDSpUtj3bp1sLCwQJ06dRAVFQVbW1t06NABenp60rdmos+REWwiIyOltkqVKmHw4MGoX78++vXrh/379yNfvnwYNWoUfvjhB+jo6Ei7pEjDtDSRmWTi5MmTwsjISIwcOVI0adJEODs7C29vbxEeHi71CQsLE3379hWWlpaZTuJHlFMZJ4nMSkBAgPjuu+/EnTt3pLY7d+4IBwcH0bFjxy9RHn0jPtwON23aJEqUKKFyhKgQQly5ckW0bt1aFCtWTBw5ckQIIURSUpJ0VN6/bcuUMxy5oRyLiIhAaGgopk+fjtmzZ2Pfvn3o27cvrl27Bl9fXzx48ACJiYk4ffo0oqOjcfToUbi4uGi7bJKBD3cBnDt3DqdOncLRo0el5a1atUK1atUQFhYmtX333Xc4duwY/vjjjy9eL8lTSkqKtB2mpqaiZMmSKFOmDHx8fHDx4kWpX6VKleDh4YEnT56gcePGOHXqFIyMjKQ5Nh/vzqLPx58oZcuiRYvw119/Sffv3LmDDh06YNWqVTAyMpLafXx80KVLF9y8eROzZ89GTEwMRo4ciTVr1qBChQraKJ1k5sMPg7Fjx6J79+7o2bMnvLy80KFDB8TFxaFw4cLSfIa0tDRpXXt7e+jq6iI9PV1b5ZNM7Nu3D+vWrQMA9OnTBw0aNEDVqlUxfPhw2NrawtvbGxcuXJD6FytWDB07dsS8efNQrVo1qZ2Th3OJtoeOKO97+PCh6Ny5s7h3755K+6+//iqsra1F27ZtpZOlZVi6dKkoXbq0GDRoEE9ERbli7ty5omDBguLs2bMiPT1dzJgxQygUCnHixAmpT82aNYW3t7cWqyS56tSpk3BwcBDu7u6iUKFC4urVq9Kyw4cPCw8PD1GhQgWxb98+8fDhQ+Hh4SGGDx8u9eHV5nMXww1lS2JiohBCiDNnzoht27ZJ7RMmTBAVK1YU48ePF1FRUSrrLF++XDx8+PBLlknfCKVSKby8vERgYKAQQojt27cLc3NzERAQIIQQIj4+XgghxL59+0SrVq3EtWvXtFYryZeLi4tQKBQqF2bNcPz4cdGtWzehUCjEd999J5ydnaUvejwDdu7jMZCULcbGxoiJiYGvry+ePXsGXV1deHh4YPLkyUhLS8PevXshhMDgwYNhZWUFAOjdu7eWqya5Sk5OxtmzZ1GvXj0cOXIEXl5emDNnDry9vfHu3TvMnj0b1atXh5ubG6ZMmYJz586hYsWK2i6bZCI1NRXJyclwcnJCsWLFsHnzZhQtWhQdO3aUTolRq1YtVKtWDX369EFaWhrq1q0LXV1dngH7C+GcG8oWhUIBc3NzDB8+HI6OjvD390dwcDAAYMaMGWjSpAlCQ0MxY8YMvHz5UsvVkpxcu3YNT58+BQAMHToUR48ehbGxMTp37ow//vgDzZo1g5+fn3TByzdv3uDChQu4c+cOLCwssG7dOhQvXlybL4FkxsDAAKampti6dSt27dqF77//HrNnz8amTZsQHx8v9UtOTkbt2rXRoEEDaa4Xg82XwXBD2SLe78JE7dq1MXToUFhYWOD3339XCThubm64fPkyBM8LSRoghMDdu3dRv359rFq1Cv369cOCBQtgYWEBAHBzc0NERASqVauG6tWrAwD++ecfdO/eHTExMRgwYAAAoGTJkmjUqJHWXgfJjxACSqVSur9mzRrUqFEDfn5+WLt2LR4/fowGDRqgXbt2Un+AZ8D+kniGYsqWjLO/xsbGwsTEBNeuXcP06dPx5s0bDB48GB4eHgDen3I8Y7cUkSYsX74co0aNQnJyMnbt2oXGjRtLZ8bevHkzpkyZAiEE9PT0YGxsDKVSiVOnTkFfX5/XiqLP9vr1a1haWqq0ZWx/W7duRWhoKAIDAwEAffv2xZEjR5Ceng5LS0ucPHmSZx/WEo7c0H969+4ddHV18ejRI9SrVw8HDhxAlSpVMGLECFhZWWHy5MnYs2cPADDYkMZkfDO2t7eHoaEhTE1NcebMGTx69Eg6fLZDhw5Yu3YtpkyZgvbt22P06NE4c+aMdL0eBhv6HAsWLMD333+vsqsJgBRsunfvjkqVKkntgYGBWLZsGRYuXIgzZ87AwMCAZ8DWFu3MY6a86lOz+MPDw4WNjY3o3bu3yiGMR44cEd26dROPHj36UiWSzH28DaampoqkpCSxdOlSUbRoUTF27Nj/3N54mC19rmXLlglDQ0OxYcOGTMseP34sKlasKBYtWiS1ZbXNcTvUHu6WIon4/6HW06dP49atWwgPD4enpycKFy6MNWvW4MKFC1izZk2mK9cmJyernMiPKKc+PPPw69evER8frzIZ2N/fH3PnzkWvXr3Qo0cPODg4oGXLlhg3bhzc3Ny0VTbJzPLlyzFw4ECsW7cO7dq1Q0xMDBITE5GcnAxra2sUKFAA9+7dQ6lSpbRdKn0Cww2p2L59O/r27StdYPDFixfo0KEDRo8ejQIFCmi7PJKxD4PNlClTcODAAdy4cQPt27dHmzZt0LRpUwDvA46/vz8qVKiAV69e4fHjx3j06BEvQEga8eDBAzg5OaF9+/bYtGkTbty4gV9++QUvXrxAREQE6tevj/79+6NFixbaLpX+BY9JI8mNGzcwdOhQzJs3D927d0dcXBzMzc1hbGzMYEO5LiPYTJgwAYGBgZgzZw4cHBzQr18/3Lt3DzExMejUqROGDBmCQoUK4erVq0hOTsbx48elq3vzMFv6XFZWVpg1axYmTJiAESNG4MCBA6hduzZat26NuLg4bNu2DePHj0ehQoU4WpiXaXOfGGnP4cOHxf379zO1Va9eXQghxK1bt0Tx4sVF7969peX379/nPmTKVYcPHxbly5cXx44dE0IIcerUKWFgYCDKlSsnqlWrJrZu3Sr1/fCyHrzEB2lScnKymDt3rtDR0RE9e/YUqamp0rILFy6I0qVLi8WLF2uxQvovPFrqGyOEwOXLl9G0aVMsXboUERER0rJnz55BCIGEhAQ0adIEjRs3xrJlywAAoaGhWLp0Kd68eaOt0kmGxEd7xYsWLYr+/fujdu3aOHDgAFq0aIHAwECEhobi/v37+P3337Fy5UoAUBml4YgNaZKhoSH69euH7du3o3fv3tDX15e21SpVqsDIyAhPnjzRcpX0bxhuvjEKhQKurq6YN28etmzZgqVLl+LBgwcAgObNmyMqKgqmpqZo3rw5AgMDpV0FISEhuHbtGg+tJY1RKpXSpPQHDx4gMTERpUqVQqdOnZCcnIwFCxZg0KBB6NatG4oUKYLy5csjPDwct27d0nLl9C3Ily8fmjZtKp0gMmNbjY6OhrGxMcqXL6/N8ug/8OvONyZjXoKPjw8AYM6cOdDV1UXv3r3h6OiI3377DTNmzMC7d+/w9u1bhIeHY+PGjVixYgVOnDghnR2W6HN8OHl4woQJOH36NEaOHIn69evD0tISiYmJeP78OUxMTKCjo4OUlBQ4ODhg1KhRaNKkiZarJzkSHxwBmsHQ0FD6f3p6Ol6+fIk+ffpAoVCgU6dOX7pEUgPDzTcmY+TlwIED0NHRQVpaGvz9/ZGcnIzRo0ejffv2SEpKwowZM7Bt2zbY2NjAwMAAYWFhqFChgparJ7n4MNgsW7YMgYGBcHV1lY54SklJgaWlJU6cOCFNGn716hVWrVoFHR0dlXBElBMRERF4/fo1ChYsCFtb2389k3BaWhrWrVuHjRs34vXr1zhz5ox0rSiOZudNPBT8GxQSEiJdbDBfvny4d+8efv/9d/zyyy8YPXo0rKysEB8fj6NHj8LBwQHW1tawtrbWdtn0lfs4kNy9exceHh6YNWsWWrZsmanf+fPnMX78eCQkJMDS0hLBwcHQ19dnsKHPtnbtWsybNw/R0dEoVKgQBg4cKI3IZPh4OwsNDcXNmzcxYMAAHp33FWC4+cYolUp06dIFCoUCGzZskNoXLlyIUaNGwcfHB7/88gtKlCihxSpJbtq2bYuxY8eiatWqUtuVK1fQpEkTHD16FKVLl87yxJDJyckQQsDIyAgKhYIfKPTZ1q5dCx8fH+nSCjNmzMCDBw9w8uRJadvKCDYxMTE4cOAA2rdvr/IYHLHJ+/j15xuT8U0kY/g/NTUVADBw4EB4e3tj9erV+P3331WOoiL6XGZmZnB2dlZpMzIywps3b3Djxg2pLeN6UqdPn8b27duho6MDY2NjKBQKKJVKBhv6LBcuXMDUqVOxaNEi9OzZExUrVsTQoUPh5OSEU6dO4ebNm4iLi5N22a9Zswa//PIL/vjjD5XHYbDJ+xhuvhH//POP9P/SpUvjzz//RHR0NAwMDJCWlgYAsLOzg4mJCcLCwmBsbKytUklGnj17BgBYvXo1DAwM8Pvvv+PAgQNITU2Fk5MTOnTogDlz5uDgwYNQKBTQ0dFBeno6pk+fjrCwMJV5ENwVRZ8rJSUFQ4YMQfPmzaW2SZMm4dChQ+jUqRM8PT3RsWNHvH79Gvr6+mjWrBlGjBjBycNfIe6W+gZcvXoVAwYMQOfOndG/f3+kpqaiQYMGePnyJY4cOQJbW1sAwOjRo1G+fHm0aNEClpaWWq6avnZ9+vQBAIwZM0bazens7IyXL19i06ZNqFOnDo4fPw4/Pz9cv34dXbp0gYGBAQ4dOoQXL17g0qVLHKkhjVIqlXjx4gVsbGwAAJ6enjh48CB2794Ne3t7HD16FNOmTcPo0aPRuXNnlTk43BX1deFXoW+AiYkJzM3NsW3bNgQFBcHAwADLli2DlZUVypYtCw8PDzRu3BgLFixA1apVGWxII5ydnbF//34sXboU4eHhAIBr166hdOnS6NKlC44dO4batWtjypQp8PT0xLp163D48GEUK1YMFy9elCZtEmmKjo6OFGwAYMSIETh79iyqVq0KGxsbNG3aFK9fv0ZUVFSmw8IZbL4uHLn5RoSHh2Ps2LGIjIxEnz590K1bN6Snp2Pu3LmIiIiAEAIDBw5EuXLltF0qyciqVaswYcIEdOzYEX369EHp0qUBAHXq1MHDhw+xfv161KlTBwDw9u1bmJiYSOty8jB9aU+fPkXXrl0xYsQIXhjzK8dwI1OXLl3C8+fPVfYth4eHY/z48Xj06BEGDhyILl26aLFCkrMPD6NduXIlJkyYgE6dOmUKOBEREVi7di2qV6+uMr8mqxOqEanjw20o4/8Z/7548QJWVlYq/RMTE9GpUyfExsbi8OHDHKn5yjHcyFB8fDyaN28OXV1djBo1Ck2bNpWWPXr0CE2aNIGJiQl69+6NX375RYuVktx86hw0y5cvx+TJk9GhQwf07dtXCjgNGjTAyZMncebMGbi6un7pckmmstoOM9qCg4OxceNGLFiwAEWKFEFSUhJ27dqFdevW4dmzZzh//jz09fU5x+Yrxzk3MpKRUwsUKIDZs2dDT08PixYtwt69e6U+Dg4OqF+/PiIjI3Ho0CHExMRoqVqSmw8/UE6dOoWwsDBcvXoVwPvJxb/99hs2bdqEwMBA3LlzBwBw+PBh9O7dO9Nh4kQ5deLECemilsOGDcPMmTMBvJ9vs3nzZnh6eqJRo0YoUqQIgPcXXX348CFKlCiBCxcuQF9fH+/evWOw+cpx5EYGMoZaM75pZHzInD17Fr/++ivy5cuH/v37S7uohg8fjhIlSqBt27YoXLiwlqsnOfhwF8CwYcOwefNmJCQkwM7ODsWKFcO+ffsAAMuWLcO0adPQsWNHeHl5qVzSg9+U6XMIIRAbGwtra2s0bdoUhQoVQnBwMI4fP44KFSogJiYGbm5u8PHxwcCBA6V1PvzbCXA7lAuGm69cxi9nWFgYdu/ejdevX6NWrVpo164dzM3NcebMGfz2229ISUlBiRIlYGJigs2bN+Pq1auws7PTdvkkAx8GmwMHDmDIkCEIDAyEubk5/v77b0ycOBH58uXDhQsXALyfg+Pt7Q1/f38MGDBAm6WTDEVHR6NEiRJIT0/H9u3b0axZM2lZVnNtspqbQ18/7pb6yikUCuzYsQMtW7bE27dv8fbtW6xbtw79+/fH69ev4ebmhrlz56Ju3boIDw/HgwcPcPjwYQYb0piMD4Pdu3dj06ZNaNSoEWrVqoUKFSrg559/xtq1a5GQkID+/fsDAHr16oVdu3ZJ94k0JSUlBZGRkTAxMYGuri5WrVolnYYAAAoVKiT9P+Ns2B+GGQYb+eDIzVfuwoUL6NixI3799Vf07t0bERERqFy5MoyNjeHi4oK1a9fC0tJSulbPx4fbEmnC69ev0aJFC1y9ehX169fHnj17VJaPHTsWJ0+exF9//YV8+fJJ7dwFQJ/rU5PYHz16BGdnZ9SvXx/z589HyZIltVAdaQtHbr4ivr6+GDdunPSNA3h/ens3Nzf07t0bjx49QsOGDeHh4YHx48fj/Pnz+OWXX/D69WsYGRkBAIMNacSH2yAAWFpaYs2aNfjxxx9x+fJlrF69WmV5qVKl8OrVKyQlJam0M9jQ5/gw2Bw5cgQbNmzA1atX8ezZMzg4OODkyZMICwvDqFGjpEnsbdq0wcKFC7VZNn0BHLn5iixcuBCDBw/GjBkzMGrUKOmX+tatWyhdujRat24tfcgolUq4uLggPDwczZs3x+bNm3ltHtKIDz9Q7t+/D4VCARMTE9ja2uLhw4fw8fFBYmIi2rVrB29vb0RFRcHLywtGRkbYs2cPh/5J40aMGIE1a9ZAT08P+fPnh62tLfz8/FC1alVcv34d9evXh4ODA1JTU/Hu3TtcvXpVungwyZSgr4JSqRRCCLF8+XKho6Mjpk6dKtLS0qTlT548EWXLlhV79uwRQgjx+vVr0alTJ7Fw4ULx9OlTrdRM8pOxHQohxMSJE0XFihVFmTJlROHChUVgYKAQQojw8HDRrFkzYWRkJEqXLi3atGkj3N3dRVJSkhBCiPT0dK3UTvLx4XYYGhoqKlWqJI4fPy5ev34tdu3aJdq0aSOcnJzEpUuXhBBC3Lt3T0yZMkVMnz5d+rv54d9Pkh+Gm6+AUqmUfpmVSqX4448/hI6Ojpg2bZr0QREdHS1cXFyEt7e3ePTokRg7dqz4/vvvRVRUlDZLJ5maMmWKsLKyEiEhISIhIUG0adNGmJubi5s3bwohhHjw4IFo3ry5cHFxEX5+ftJ6ycnJWqqY5GjNmjViwIABom/fvirt58+fF02aNBFeXl4iISFBCKEaiBhs5I/7Kb4SCoUCBw8exPDhw1GlShXpmj0zZ86EEAIWFhbo0qULjh49Cjc3N6xduxYBAQGwtrbWdukkAx/OsVEqlTh37hz8/PzQuHFjhIaG4siRI5gxYwbKlSuHtLQ0ODo6Yt68ebCxscHevXsRHBwMADA0NNTWSyAZEB/Noti5cycWL16MK1euICUlRWqvWrUqateujRMnTiA9PR2A6pFQvGbZN0Db6YqyZ/v27cLY2FhMnTpVnD9/XgghRGBgoLSLSgghUlJSxM2bN0VoaKh48uSJNsslmZowYYKYOXOmKFq0qLhz544ICwsT+fPnF0uXLhVCCPH27Vsxbtw48ejRIyGEEHfv3hUtWrQQVatWFcHBwdosnb5yH468rF+/Xqxdu1YIIcSAAQOEubm5WLx4sYiNjZX6hISEiDJlykjbIn1bGG6+Anfu3BGOjo5iyZIlmZYtW7ZM2kVFpGkfzo/ZtGmTsLe3Fzdu3BBdu3YV7u7uwsTERKxcuVLq8+zZM1G7dm2xdu1aad1bt26Jn3/+WURERHzx+kkePtwOb9y4IVxdXUWlSpXErl27hBBCeHl5iVKlSonp06eL8PBwER4eLho2bCjq1q2rEoro28Gxua/A48ePoa+vr3KmzYwjVvr27Yt8+fKhW7duMDQ0xIgRI7RYKclNxlFRR48exZEjRzB8+HCUL19eOjlkw4YN0bNnTwDvL9jau3dv6OrqonPnztDR0YFSqUSZMmWwYcMGHp1COZaxHY4cORIPHz6EsbExbt++jaFDh+Ldu3cICgpCz549MX78eCxcuBA1a9ZE/vz5sXnzZigUik+eC4fki+HmK5CQkKByfhClUintPz5y5AiqVKmCzZs3q1ynh0hTIiMj0atXL0RHR2Ps2LEAgH79+uH+/fs4fPgwXF1dUapUKTx+/BjJyck4f/48dHV1VU7QxzkO9LmCgoKwYsUKHDp0CI6OjkhJSYGXlxd8fX2ho6ODVatWwcTEBFu2bEGTJk3QsWNHGBoaIjU1FQYGBtoun74wRtmvQKVKlfDy5UsEBgYCeP8tJiPc7Nq1Cxs2bEDbtm1RtmxZbZZJMmVra4vg4GDY2Njgzz//xMWLF6Grq4s5c+ZgypQpaNCgAWxtbdGhQ4dPXlWZ57ahzxUeHo4KFSrAxcUFZmZmsLW1xapVq6Crq4uhQ4dix44dWLRoERo1aoT58+dj9+7diI+PZ7D5RvHr1FfA0dERixYtQr9+/ZCWlgZPT0/o6uoiKCgIQUFBOH36NM/0SrnK2dkZ27dvh5eXFwICAjBw4EA4OzujVatWaNWqlUrf9PR0jtSQxoj/v5iloaEhkpOTkZqaCiMjI6SlpaFo0aLw9fVFixYt4O/vD2NjY2zYsAGdO3fGiBEjoKenh/bt22v7JZAW8AzFXwmlUont27fD29sb+fLlg5GREXR1dbFx40a4urpquzz6Rly+fBm9e/dGlSpVMHjwYJQvX17bJdE34vr163B1dcVvv/2GiRMnSu0hISFYvnw53rx5g/T0dBw5cgQA0KNHD/z2228oUaKEliombWK4+cr8888/iIiIgEKhgKOjI2xsbLRdEn1jLl++DG9vbxQvXhyzZ8+Go6Ojtkuib0RQUBD69u2LIUOGoEOHDrCwsMCgQYNQo0YNtGnTBuXLl8fevXvRtGlTbZdKWsZwQ0RqO3fuHAICArBixQoehUJf1Pbt2/HLL7/AwMAAQghYW1vj1KlTiIqKwo8//oht27bB2dlZ22WSljHcEFGOZMyF4GG29KU9e/YMT548QVpaGmrWrAkdHR2MGTMGO3fuRFhYGGxtbbVdImkZww0R5VhGwCHSlps3b2LWrFn466+/cPDgQbi4uGi7JMoDeEgDEeUYgw1p07t375Camgpra2scPXqUE9xJwpEbIiL6qqWlpfEM2KSC4YaIiIhkhbMAiYiISFYYboiIiEhWGG6IiIhIVhhuiIiISFYYbohI9o4cOQKFQoGYmJhsr+Pg4AB/f/9cq4mIcg/DDRFpXffu3aFQKNCvX79My3x8fKBQKNC9e/cvXxgRfZUYbogoT7C3t8emTZuQlJQktSUnJ2PDhg0oVqyYFisjoq8Nww0R5QmVK1eGvb09goODpbbg4GAUK1YMrq6uUltKSgoGDRoEa2trGBkZoVatWjh//rzKY/3111/47rvvYGxsjPr16+PRo0eZnu/EiROoXbs2jI2NYW9vj0GDBiExMTHXXh8RfTkMN0SUZ/Ts2ROrV6+W7q9atQo9evRQ6TNq1Chs374da9aswaVLl+Dk5AR3d3e8fv0aAPDkyRO0bdsWLVu2xJUrV9C7d2/8+uuvKo9x//59NGnSBD/99BOuXbuGzZs348SJExgwYEDuv0giynUMN0SUZ3Tt2hUnTpxAREQEIiIicPLkSXTt2lVanpiYiKVLl2LOnDlo2rQpypUrh+XLl8PY2BgrV64EACxduhQlS5bEvHnzULp0aXTp0iXTfB1fX1906dIFQ4YMQalSpVCjRg38/vvvWLt2LZKTk7/kSyaiXMALZxJRnmFlZYXmzZsjKCgIQgg0b94chQoVkpbfv38faWlpqFmzptSmr6+PH374Abdu3QIA3Lp1C9WqVVN53OrVq6vcv3r1Kq5du4b169dLbUIIKJVKPHz4EGXLls2Nl0dEXwjDDRHlKT179pR2Dy1evDhXniMhIQHe3t4YNGhQpmWcvEz09WO4IaI8pUmTJkhNTYVCoYC7u7vKspIlS8LAwAAnT55E8eLFAby/IvT58+cxZMgQAEDZsmWxe/dulfXOnDmjcr9y5cr4+++/4eTklHsvhIi0hnNuiChP0dXVxa1bt/D3339DV1dXZVm+fPnQv39/jBw5Evv378fff/+NPn364O3bt+jVqxcAoF+/frh37x5GjhyJO3fuYMOGDQgKClJ5nNGjR+PUqVMYMGAArly5gnv37mHXrl2cUEwkEww3RJTnmJqawtTUNMtlM2fOxE8//YRu3bqhcuXKCA8PR0hICCwsLAC83620fft27Ny5E5UqVUJAQABmzJih8hjOzs44evQo7t69i9q1a8PV1RUTJkxAkSJFcv21EVHuUwghhLaLICIiItIUjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs/B+XLE52CERTBAAAAABJRU5ErkJggg==",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "## calculate avg response time\n",
+ "unique_models = set(unique_result[\"response\"]['model'] for unique_result in result[0][\"results\"])\n",
+ "model_dict = {model: {\"response_time\": []} for model in unique_models}\n",
+ "for iteration in result:\n",
+ " for completion_result in iteration[\"results\"]:\n",
+ " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n",
+ "\n",
+ "avg_response_time = {}\n",
+ "for model, data in model_dict.items():\n",
+ " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n",
+ "\n",
+ "models = list(avg_response_time.keys())\n",
+ "response_times = list(avg_response_time.values())\n",
+ "\n",
+ "plt.bar(models, response_times)\n",
+ "plt.xlabel('Model', fontsize=10)\n",
+ "plt.ylabel('Average Response Time')\n",
+ "plt.title('Average Response Times for each Model')\n",
+ "\n",
+ "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n",
+ "plt.show()"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb b/cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb
new file mode 100644
index 000000000..7df1c47eb
--- /dev/null
+++ b/cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb
@@ -0,0 +1,422 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "BmX0b5Ueh91v"
+ },
+ "source": [
+ "# LiteLLM - Azure OpenAI + OpenAI Calls\n",
+ "This notebook covers the following for Azure OpenAI + OpenAI:\n",
+ "* Completion - Quick start\n",
+ "* Completion - Streaming\n",
+ "* Completion - Azure, OpenAI in separate threads\n",
+ "* Completion - Stress Test 10 requests in parallel\n",
+ "* Completion - Azure, OpenAI in the same thread"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "iHq4d0dpfawS"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "mnveHO5dfcB0"
+ },
+ "outputs": [],
+ "source": [
+ "import os"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eo88QUdbiDIE"
+ },
+ "source": [
+ "## Completion - Quick start"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "5OSosWNCfc_2",
+ "outputId": "c52344b1-2458-4695-a7eb-a9b076893348"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Openai Response\n",
+ "\n",
+ "{\n",
+ " \"id\": \"chatcmpl-7yjVOEKCPw2KdkfIaM3Ao1tIXp8EM\",\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"created\": 1694708958,\n",
+ " \"model\": \"gpt-3.5-turbo-0613\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"role\": \"assistant\",\n",
+ " \"content\": \"I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you?\"\n",
+ " },\n",
+ " \"finish_reason\": \"stop\"\n",
+ " }\n",
+ " ],\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 13,\n",
+ " \"completion_tokens\": 26,\n",
+ " \"total_tokens\": 39\n",
+ " }\n",
+ "}\n",
+ "Azure Response\n",
+ "\n",
+ "{\n",
+ " \"id\": \"chatcmpl-7yjVQ6m2R2HRtnKHRRFp6JzL4Fjez\",\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"created\": 1694708960,\n",
+ " \"model\": \"gpt-35-turbo\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"index\": 0,\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"message\": {\n",
+ " \"role\": \"assistant\",\n",
+ " \"content\": \"Hello there! As an AI language model, I don't have feelings but I'm functioning well. How can I assist you today?\"\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"usage\": {\n",
+ " \"completion_tokens\": 27,\n",
+ " \"prompt_tokens\": 14,\n",
+ " \"total_tokens\": 41\n",
+ " }\n",
+ "}\n"
+ ]
+ }
+ ],
+ "source": [
+ "from litellm import completion\n",
+ "\n",
+ "# openai configs\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
+ "\n",
+ "# azure openai configs\n",
+ "os.environ[\"AZURE_API_KEY\"] = \"\"\n",
+ "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
+ "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
+ "\n",
+ "\n",
+ "# openai call\n",
+ "response = completion(\n",
+ " model = \"gpt-3.5-turbo\",\n",
+ " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
+ ")\n",
+ "print(\"Openai Response\\n\")\n",
+ "print(response)\n",
+ "\n",
+ "\n",
+ "\n",
+ "# azure call\n",
+ "response = completion(\n",
+ " model = \"azure/your-azure-deployment\",\n",
+ " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
+ ")\n",
+ "print(\"Azure Response\\n\")\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "dQMkM-diiKdE"
+ },
+ "source": [
+ "## Completion - Streaming"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "uVvJDVn4g1i1"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from litellm import completion\n",
+ "\n",
+ "# openai configs\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
+ "\n",
+ "# azure openai configs\n",
+ "os.environ[\"AZURE_API_KEY\"] = \"\"\n",
+ "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
+ "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
+ "\n",
+ "\n",
+ "# openai call\n",
+ "response = completion(\n",
+ " model = \"gpt-3.5-turbo\",\n",
+ " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
+ " stream=True\n",
+ ")\n",
+ "print(\"OpenAI Streaming response\")\n",
+ "for chunk in response:\n",
+ " print(chunk)\n",
+ "\n",
+ "# azure call\n",
+ "response = completion(\n",
+ " model = \"azure/your-azure-deployment\",\n",
+ " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
+ " stream=True\n",
+ ")\n",
+ "print(\"Azure Streaming response\")\n",
+ "for chunk in response:\n",
+ " print(chunk)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "4xrOPnt-oqwm"
+ },
+ "source": [
+ "## Completion - Azure, OpenAI in separate threads"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "V5b5taJPjvC3"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import threading\n",
+ "from litellm import completion\n",
+ "\n",
+ "# Function to make a completion call\n",
+ "def make_completion(model, messages):\n",
+ " response = completion(\n",
+ " model=model,\n",
+ " messages=messages\n",
+ " )\n",
+ "\n",
+ " print(f\"Response for {model}: {response}\")\n",
+ "\n",
+ "# openai configs\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
+ "\n",
+ "# azure openai configs\n",
+ "os.environ[\"AZURE_API_KEY\"] = \"\"\n",
+ "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
+ "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
+ "\n",
+ "# Define the messages for the completions\n",
+ "messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
+ "\n",
+ "# Create threads for making the completions\n",
+ "thread1 = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\", messages))\n",
+ "thread2 = threading.Thread(target=make_completion, args=(\"azure/your-azure-deployment\", messages))\n",
+ "\n",
+ "# Start both threads\n",
+ "thread1.start()\n",
+ "thread2.start()\n",
+ "\n",
+ "# Wait for both threads to finish\n",
+ "thread1.join()\n",
+ "thread2.join()\n",
+ "\n",
+ "print(\"Both completions are done.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "lx8DbMBqoAoN"
+ },
+ "source": [
+ "## Completion - Stress Test 10 requests in parallel\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "pHYANOlOkoDh"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import threading\n",
+ "from litellm import completion\n",
+ "\n",
+ "# Function to make a completion call\n",
+ "def make_completion(model, messages):\n",
+ " response = completion(\n",
+ " model=model,\n",
+ " messages=messages\n",
+ " )\n",
+ "\n",
+ " print(f\"Response for {model}: {response}\")\n",
+ "\n",
+ "# Set your API keys\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
+ "os.environ[\"AZURE_API_KEY\"] = \"\"\n",
+ "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
+ "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
+ "\n",
+ "# Define the messages for the completions\n",
+ "messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
+ "\n",
+ "# Create and start 10 threads for making completions\n",
+ "threads = []\n",
+ "for i in range(10):\n",
+ " thread = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\" if i % 2 == 0 else \"azure/your-azure-deployment\", messages))\n",
+ " threads.append(thread)\n",
+ " thread.start()\n",
+ "\n",
+ "# Wait for all threads to finish\n",
+ "for thread in threads:\n",
+ " thread.join()\n",
+ "\n",
+ "print(\"All completions are done.\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "yB2NDOO4oxrp"
+ },
+ "source": [
+ "## Completion - Azure, OpenAI in the same thread"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "HTBqwzxpnxab",
+ "outputId": "f3bc0efe-e4d5-44d5-a193-97d178cfbe14"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "OpenAI Response: {\n",
+ " \"id\": \"chatcmpl-7yjzrDeOeVeSrQ00tApmTxEww3vBS\",\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"created\": 1694710847,\n",
+ " \"model\": \"gpt-3.5-turbo-0613\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"role\": \"assistant\",\n",
+ " \"content\": \"Hello! I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you today?\"\n",
+ " },\n",
+ " \"finish_reason\": \"stop\"\n",
+ " }\n",
+ " ],\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 13,\n",
+ " \"completion_tokens\": 29,\n",
+ " \"total_tokens\": 42\n",
+ " }\n",
+ "}\n",
+ "Azure OpenAI Response: {\n",
+ " \"id\": \"chatcmpl-7yjztAQ0gK6IMQt7cvLroMSOoXkeu\",\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"created\": 1694710849,\n",
+ " \"model\": \"gpt-35-turbo\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"index\": 0,\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"message\": {\n",
+ " \"role\": \"assistant\",\n",
+ " \"content\": \"As an AI language model, I don't have feelings but I'm functioning properly. Thank you for asking! How can I assist you today?\"\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"usage\": {\n",
+ " \"completion_tokens\": 29,\n",
+ " \"prompt_tokens\": 14,\n",
+ " \"total_tokens\": 43\n",
+ " }\n",
+ "}\n"
+ ]
+ }
+ ],
+ "source": [
+ "import os\n",
+ "from litellm import completion\n",
+ "\n",
+ "# Function to make both OpenAI and Azure completions\n",
+ "def make_completions():\n",
+ " # Set your OpenAI API key\n",
+ " os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
+ "\n",
+ " # OpenAI completion\n",
+ " openai_response = completion(\n",
+ " model=\"gpt-3.5-turbo\",\n",
+ " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
+ " )\n",
+ "\n",
+ " print(\"OpenAI Response:\", openai_response)\n",
+ "\n",
+ " # Set your Azure OpenAI API key and configuration\n",
+ " os.environ[\"AZURE_API_KEY\"] = \"\"\n",
+ " os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
+ " os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
+ "\n",
+ " # Azure OpenAI completion\n",
+ " azure_response = completion(\n",
+ " model=\"azure/your-azure-deployment\",\n",
+ " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
+ " )\n",
+ "\n",
+ " print(\"Azure OpenAI Response:\", azure_response)\n",
+ "\n",
+ "# Call the function to make both completions in one thread\n",
+ "make_completions()\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/cookbook/LiteLLM_Bedrock.ipynb b/cookbook/LiteLLM_Bedrock.ipynb
new file mode 100644
index 000000000..eed603639
--- /dev/null
+++ b/cookbook/LiteLLM_Bedrock.ipynb
@@ -0,0 +1,310 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "fNkMBurtxawJ"
+ },
+ "source": [
+ "# LiteLLM Bedrock Usage\n",
+ "Important Note: For Bedrock Requests you need to ensure you have `pip install boto3>=1.28.57`, boto3 supports bedrock from `boto3>=1.28.57` and higher "
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "htAufI28xeSy"
+ },
+ "source": [
+ "## Pre-Requisites"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "jT5GbPjAuDTp"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm\n",
+ "!pip install boto3>=1.28.57 # this version onwards has bedrock support"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "H4Vu4er2xnfI"
+ },
+ "source": [
+ "## Set Bedrock/AWS Credentials"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "id": "CtTrBthWxp-t"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "os.environ[\"AWS_ACCESS_KEY_ID\"] = \"\" # Access key\n",
+ "os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"\" # Secret access key\n",
+ "os.environ[\"AWS_REGION_NAME\"] = \"\""
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ycRK9NUdx1EI"
+ },
+ "source": [
+ "## Anthropic Requests"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "tgkuoHa5uLOy",
+ "outputId": "27a78e86-c6a7-4bcc-8559-0813cb978426"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Claude instant 1, response\n",
+ "{\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \" I'm doing well, thanks for asking!\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"id\": \"chatcmpl-4f2e64a1-56d2-43f2-90d3-60ffd6f5086d\",\n",
+ " \"created\": 1696256761.3265705,\n",
+ " \"model\": \"anthropic.claude-instant-v1\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 11,\n",
+ " \"completion_tokens\": 9,\n",
+ " \"total_tokens\": 20\n",
+ " },\n",
+ " \"finish_reason\": \"stop_sequence\"\n",
+ "}\n",
+ "Claude v2, response\n",
+ "{\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \" I'm doing well, thanks for asking!\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"id\": \"chatcmpl-34f59b33-f94e-40c2-8bdb-f4af0813405e\",\n",
+ " \"created\": 1696256762.2137017,\n",
+ " \"model\": \"anthropic.claude-v2\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 11,\n",
+ " \"completion_tokens\": 9,\n",
+ " \"total_tokens\": 20\n",
+ " },\n",
+ " \"finish_reason\": \"stop_sequence\"\n",
+ "}\n"
+ ]
+ }
+ ],
+ "source": [
+ "from litellm import completion\n",
+ "\n",
+ "response = completion(\n",
+ " model=\"bedrock/anthropic.claude-instant-v1\",\n",
+ " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
+ ")\n",
+ "print(\"Claude instant 1, response\")\n",
+ "print(response)\n",
+ "\n",
+ "\n",
+ "response = completion(\n",
+ " model=\"bedrock/anthropic.claude-v2\",\n",
+ " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
+ ")\n",
+ "print(\"Claude v2, response\")\n",
+ "print(response)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "HnM-HtM3yFMT"
+ },
+ "source": [
+ "## Anthropic Requests - With Streaming"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "_JZvg2yovRsU"
+ },
+ "outputs": [],
+ "source": [
+ "from litellm import completion\n",
+ "\n",
+ "response = completion(\n",
+ " model=\"bedrock/anthropic.claude-instant-v1\",\n",
+ " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
+ " stream=True,\n",
+ ")\n",
+ "print(\"Claude instant 1, response\")\n",
+ "for chunk in response:\n",
+ " print(chunk)\n",
+ "\n",
+ "\n",
+ "response = completion(\n",
+ " model=\"bedrock/anthropic.claude-v2\",\n",
+ " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
+ " stream=True\n",
+ ")\n",
+ "print(\"Claude v2, response\")\n",
+ "print(response)\n",
+ "for chunk in response:\n",
+ " print(chunk)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "zj1U1mh9zEhP"
+ },
+ "source": [
+ "## A121 Requests"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "6wK6MZLovU7r",
+ "outputId": "4cf80c04-f15d-4066-b4c7-113b551538de"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "J2 ultra response\n",
+ "{\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \"\\nHi, I'm doing well, thanks for asking! How about you?\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"id\": \"chatcmpl-f2de678f-0e70-4e36-a01f-8b184c2e4d50\",\n",
+ " \"created\": 1696257116.044311,\n",
+ " \"model\": \"ai21.j2-ultra\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 6,\n",
+ " \"completion_tokens\": 16,\n",
+ " \"total_tokens\": 22\n",
+ " }\n",
+ "}\n",
+ "J2 mid response\n",
+ "{\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \"\\nGood. And you?\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"id\": \"chatcmpl-420d6bf9-36d8-484b-93b4-4c9e00f7ce2e\",\n",
+ " \"created\": 1696257116.5756805,\n",
+ " \"model\": \"ai21.j2-mid\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 6,\n",
+ " \"completion_tokens\": 6,\n",
+ " \"total_tokens\": 12\n",
+ " }\n",
+ "}\n"
+ ]
+ }
+ ],
+ "source": [
+ "response = completion(\n",
+ " model=\"bedrock/ai21.j2-ultra\",\n",
+ " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
+ ")\n",
+ "print(\"J2 ultra response\")\n",
+ "print(response)\n",
+ "\n",
+ "response = completion(\n",
+ " model=\"bedrock/ai21.j2-mid\",\n",
+ " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
+ ")\n",
+ "print(\"J2 mid response\")\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Y5gGZIwzzSON"
+ },
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/LiteLLM_CometAPI.ipynb b/cookbook/LiteLLM_CometAPI.ipynb
new file mode 100644
index 000000000..0a7ab581a
--- /dev/null
+++ b/cookbook/LiteLLM_CometAPI.ipynb
@@ -0,0 +1,474 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "iFEmsVJI_2BR"
+ },
+ "source": [
+ "# LiteLLM CometAPI Cookbook"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "cBlUhCEP_xj4"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: litellm in /Users/xmx/.miniforge3/lib/python3.12/site-packages (1.78.2)\n",
+ "Requirement already satisfied: aiohttp>=3.10 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.11.18)\n",
+ "Requirement already satisfied: click in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.3.0)\n",
+ "Requirement already satisfied: fastuuid>=0.13.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.13.3)\n",
+ "Requirement already satisfied: httpx>=0.23.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.28.1)\n",
+ "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n",
+ "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n",
+ "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n",
+ "Requirement already satisfied: openai>=2.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n",
+ "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n",
+ "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n",
+ "Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n",
+ "Requirement already satisfied: tokenizers in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.22.1)\n",
+ "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (2.6.1)\n",
+ "Requirement already satisfied: aiosignal>=1.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.4.0)\n",
+ "Requirement already satisfied: attrs>=17.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (25.3.0)\n",
+ "Requirement already satisfied: frozenlist>=1.1.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.6.0)\n",
+ "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (6.6.3)\n",
+ "Requirement already satisfied: propcache>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (0.3.1)\n",
+ "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.20.0)\n",
+ "Requirement already satisfied: anyio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (4.11.0)\n",
+ "Requirement already satisfied: certifi in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (2025.10.5)\n",
+ "Requirement already satisfied: httpcore==1.* in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (1.0.9)\n",
+ "Requirement already satisfied: idna in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (3.10)\n",
+ "Requirement already satisfied: h11>=0.16 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.23.0->litellm) (0.16.0)\n",
+ "Requirement already satisfied: zipp>=3.20 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n",
+ "Requirement already satisfied: MarkupSafe>=2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.3)\n",
+ "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n",
+ "Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n",
+ "Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n",
+ "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.9.0)\n",
+ "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (0.11.0)\n",
+ "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.3.1)\n",
+ "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.67.1)\n",
+ "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.15.0)\n",
+ "Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n",
+ "Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n",
+ "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n",
+ "Requirement already satisfied: regex>=2022.1.18 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2025.9.18)\n",
+ "Requirement already satisfied: requests>=2.26.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2.32.2)\n",
+ "Requirement already satisfied: huggingface-hub<2.0,>=0.16.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tokenizers->litellm) (0.25.2)\n",
+ "Requirement already satisfied: filelock in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (3.15.4)\n",
+ "Requirement already satisfied: fsspec>=2023.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (2025.9.0)\n",
+ "Requirement already satisfied: packaging>=20.9 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (25.0)\n",
+ "Requirement already satisfied: pyyaml>=5.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (6.0.3)\n",
+ "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (3.4.0)\n",
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (1.26.20)\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Completion"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "p-MQqWOT_1a7"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "os.environ['COMETAPI_KEY'] = \"Your_CometAPI_Key_Here\"\n",
+ "api_key = os.getenv('COMETAPI_KEY')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Ze8JqMqWAARO",
+ "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "ModelResponse(id='msg_017L3DDDit8AkEgHRe2DQBc9', created=1760589916, model='claude-sonnet-4-5-20250929', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='I\\'ll create a simple Python script that says hi.\\n\\n\\nhello.py \\n#!/usr/bin/env python3\\n\"\"\"\\nA simple script that says hi!\\n\"\"\"\\n\\ndef say_hi(name=None):\\n \"\"\"Say hi to someone, or just say hi generally.\"\"\"\\n if name:\\n print(f\"Hi, {name}!\")\\n else:\\n print(\"Hi!\")\\n\\nif __name__ == \"__main__\":\\n # Say hi generally\\n say_hi()\\n \\n # Say hi to someone specific\\n say_hi(\"World\")\\n \\n \\n\\nI\\'ve created a simple Python script called `hello.py` that:\\n\\n1. Defines a `say_hi()` function that can optionally take a name parameter\\n2. Prints \"Hi!\" if no name is provided\\n3. Prints \"Hi, [name]!\" if a name is provided\\n4. Demonstrates both usages when run\\n\\nYou can run it with:\\n```bash\\npython hello.py\\n```\\n\\nThis will output:\\n```\\nHi!\\nHi, World!\\n```\\n\\nWould you like me to create versions in other programming languages, or modify this in any way?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields=None), provider_specific_fields={})], usage=Usage(completion_tokens=290, prompt_tokens=26, total_tokens=316, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=None, text_tokens=None, image_tokens=None, cached_tokens_details={})))"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from litellm import completion\n",
+ "response = completion(\n",
+ " model=\"cometapi/claude-sonnet-4-5-20250929\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+ ")\n",
+ "response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "-LnhELrnAM_J",
+ "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "ModelResponse(id='chatcmpl-CRA9Uo6nsQ9C7kMJv1J4kyNDFJym7', created=1760589916, model='gpt-5-chat-latest', object='chat.completion', system_fingerprint='fp_2da73a467a', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Sure! I can help you write a simple code that prints out \"Hi\" in different programming languages. \\n\\nHereās an example in **Python**:\\n\\n```python\\n# Simple Python program to say \"Hi\"\\nprint(\"Hi\")\\n```\\n\\nExample in **JavaScript**:\\n\\n```javascript\\n// Simple JavaScript program to say \"Hi\"\\nconsole.log(\"Hi\");\\n```\\n\\nExample in **C**:\\n\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi\\\\n\");\\n return 0;\\n}\\n```\\n\\nExample in **Java**:\\n\\n```java\\npublic class SayHi {\\n public static void main(String[] args) {\\n System.out.println(\"Hi\");\\n }\\n}\\n```\\n\\nWhich language would you like me to focus on, or do you want me to make it interactive so the program greets the user by name?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={})], usage=Usage(completion_tokens=174, prompt_tokens=12, total_tokens=186, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)))"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "response = completion(\n",
+ " model=\"cometapi/gpt-5-chat-latest\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+ ")\n",
+ "response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "dJBOUYdwCEn1",
+ "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "ModelResponse(id='02176058998406949c23b3bf3d52941de23f13565a086747738f0', created=1760589991, model='deepseek-v3.2-exp', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='Here are several ways to say \"hi\" in different programming languages:\\n\\n## Python\\n```python\\nprint(\"Hi!\")\\n```\\n\\n## JavaScript (Browser)\\n```javascript\\nconsole.log(\"Hi!\");\\n// or\\nalert(\"Hi!\");\\n```\\n\\n## JavaScript (Node.js)\\n```javascript\\nconsole.log(\"Hi!\");\\n```\\n\\n## Java\\n```java\\npublic class Hello {\\n public static void main(String[] args) {\\n System.out.println(\"Hi!\");\\n }\\n}\\n```\\n\\n## C\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi!\\\\n\");\\n return 0;\\n}\\n```\\n\\n## C++\\n```cpp\\n#include \\n\\nint main() {\\n std::cout << \"Hi!\" << std::endl;\\n return 0;\\n}\\n```\\n\\n## C#\\n```csharp\\nusing System;\\n\\nclass Program {\\n static void Main() {\\n Console.WriteLine(\"Hi!\");\\n }\\n}\\n```\\n\\n## PHP\\n```php\\n\\n```\\n\\n## Ruby\\n```ruby\\nputs \"Hi!\"\\n```\\n\\n## Go\\n```go\\npackage main\\n\\nimport \"fmt\"\\n\\nfunc main() {\\n fmt.Println(\"Hi!\")\\n}\\n```\\n\\n## Rust\\n```rust\\nfn main() {\\n println!(\"Hi!\");\\n}\\n```\\n\\n## Swift\\n```swift\\nprint(\"Hi!\")\\n```\\n\\n## Kotlin\\n```kotlin\\nfun main() {\\n println(\"Hi!\")\\n}\\n```\\n\\n## HTML (webpage)\\n```html\\n\\n\\n\\n Hi Page \\n\\n\\n Hi! \\n\\n\\n```\\n\\nThe Python version is probably the simplest if you\\'re just getting started!', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}), provider_specific_fields={})], usage=Usage(completion_tokens=347, prompt_tokens=10, total_tokens=357, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=None, audio_tokens=None, reasoning_tokens=0, rejected_prediction_tokens=None, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None)), service_tier='default')"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "response = completion(\n",
+ " model=\"cometapi/deepseek-v3.2-exp\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+ ")\n",
+ "response"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Streaming"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Streaming Responses"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "I'm\n",
+ " doing\n",
+ " well\n",
+ " ā\n",
+ " thanks\n",
+ " for\n",
+ " asking\n",
+ "!\n",
+ " How\n",
+ " can\n",
+ " I\n",
+ " help\n",
+ " you\n",
+ " today\n",
+ "?\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "messages = [{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}]\n",
+ "response = completion(model=\"cometapi/gpt-5-mini\", messages=messages, stream=True)\n",
+ "for part in response:\n",
+ " print(part.choices[0].delta.content or \"\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Async Completion"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "ModelResponse(id='chatcmpl-CRAAkmfczlmCEnCM55D9CKexbRenn', created=1760589994, model='gpt-5-mini-2025-08-07', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"I'm doing well, thanks ā how are you? How can I help today?\", role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'protected_material_code': {'filtered': False, 'detected': False}, 'protected_material_text': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}})], usage=Usage(completion_tokens=26, prompt_tokens=12, total_tokens=38, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)), prompt_filter_results=[{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}])\n"
+ ]
+ }
+ ],
+ "source": [
+ "from litellm import acompletion\n",
+ "import asyncio\n",
+ "\n",
+ "async def test_get_response():\n",
+ " user_message = \"Hello, how are you?\"\n",
+ " messages = [{\"content\": user_message, \"role\": \"user\"}]\n",
+ " response = await acompletion(model=\"cometapi/gpt-5-mini\", messages=messages)\n",
+ " return response\n",
+ "\n",
+ "response = await test_get_response()\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Async Streaming"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "test acompletion + streaming\n",
+ "response: \n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='Hi', role='assistant', function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' ā', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='ām', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' doing', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' well', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=',', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' thanks', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='!', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' How', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' are', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' What', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' can', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' help', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' with', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' today', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n",
+ "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(provider_specific_fields=None, content=None, role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None)\n"
+ ]
+ }
+ ],
+ "source": [
+ "from litellm import acompletion\n",
+ "import asyncio, os, traceback\n",
+ "\n",
+ "async def completion_call():\n",
+ " try:\n",
+ " print(\"test acompletion + streaming\")\n",
+ " response = await acompletion(\n",
+ " model=\"cometapi/gpt-5-mini\", \n",
+ " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}], \n",
+ " stream=True\n",
+ " )\n",
+ " print(f\"response: {response}\")\n",
+ " async for chunk in response:\n",
+ " print(chunk)\n",
+ " except:\n",
+ " print(f\"error occurred: {traceback.format_exc()}\")\n",
+ " pass\n",
+ "\n",
+ "await completion_call()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Embedding"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n"
+ ]
+ }
+ ],
+ "source": [
+ "import litellm\n",
+ "\n",
+ "\n",
+ "async def main():\n",
+ " response = await litellm.aembedding(\n",
+ " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"openai\" + the model name from ai/ml api\n",
+ " api_key=api_key, # your aiml api-key\n",
+ " api_base=\"https://api.cometapi.com/v1\", # š the URL has changed from v2 to v1\n",
+ " input=\"Your text string\",\n",
+ " )\n",
+ " print(response)\n",
+ "\n",
+ "await main()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n"
+ ]
+ }
+ ],
+ "source": [
+ "import litellm\n",
+ "\n",
+ "\n",
+ "async def main():\n",
+ " response = await litellm.aembedding(\n",
+ " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n",
+ " api_key=api_key, # your CometAPI api-key\n",
+ " api_base=\"https://api.cometapi.com/v1\",\n",
+ " input=\"Your text string\",\n",
+ " )\n",
+ " print(response)\n",
+ "\n",
+ "\n",
+ "await main()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Async Image Generation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "ImageResponse(created=1760591151, background=None, data=[ImageObject(b64_json=None, revised_prompt=\"Generate an image of an adorable baby sea otter. It should be floating on its back in a calm, clear ocean, playfully grasping a colorful shell in its small paws. The sun is setting in the background, casting a peaceful orange and purple hue across the sky and reflecting upon the ocean waves. The otter's fur is a deep, rich brown and appears silky and wet, with glints of sunlight catching on it. Its eyes are bright, expressing joy and curiosity as it examines its newfound treasure.\", url='https://oaidalleapiprodscus.blob.core.windows.net/private/org-OKnsK88id12jfvnKByup1O0l/user-3GxuMyEg9YMU8LFCPHi31prf/img-7PUEF8Wb6thGDAuZWLJjSnfP.png?st=2025-10-16T04%3A05%3A51Z&se=2025-10-16T06%3A05%3A51Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=38e27a3b-6174-4d3e-90ac-d7d9ad49543f&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2025-10-16T02%3A51%3A01Z&ske=2025-10-17T02%3A51%3A01Z&sks=b&skv=2024-08-04&sig=IZKG2VE%2B6VdOe5Tq0Zk/5bVyGK/oK/yO8g%2BDX4krpug%3D')], output_format=None, quality=None, size=None, usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0, completion_tokens_details=None, prompt_tokens_details=None, input_tokens=0, input_tokens_details={'image_tokens': 0, 'text_tokens': 0}, output_tokens=0))\n"
+ ]
+ }
+ ],
+ "source": [
+ "import asyncio\n",
+ "\n",
+ "import litellm\n",
+ "\n",
+ "\n",
+ "async def main():\n",
+ " response = await litellm.aimage_generation(\n",
+ " model=\"cometapi/dall-e-3\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n",
+ " api_key=api_key, # your cometapi api-key\n",
+ " api_base=\"https://api.cometapi.com/v1\",\n",
+ " prompt=\"A cute baby sea otter\",\n",
+ " )\n",
+ " print(response)\n",
+ "\n",
+ "\n",
+ "await main()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "base",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.8"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/LiteLLM_Comparing_LLMs.ipynb b/cookbook/LiteLLM_Comparing_LLMs.ipynb
new file mode 100644
index 000000000..0b2e4e8c7
--- /dev/null
+++ b/cookbook/LiteLLM_Comparing_LLMs.ipynb
@@ -0,0 +1,441 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "L-W4C3SgClxl"
+ },
+ "source": [
+ "## Comparing LLMs on a Test Set using LiteLLM\n",
+ "LiteLLM allows you to use any LLM as a drop in replacement for `gpt-3.5-turbo`\n",
+ "\n",
+ "This notebook walks through how you can compare GPT-4 vs Claude-2 on a given test set using litellm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "fBkbl4Qo9pvz"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {
+ "id": "tzS-AXWK8lJC"
+ },
+ "outputs": [],
+ "source": [
+ "from litellm import completion\n",
+ "\n",
+ "# init your test set questions\n",
+ "questions = [\n",
+ " \"how do i call completion() using LiteLLM\",\n",
+ " \"does LiteLLM support VertexAI\",\n",
+ " \"how do I set my keys on replicate llama2?\",\n",
+ "]\n",
+ "\n",
+ "\n",
+ "# set your prompt\n",
+ "prompt = \"\"\"\n",
+ "You are a coding assistant helping users using litellm.\n",
+ "litellm is a light package to simplify calling OpenAI, Azure, Cohere, Anthropic, Huggingface API Endpoints. It manages:\n",
+ "\n",
+ "\"\"\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "id": "vMlqi40x-KAA"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "os.environ['OPENAI_API_KEY'] = \"\"\n",
+ "os.environ['ANTHROPIC_API_KEY'] = \"\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "-HOzUfpK-H8J"
+ },
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Ktn25dfKEJF1"
+ },
+ "source": [
+ "## Calling gpt-3.5-turbo and claude-2 on the same questions\n",
+ "\n",
+ "## LiteLLM `completion()` allows you to call all LLMs in the same format\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "DhXwRlc-9DED"
+ },
+ "outputs": [],
+ "source": [
+ "results = [] # for storing results\n",
+ "\n",
+ "models = ['gpt-3.5-turbo', 'claude-2'] # define what models you're testing, see: https://docs.litellm.ai/docs/providers\n",
+ "for question in questions:\n",
+ " row = [question]\n",
+ " for model in models:\n",
+ " print(\"Calling:\", model, \"question:\", question)\n",
+ " response = completion( # using litellm.completion\n",
+ " model=model,\n",
+ " messages=[\n",
+ " {'role': 'system', 'content': prompt},\n",
+ " {'role': 'user', 'content': question}\n",
+ " ]\n",
+ " )\n",
+ " answer = response.choices[0].message['content']\n",
+ " row.append(answer)\n",
+ " print(print(\"Calling:\", model, \"answer:\", answer))\n",
+ "\n",
+ " results.append(row) # save results\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "RkEXhXxCDN77"
+ },
+ "source": [
+ "## Visualizing Results"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 761
+ },
+ "id": "42hrmW6q-n4s",
+ "outputId": "b763bf39-72b9-4bea-caf6-de6b2412f86d"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.google.colaboratory.module+javascript": "\n import \"https://ssl.gstatic.com/colaboratory/data_table/881c4a0d49046431/data_table.js\";\n\n const table = window.createDataTable({\n data: [[{\n 'v': 0,\n 'f': \"0\",\n },\n\"how do i call completion() using LiteLLM\",\n\"To call the `completion()` function using LiteLLM, you need to follow these steps:\\n\\n1. Install the `litellm` package by running `pip install litellm` in your terminal.\\n2. Import the `Completion` class from the `litellm` module.\\n3. Initialize an instance of the `Completion` class by providing the required parameters like the API endpoint URL and your API key.\\n4. Call the `complete()` method on the `Completion` instance and pass the text prompt as a string.\\n5. Retrieve the generated completion from the response object and use it as desired.\\n\\nHere's an example:\\n\\n```python\\nfrom litellm.completion import Completion\\n\\n# Initialize the Completion client\\ncompletion_client = Completion(\\n model_name='gpt-3.5-turbo',\\n api_key='your_api_key',\\n endpoint='https://your_endpoint_url'\\n)\\n\\n# Call the completion() method\\nresponse = completion_client.complete(\\\"Once upon a time\\\")\\n\\n# Retrieve the generated completion\\ncompletion = response['choices'][0]['text']\\n\\nprint(completion)\\n```\\n\\nMake sure to replace `'gpt-3.5-turbo'` with the desired model name, `'your_api_key'` with your actual API key, and `'https://your_endpoint_url'` with the correct API endpoint URL provided by your service provider.\\n\\nNote: The above example assumes you have a valid API key and endpoint URL for the OpenAI GPT-3.5-turbo model. Make sure to obtain the necessary credentials according to the API you are using.\",\n\" Here is how you can call the completion() method using LiteLLM:\\n\\nFirst, import LiteLLM:\\n\\n```python\\nimport litellm as lm\\n```\\n\\nThen create a LiteLLM object, specifying the API you want to use (e.g. \\\"openai\\\"):\\n\\n```python \\nai = lm.LiteLLM(\\\"openai\\\")\\n```\\n\\nNow you can call the completion() method on the ai object:\\n\\n```python\\nresponse = ai.completion(\\n prompt=\\\"Hello\\\", \\n model=\\\"text-davinci-003\\\",\\n max_tokens=100\\n)\\n```\\n\\nThe completion() method takes parameters like:\\n\\n- prompt (str): The prompt text to complete \\n- model (str): The AI model to use\\n- max_tokens (int): The maximum number of tokens to generate\\n\\nIt returns a Python dictionary with the AI's response.\\n\\nYou can then access the generated text using:\\n\\n```python\\nprint(response[\\\"choices\\\"][0][\\\"text\\\"]) \\n```\\n\\nSo LiteLLM provides a simple unified interface to call the underlying AI APIs. The completion() method works similarly for OpenAI\"],\n [{\n 'v': 1,\n 'f': \"1\",\n },\n\"does LiteLLM support VertexAI\",\n\"Yes, LiteLLM does support Google Cloud Vertex AI. It provides convenient wrappers and simplified functions to call Vertex AI API endpoints for natural language processing tasks such as text classification, entity extraction, sentiment analysis, etc. You can easily integrate LiteLLM with Vertex AI in your code to leverage its capabilities.\",\n\" Unfortunately, LiteLLM does not currently support VertexAI. LiteLLM focuses on providing a simple interface to call the APIs of services like OpenAI, Azure, Cohere, Anthropic, and Hugging Face. \\n\\nVertexAI is Google's managed machine learning platform. Integrating VertexAI would require additional development work to wrap the VertexAI SDK in a simple interface like LiteLLM provides for other services. \\n\\nHowever, LiteLLM is open source, so it is possible for someone to contribute support for VertexAI. The maintainers would likely welcome a pull request to add VertexAI as an option if implemented well. But out of the box, LiteLLM does not have built-in support for calling VertexAI APIs.\\n\\nThe key services LiteLLM supports are:\\n\\n- OpenAI (GPT, Codex, DALL-E)\\n- Azure Cognitive Services (Text Analytics, Computer Vision, Speech) \\n- Cohere\\n- Anthropic AI\\n- Hugging Face Transformers\\n\\nSo while it doesn't cover every ML API provider, it does make it easy to use the most popular natural language, speech, and vision APIs through a simple interface. Adding VertexAI\"],\n [{\n 'v': 2,\n 'f': \"2\",\n },\n\"how do I set my keys on replicate llama2?\",\n\"To set your keys on Replicate Llama2, follow these steps:\\n\\n1. Open the Llama2 dashboard in your browser.\\n2. Click on the \\\"Settings\\\" tab in the top menu.\\n3. Scroll down to the \\\"API Keys\\\" section.\\n4. Click on the \\\"Add a Key\\\" button.\\n5. Enter a name for your API key to help you identify it later.\\n6. Select the provider for your API key from the dropdown menu. For example, you can select \\\"OpenAI\\\" for OpenAI GPT-3 access.\\n7. Enter your API key in the provided input field. Make sure to copy it correctly.\\n8. Click on the \\\"Save\\\" button to save your API key.\\n\\nNote: The actual steps may vary slightly depending on the platform or interface you are using to access Llama2.\",\n\" Here are the steps to set your API keys on Replicate for litellm:\\n\\n1. Go to your Replicate project settings and select the Environment tab.\\n\\n2. Under Environment Variables, click Add Variable.\\n\\n3. Add variables for the API keys you want to use. The variable names should match the ones used in litellm:\\n\\n- `OPENAI_API_KEY` for OpenAI \\n- `AZURE_API_KEY` for Azure Cognitive Services\\n- `COHERE_API_KEY` for Cohere\\n- `ANTHROPIC_API_KEY` for Anthropic\\n- `HUGGINGFACE_API_KEY` for Hugging Face\\n\\n4. Set the value to your actual API key for each service. Make sure to treat the values as secrets.\\n\\n5. Make sure your litellm code is referencing the environment variable names, for example:\\n\\n```python\\nimport litellm as lm\\n\\nlm.auth(openai_key=os.getenv(\\\"OPENAI_API_KEY\\\")) \\n```\\n\\n6. Restart your Replicate runtime to load the new environment variables.\\n\\nNow litellm will use your\"]],\n columns: [[\"number\", \"index\"], [\"string\", \"Question\"], [\"string\", \"gpt-3.5-turbo\"], [\"string\", \"claude-2\"]],\n columnOptions: [{\"width\": \"1px\", \"className\": \"index_column\"}],\n rowsPerPage: 25,\n helpUrl: \"https://colab.research.google.com/notebooks/data_table.ipynb\",\n suppressOutputScrolling: true,\n minimumWidth: undefined,\n });\n\n function appendQuickchartButton(parentElement) {\n let quickchartButtonContainerElement = document.createElement('div');\n quickchartButtonContainerElement.innerHTML = `\n\n
\n \n\n \n \n \n \n \n \n\n\n \n
`;\n parentElement.appendChild(quickchartButtonContainerElement);\n }\n\n appendQuickchartButton(table);\n ",
+ "text/html": [
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " Question \n",
+ " gpt-3.5-turbo \n",
+ " claude-2 \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 \n",
+ " how do i call completion() using LiteLLM \n",
+ " To call the `completion()` function using Lite... \n",
+ " Here is how you can call the completion() met... \n",
+ " \n",
+ " \n",
+ " 1 \n",
+ " does LiteLLM support VertexAI \n",
+ " Yes, LiteLLM does support Google Cloud Vertex ... \n",
+ " Unfortunately, LiteLLM does not currently sup... \n",
+ " \n",
+ " \n",
+ " 2 \n",
+ " how do I set my keys on replicate llama2? \n",
+ " To set your keys on Replicate Llama2, follow t... \n",
+ " Here are the steps to set your API keys on Re... \n",
+ " \n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
+ ],
+ "text/plain": [
+ " Question \\\n",
+ "0 how do i call completion() using LiteLLM \n",
+ "1 does LiteLLM support VertexAI \n",
+ "2 how do I set my keys on replicate llama2? \n",
+ "\n",
+ " gpt-3.5-turbo \\\n",
+ "0 To call the `completion()` function using Lite... \n",
+ "1 Yes, LiteLLM does support Google Cloud Vertex ... \n",
+ "2 To set your keys on Replicate Llama2, follow t... \n",
+ "\n",
+ " claude-2 \n",
+ "0 Here is how you can call the completion() met... \n",
+ "1 Unfortunately, LiteLLM does not currently sup... \n",
+ "2 Here are the steps to set your API keys on Re... "
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Create a table to visualize results\n",
+ "import pandas as pd\n",
+ "\n",
+ "columns = ['Question'] + models\n",
+ "df = pd.DataFrame(results, columns=columns)\n",
+ "\n",
+ "df"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/cookbook/LiteLLM_Completion_Cost.ipynb b/cookbook/LiteLLM_Completion_Cost.ipynb
new file mode 100644
index 000000000..b8f5eb36a
--- /dev/null
+++ b/cookbook/LiteLLM_Completion_Cost.ipynb
@@ -0,0 +1,241 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Use LiteLLM to calculate costs for all your completion calls\n",
+ "In this notebook we'll use `litellm.completion_cost` to get completion costs"
+ ],
+ "metadata": {
+ "id": "BgWr0PsUR3vV"
+ }
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ViczFTjsDzSI"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm==0.1.549 # use 0.1.549 or later"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Calculating costs for gpt-3.5 turbo completion()"
+ ],
+ "metadata": {
+ "id": "k_1CWUwmSNtj"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "from litellm import completion, completion_cost\n",
+ "import os\n",
+ "os.environ['OPENAI_API_KEY'] = \"\"\n",
+ "\n",
+ "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
+ "response = completion(\n",
+ " model=\"gpt-3.5-turbo\",\n",
+ " messages=messages,\n",
+ ")\n",
+ "\n",
+ "print(response)\n",
+ "\n",
+ "cost = completion_cost(completion_response=response)\n",
+ "formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n",
+ "print(formatted_string)\n"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Tp0fyk-jD0pP",
+ "outputId": "ce885fb3-3237-41b2-9d8b-3fb30bba498b"
+ },
+ "execution_count": 6,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "got response\n",
+ "{\n",
+ " \"id\": \"chatcmpl-7vyCApIZaCxP36kb9meUMN2DFSJPh\",\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"created\": 1694050442,\n",
+ " \"model\": \"gpt-3.5-turbo-0613\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"role\": \"assistant\",\n",
+ " \"content\": \"Hello! I'm an AI and I don't have feelings, but I'm here to help you. How can I assist you today?\"\n",
+ " },\n",
+ " \"finish_reason\": \"stop\"\n",
+ " }\n",
+ " ],\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 13,\n",
+ " \"completion_tokens\": 28,\n",
+ " \"total_tokens\": 41\n",
+ " }\n",
+ "}\n",
+ "Cost for completion call: $0.0000755000\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Calculating costs for Together Computer completion()"
+ ],
+ "metadata": {
+ "id": "AjDs4G-uS6PS"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "from litellm import completion, completion_cost\n",
+ "import os\n",
+ "os.environ['TOGETHERAI_API_KEY'] = \"\"\n",
+ "\n",
+ "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
+ "response = completion(\n",
+ " model=\"togethercomputer/llama-2-70b-chat\",\n",
+ " messages=messages,\n",
+ ")\n",
+ "\n",
+ "print(response)\n",
+ "\n",
+ "cost = completion_cost(completion_response=response)\n",
+ "formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n",
+ "print(formatted_string)\n"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "jMPsUV-KEa6a",
+ "outputId": "7a69b291-f149-4b9c-8a78-9c8142bac759"
+ },
+ "execution_count": 7,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "{\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \"Hello! I'm doing well, thanks for asking. I hope you're having a great\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"created\": 1694050771.2821715,\n",
+ " \"model\": \"togethercomputer/llama-2-70b-chat\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 12,\n",
+ " \"completion_tokens\": 18,\n",
+ " \"total_tokens\": 30\n",
+ " }\n",
+ "}\n",
+ "Cost for completion call: $0.0000900000\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Calculating costs for Replicate Llama2 completion()"
+ ],
+ "metadata": {
+ "id": "vEa4s6-7TANS"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "from litellm import completion, completion_cost\n",
+ "import os\n",
+ "os.environ['REPLICATE_API_KEY'] = \"\"\n",
+ "\n",
+ "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
+ "response = completion(\n",
+ " model=\"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n",
+ " messages=messages,\n",
+ ")\n",
+ "\n",
+ "print(response)\n",
+ "\n",
+ "cost = completion_cost(completion_response=response)\n",
+ "formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n",
+ "print(formatted_string)\n"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Xf1TKRDuS1bR",
+ "outputId": "cfb2b484-a6e5-41ad-86c5-7e66aba27648"
+ },
+ "execution_count": 8,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "{\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \" Hello! I'm doing well, thanks for asking. How about you? Is there anything you need help with today?\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"created\": 1694050893.4534576,\n",
+ " \"model\": \"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 6,\n",
+ " \"completion_tokens\": 24,\n",
+ " \"total_tokens\": 30\n",
+ " },\n",
+ " \"ended\": 1694050896.6689413\n",
+ "}\n",
+ "total_replicate_run_time 3.2154836654663086\n",
+ "Cost for completion call: $0.0045016771\n"
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/cookbook/LiteLLM_HuggingFace.ipynb b/cookbook/LiteLLM_HuggingFace.ipynb
new file mode 100644
index 000000000..bf8482a5f
--- /dev/null
+++ b/cookbook/LiteLLM_HuggingFace.ipynb
@@ -0,0 +1,252 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "9dKM5k8qsMIj"
+ },
+ "source": [
+ "## LiteLLM Hugging Face\n",
+ "\n",
+ "Docs for huggingface: https://docs.litellm.ai/docs/providers/huggingface\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "BVDdmCp-o97j"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "yp5UXRqtpu9f"
+ },
+ "source": [
+ "## Serverless Inference Providers\n",
+ "\n",
+ "Read more about Inference Providers here: https://huggingface.co/blog/inference-providers.\n",
+ "\n",
+ "In order to use litellm with Hugging Face Inference Providers, you need to set `model=huggingface//`.\n",
+ "\n",
+ "Example: `huggingface/together/deepseek-ai/DeepSeek-R1` to run DeepSeek-R1 (https://huggingface.co/deepseek-ai/DeepSeek-R1) through Together AI.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Pi5Oww8gpCUm",
+ "outputId": "659a67c7-f90d-4c06-b94e-2c4aa92d897a"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from litellm import completion\n",
+ "\n",
+ "# You can create a HF token here: https://huggingface.co/settings/tokens\n",
+ "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
+ "\n",
+ "# Call DeepSeek-R1 model through Together AI\n",
+ "response = completion(\n",
+ " model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
+ " messages=[{\"content\": \"How many r's are in the word `strawberry`?\", \"role\": \"user\"}],\n",
+ ")\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "EU0UubrKzTFe"
+ },
+ "source": [
+ "## Streaming\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "y-QfIvA-uJKX",
+ "outputId": "b007bb98-00d0-44a4-8264-c8a2caed6768"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from litellm import completion\n",
+ "\n",
+ "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
+ "\n",
+ "response = completion(\n",
+ " model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
+ " messages=[\n",
+ " {\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"How many r's are in the word `strawberry`?\",\n",
+ " \n",
+ " }\n",
+ " ],\n",
+ " stream=True,\n",
+ ")\n",
+ "\n",
+ "for chunk in response:\n",
+ " print(chunk)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## With images as input\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from litellm import completion\n",
+ "\n",
+ "# Set your Hugging Face Token\n",
+ "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
+ "\n",
+ "messages = [\n",
+ " {\n",
+ " \"role\": \"user\",\n",
+ " \"content\": [\n",
+ " {\"type\": \"text\", \"text\": \"What's in this image?\"},\n",
+ " {\n",
+ " \"type\": \"image_url\",\n",
+ " \"image_url\": {\n",
+ " \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n",
+ " },\n",
+ " },\n",
+ " ],\n",
+ " }\n",
+ "]\n",
+ "\n",
+ "response = completion(\n",
+ " model=\"huggingface/sambanova/meta-llama/Llama-3.3-70B-Instruct\",\n",
+ " messages=messages,\n",
+ ")\n",
+ "print(response.choices[0])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Tools - Function Calling\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from litellm import completion\n",
+ "\n",
+ "\n",
+ "# Set your Hugging Face Token\n",
+ "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
+ "\n",
+ "tools = [\n",
+ " {\n",
+ " \"type\": \"function\",\n",
+ " \"function\": {\n",
+ " \"name\": \"get_current_weather\",\n",
+ " \"description\": \"Get the current weather in a given location\",\n",
+ " \"parameters\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": {\n",
+ " \"location\": {\n",
+ " \"type\": \"string\",\n",
+ " \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
+ " },\n",
+ " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
+ " },\n",
+ " \"required\": [\"location\"],\n",
+ " },\n",
+ " },\n",
+ " }\n",
+ "]\n",
+ "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n",
+ "\n",
+ "response = completion(\n",
+ " model=\"huggingface/sambanova/meta-llama/Llama-3.1-8B-Instruct\", messages=messages, tools=tools, tool_choice=\"auto\"\n",
+ ")\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Hugging Face Dedicated Inference Endpoints\n",
+ "\n",
+ "Steps to use\n",
+ "\n",
+ "- Create your own Hugging Face dedicated endpoint here: https://ui.endpoints.huggingface.co/\n",
+ "- Set `api_base` to your deployed api base\n",
+ "- set the model to `huggingface/tgi` so that litellm knows it's a huggingface Deployed Inference Endpoint.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import litellm\n",
+ "\n",
+ "\n",
+ "response = litellm.completion(\n",
+ " model=\"huggingface/tgi\",\n",
+ " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}],\n",
+ " api_base=\"https://my-endpoint.endpoints.huggingface.cloud/v1/\",\n",
+ ")\n",
+ "print(response)"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.0"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb b/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb
new file mode 100644
index 000000000..8fa7d0b98
--- /dev/null
+++ b/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb
@@ -0,0 +1,97 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "iFEmsVJI_2BR"
+ },
+ "source": [
+ "# LiteLLM NovitaAI Cookbook"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "cBlUhCEP_xj4"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "p-MQqWOT_1a7"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "os.environ['NOVITA_API_KEY'] = \"\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Ze8JqMqWAARO"
+ },
+ "outputs": [],
+ "source": [
+ "from litellm import completion\n",
+ "response = completion(\n",
+ " model=\"novita/deepseek/deepseek-r1\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+ ")\n",
+ "response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "-LnhELrnAM_J"
+ },
+ "outputs": [],
+ "source": [
+ "response = completion(\n",
+ " model=\"novita/deepseek/deepseek-r1\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+ ")\n",
+ "response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "dJBOUYdwCEn1"
+ },
+ "outputs": [],
+ "source": [
+ "response = completion(\n",
+ " model=\"mistralai/mistral-7b-instruct\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+ ")\n",
+ "response"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/LiteLLM_OpenRouter.ipynb b/cookbook/LiteLLM_OpenRouter.ipynb
new file mode 100644
index 000000000..6444b23b2
--- /dev/null
+++ b/cookbook/LiteLLM_OpenRouter.ipynb
@@ -0,0 +1,179 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "iFEmsVJI_2BR"
+ },
+ "source": [
+ "# LiteLLM OpenRouter Cookbook"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "cBlUhCEP_xj4"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "id": "p-MQqWOT_1a7"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "os.environ['OPENROUTER_API_KEY'] = \"\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Ze8JqMqWAARO",
+ "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ " JSON: {\n",
+ " \"id\": \"gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv\",\n",
+ " \"model\": \"chat-bison@001\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"message\": {\n",
+ " \"role\": \"assistant\",\n",
+ " \"content\": \"```\\n#include \\n\\nint main() {\\n printf(\\\"Hi!\\\\n\\\");\\n return 0;\\n}\\n```\"\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"response_ms\": 7817.777999999999\n",
+ "}"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from litellm import completion\n",
+ "response = completion(\n",
+ " model=\"openrouter/google/palm-2-chat-bison\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+ ")\n",
+ "response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "-LnhELrnAM_J",
+ "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ " JSON: {\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"message\": {\n",
+ " \"role\": \"assistant\",\n",
+ " \"content\": \" Here is some simple code to print \\\"Hi\\\":\\n\\n```python\\nprint(\\\"Hi\\\")\\n```\\n\\nThis uses the print() function in Python to output the text \\\"Hi\\\".\"\n",
+ " },\n",
+ " \"finish_reason\": \"stop_sequence\"\n",
+ " }\n",
+ " ],\n",
+ " \"model\": \"claude-2.0\",\n",
+ " \"id\": \"gen-IiuV7ZNimDufVeutBHrl8ajPuzEh\",\n",
+ " \"response_ms\": 8112.443000000001\n",
+ "}"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "response = completion(\n",
+ " model=\"openrouter/anthropic/claude-2\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+ ")\n",
+ "response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "dJBOUYdwCEn1",
+ "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ " JSON: {\n",
+ " \"id\": \"gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl\",\n",
+ " \"model\": \"togethercomputer/llama-2-70b-chat\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"message\": {\n",
+ " \"role\": \"assistant\",\n",
+ " \"content\": \"*gives a sly smile as they type*\\n\\nHey there, handsome. \\ud83d\\ude0f\\n\\nWhat brings you to my neck of the woods today? \\ud83d\\ude18\"\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"response_ms\": 9618.775\n",
+ "}"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "response = completion(\n",
+ " model=\"openrouter/meta-llama/llama-2-70b-chat\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
+ ")\n",
+ "response"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/LiteLLM_Petals.ipynb b/cookbook/LiteLLM_Petals.ipynb
new file mode 100644
index 000000000..aacc22dd1
--- /dev/null
+++ b/cookbook/LiteLLM_Petals.ipynb
@@ -0,0 +1,568 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "dwGtLi_tvM6N"
+ },
+ "source": [
+ "# Using LiteLLM with Petals"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "bdlgaWQqDpzj"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm # 0.1.715 and upwards"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "5Id2QKwOEH8X"
+ },
+ "outputs": [],
+ "source": [
+ "# install petals\n",
+ "!pip install git+https://github.com/bigscience-workshop/petals"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "k42fldw3veSN"
+ },
+ "source": [
+ "## petals-team/StableBeluga2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "tIHcEHdSDqju",
+ "outputId": "485dbf54-395c-433a-bbf4-8eb70a9fa624"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "You are using the default legacy behaviour of the . If you see this, DO NOT PANIC! This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
+ "Sep 19 18:39:50.634 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n",
+ "Sep 19 18:39:50.639 [\u001b[1m\u001b[34mINFO\u001b[0m] Using DHT prefix: StableBeluga2-hf\n",
+ "Sep 19 18:40:13.920 [\u001b[1m\u001b[34mINFO\u001b[0m] Route found: 0:40 via ā¦HfQWVM => 40:80 via ā¦Zj98Se\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \"Hello, how are you?\\nI'm doing well, thank you. I'm just getting ready to go to the gym.\\nOh, that's great. I'm trying to get back into a workout routine myself.\\nYeah,\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"id\": \"chatcmpl-f09d79b3-c1d1-49b7-b55f-cd8dfa1043bf\",\n",
+ " \"created\": 1695148897.473613,\n",
+ " \"model\": \"petals-team/StableBeluga2\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 6,\n",
+ " \"completion_tokens\": 45,\n",
+ " \"total_tokens\": 51\n",
+ " }\n",
+ "}\n"
+ ]
+ }
+ ],
+ "source": [
+ "from litellm import completion\n",
+ "\n",
+ "response = completion(model=\"petals/petals-team/StableBeluga2\", messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}], max_tokens=50)\n",
+ "\n",
+ "print(response)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "J8DubRnHvh_j"
+ },
+ "source": [
+ "## huggyllama/llama-65b"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 538,
+ "referenced_widgets": [
+ "2fec5cc400424671a3d517327117d18a",
+ "3687c76fe84d464baaf35366b21e83b3",
+ "c29d4460dbaa441cae110b58e0014151",
+ "6560449a38bf4a7bacd97ccaacf01c4c",
+ "5fbd6ae281984d28ba59ebfd0279eda7",
+ "323e30e275434aeea241163e5f1f9031",
+ "48f4adec51c94f9da6e4c4564daeff84",
+ "2a672981a44b4a7fb30674f97f4c10c6",
+ "d75ae8d22ea74840b4c80c8f386384c4",
+ "54c06312ecff4e7588665e8b0cb7118b",
+ "300078a9d1a6483fba81a4be63793ff7"
+ ]
+ },
+ "id": "IlTCJwDsNvgF",
+ "outputId": "2e84d125-d982-48ed-8a92-6ca438a50d0c"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Sep 19 18:41:37.912 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n",
+ "Sep 19 18:41:37.914 [\u001b[1m\u001b[34mINFO\u001b[0m] Using DHT prefix: llama-65b-hf\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "2fec5cc400424671a3d517327117d18a",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Loading checkpoint shards: 0%| | 0/2 [00:00, ?it/s]"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/usr/local/lib/python3.10/dist-packages/transformers/generation/configuration_utils.py:362: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.2` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
+ " warnings.warn(\n",
+ "Sep 19 18:41:48.396 [\u001b[1m\u001b[34mINFO\u001b[0m] Route found: 0:80 via ā¦g634yJ\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \"Hello, how are you?\\nI'm fine, thank you. And\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"id\": \"chatcmpl-3496e6eb-2a27-4f94-8d75-70648eacd88f\",\n",
+ " \"created\": 1695148912.9116046,\n",
+ " \"model\": \"huggyllama/llama-65b\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 6,\n",
+ " \"completion_tokens\": 14,\n",
+ " \"total_tokens\": 20\n",
+ " }\n",
+ "}\n"
+ ]
+ }
+ ],
+ "source": [
+ "response = completion(model=\"petals/huggyllama/llama-65b\", messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}], temperature=0.2, max_tokens=10)\n",
+ "\n",
+ "print(response)"
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "gpuType": "T4",
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ },
+ "widgets": {
+ "application/vnd.jupyter.widget-state+json": {
+ "2a672981a44b4a7fb30674f97f4c10c6": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "2fec5cc400424671a3d517327117d18a": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_3687c76fe84d464baaf35366b21e83b3",
+ "IPY_MODEL_c29d4460dbaa441cae110b58e0014151",
+ "IPY_MODEL_6560449a38bf4a7bacd97ccaacf01c4c"
+ ],
+ "layout": "IPY_MODEL_5fbd6ae281984d28ba59ebfd0279eda7"
+ }
+ },
+ "300078a9d1a6483fba81a4be63793ff7": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "323e30e275434aeea241163e5f1f9031": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "3687c76fe84d464baaf35366b21e83b3": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_323e30e275434aeea241163e5f1f9031",
+ "placeholder": "ā",
+ "style": "IPY_MODEL_48f4adec51c94f9da6e4c4564daeff84",
+ "value": "Loading checkpoint shards: 100%"
+ }
+ },
+ "48f4adec51c94f9da6e4c4564daeff84": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "54c06312ecff4e7588665e8b0cb7118b": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "5fbd6ae281984d28ba59ebfd0279eda7": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "6560449a38bf4a7bacd97ccaacf01c4c": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_54c06312ecff4e7588665e8b0cb7118b",
+ "placeholder": "ā",
+ "style": "IPY_MODEL_300078a9d1a6483fba81a4be63793ff7",
+ "value": " 2/2 [00:00<00:00, 2.36it/s]"
+ }
+ },
+ "c29d4460dbaa441cae110b58e0014151": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_2a672981a44b4a7fb30674f97f4c10c6",
+ "max": 2,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_d75ae8d22ea74840b4c80c8f386384c4",
+ "value": 2
+ }
+ },
+ "d75ae8d22ea74840b4c80c8f386384c4": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ }
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/LiteLLM_PromptLayer.ipynb b/cookbook/LiteLLM_PromptLayer.ipynb
new file mode 100644
index 000000000..8fd549410
--- /dev/null
+++ b/cookbook/LiteLLM_PromptLayer.ipynb
@@ -0,0 +1,130 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "9AxeMfikUw2A"
+ },
+ "source": [
+ "# Using LiteLLM with PromptLayer\n",
+ "Promptlayer allows you to track requests, responses and prompts\n",
+ "\n",
+ "LiteLLM allows you to use any litellm supported model and send data to promptlayer\n",
+ "\n",
+ "Getting started docs: https://docs.litellm.ai/docs/observability/promptlayer_integration"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "VwgSvAcVCiJX"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "id": "r8QSgKbXFhpe"
+ },
+ "outputs": [],
+ "source": [
+ "import litellm\n",
+ "from litellm import completion\n",
+ "import os\n",
+ "os.environ['OPENAI_API_KEY'] = \"\"\n",
+ "os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
+ "os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
+ "\n",
+ "# Set Promptlayer as a success callback\n",
+ "litellm.success_callback =['promptlayer']\n",
+ "\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gaklMZhxVFBv"
+ },
+ "source": [
+ "## Call OpenAI with LiteLLM x PromptLayer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "NOZL7MWiTFct",
+ "outputId": "039af693-c1d6-40ee-a081-0a494cf27c6a"
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "result = completion(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": \"gm this is ishaan\"}])\n",
+ "print(result)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Qt91CjpeVJ32"
+ },
+ "source": [
+ "## Call Replicate-CodeLlama with LiteLLM x PromptLayer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "dTwhEKelDy_J",
+ "outputId": "751f7883-390f-47bd-9aa4-3b1523bd1af2"
+ },
+ "outputs": [],
+ "source": [
+ "model=\"replicate/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db\"\n",
+ "\n",
+ "result = completion(model=model, messages=[{\"role\": \"user\", \"content\": \"gm this is ishaan\"}])\n",
+ "print(result)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "qk-k6t8eVukF"
+ },
+ "source": [
+ "## View Logs on PromptLayer\n",
+ ""
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/LiteLLM_User_Based_Rate_Limits.ipynb b/cookbook/LiteLLM_User_Based_Rate_Limits.ipynb
new file mode 100644
index 000000000..29d36e461
--- /dev/null
+++ b/cookbook/LiteLLM_User_Based_Rate_Limits.ipynb
@@ -0,0 +1,224 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "JRCXfhACct4Y"
+ },
+ "source": [
+ "## User Based Rate Limiting Using LiteLLM\n",
+ "- LiteLLM allows you to set budgets per user\n",
+ "- Check if a given user has cross their allocated budget\n",
+ "\n",
+ "In this notebook we create a $0.0002 daily budget per user and make completion calls using the litellm budget manager"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "fl1kcLG8aaIV"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm uuid"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "zqRrpoQ3c6oQ"
+ },
+ "source": [
+ "## Imports & Env variables"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "id": "CSkz8bmwdD3w"
+ },
+ "outputs": [],
+ "source": [
+ "import uuid\n",
+ "import os\n",
+ "os.environ['OPENAI_API_KEY'] = \"\""
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ktqe3gSmdFQ4"
+ },
+ "source": [
+ "## completion() with the budget manager\n",
+ "\n",
+ "This code does the following\n",
+ "- Initializes a litellm.BudgetManager()\n",
+ "- Checks if a budget exists for a user\n",
+ " - Creates a $0.0002 budget if the user does not exisr\n",
+ "- Makes a `litellm.completion()` request only if the user is under their budget"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "pUN48YvmaiRU",
+ "outputId": "082d6a8b-9aef-4794-9eac-7ba9823ea373"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "No budget exists for user: 29af95f8-c3c6-4c8c-b080-8b2d18d25432\n",
+ "\n",
+ "Creating a budget for user: 29af95f8-c3c6-4c8c-b080-8b2d18d25432, daily budget $0.0002\n",
+ "\n",
+ "User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $0, budget for user: $0.0002\n",
+ "\n",
+ "{\n",
+ " \"id\": \"chatcmpl-7yAUkHQV8xdfldzzZnnnuVU8pl31b\",\n",
+ " \"object\": \"chat.completion\",\n",
+ " \"created\": 1694574378,\n",
+ " \"model\": \"gpt-3.5-turbo-0613\",\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"role\": \"assistant\",\n",
+ " \"content\": \"Hello! I'm an AI, so I don't have emotions, but I'm here to assist you. How can I help you today?\"\n",
+ " },\n",
+ " \"finish_reason\": \"stop\"\n",
+ " }\n",
+ " ],\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 14,\n",
+ " \"completion_tokens\": 29,\n",
+ " \"total_tokens\": 43\n",
+ " }\n",
+ "}\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "{'status': 'success'}"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from litellm import BudgetManager, completion\n",
+ "\n",
+ "# Initializes a litellm.BudgetManager()\n",
+ "budget_manager = BudgetManager(project_name=\"liteLLM_project\", client_type=\"hosted\") # see https://docs.litellm.ai/docs/budget_manager\n",
+ "\n",
+ "user_id = str(uuid.uuid4()) # create a new user id\n",
+ "daily_budget = 0.0002\n",
+ "\n",
+ "# Checks if a budget exists for a user\n",
+ "if not budget_manager.is_valid_user(user_id):\n",
+ " # Creates a $0.0002 budget if the user does not exisr\n",
+ " print(f\"No budget exists for user: {user_id}\\n\")\n",
+ " print(f\"Creating a budget for user: {user_id}, daily budget ${daily_budget}\\n\")\n",
+ " budget_manager.create_budget(total_budget=daily_budget, user=user_id, duration=\"daily\") # duration can be daily, weekly, monthly\n",
+ "\n",
+ "\n",
+ "# Makes a `litellm.completion()` request only if the user is under their budget\n",
+ "current_spend_for_user = budget_manager.get_current_cost(user=user_id)\n",
+ "budget_for_user = budget_manager.get_total_budget(user_id)\n",
+ "print(f\"User: {user_id} has spent ${current_spend_for_user}, budget for user: ${budget_for_user}\\n\")\n",
+ "\n",
+ "if current_spend_for_user <= budget_for_user:\n",
+ " response = completion(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}])\n",
+ " budget_manager.update_cost(completion_obj=response, user=user_id)\n",
+ "else:\n",
+ " response = \"Sorry - no budget!\"\n",
+ "\n",
+ "print(response)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "yMOirNoBfmmc"
+ },
+ "source": [
+ "## Make 10 calls to cross the budget per user\n",
+ "- Code fails after user crossed their budget"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "apKF3H-xbFXc",
+ "outputId": "1c6ef0fe-e27e-4ead-adc6-2c7eb0214e44"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $7.9e-05, budget for user: $0.0002\n",
+ "\n",
+ "User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $0.00015999999999999999, budget for user: $0.0002\n",
+ "\n",
+ "User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has spent $0.00023899999999999998, budget for user: $0.0002\n",
+ "\n",
+ "User: 29af95f8-c3c6-4c8c-b080-8b2d18d25432 has exceeded budget, current spend $0.00023899999999999998, budget for user: $0.0002\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "user_id = \"29af95f8-c3c6-4c8c-b080-8b2d18d25432\" # set in the previous cell\n",
+ "\n",
+ "for _ in range(10):\n",
+ " # check if a given call can be made\n",
+ " current_spend_for_user = budget_manager.get_current_cost(user=user_id)\n",
+ " budget_for_user = budget_manager.get_total_budget(user_id)\n",
+ " print(f\"User: {user_id} has spent ${current_spend_for_user}, budget for user: ${budget_for_user}\\n\")\n",
+ " if current_spend_for_user <= budget_for_user:\n",
+ " response = completion(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}])\n",
+ " budget_manager.update_cost(completion_obj=response, user=user_id)\n",
+ " else:\n",
+ " response = \"Sorry - no budget!\"\n",
+ " print(f\"User: {user_id} has exceeded budget, current spend ${current_spend_for_user}, budget for user: ${budget_for_user}\\n\")\n",
+ " break # no more requests\n",
+ "\n",
+ " # print(response)"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/LiteLLM_batch_completion.ipynb b/cookbook/LiteLLM_batch_completion.ipynb
new file mode 100644
index 000000000..b0d33d62f
--- /dev/null
+++ b/cookbook/LiteLLM_batch_completion.ipynb
@@ -0,0 +1,163 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MbLbs1tbISk-"
+ },
+ "source": [
+ "# LiteLLM Batch Completions Example\n",
+ "\n",
+ "* This tutorial walks through using `batch_completion`\n",
+ "* Docs: https://docs.litellm.ai/docs/completion/batching"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Ty6-ko_aDlPF"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "KGhNJRUCIh1j"
+ },
+ "source": [
+ "## Import Batch Completion"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "LOtI43snDrSK"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from litellm import batch_completion\n",
+ "\n",
+ "# set your API_KEY\n",
+ "os.environ['ANTHROPIC_API_KEY'] = \"\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Xhv92NBaIpaw"
+ },
+ "source": [
+ "## Calling `litellm.batch_completion`\n",
+ "\n",
+ "In the batch_completion method, you provide a list of messages where each sub-list of messages is passed to litellm.completion(), allowing you to process multiple prompts efficiently in a single API call."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "yY7GIRLsDywu",
+ "outputId": "009ea67f-95d5-462b-947f-b0d21e60c5bb"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[ JSON: {\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \" Good morning!\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"created\": 1694030351.309254,\n",
+ " \"model\": \"claude-2\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 11,\n",
+ " \"completion_tokens\": 3,\n",
+ " \"total_tokens\": 14\n",
+ " }\n",
+ " },\n",
+ " JSON: {\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \" I'm an AI assistant created by Anthropic. I don't actually have a concept of the current time.\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"created\": 1694030352.1215081,\n",
+ " \"model\": \"claude-2\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 13,\n",
+ " \"completion_tokens\": 22,\n",
+ " \"total_tokens\": 35\n",
+ " }\n",
+ " }]"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import os\n",
+ "\n",
+ "os.environ['ANTHROPIC_API_KEY'] = \"\"\n",
+ "\n",
+ "\n",
+ "responses = batch_completion(\n",
+ " model=\"claude-2\",\n",
+ " messages = [\n",
+ " [\n",
+ " {\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"good morning? \"\n",
+ " }\n",
+ " ],\n",
+ " [\n",
+ " {\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"what's the time? \"\n",
+ " }\n",
+ " ]\n",
+ " ]\n",
+ ")\n",
+ "responses"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb b/cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb
new file mode 100644
index 000000000..740e7c7a4
--- /dev/null
+++ b/cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb
@@ -0,0 +1,565 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "kccfk0mHZ4Ad"
+ },
+ "source": [
+ "# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
+ "\n",
+ "Covers:\n",
+ "\n",
+ "* /chat/completion\n",
+ "* /embedding\n",
+ "\n",
+ "\n",
+ "These are **selected examples**. LiteLLM Proxy is **OpenAI-Compatible**, it works with any project that calls OpenAI. Just change the `base_url`, `api_key` and `model`.\n",
+ "\n",
+ "For more examples, [go here](https://docs.litellm.ai/docs/proxy/user_keys)\n",
+ "\n",
+ "To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
+ "\n",
+ "To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nmSClzCPaGH6"
+ },
+ "source": [
+ "## /chat/completion\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "_vqcjwOVaKpO"
+ },
+ "source": [
+ "### OpenAI Python SDK"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "x1e_Ok3KZzeP"
+ },
+ "outputs": [],
+ "source": [
+ "import openai\n",
+ "client = openai.OpenAI(\n",
+ " api_key=\"anything\",\n",
+ " base_url=\"http://0.0.0.0:4000\"\n",
+ ")\n",
+ "\n",
+ "# request sent to model set on litellm proxy, `litellm --model`\n",
+ "response = client.chat.completions.create(\n",
+ " model=\"gpt-3.5-turbo\",\n",
+ " messages = [\n",
+ " {\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"this is a test request, write a short poem\"\n",
+ " }\n",
+ " ],\n",
+ " extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params\n",
+ " \"metadata\": { # š use for logging additional params (e.g. to langfuse)\n",
+ " \"generation_name\": \"ishaan-generation-openai-client\",\n",
+ " \"generation_id\": \"openai-client-gen-id22\",\n",
+ " \"trace_id\": \"openai-client-trace-id22\",\n",
+ " \"trace_user_id\": \"openai-client-user-id2\"\n",
+ " }\n",
+ " }\n",
+ ")\n",
+ "\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "AqkyKk9Scxgj"
+ },
+ "source": [
+ "## Function Calling"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "wDg10VqLczE1"
+ },
+ "outputs": [],
+ "source": [
+ "from openai import OpenAI\n",
+ "client = OpenAI(\n",
+ " api_key=\"sk-1234\", # [OPTIONAL] set if you set one on proxy, else set \"\"\n",
+ " base_url=\"http://0.0.0.0:4000\",\n",
+ ")\n",
+ "\n",
+ "tools = [\n",
+ " {\n",
+ " \"type\": \"function\",\n",
+ " \"function\": {\n",
+ " \"name\": \"get_current_weather\",\n",
+ " \"description\": \"Get the current weather in a given location\",\n",
+ " \"parameters\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": {\n",
+ " \"location\": {\n",
+ " \"type\": \"string\",\n",
+ " \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
+ " },\n",
+ " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
+ " },\n",
+ " \"required\": [\"location\"],\n",
+ " },\n",
+ " }\n",
+ " }\n",
+ "]\n",
+ "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n",
+ "completion = client.chat.completions.create(\n",
+ " model=\"gpt-4o\", # use 'model_name' from config.yaml\n",
+ " messages=messages,\n",
+ " tools=tools,\n",
+ " tool_choice=\"auto\"\n",
+ ")\n",
+ "\n",
+ "print(completion)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "YYoxLloSaNWW"
+ },
+ "source": [
+ "### Azure OpenAI Python SDK"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "yA1XcgowaSRy"
+ },
+ "outputs": [],
+ "source": [
+ "import openai\n",
+ "client = openai.AzureOpenAI(\n",
+ " api_key=\"anything\",\n",
+ " base_url=\"http://0.0.0.0:4000\"\n",
+ ")\n",
+ "\n",
+ "# request sent to model set on litellm proxy, `litellm --model`\n",
+ "response = client.chat.completions.create(\n",
+ " model=\"gpt-3.5-turbo\",\n",
+ " messages = [\n",
+ " {\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"this is a test request, write a short poem\"\n",
+ " }\n",
+ " ],\n",
+ " extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params\n",
+ " \"metadata\": { # š use for logging additional params (e.g. to langfuse)\n",
+ " \"generation_name\": \"ishaan-generation-openai-client\",\n",
+ " \"generation_id\": \"openai-client-gen-id22\",\n",
+ " \"trace_id\": \"openai-client-trace-id22\",\n",
+ " \"trace_user_id\": \"openai-client-user-id2\"\n",
+ " }\n",
+ " }\n",
+ ")\n",
+ "\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "yl9qhDvnaTpL"
+ },
+ "source": [
+ "### Langchain Python"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "5MUZgSquaW5t"
+ },
+ "outputs": [],
+ "source": [
+ "from langchain.chat_models import ChatOpenAI\n",
+ "from langchain.prompts.chat import (\n",
+ " ChatPromptTemplate,\n",
+ " HumanMessagePromptTemplate,\n",
+ " SystemMessagePromptTemplate,\n",
+ ")\n",
+ "from langchain.schema import HumanMessage, SystemMessage\n",
+ "import os\n",
+ "\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"anything\"\n",
+ "\n",
+ "chat = ChatOpenAI(\n",
+ " openai_api_base=\"http://0.0.0.0:4000\",\n",
+ " model = \"gpt-3.5-turbo\",\n",
+ " temperature=0.1,\n",
+ " extra_body={\n",
+ " \"metadata\": {\n",
+ " \"generation_name\": \"ishaan-generation-langchain-client\",\n",
+ " \"generation_id\": \"langchain-client-gen-id22\",\n",
+ " \"trace_id\": \"langchain-client-trace-id22\",\n",
+ " \"trace_user_id\": \"langchain-client-user-id2\"\n",
+ " }\n",
+ " }\n",
+ ")\n",
+ "\n",
+ "messages = [\n",
+ " SystemMessage(\n",
+ " content=\"You are a helpful assistant that im using to make a test request to.\"\n",
+ " ),\n",
+ " HumanMessage(\n",
+ " content=\"test from litellm. tell me why it's amazing in 1 sentence\"\n",
+ " ),\n",
+ "]\n",
+ "response = chat(messages)\n",
+ "\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "B9eMgnULbRaz"
+ },
+ "source": [
+ "### Curl"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "VWCCk5PFcmhS"
+ },
+ "source": [
+ "\n",
+ "\n",
+ "```\n",
+ "curl -X POST 'http://0.0.0.0:4000/chat/completions' \\\n",
+ " -H 'Content-Type: application/json' \\\n",
+ " -d '{\n",
+ " \"model\": \"gpt-3.5-turbo\",\n",
+ " \"messages\": [\n",
+ " {\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"what llm are you\"\n",
+ " }\n",
+ " ],\n",
+ " \"metadata\": {\n",
+ " \"generation_name\": \"ishaan-test-generation\",\n",
+ " \"generation_id\": \"gen-id22\",\n",
+ " \"trace_id\": \"trace-id22\",\n",
+ " \"trace_user_id\": \"user-id2\"\n",
+ " }\n",
+ "}'\n",
+ "```\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "drBAm2e1b6xe"
+ },
+ "source": [
+ "### LlamaIndex"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "d0bZcv8fb9mL"
+ },
+ "outputs": [],
+ "source": [
+ "import os, dotenv\n",
+ "\n",
+ "from llama_index.llms import AzureOpenAI\n",
+ "from llama_index.embeddings import AzureOpenAIEmbedding\n",
+ "from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext\n",
+ "\n",
+ "llm = AzureOpenAI(\n",
+ " engine=\"azure-gpt-3.5\", # model_name on litellm proxy\n",
+ " temperature=0.0,\n",
+ " azure_endpoint=\"http://0.0.0.0:4000\", # litellm proxy endpoint\n",
+ " api_key=\"sk-1234\", # litellm proxy API Key\n",
+ " api_version=\"2023-07-01-preview\",\n",
+ ")\n",
+ "\n",
+ "embed_model = AzureOpenAIEmbedding(\n",
+ " deployment_name=\"azure-embedding-model\",\n",
+ " azure_endpoint=\"http://0.0.0.0:4000\",\n",
+ " api_key=\"sk-1234\",\n",
+ " api_version=\"2023-07-01-preview\",\n",
+ ")\n",
+ "\n",
+ "\n",
+ "documents = SimpleDirectoryReader(\"llama_index_data\").load_data()\n",
+ "service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)\n",
+ "index = VectorStoreIndex.from_documents(documents, service_context=service_context)\n",
+ "\n",
+ "query_engine = index.as_query_engine()\n",
+ "response = query_engine.query(\"What did the author do growing up?\")\n",
+ "print(response)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "xypvNdHnb-Yy"
+ },
+ "source": [
+ "### Langchain JS"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "R55mK2vCcBN2"
+ },
+ "outputs": [],
+ "source": [
+ "import { ChatOpenAI } from \"@langchain/openai\";\n",
+ "\n",
+ "\n",
+ "const model = new ChatOpenAI({\n",
+ " modelName: \"gpt-4\",\n",
+ " openAIApiKey: \"sk-1234\",\n",
+ " modelKwargs: {\"metadata\": \"hello world\"} // š PASS Additional params here\n",
+ "}, {\n",
+ " basePath: \"http://0.0.0.0:4000\",\n",
+ "});\n",
+ "\n",
+ "const message = await model.invoke(\"Hi there!\");\n",
+ "\n",
+ "console.log(message);\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nC4bLifCcCiW"
+ },
+ "source": [
+ "### OpenAI JS"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "MICH8kIMcFpg"
+ },
+ "outputs": [],
+ "source": [
+ "const { OpenAI } = require('openai');\n",
+ "\n",
+ "const openai = new OpenAI({\n",
+ " apiKey: \"sk-1234\", // This is the default and can be omitted\n",
+ " baseURL: \"http://0.0.0.0:4000\"\n",
+ "});\n",
+ "\n",
+ "async function main() {\n",
+ " const chatCompletion = await openai.chat.completions.create({\n",
+ " messages: [{ role: 'user', content: 'Say this is a test' }],\n",
+ " model: 'gpt-3.5-turbo',\n",
+ " }, {\"metadata\": {\n",
+ " \"generation_name\": \"ishaan-generation-openaijs-client\",\n",
+ " \"generation_id\": \"openaijs-client-gen-id22\",\n",
+ " \"trace_id\": \"openaijs-client-trace-id22\",\n",
+ " \"trace_user_id\": \"openaijs-client-user-id2\"\n",
+ " }});\n",
+ "}\n",
+ "\n",
+ "main();\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "D1Q07pEAcGTb"
+ },
+ "source": [
+ "### Anthropic SDK"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "qBjFcAvgcI3t"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "\n",
+ "from anthropic import Anthropic\n",
+ "\n",
+ "client = Anthropic(\n",
+ " base_url=\"http://localhost:4000\", # proxy endpoint\n",
+ " api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
+ ")\n",
+ "\n",
+ "message = client.messages.create(\n",
+ " max_tokens=1024,\n",
+ " messages=[\n",
+ " {\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"Hello, Claude\",\n",
+ " }\n",
+ " ],\n",
+ " model=\"claude-3-opus-20240229\",\n",
+ ")\n",
+ "print(message.content)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "dFAR4AJGcONI"
+ },
+ "source": [
+ "## /embeddings"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "lgNoM281cRzR"
+ },
+ "source": [
+ "### OpenAI Python SDK"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "NY3DJhPfcQhA"
+ },
+ "outputs": [],
+ "source": [
+ "import openai\n",
+ "from openai import OpenAI\n",
+ "\n",
+ "# set base_url to your proxy server\n",
+ "# set api_key to send to proxy server\n",
+ "client = OpenAI(api_key=\"\", base_url=\"http://0.0.0.0:4000\")\n",
+ "\n",
+ "response = client.embeddings.create(\n",
+ " input=[\"hello from litellm\"],\n",
+ " model=\"text-embedding-ada-002\"\n",
+ ")\n",
+ "\n",
+ "print(response)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "hmbg-DW6cUZs"
+ },
+ "source": [
+ "### Langchain Embeddings"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "lX2S8Nl1cWVP"
+ },
+ "outputs": [],
+ "source": [
+ "from langchain.embeddings import OpenAIEmbeddings\n",
+ "\n",
+ "embeddings = OpenAIEmbeddings(model=\"sagemaker-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n",
+ "\n",
+ "\n",
+ "text = \"This is a test document.\"\n",
+ "\n",
+ "query_result = embeddings.embed_query(text)\n",
+ "\n",
+ "print(f\"SAGEMAKER EMBEDDINGS\")\n",
+ "print(query_result[:5])\n",
+ "\n",
+ "embeddings = OpenAIEmbeddings(model=\"bedrock-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n",
+ "\n",
+ "text = \"This is a test document.\"\n",
+ "\n",
+ "query_result = embeddings.embed_query(text)\n",
+ "\n",
+ "print(f\"BEDROCK EMBEDDINGS\")\n",
+ "print(query_result[:5])\n",
+ "\n",
+ "embeddings = OpenAIEmbeddings(model=\"bedrock-titan-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n",
+ "\n",
+ "text = \"This is a test document.\"\n",
+ "\n",
+ "query_result = embeddings.embed_query(text)\n",
+ "\n",
+ "print(f\"TITAN EMBEDDINGS\")\n",
+ "print(query_result[:5])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "oqGbWBCQcYfd"
+ },
+ "source": [
+ "### Curl Request"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "7rkIMV9LcdwQ"
+ },
+ "source": [
+ "\n",
+ "\n",
+ "```curl\n",
+ "curl -X POST 'http://0.0.0.0:4000/embeddings' \\\n",
+ " -H 'Content-Type: application/json' \\\n",
+ " -d ' {\n",
+ " \"model\": \"text-embedding-ada-002\",\n",
+ " \"input\": [\"write a litellm poem\"]\n",
+ " }'\n",
+ "```\n",
+ "\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/Parallel_function_calling.ipynb b/cookbook/Parallel_function_calling.ipynb
new file mode 100644
index 000000000..cb7fbafac
--- /dev/null
+++ b/cookbook/Parallel_function_calling.ipynb
@@ -0,0 +1,478 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "source": [
+ "This is a tutorial on using Parallel function calling with LiteLLM"
+ ],
+ "metadata": {
+ "id": "gHwFJ-srdnku"
+ }
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "RrtHuVHlZmUe"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "This tutorial walks through the steps doing parallel function calling using\n",
+ " - OpenAI\n",
+ " - Azure OpenAI"
+ ],
+ "metadata": {
+ "id": "sG5ANaazjU0g"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# set openai api key\n",
+ "import os\n",
+ "os.environ['OPENAI_API_KEY'] = \"\" # litellm reads OPENAI_API_KEY from .env and sends the request"
+ ],
+ "metadata": {
+ "id": "l4GQ-M5yZ5UW"
+ },
+ "execution_count": 3,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "\n",
+ "# OpenAI gpt-3.5-turbo-1106\n",
+ "## Step 1: send the conversation and available functions to the model"
+ ],
+ "metadata": {
+ "id": "AxgR2fCgaRoW"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import litellm\n",
+ "import json\n",
+ "# Example dummy function hard coded to return the same weather\n",
+ "# In production, this could be your backend API or an external API\n",
+ "def get_current_weather(location, unit=\"fahrenheit\"):\n",
+ " \"\"\"Get the current weather in a given location\"\"\"\n",
+ " if \"tokyo\" in location.lower():\n",
+ " return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n",
+ " elif \"san francisco\" in location.lower():\n",
+ " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n",
+ " elif \"paris\" in location.lower():\n",
+ " return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n",
+ " else:\n",
+ " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n",
+ "\n",
+ "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n",
+ "tools = [\n",
+ " {\n",
+ " \"type\": \"function\",\n",
+ " \"function\": {\n",
+ " \"name\": \"get_current_weather\",\n",
+ " \"description\": \"Get the current weather in a given location\",\n",
+ " \"parameters\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": {\n",
+ " \"location\": {\n",
+ " \"type\": \"string\",\n",
+ " \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
+ " },\n",
+ " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
+ " },\n",
+ " \"required\": [\"location\"],\n",
+ " },\n",
+ " },\n",
+ " }\n",
+ "]\n",
+ "\n",
+ "response = litellm.completion(\n",
+ " model=\"gpt-3.5-turbo-1106\",\n",
+ " messages=messages,\n",
+ " tools=tools,\n",
+ " tool_choice=\"auto\", # auto is default, but we'll be explicit\n",
+ ")\n",
+ "print(\"\\nLLM Response1:\\n\", response)\n",
+ "response_message = response.choices[0].message\n",
+ "tool_calls = response.choices[0].message.tool_calls\n",
+ "print(\"\\nTool Choice:\\n\", tool_calls)\n"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Y3qteFo8ZrZP",
+ "outputId": "ee6c1183-55c1-4111-cdc0-967b8fed9db3"
+ },
+ "execution_count": 18,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "\n",
+ "LLM Response1:\n",
+ " ModelResponse(id='chatcmpl-8MNdPbrhtnwiPK1x3PEoGwrH144TW', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]))], created=1700344759, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 77, 'prompt_tokens': 88, 'total_tokens': 165}, _response_ms=1049.913)\n",
+ "\n",
+ "Tool Choice:\n",
+ " [ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Step 2 - Parse the Model Response and Execute Functions"
+ ],
+ "metadata": {
+ "id": "tD4lJQ40cU44"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Check if the model wants to call a function\n",
+ "if tool_calls:\n",
+ " # Execute the functions and prepare responses\n",
+ " available_functions = {\n",
+ " \"get_current_weather\": get_current_weather,\n",
+ " }\n",
+ "\n",
+ " messages.append(response_message) # Extend conversation with assistant's reply\n",
+ "\n",
+ " for tool_call in tool_calls:\n",
+ " print(f\"\\nExecuting tool call\\n{tool_call}\")\n",
+ " function_name = tool_call.function.name\n",
+ " function_to_call = available_functions[function_name]\n",
+ " function_args = json.loads(tool_call.function.arguments)\n",
+ " # calling the get_current_weather() function\n",
+ " function_response = function_to_call(\n",
+ " location=function_args.get(\"location\"),\n",
+ " unit=function_args.get(\"unit\"),\n",
+ " )\n",
+ " print(f\"Result from tool call\\n{function_response}\\n\")\n",
+ "\n",
+ " # Extend conversation with function response\n",
+ " messages.append(\n",
+ " {\n",
+ " \"tool_call_id\": tool_call.id,\n",
+ " \"role\": \"tool\",\n",
+ " \"name\": function_name,\n",
+ " \"content\": function_response,\n",
+ " }\n",
+ " )\n"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "af4oXQvicV_n",
+ "outputId": "abf6ac3e-4a21-4a4f-b8d7-809b763d0632"
+ },
+ "execution_count": 21,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "\n",
+ "Executing tool call\n",
+ "ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n",
+ "Result from tool call\n",
+ "{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n",
+ "\n",
+ "\n",
+ "Executing tool call\n",
+ "ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n",
+ "Result from tool call\n",
+ "{\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"}\n",
+ "\n",
+ "\n",
+ "Executing tool call\n",
+ "ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n",
+ "Result from tool call\n",
+ "{\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"}\n",
+ "\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Step 3 - Second litellm.completion() call"
+ ],
+ "metadata": {
+ "id": "E3OL1fqUdFdv"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "second_response = litellm.completion(\n",
+ " model=\"gpt-3.5-turbo-1106\",\n",
+ " messages=messages,\n",
+ ")\n",
+ "print(\"Second Response\\n\", second_response)\n",
+ "print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "8KYB2n-jc1_f",
+ "outputId": "6c6448ae-1c09-43ae-eb90-208b118e6179"
+ },
+ "execution_count": 26,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Second Response\n",
+ " ModelResponse(id='chatcmpl-8MNhat166ZqjO6egXcUh85Pd0s7KV', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\", role='assistant'))], created=1700345018, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 28, 'prompt_tokens': 465, 'total_tokens': 493}, _response_ms=999.246)\n",
+ "Second Response Message\n",
+ " The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Using Azure OpenAI"
+ ],
+ "metadata": {
+ "id": "1cIIFEvXjofp"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# set Azure env variables\n",
+ "import os\n",
+ "os.environ['AZURE_API_KEY'] = \"\" # litellm reads AZURE_API_KEY from .env and sends the request\n",
+ "os.environ['AZURE_API_BASE'] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
+ "os.environ['AZURE_API_VERSION'] = \"2023-07-01-preview\""
+ ],
+ "metadata": {
+ "id": "lG9mUnModeeE"
+ },
+ "execution_count": 32,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Step 1"
+ ],
+ "metadata": {
+ "id": "17S-Ysksj-E_"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import litellm\n",
+ "import json\n",
+ "# Example dummy function hard coded to return the same weather\n",
+ "# In production, this could be your backend API or an external API\n",
+ "def get_current_weather(location, unit=\"fahrenheit\"):\n",
+ " \"\"\"Get the current weather in a given location\"\"\"\n",
+ " if \"tokyo\" in location.lower():\n",
+ " return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n",
+ " elif \"san francisco\" in location.lower():\n",
+ " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n",
+ " elif \"paris\" in location.lower():\n",
+ " return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n",
+ " else:\n",
+ " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n",
+ "\n",
+ "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n",
+ "tools = [\n",
+ " {\n",
+ " \"type\": \"function\",\n",
+ " \"function\": {\n",
+ " \"name\": \"get_current_weather\",\n",
+ " \"description\": \"Get the current weather in a given location\",\n",
+ " \"parameters\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": {\n",
+ " \"location\": {\n",
+ " \"type\": \"string\",\n",
+ " \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
+ " },\n",
+ " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
+ " },\n",
+ " \"required\": [\"location\"],\n",
+ " },\n",
+ " },\n",
+ " }\n",
+ "]\n",
+ "\n",
+ "response = litellm.completion(\n",
+ " model=\"azure/chatgpt-functioncalling\", # model = azure/\n",
+ " messages=messages,\n",
+ " tools=tools,\n",
+ " tool_choice=\"auto\", # auto is default, but we'll be explicit\n",
+ ")\n",
+ "print(\"\\nLLM Response1:\\n\", response)\n",
+ "response_message = response.choices[0].message\n",
+ "tool_calls = response.choices[0].message.tool_calls\n",
+ "print(\"\\nTool Choice:\\n\", tool_calls)\n"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "boAIHLEXj80m",
+ "outputId": "00afcf09-5b6b-4805-c374-ba089cc6eb43"
+ },
+ "execution_count": 33,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "\n",
+ "LLM Response1:\n",
+ " ModelResponse(id='chatcmpl-8MOBPvEnqG7qitkmVqZmCrzSGEmDj', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]))], created=1700346867, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 19, 'prompt_tokens': 88, 'total_tokens': 107}, _response_ms=833.4319999999999)\n",
+ "\n",
+ "Tool Choice:\n",
+ " [ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Step 2"
+ ],
+ "metadata": {
+ "id": "hqh1y1IMkmGO"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Check if the model wants to call a function\n",
+ "if tool_calls:\n",
+ " # Execute the functions and prepare responses\n",
+ " available_functions = {\n",
+ " \"get_current_weather\": get_current_weather,\n",
+ " }\n",
+ "\n",
+ " messages.append(response_message) # Extend conversation with assistant's reply\n",
+ "\n",
+ " for tool_call in tool_calls:\n",
+ " print(f\"\\nExecuting tool call\\n{tool_call}\")\n",
+ " function_name = tool_call.function.name\n",
+ " function_to_call = available_functions[function_name]\n",
+ " function_args = json.loads(tool_call.function.arguments)\n",
+ " # calling the get_current_weather() function\n",
+ " function_response = function_to_call(\n",
+ " location=function_args.get(\"location\"),\n",
+ " unit=function_args.get(\"unit\"),\n",
+ " )\n",
+ " print(f\"Result from tool call\\n{function_response}\\n\")\n",
+ "\n",
+ " # Extend conversation with function response\n",
+ " messages.append(\n",
+ " {\n",
+ " \"tool_call_id\": tool_call.id,\n",
+ " \"role\": \"tool\",\n",
+ " \"name\": function_name,\n",
+ " \"content\": function_response,\n",
+ " }\n",
+ " )\n"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "FGu7DY7PkOiG",
+ "outputId": "96d39ae7-7fc8-4dd8-c82f-5ee9a486724c"
+ },
+ "execution_count": 34,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "\n",
+ "Executing tool call\n",
+ "ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')\n",
+ "Result from tool call\n",
+ "{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n",
+ "\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Step 3"
+ ],
+ "metadata": {
+ "id": "4MjYyeajkpBl"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "second_response = litellm.completion(\n",
+ " model=\"azure/chatgpt-functioncalling\",\n",
+ " messages=messages,\n",
+ ")\n",
+ "print(\"Second Response\\n\", second_response)\n",
+ "print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "qHgXyZq1kqGn",
+ "outputId": "61a30470-d7f5-484d-c42b-681c9b60b34a"
+ },
+ "execution_count": 36,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Second Response\n",
+ " ModelResponse(id='chatcmpl-8MOC90vwZ2LHX0DE796XYtsOxdGcc', choices=[Choices(finish_reason='stop', index=0, message=Message(content='The current weather in San Francisco is 72°F.', role='assistant'))], created=1700346913, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 11, 'prompt_tokens': 69, 'total_tokens': 80}, _response_ms=824.882)\n",
+ "Second Response Message\n",
+ " The current weather in San Francisco is 72°F.\n"
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/cookbook/Proxy_Batch_Users.ipynb b/cookbook/Proxy_Batch_Users.ipynb
new file mode 100644
index 000000000..c362ab8f8
--- /dev/null
+++ b/cookbook/Proxy_Batch_Users.ipynb
@@ -0,0 +1,205 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "680oRk1af-xJ"
+ },
+ "source": [
+ "# Environment Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "X7TgJFn8f88p"
+ },
+ "outputs": [],
+ "source": [
+ "import csv\n",
+ "from typing import Optional\n",
+ "import httpx\n",
+ "import json\n",
+ "import asyncio\n",
+ "\n",
+ "proxy_base_url = \"http://0.0.0.0:4000\" # š SET TO PROXY URL\n",
+ "master_key = \"sk-1234\" # š SET TO PROXY MASTER KEY"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "rauw8EOhgBz5"
+ },
+ "outputs": [],
+ "source": [
+ "## GLOBAL HTTP CLIENT ## - faster http calls\n",
+ "class HTTPHandler:\n",
+ " def __init__(self, concurrent_limit=1000):\n",
+ " # Create a client with a connection pool\n",
+ " self.client = httpx.AsyncClient(\n",
+ " limits=httpx.Limits(\n",
+ " max_connections=concurrent_limit,\n",
+ " max_keepalive_connections=concurrent_limit,\n",
+ " )\n",
+ " )\n",
+ "\n",
+ " async def close(self):\n",
+ " # Close the client when you're done with it\n",
+ " await self.client.aclose()\n",
+ "\n",
+ " async def get(\n",
+ " self, url: str, params: Optional[dict] = None, headers: Optional[dict] = None\n",
+ " ):\n",
+ " response = await self.client.get(url, params=params, headers=headers)\n",
+ " return response\n",
+ "\n",
+ " async def post(\n",
+ " self,\n",
+ " url: str,\n",
+ " data: Optional[dict] = None,\n",
+ " params: Optional[dict] = None,\n",
+ " headers: Optional[dict] = None,\n",
+ " ):\n",
+ " try:\n",
+ " response = await self.client.post(\n",
+ " url, data=data, params=params, headers=headers\n",
+ " )\n",
+ " return response\n",
+ " except Exception as e:\n",
+ " raise e\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "7LXN8zaLgOie"
+ },
+ "source": [
+ "# Import Sheet\n",
+ "\n",
+ "\n",
+ "Format: | ID | Name | Max Budget |"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "oiED0usegPGf"
+ },
+ "outputs": [],
+ "source": [
+ "async def import_sheet():\n",
+ " tasks = []\n",
+ " http_client = HTTPHandler()\n",
+ " with open('my-batch-sheet.csv', 'r') as file:\n",
+ " csv_reader = csv.DictReader(file)\n",
+ " for row in csv_reader:\n",
+ " task = create_user(client=http_client, user_id=row['ID'], max_budget=row['Max Budget'], user_name=row['Name'])\n",
+ " tasks.append(task)\n",
+ " # print(f\"ID: {row['ID']}, Name: {row['Name']}, Max Budget: {row['Max Budget']}\")\n",
+ "\n",
+ " keys = await asyncio.gather(*tasks)\n",
+ "\n",
+ " with open('my-batch-sheet_new.csv', 'w', newline='') as new_file:\n",
+ " fieldnames = ['ID', 'Name', 'Max Budget', 'keys']\n",
+ " csv_writer = csv.DictWriter(new_file, fieldnames=fieldnames)\n",
+ " csv_writer.writeheader()\n",
+ "\n",
+ " with open('my-batch-sheet.csv', 'r') as file:\n",
+ " csv_reader = csv.DictReader(file)\n",
+ " for i, row in enumerate(csv_reader):\n",
+ " row['keys'] = keys[i] # Add the 'keys' value from the corresponding task result\n",
+ " csv_writer.writerow(row)\n",
+ "\n",
+ " await http_client.close()\n",
+ "\n",
+ "asyncio.run(import_sheet())"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "E7M0Li_UgJeZ"
+ },
+ "source": [
+ "# Create Users + Keys\n",
+ "\n",
+ "- Creates a user\n",
+ "- Creates a key with max budget"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "NZudRFujf7j-"
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "async def create_key_with_alias(client: HTTPHandler, user_id: str, max_budget: float):\n",
+ " global proxy_base_url\n",
+ " if not proxy_base_url.endswith(\"/\"):\n",
+ " proxy_base_url += \"/\"\n",
+ " url = proxy_base_url + \"key/generate\"\n",
+ "\n",
+ " # call /key/generate\n",
+ " print(\"CALLING /KEY/GENERATE\")\n",
+ " response = await client.post(\n",
+ " url=url,\n",
+ " headers={\"Authorization\": f\"Bearer {master_key}\"},\n",
+ " data=json.dumps({\n",
+ " \"user_id\": user_id,\n",
+ " \"key_alias\": f\"{user_id}-key\",\n",
+ " \"max_budget\": max_budget # š KEY CHANGE: SETS MAX BUDGET PER KEY\n",
+ " })\n",
+ " )\n",
+ " print(f\"response: {response.text}\")\n",
+ " return response.json()[\"key\"]\n",
+ "\n",
+ "async def create_user(client: HTTPHandler, user_id: str, max_budget: float, user_name: str):\n",
+ " \"\"\"\n",
+ " - call /user/new\n",
+ " - create key for user\n",
+ " \"\"\"\n",
+ " global proxy_base_url\n",
+ " if not proxy_base_url.endswith(\"/\"):\n",
+ " proxy_base_url += \"/\"\n",
+ " url = proxy_base_url + \"user/new\"\n",
+ "\n",
+ " # call /user/new\n",
+ " await client.post(\n",
+ " url=url,\n",
+ " headers={\"Authorization\": f\"Bearer {master_key}\"},\n",
+ " data=json.dumps({\n",
+ " \"user_id\": user_id,\n",
+ " \"user_alias\": user_name,\n",
+ " \"auto_create_key\": False,\n",
+ " # \"max_budget\": max_budget # š [OPTIONAL] Sets max budget per user (if you want to set a max budget across keys)\n",
+ " })\n",
+ " )\n",
+ "\n",
+ " # create key for user\n",
+ " return await create_key_with_alias(client=client, user_id=user_id, max_budget=max_budget)\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/cookbook/TogetherAI_liteLLM.ipynb b/cookbook/TogetherAI_liteLLM.ipynb
new file mode 100644
index 000000000..d47009149
--- /dev/null
+++ b/cookbook/TogetherAI_liteLLM.ipynb
@@ -0,0 +1,1006 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "WemkFEdDAnJL"
+ },
+ "source": [
+ "## liteLLM Together AI Tutorial\n",
+ "https://together.ai/\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "pc6IO4V99O25",
+ "outputId": "2d69da44-010b-41c2-b38b-5b478576bb8b"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Collecting litellm\n",
+ " Downloading litellm-0.1.482-py3-none-any.whl (69 kB)\n",
+ "\u001b[2K \u001b[90māāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā\u001b[0m \u001b[32m69.3/69.3 kB\u001b[0m \u001b[31m757.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: importlib-metadata<7.0.0,>=6.8.0 in /usr/local/lib/python3.10/dist-packages (from litellm) (6.8.0)\n",
+ "Collecting openai<0.28.0,>=0.27.8 (from litellm)\n",
+ " Downloading openai-0.27.9-py3-none-any.whl (75 kB)\n",
+ "\u001b[2K \u001b[90māāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā\u001b[0m \u001b[32m75.5/75.5 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting python-dotenv<2.0.0,>=1.0.0 (from litellm)\n",
+ " Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
+ "Collecting tiktoken<0.5.0,>=0.4.0 (from litellm)\n",
+ " Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
+ "\u001b[2K \u001b[90māāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0.0,>=6.8.0->litellm) (3.16.2)\n",
+ "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (2.31.0)\n",
+ "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (4.66.1)\n",
+ "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (3.8.5)\n",
+ "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm) (2023.6.3)\n",
+ "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.2.0)\n",
+ "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.4)\n",
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2.0.4)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2023.7.22)\n",
+ "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (23.1.0)\n",
+ "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (6.0.4)\n",
+ "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (4.0.3)\n",
+ "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.9.2)\n",
+ "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.4.0)\n",
+ "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.3.1)\n",
+ "Installing collected packages: python-dotenv, tiktoken, openai, litellm\n",
+ "Successfully installed litellm-0.1.482 openai-0.27.9 python-dotenv-1.0.0 tiktoken-0.4.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install litellm"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "TMI3739_9q97"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "from litellm import completion\n",
+ "os.environ[\"TOGETHERAI_API_KEY\"] = \"\" #@param\n",
+ "user_message = \"Hello, whats the weather in San Francisco??\"\n",
+ "messages = [{ \"content\": user_message,\"role\": \"user\"}]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "bEqJ2HHjBJqq"
+ },
+ "source": [
+ "## Calling togethercomputer/llama-2-70b-chat\n",
+ "https://api.together.xyz/playground/chat?model=togethercomputer%2Fllama-2-70b-chat"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Jrrt8puj523f",
+ "outputId": "24494dea-816f-47a6-ade4-1b04f2e9085b"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{\n",
+ " 'choices': [\n",
+ "{\n",
+ " 'finish_reason': 'stop',\n",
+ " 'index': 0,\n",
+ " 'message': {\n",
+ " 'role': 'assistant',\n",
+ " 'content': \"\n",
+ "\n",
+ "I'm not able to provide real-time weather information. However, I can suggest some ways for you to find out the current weather in San Francisco.\n",
+ "\n",
+ "1. Check online weather websites: There are many websites that provide up-to-date weather information, such as AccuWeather, Weather.com, or the National Weather Service. You can enter \"San Francisco\" in the search bar and get the current weather conditions, forecast, and radar imagery.\n",
+ "2. Use a weather app: You can download a weather app on your smartphone that provides real-time weather information. Some popular weather apps include Dark Sky, Weather Underground, and The Weather Channel.\n",
+ "3. Tune into local news: You can watch local news channels or listen to local radio stations to get the latest weather forecast and current conditions.\n",
+ "4. Check social media: Follow local weather accounts on social media platforms like Twitter or Facebook to\"\n",
+ "}\n",
+ "}\n",
+ " ],\n",
+ " 'created': 1692323365.8261144,\n",
+ " 'model': 'togethercomputer/llama-2-70b-chat',\n",
+ " 'usage': {'prompt_tokens': 9, 'completion_tokens': 176, 'total_tokens': 185}\n",
+ "}\n"
+ ]
+ }
+ ],
+ "source": [
+ "model_name = \"togethercomputer/llama-2-70b-chat\"\n",
+ "response = completion(model=model_name, messages=messages, max_tokens=200)\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "GIUevHlMvPb8",
+ "outputId": "ad930a12-16e3-4400-fff4-38151e4f6da5"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[92mHere's your LiteLLM Dashboard š \u001b[94m\u001b[4mhttps://admin.litellm.ai/6c0f0403-becb-44af-9724-7201c7d381d0\u001b[0m\n",
+ "{\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \"\\nI'm in San Francisco, and I'm not sure what the weather is like.\\nI'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and\",\n",
+ " \"role\": \"assistant\"\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"created\": 1692934243.8663018,\n",
+ " \"model\": \"togethercomputer/CodeLlama-34b-Instruct\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 9,\n",
+ " \"completion_tokens\": 178,\n",
+ " \"total_tokens\": 187\n",
+ " }\n",
+ "}\n"
+ ]
+ }
+ ],
+ "source": [
+ "model_name = \"togethercomputer/CodeLlama-34b-Instruct\"\n",
+ "response = completion(model=model_name, messages=messages, max_tokens=200)\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "sfWtgf-mBQcM"
+ },
+ "source": [
+ "## With Streaming"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "background_save": true,
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "wuBhlZtC6MH5",
+ "outputId": "8f4a408c-25eb-4434-cdd4-7b4ae4f6d3aa"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Com'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'bin'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ator'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' ('}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ')'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' two'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' popular'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' gained'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' recognition'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' effect'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'iveness'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'urt'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'uring'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' scaling'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' early'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'stage'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ities'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' also'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' distinct'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' differences'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' set'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' them'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' apart'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' In'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' this'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' ess'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ay'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' we'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' will'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' explore'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' key'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' features'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' discuss'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' which'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' might'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Com'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'bin'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ator'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' one'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' most'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' successful'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' world'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' port'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'folio'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' includes'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Air'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'b'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'nb'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Drop'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'box'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Red'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'dit'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' F'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ounded'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' '}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '2'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '5'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ed'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' over'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' '}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '9'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' start'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ups'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' combined'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' valu'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ation'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' over'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' $'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' billion'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' The'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' known'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' inten'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'se'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' three'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'month'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' boot'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' camp'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'style'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' format'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' where'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' work'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' closely'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' experienced'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' ment'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ors'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' develop'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' products'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' ref'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ine'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' business'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' models'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' prepare'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ra'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ising'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' software'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' technology'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' internet'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' start'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ups'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' strong'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' track'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' record'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' ident'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ifying'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'urt'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'uring'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' successful'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' these'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' spaces'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' other'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' hand'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' relatively'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' new'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ator'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' was'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' founded'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' '}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '2'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '7'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' While'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' it'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' not'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' same'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' level'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' brand'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' recognition'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' as'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' quickly'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' gained'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' reputation'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' unique'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' approach'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceleration'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' The'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'es'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' supporting'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' under'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 're'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'present'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ed'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' particularly'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' women'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' people'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' color'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' provides'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' range'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' support'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' help'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' these'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' succeed'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' designed'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' more'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' flexible'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' personal'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ized'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' than'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' traditional'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' connecting'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' ment'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ors'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' tail'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ored'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' needs'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'One'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' key'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' difference'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' between'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' type'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' support'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'es'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' primarily'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' software'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' technology'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' internet'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' start'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ups'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' while'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' bro'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ader'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' includes'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' range'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' indust'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ries'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' such'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' as'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' health'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'care'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fin'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ance'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' consumer'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' products'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' This'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' means'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' non'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'tech'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' industry'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'An'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'other'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' difference'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' between'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' two'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' programs'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' approach'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' provides'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' seed'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' all'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' port'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'folio'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' typically'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' range'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' $'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' $'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '2'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' In'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' contrast'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' does'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' not'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' provide'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' port'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'folio'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' but'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' instead'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'es'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' connecting'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' invest'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ors'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' help'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' them'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' raise'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' capital'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' This'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' means'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' looking'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' option'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'So'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' which'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' right'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '?'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' It'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' ultimately'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' depends'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' needs'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' goals'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' If'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' non'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'tech'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' industry'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' bro'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ader'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Additionally'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 're'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' looking'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' more'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' personal'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ized'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' flexible'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' approach'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceleration'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' choice'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' On'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' other'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' hand'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' software'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' technology'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' or'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' internet'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' space'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 're'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' looking'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' seed'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'In'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' conclusion'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' both'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' excellent'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' provide'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' valuable'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' support'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' early'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'stage'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' While'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' share'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' some'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' similar'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 'ities'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' also'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' distinct'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' differences'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' set'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' them'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' apart'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' By'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' considering'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' needs'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' goals'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' determine'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' which'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' best'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': ' business'}}]}\n",
+ "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n"
+ ]
+ }
+ ],
+ "source": [
+ "user_message = \"Write 1page essay on YC + liteLLM\"\n",
+ "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n",
+ "\n",
+ "\n",
+ "async def parse_stream(stream):\n",
+ " async for elem in stream:\n",
+ " print(elem)\n",
+ " return\n",
+ "\n",
+ "stream = completion(model=\"togethercomputer/llama-2-70b-chat\", messages=messages, stream=True, max_tokens=800)\n",
+ "print(stream)\n",
+ "\n",
+ "# Await the asynchronous function directly in the notebook cell\n",
+ "await parse_stream(stream)\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb b/cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb
new file mode 100644
index 000000000..0c3ff97a3
--- /dev/null
+++ b/cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb
@@ -0,0 +1,157 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eKXncoQbU_2j"
+ },
+ "source": [
+ "# Using Nemo-Guardrails with LiteLLM Server\n",
+ "\n",
+ "[Call Bedrock, TogetherAI, Huggingface, etc. on the server](https://docs.litellm.ai/docs/providers)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ZciYaLwvuFbu"
+ },
+ "source": [
+ "## Using with Bedrock\n",
+ "\n",
+ "`docker run -e PORT=8000 -e AWS_ACCESS_KEY_ID= -e AWS_SECRET_ACCESS_KEY= -p 8000:8000 ghcr.io/berriai/litellm:latest`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "vOUwGSJ2Vsy3"
+ },
+ "outputs": [],
+ "source": [
+ "pip install nemoguardrails langchain"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "xXEJNxe7U0IN"
+ },
+ "outputs": [],
+ "source": [
+ "from langchain.chat_models import ChatOpenAI\n",
+ "\n",
+ "llm = ChatOpenAI(model_name=\"anthropic.claude-v2\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-fake-key\")\n",
+ "\n",
+ "from nemoguardrails import LLMRails, RailsConfig\n",
+ "\n",
+ "config = RailsConfig.from_path(\"./config.yml\")\n",
+ "app = LLMRails(config, llm=llm)\n",
+ "\n",
+ "new_message = app.generate(messages=[{\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"Hello! What can you do for me?\"\n",
+ "}])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "vz5n00qyuKjp"
+ },
+ "source": [
+ "## Using with TogetherAI\n",
+ "\n",
+ "1. You can either set this in the server environment:\n",
+ "`docker run -e PORT=8000 -e TOGETHERAI_API_KEY= -p 8000:8000 ghcr.io/berriai/litellm:latest`\n",
+ "\n",
+ "2. **Or** Pass this in as the api key `(...openai_api_key=\"\")`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "XK1sk-McuhpE"
+ },
+ "outputs": [],
+ "source": [
+ "from langchain.chat_models import ChatOpenAI\n",
+ "\n",
+ "llm = ChatOpenAI(model_name=\"together_ai/togethercomputer/CodeLlama-13b-Instruct\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-together-ai-api-key\")\n",
+ "\n",
+ "from nemoguardrails import LLMRails, RailsConfig\n",
+ "\n",
+ "config = RailsConfig.from_path(\"./config.yml\")\n",
+ "app = LLMRails(config, llm=llm)\n",
+ "\n",
+ "new_message = app.generate(messages=[{\n",
+ " \"role\": \"user\",\n",
+ " \"content\": \"Hello! What can you do for me?\"\n",
+ "}])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "8A1KWKnzuxAS"
+ },
+ "source": [
+ "### CONFIG.YML\n",
+ "\n",
+ "save this example `config.yml` in your current directory"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "NKN1GmSvu0Cx"
+ },
+ "outputs": [],
+ "source": [
+ "# instructions:\n",
+ "# - type: general\n",
+ "# content: |\n",
+ "# Below is a conversation between a bot and a user about the recent job reports.\n",
+ "# The bot is factual and concise. If the bot does not know the answer to a\n",
+ "# question, it truthfully says it does not know.\n",
+ "\n",
+ "# sample_conversation: |\n",
+ "# user \"Hello there!\"\n",
+ "# express greeting\n",
+ "# bot express greeting\n",
+ "# \"Hello! How can I assist you today?\"\n",
+ "# user \"What can you do for me?\"\n",
+ "# ask about capabilities\n",
+ "# bot respond about capabilities\n",
+ "# \"I am an AI assistant that helps answer mathematical questions. My core mathematical skills are powered by wolfram alpha.\"\n",
+ "# user \"What's 2+2?\"\n",
+ "# ask math question\n",
+ "# bot responds to math question\n",
+ "# \"2+2 is equal to 4.\"\n",
+ "\n",
+ "# models:\n",
+ "# - type: main\n",
+ "# engine: openai\n",
+ "# model: claude-instant-1"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/cookbook/VLLM_Model_Testing.ipynb b/cookbook/VLLM_Model_Testing.ipynb
new file mode 100644
index 000000000..0cacac661
--- /dev/null
+++ b/cookbook/VLLM_Model_Testing.ipynb
@@ -0,0 +1,404 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "provenance": [],
+ "machine_shape": "hm",
+ "gpuType": "V100"
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ },
+ "language_info": {
+ "name": "python"
+ },
+ "accelerator": "GPU"
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Set up Environment"
+ ],
+ "metadata": {
+ "id": "vDOm5wfjdFLP"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!pip install --upgrade litellm"
+ ],
+ "metadata": {
+ "id": "Bx6mAA6MHiy_"
+ },
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "zIYv7JTyxSxR",
+ "outputId": "53890320-f9fa-4bf4-8362-0f17f52c6ed4"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Successfully installed fastapi-0.103.1 h11-0.14.0 huggingface-hub-0.16.4 ninja-1.11.1 pydantic-1.10.12 ray-2.6.3 safetensors-0.3.3 sentencepiece-0.1.99 starlette-0.27.0 tokenizers-0.13.3 transformers-4.33.1 uvicorn-0.23.2 vllm-0.1.4 xformers-0.0.21\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install vllm"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Load the Logs"
+ ],
+ "metadata": {
+ "id": "RMcoAni6WKEx"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import pandas as pd"
+ ],
+ "metadata": {
+ "id": "zchxB8c7WJe5"
+ },
+ "execution_count": 4,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# path of the csv file\n",
+ "file_path = 'Model-prompts-example.csv'\n",
+ "\n",
+ "# load the csv file as a pandas DataFrame\n",
+ "data = pd.read_csv(file_path)\n",
+ "\n",
+ "data.head()"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 81
+ },
+ "id": "aKcWr015WNPm",
+ "outputId": "6e226773-333f-46a2-9fc8-4f54f309d204"
+ },
+ "execution_count": 6,
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ " Success Timestamp Input \\\n",
+ "0 True 1694041195 This is the templated query input \n",
+ "\n",
+ " Output RunId (Wandb Runid) \\\n",
+ "0 This is the query output from the model 8hlumwuk \n",
+ "\n",
+ " Model ID (or Name) \n",
+ "0 OpenAI/Turbo-3.5 "
+ ],
+ "text/html": [
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " Success \n",
+ " Timestamp \n",
+ " Input \n",
+ " Output \n",
+ " RunId (Wandb Runid) \n",
+ " Model ID (or Name) \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 \n",
+ " True \n",
+ " 1694041195 \n",
+ " This is the templated query input \n",
+ " This is the query output from the model \n",
+ " 8hlumwuk \n",
+ " OpenAI/Turbo-3.5 \n",
+ " \n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
+ ]
+ },
+ "metadata": {},
+ "execution_count": 6
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "input_texts = data['Input'].values"
+ ],
+ "metadata": {
+ "id": "0DbL-kirWUyn"
+ },
+ "execution_count": 7,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "messages = [[{\"role\": \"user\", \"content\": input_text}] for input_text in input_texts]"
+ ],
+ "metadata": {
+ "id": "cqpAvy8hWXyC"
+ },
+ "execution_count": 8,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Running Inference"
+ ],
+ "metadata": {
+ "id": "SugCyom0Xy8U"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "from litellm import batch_completion\n",
+ "model_name = \"facebook/opt-125m\"\n",
+ "provider = \"vllm\"\n",
+ "response_list = batch_completion(\n",
+ " model=model_name,\n",
+ " custom_llm_provider=provider, # can easily switch to huggingface, replicate, together ai, sagemaker, etc.\n",
+ " messages=messages,\n",
+ " temperature=0.2,\n",
+ " max_tokens=80,\n",
+ " )"
+ ],
+ "metadata": {
+ "id": "qpikx3uxHns3"
+ },
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "response_list"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "QDPikHtwKJJ2",
+ "outputId": "06f47c44-e258-452a-f9db-232a5b6d2810"
+ },
+ "execution_count": 10,
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ "[ JSON: {\n",
+ " \"choices\": [\n",
+ " {\n",
+ " \"finish_reason\": \"stop\",\n",
+ " \"index\": 0,\n",
+ " \"message\": {\n",
+ " \"content\": \".\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is\",\n",
+ " \"role\": \"assistant\",\n",
+ " \"logprobs\": null\n",
+ " }\n",
+ " }\n",
+ " ],\n",
+ " \"created\": 1694053363.6139505,\n",
+ " \"model\": \"facebook/opt-125m\",\n",
+ " \"usage\": {\n",
+ " \"prompt_tokens\": 9,\n",
+ " \"completion_tokens\": 80,\n",
+ " \"total_tokens\": 89\n",
+ " }\n",
+ " }]"
+ ]
+ },
+ "metadata": {},
+ "execution_count": 10
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "response_values = [response['choices'][0]['message']['content'] for response in response_list]"
+ ],
+ "metadata": {
+ "id": "SYqTcCiJbQDF"
+ },
+ "execution_count": 11,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "response_values"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "wqs-Oy9FbiPo",
+ "outputId": "16a6a7b7-97c8-4b5b-eff8-09ea5eb5ad06"
+ },
+ "execution_count": 12,
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ "['.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is']"
+ ]
+ },
+ "metadata": {},
+ "execution_count": 12
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "data[f\"{model_name}_output\"] = response_values"
+ ],
+ "metadata": {
+ "id": "mElNbBehbkrz"
+ },
+ "execution_count": 13,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "data.to_csv('model_responses.csv', index=False)"
+ ],
+ "metadata": {
+ "id": "F06NXssDc45k"
+ },
+ "execution_count": 14,
+ "outputs": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
new file mode 100644
index 000000000..b2d81be25
--- /dev/null
+++ b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
@@ -0,0 +1,295 @@
+# Claude Code with LiteLLM Quickstart
+
+This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
+
+> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
+
+## Video Walkthrough
+
+Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
+
+## Prerequisites
+
+- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
+- API keys for your chosen providers
+
+## Installation
+
+First, install LiteLLM with proxy support:
+
+```bash
+pip install 'litellm[proxy]'
+```
+
+## Step 1: Setup config.yaml
+
+Create a secure configuration using environment variables:
+
+```yaml
+model_list:
+ # Claude models
+ - model_name: claude-3-5-sonnet-20241022
+ litellm_params:
+ model: anthropic/claude-3-5-sonnet-20241022
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ - model_name: claude-3-5-haiku-20241022
+ litellm_params:
+ model: anthropic/claude-3-5-haiku-20241022
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+
+litellm_settings:
+ master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+Set your environment variables:
+
+```bash
+export ANTHROPIC_API_KEY="your-anthropic-api-key"
+export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
+```
+
+## Step 2: Start Proxy
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+## Step 3: Verify Setup
+
+Test that your proxy is working correctly:
+
+```bash
+curl -X POST http://0.0.0.0:4000/v1/messages \
+-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+-H "Content-Type: application/json" \
+-d '{
+ "model": "claude-3-5-sonnet-20241022",
+ "max_tokens": 1000,
+ "messages": [{"role": "user", "content": "What is the capital of France?"}]
+}'
+```
+
+## Step 4: Configure Claude Code
+
+### Method 1: Unified Endpoint (Recommended)
+
+Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
+
+```bash
+export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
+export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
+```
+
+> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
+
+### Method 2: Provider-specific Pass-through Endpoint
+
+Alternatively, use the Anthropic pass-through endpoint:
+
+```bash
+export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
+export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
+```
+
+## Step 5: Use Claude Code
+
+### Choosing Your Model
+
+You have two options for specifying which model Claude Code uses:
+
+#### Option 1: Command Line / Session Model Selection
+
+Specify the model directly when starting Claude Code or during a session:
+
+```bash
+# Specify model at startup
+claude --model claude-3-5-sonnet-20241022
+
+# Or change model during a session
+/model claude-3-5-haiku-20241022
+```
+
+This method uses the exact model you specify.
+
+#### Option 2: Environment Variables
+
+Configure default models using environment variables:
+
+```bash
+# Tell Claude Code which models to use by default
+export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
+export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
+export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229
+
+claude # Will use the models specified above
+```
+
+**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model.
+
+**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line).
+
+### Using 1M Context Window
+
+Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models:
+
+```bash
+# Use Sonnet 4.5 with 1M context (requires quotes for shell)
+claude --model 'claude-sonnet-4-5-20250929[1m]'
+
+# Inside a Claude Code session (no quotes needed)
+/model claude-sonnet-4-5-20250929[1m]
+```
+
+**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets.
+
+Alternatively, set as default with environment variables:
+
+```bash
+export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]'
+claude
+```
+
+**How it works:**
+- Claude Code strips the `[1m]` suffix before sending to LiteLLM
+- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07`
+- Your LiteLLM config should **NOT** include `[1m]` in model names
+
+**Verify 1M context is active:**
+```bash
+/context
+# Should show: 21k/1000k tokens (2%)
+```
+
+**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate.
+
+## Troubleshooting
+
+Common issues and solutions:
+
+**Claude Code not connecting:**
+- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
+- Check that `ANTHROPIC_BASE_URL` is set correctly
+- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
+
+**Authentication errors:**
+- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
+- Check that your API keys are valid and have sufficient credits
+- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
+
+**Model not found:**
+- Check what model Claude Code is requesting in LiteLLM logs
+- Ensure your `config.yaml` has a matching `model_name` entry
+- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL`
+
+**1M context not working (showing 200k instead of 1000k):**
+- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]`
+- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request
+- Ensure your model supports 1M context (only certain Claude models do)
+- Your LiteLLM config should **NOT** include `[1m]` in the `model_name`
+
+## Using Multiple Models and Providers
+
+You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers:
+
+```yaml
+model_list:
+ # OpenAI models
+ - model_name: codex-mini
+ litellm_params:
+ model: openai/codex-mini
+ api_key: os.environ/OPENAI_API_KEY
+ api_base: https://api.openai.com/v1
+
+ - model_name: o3-pro
+ litellm_params:
+ model: openai/o3-pro
+ api_key: os.environ/OPENAI_API_KEY
+ api_base: https://api.openai.com/v1
+
+ - model_name: gpt-4o
+ litellm_params:
+ model: openai/gpt-4o
+ api_key: os.environ/OPENAI_API_KEY
+ api_base: https://api.openai.com/v1
+
+ # Anthropic models
+ - model_name: claude-3-5-sonnet-20241022
+ litellm_params:
+ model: anthropic/claude-3-5-sonnet-20241022
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ - model_name: claude-3-5-haiku-20241022
+ litellm_params:
+ model: anthropic/claude-3-5-haiku-20241022
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ # AWS Bedrock
+ - model_name: claude-bedrock
+ litellm_params:
+ model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: us-east-1
+
+litellm_settings:
+ master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`.
+
+Switch between models seamlessly:
+
+```bash
+# Use environment variables to set defaults
+export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
+export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
+
+# Or specify directly
+claude --model claude-3-5-sonnet-20241022 # Complex reasoning
+claude --model claude-3-5-haiku-20241022 # Fast responses
+claude --model claude-bedrock # Bedrock deployment
+```
+
+## Default Models Used by Claude Code
+
+If you **don't** set environment variables, Claude Code uses these default model names:
+
+| Purpose | Default Model Name (v2.1.14) |
+|---------|------------------------------|
+| Main model | `claude-sonnet-4-5-20250929` |
+| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` |
+| Planning mode | `claude-opus-4-5-20251101` |
+
+Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables:
+
+```yaml
+model_list:
+ - model_name: claude-sonnet-4-5-20250929
+ litellm_params:
+ # Can be any provider - Anthropic, Bedrock, Vertex AI, etc.
+ model: anthropic/claude-sonnet-4-5-20250929
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ - model_name: claude-haiku-4-5-20251001
+ litellm_params:
+ model: anthropic/claude-haiku-4-5-20251001
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ - model_name: claude-opus-4-5-20251101
+ litellm_params:
+ model: anthropic/claude-opus-4-5-20251101
+ api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting.
+
+## Additional Resources
+
+- [LiteLLM Documentation](https://docs.litellm.ai/)
+- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
+- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)
+
diff --git a/cookbook/ai_coding_tool_guides/index.json b/cookbook/ai_coding_tool_guides/index.json
new file mode 100644
index 000000000..3e71670d6
--- /dev/null
+++ b/cookbook/ai_coding_tool_guides/index.json
@@ -0,0 +1,134 @@
+[{
+ "title": "Claude Code Quickstart",
+ "description": "This is a quickstart guide to using Claude Code with LiteLLM.",
+ "url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
+ "date": "2026-01-15",
+ "version": "1.0.0",
+ "tags": [
+ "Claude Code",
+ "LiteLLM"
+ ]
+},
+{
+ "title": "Claude Code with MCPs",
+ "description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
+ "url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
+ "date": "2026-01-15",
+ "version": "1.0.0",
+ "tags": [
+ "Claude Code",
+ "LiteLLM",
+ "MCP"
+ ]
+},
+{
+ "title": "Claude Code with Non-Anthropic Models",
+ "description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
+ "url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
+ "date": "2026-01-16",
+ "version": "1.0.0",
+ "tags": [
+ "Claude Code",
+ "LiteLLM",
+ "OpenAI",
+ "Gemini"
+ ]
+},
+{
+ "title": "Cursor Quickstart",
+ "description": "This is a quickstart guide to using Cursor with LiteLLM.",
+ "url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
+ "date": "2026-01-16",
+ "version": "1.0.0",
+ "tags": [
+ "Cursor",
+ "LiteLLM",
+ "Quickstart"
+ ]
+},
+{
+ "title": "Github Copilot Quickstart",
+ "description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
+ "url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
+ "date": "2026-01-16",
+ "version": "1.0.0",
+ "tags": [
+ "Github Copilot",
+ "LiteLLM",
+ "Quickstart"
+ ]
+},
+{
+ "title": "LiteLLM Gemini CLI Quickstart",
+ "description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
+ "url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
+ "date": "2026-01-16",
+ "version": "1.0.0",
+ "tags": [
+ "Gemini CLI",
+ "Gemini",
+ "LiteLLM",
+ "Quickstart"
+ ]
+},
+{
+ "title": "OpenAI Codex CLI Quickstart",
+ "description": "This is a quickstart guide to using OpenAI Codex CLI.",
+ "url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
+ "date": "2026-01-16",
+ "version": "1.0.0",
+ "tags": [
+ "OpenAI Codex CLI",
+ "OpenAI",
+ "LiteLLM",
+ "Quickstart"
+ ]
+},
+{
+ "title": "OpenWebUI Quickstart",
+ "description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
+ "url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
+ "date": "2026-01-16",
+ "version": "1.0.0",
+ "tags": [
+ "OpenWebUI",
+ "LiteLLM",
+ "Quickstart"
+ ]
+},
+{
+ "title": "AI Coding Tool Usage Tracking",
+ "description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.",
+ "url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding",
+ "date": "2026-01-17",
+ "version": "1.0.0",
+ "tags": [
+ "Claude Code",
+ "Gemini CLI",
+ "OpenAI Codex",
+ "LiteLLM"
+ ]
+},
+{
+ "title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)",
+ "description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
+ "url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
+ "date": "2026-01-17",
+ "version": "1.0.0",
+ "tags": [
+ "Claude Code",
+ "LiteLLM",
+ "Web Search"
+ ]
+},
+{
+ "title": "Track Claude Code Usage per user via Custom Headers",
+ "description": "This is a guide for tracking claude code user usage by passing a customer ID header.",
+ "url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking",
+ "date": "2026-01-17",
+ "version": "1.0.0",
+ "tags": [
+ "Claude Code",
+ "LiteLLM"
+ ]
+}]
\ No newline at end of file
diff --git a/cookbook/anthropic_agent_sdk/README.md b/cookbook/anthropic_agent_sdk/README.md
new file mode 100644
index 000000000..294d949e2
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/README.md
@@ -0,0 +1,144 @@
+# Claude Agent SDK with LiteLLM Gateway
+
+A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
+
+## Quick Start
+
+### 1. Install dependencies
+
+```bash
+pip install anthropic claude-agent-sdk litellm
+```
+
+### 2. Start LiteLLM proxy
+
+```bash
+# Simple start with Claude
+litellm --model claude-sonnet-4-20250514
+
+# Or with a config file
+litellm --config config.yaml
+```
+
+### 3. Run the chat
+
+**Basic Agent (no MCP):**
+
+```bash
+python main.py
+```
+
+**Agent with MCP (DeepWiki2 for research):**
+
+```bash
+python agent_with_mcp.py
+```
+
+If MCP connection fails, you can disable it:
+
+```bash
+USE_MCP=false python agent_with_mcp.py
+```
+
+That's it! You can now chat with the agent in your terminal.
+
+### Chat Commands
+
+While chatting, you can use these commands:
+- `models` - List all available models (fetched from your LiteLLM proxy)
+- `model` - Switch to a different model
+- `clear` - Start a new conversation
+- `quit` or `exit` - End the chat
+
+The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
+
+## Configuration
+
+Set these environment variables if needed:
+
+```bash
+export LITELLM_PROXY_URL="http://localhost:4000"
+export LITELLM_API_KEY="sk-1234"
+export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
+```
+
+Or just use the defaults - it'll connect to `http://localhost:4000` by default.
+
+## Files
+
+- `main.py` - Basic interactive agent without MCP
+- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2)
+- `common.py` - Shared utilities and functions
+- `config.example.yaml` - Example LiteLLM configuration
+- `requirements.txt` - Python dependencies
+
+## Example Config File
+
+If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
+
+```yaml
+model_list:
+ - model_name: bedrock-claude-sonnet-4
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-sonnet-4.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+ aws_region_name: "us-east-1"
+```
+
+Then start LiteLLM with: `litellm --config config.yaml`
+
+## How It Works
+
+The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
+
+```python
+# Point to LiteLLM gateway (not Anthropic)
+os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
+os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
+
+# Use any model configured in LiteLLM
+options = ClaudeAgentOptions(
+ model="bedrock-claude-sonnet-4", # or gpt-4, or anything else
+ system_prompt="You are a helpful assistant.",
+ max_turns=50,
+)
+```
+
+Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
+
+## Why Use This?
+
+- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
+- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
+- **Rate limiting**: Set budgets and limits on your agent usage
+- **Load balancing**: Distribute requests across multiple API keys or regions
+- **Fallbacks**: Automatically retry with a different model if one fails
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model your-model`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Authentication errors?**
+- Verify your LiteLLM API key is correct
+- Make sure the model is configured in your LiteLLM setup
+
+**Model not found?**
+- Check the model name matches what's in your LiteLLM config
+- Run `litellm --model your-model` to test it works
+
+**Agent with MCP stuck or failing?**
+- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2`
+- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py`
+- Or use the basic agent: `python main.py`
+
+## Learn More
+
+- [LiteLLM Docs](https://docs.litellm.ai/)
+- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
+- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)
diff --git a/cookbook/anthropic_agent_sdk/agent_with_mcp.py b/cookbook/anthropic_agent_sdk/agent_with_mcp.py
new file mode 100644
index 000000000..8a7513c78
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/agent_with_mcp.py
@@ -0,0 +1,144 @@
+"""
+Interactive Claude Agent SDK CLI with MCP Support
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy,
+with MCP (Model Context Protocol) server integration for enhanced capabilities.
+"""
+
+import asyncio
+import os
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+ Config,
+ fetch_available_models,
+ setup_litellm_env,
+ print_header,
+ handle_model_list,
+ handle_model_switch,
+ stream_response,
+)
+
+
+async def interactive_chat_with_mcp():
+ """
+ Interactive CLI chat with the agent and MCP server
+ """
+ config = Config()
+
+ # Configure Anthropic SDK to point to LiteLLM gateway
+ litellm_base_url = setup_litellm_env(config)
+
+ # Fetch available models from proxy
+ available_models = await fetch_available_models(
+ litellm_base_url, config.LITELLM_API_KEY
+ )
+
+ current_model = config.LITELLM_MODEL
+
+ # MCP server configuration
+ mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
+ use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
+
+ if not use_mcp:
+ print("ā ļø MCP disabled via USE_MCP=false")
+
+ print_header(litellm_base_url, current_model, has_mcp=use_mcp)
+
+ while True:
+ # Configure agent options
+ if use_mcp:
+ try:
+ # Try with MCP server (HTTP transport)
+ # Using McpHttpServerConfig format from Agent SDK
+ options = ClaudeAgentOptions(
+ system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.",
+ model=current_model,
+ max_turns=50,
+ mcp_servers={
+ "deepwiki2": {
+ "type": "http",
+ "url": mcp_server_url,
+ "headers": {
+ "Authorization": f"Bearer {config.LITELLM_API_KEY}"
+ },
+ }
+ },
+ )
+ except Exception as e:
+ print(f"ā ļø Warning: Could not configure MCP server: {e}")
+ print("Continuing without MCP...\n")
+ use_mcp = False
+ options = ClaudeAgentOptions(
+ system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+ model=current_model,
+ max_turns=50,
+ )
+ else:
+ # Without MCP
+ options = ClaudeAgentOptions(
+ system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+ model=current_model,
+ max_turns=50,
+ )
+
+ # Create agent client
+ try:
+ async with ClaudeSDKClient(options=options) as client:
+ conversation_active = True
+
+ while conversation_active:
+ # Get user input
+ try:
+ user_input = input("\nš¤ You: ").strip()
+ except (EOFError, KeyboardInterrupt):
+ print("\n\nš Goodbye!")
+ return
+
+ # Handle commands
+ if user_input.lower() in ["quit", "exit"]:
+ print("\nš Goodbye!")
+ return
+
+ if user_input.lower() == "clear":
+ print("\nš Starting new conversation...\n")
+ conversation_active = False
+ continue
+
+ if user_input.lower() == "models":
+ handle_model_list(available_models, current_model)
+ continue
+
+ if user_input.lower() == "model":
+ new_model, should_restart = handle_model_switch(
+ available_models, current_model
+ )
+ if should_restart:
+ current_model = new_model
+ conversation_active = False
+ continue
+
+ if not user_input:
+ continue
+
+ # Stream response from agent
+ await stream_response(client, user_input)
+
+ except Exception as e:
+ print(f"\nā Error creating agent client: {e}")
+ print("This might be an MCP configuration issue. Try running without MCP:")
+ print(" USE_MCP=false python agent_with_mcp.py")
+ print("\nOr use the basic agent:")
+ print(" python main.py")
+ return
+
+
+def main():
+ """Run interactive chat with MCP"""
+ try:
+ asyncio.run(interactive_chat_with_mcp())
+ except KeyboardInterrupt:
+ print("\n\nš Goodbye!")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/cookbook/anthropic_agent_sdk/common.py b/cookbook/anthropic_agent_sdk/common.py
new file mode 100644
index 000000000..a2555ed33
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/common.py
@@ -0,0 +1,164 @@
+"""
+Common utilities for Claude Agent SDK examples
+"""
+
+import os
+import httpx
+
+
+class Config:
+ """Configuration for LiteLLM Gateway connection"""
+
+ # LiteLLM proxy URL (default to local instance)
+ LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
+
+ # LiteLLM API key (master key or virtual key)
+ LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+
+ # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
+ LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
+
+
+async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
+ """
+ Fetch available models from LiteLLM proxy /models endpoint
+ """
+ try:
+ async with httpx.AsyncClient() as client:
+ response = await client.get(
+ f"{base_url}/models",
+ headers={"Authorization": f"Bearer {api_key}"},
+ timeout=10.0,
+ )
+ response.raise_for_status()
+ data = response.json()
+ return [model["id"] for model in data.get("data", [])]
+ except Exception as e:
+ print(f"ā ļø Warning: Could not fetch models from proxy: {e}")
+ print("Using default model list...")
+ # Fallback to default models
+ return [
+ "bedrock-claude-sonnet-3.5",
+ "bedrock-claude-sonnet-4",
+ "bedrock-claude-sonnet-4.5",
+ "bedrock-claude-opus-4.5",
+ "bedrock-nova-premier",
+ ]
+
+
+def setup_litellm_env(config: Config):
+ """
+ Configure environment variables to point Agent SDK to LiteLLM
+ """
+ litellm_base_url = config.LITELLM_PROXY_URL.rstrip("/")
+ os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
+ os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
+ return litellm_base_url
+
+
+def print_header(base_url: str, current_model: str, has_mcp: bool = False):
+ """
+ Print the chat header
+ """
+ mcp_indicator = " + MCP" if has_mcp else ""
+ print("=" * 70)
+ print(f"š¤ Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat")
+ print("=" * 70)
+ print(f"š Connected to: {base_url}")
+ print(f"š¦ Current model: {current_model}")
+ if has_mcp:
+ print("š MCP: deepwiki2 enabled")
+ print("\nType your messages below. Commands:")
+ print(" - 'quit' or 'exit' to end the conversation")
+ print(" - 'clear' to start a new conversation")
+ print(" - 'model' to switch models")
+ print(" - 'models' to list available models")
+ print("=" * 70)
+ print()
+
+
+def handle_model_list(available_models: list[str], current_model: str):
+ """
+ Display available models
+ """
+ print("\nš Available models:")
+ for i, model in enumerate(available_models, 1):
+ marker = "ā" if model == current_model else " "
+ print(f" {marker} {i}. {model}")
+
+
+def handle_model_switch(
+ available_models: list[str], current_model: str
+) -> tuple[str, bool]:
+ """
+ Handle model switching
+
+ Returns:
+ tuple: (new_model, should_restart_conversation)
+ """
+ print("\nš Select a model:")
+ for i, model in enumerate(available_models, 1):
+ marker = "ā" if model == current_model else " "
+ print(f" {marker} {i}. {model}")
+
+ try:
+ choice = input("\nEnter number (or press Enter to cancel): ").strip()
+ if choice:
+ idx = int(choice) - 1
+ if 0 <= idx < len(available_models):
+ new_model = available_models[idx]
+ print(f"\nā
Switched to: {new_model}")
+ print("š Starting new conversation with new model...\n")
+ return new_model, True
+ else:
+ print("ā Invalid choice")
+ except (ValueError, IndexError):
+ print("ā Invalid input")
+
+ return current_model, False
+
+
+async def stream_response(client, user_input: str):
+ """
+ Stream response from the agent
+ """
+ print("\nš¤ Assistant: ", end="", flush=True)
+
+ try:
+ await client.query(user_input)
+
+ # Show loading indicator
+ print("ā³ thinking...", end="", flush=True)
+
+ # Stream the response
+ first_chunk = True
+ async for msg in client.receive_response():
+ # Clear loading indicator on first message
+ if first_chunk:
+ print("\rš¤ Assistant: ", end="", flush=True)
+ first_chunk = False
+
+ # Handle different message types
+ if hasattr(msg, "type"):
+ if msg.type == "content_block_delta":
+ # Streaming text delta
+ if hasattr(msg, "delta") and hasattr(msg.delta, "text"):
+ print(msg.delta.text, end="", flush=True)
+ elif msg.type == "content_block_start":
+ # Start of content block
+ if hasattr(msg, "content_block") and hasattr(
+ msg.content_block, "text"
+ ):
+ print(msg.content_block.text, end="", flush=True)
+
+ # Fallback to original content handling
+ if hasattr(msg, "content"):
+ for content_block in msg.content:
+ if hasattr(content_block, "text"):
+ print(content_block.text, end="", flush=True)
+
+ print() # New line after response
+
+ except Exception as e:
+ print(f"\r\nā Error: {e}")
+ print("Please check your LiteLLM gateway is running and configured correctly.")
diff --git a/cookbook/anthropic_agent_sdk/config.example.yaml b/cookbook/anthropic_agent_sdk/config.example.yaml
new file mode 100644
index 000000000..eb1984fc4
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/config.example.yaml
@@ -0,0 +1,25 @@
+model_list:
+ - model_name: bedrock-claude-sonnet-3.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-sonnet-4
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-sonnet-4.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-claude-opus-4.5
+ litellm_params:
+ model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
+ aws_region_name: "us-east-1"
+
+ - model_name: bedrock-nova-premier
+ litellm_params:
+ model: "bedrock/amazon.nova-premier-v1:0"
+ aws_region_name: "us-east-1"
diff --git a/cookbook/anthropic_agent_sdk/main.py b/cookbook/anthropic_agent_sdk/main.py
new file mode 100644
index 000000000..506c6fa07
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/main.py
@@ -0,0 +1,99 @@
+"""
+Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
+LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
+through the Claude Agent SDK by pointing it to the LiteLLM gateway.
+"""
+
+import asyncio
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+ Config,
+ fetch_available_models,
+ setup_litellm_env,
+ print_header,
+ handle_model_list,
+ handle_model_switch,
+ stream_response,
+)
+
+
+async def interactive_chat():
+ """
+ Interactive CLI chat with the agent
+ """
+ config = Config()
+
+ # Configure Anthropic SDK to point to LiteLLM gateway
+ litellm_base_url = setup_litellm_env(config)
+
+ # Fetch available models from proxy
+ available_models = await fetch_available_models(
+ litellm_base_url, config.LITELLM_API_KEY
+ )
+
+ current_model = config.LITELLM_MODEL
+
+ print_header(litellm_base_url, current_model)
+
+ while True:
+ # Configure agent options for each conversation
+ options = ClaudeAgentOptions(
+ system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+ model=current_model,
+ max_turns=50,
+ )
+
+ # Create agent client
+ async with ClaudeSDKClient(options=options) as client:
+ conversation_active = True
+
+ while conversation_active:
+ # Get user input
+ try:
+ user_input = input("\nš¤ You: ").strip()
+ except (EOFError, KeyboardInterrupt):
+ print("\n\nš Goodbye!")
+ return
+
+ # Handle commands
+ if user_input.lower() in ["quit", "exit"]:
+ print("\nš Goodbye!")
+ return
+
+ if user_input.lower() == "clear":
+ print("\nš Starting new conversation...\n")
+ conversation_active = False
+ continue
+
+ if user_input.lower() == "models":
+ handle_model_list(available_models, current_model)
+ continue
+
+ if user_input.lower() == "model":
+ new_model, should_restart = handle_model_switch(
+ available_models, current_model
+ )
+ if should_restart:
+ current_model = new_model
+ conversation_active = False
+ continue
+
+ if not user_input:
+ continue
+
+ # Stream response from agent
+ await stream_response(client, user_input)
+
+
+def main():
+ """Run interactive chat"""
+ try:
+ asyncio.run(interactive_chat())
+ except KeyboardInterrupt:
+ print("\n\nš Goodbye!")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/cookbook/anthropic_agent_sdk/requirements.txt b/cookbook/anthropic_agent_sdk/requirements.txt
new file mode 100644
index 000000000..1e810bb7d
--- /dev/null
+++ b/cookbook/anthropic_agent_sdk/requirements.txt
@@ -0,0 +1,2 @@
+claude-agent-sdk
+httpx>=0.27.0
diff --git a/cookbook/benchmark/benchmark.py b/cookbook/benchmark/benchmark.py
new file mode 100644
index 000000000..b38d185a1
--- /dev/null
+++ b/cookbook/benchmark/benchmark.py
@@ -0,0 +1,90 @@
+from litellm import completion, completion_cost
+import time
+import click
+from tqdm import tqdm
+from tabulate import tabulate
+from termcolor import colored
+import os
+
+
+# Define the list of models to benchmark
+# select any LLM listed here: https://docs.litellm.ai/docs/providers
+models = ["gpt-3.5-turbo", "claude-2"]
+
+# Enter LLM API keys
+# https://docs.litellm.ai/docs/providers
+os.environ["OPENAI_API_KEY"] = ""
+os.environ["ANTHROPIC_API_KEY"] = ""
+
+# List of questions to benchmark (replace with your questions)
+questions = ["When will BerriAI IPO?", "When will LiteLLM hit $100M ARR?"]
+
+# Enter your system prompt here
+system_prompt = """
+You are LiteLLMs helpful assistant
+"""
+
+
+@click.command()
+@click.option(
+ "--system-prompt",
+ default="You are a helpful assistant that can answer questions.",
+ help="System prompt for the conversation.",
+)
+def main(system_prompt):
+ for question in questions:
+ data = [] # Data for the current question
+
+ with tqdm(total=len(models)) as pbar:
+ for model in models:
+ colored_description = colored(
+ f"Running question: {question} for model: {model}", "green"
+ )
+ pbar.set_description(colored_description)
+ start_time = time.time()
+
+ response = completion(
+ model=model,
+ max_tokens=500,
+ messages=[
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": question},
+ ],
+ )
+
+ end = time.time()
+ total_time = end - start_time
+ cost = completion_cost(completion_response=response)
+ raw_response = response["choices"][0]["message"]["content"]
+
+ data.append(
+ {
+ "Model": colored(model, "light_blue"),
+ "Response": raw_response, # Colorize the response
+ "ResponseTime": colored(f"{total_time:.2f} seconds", "red"),
+ "Cost": colored(f"${cost:.6f}", "green"), # Colorize the cost
+ }
+ )
+
+ pbar.update(1)
+
+ # Separate headers from the data
+ headers = ["Model", "Response", "Response Time (seconds)", "Cost ($)"]
+ colwidths = [15, 80, 15, 10]
+
+ # Create a nicely formatted table for the current question
+ table = tabulate(
+ [list(d.values()) for d in data],
+ headers,
+ tablefmt="grid",
+ maxcolwidths=colwidths,
+ )
+
+ # Print the table for the current question
+ colored_question = colored(question, "green")
+ click.echo(f"\nBenchmark Results for '{colored_question}':")
+ click.echo(table) # Display the formatted table
+
+
+if __name__ == "__main__":
+ main()
diff --git a/cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py b/cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py
new file mode 100644
index 000000000..daa38dda5
--- /dev/null
+++ b/cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py
@@ -0,0 +1,30 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import litellm
+
+from autoevals.llm import *
+
+###################
+
+# litellm completion call
+question = "which country has the highest population"
+response = litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": question}],
+)
+print(response)
+# use the auto eval Factuality() evaluator
+
+print("calling evaluator")
+evaluator = Factuality()
+result = evaluator(
+ output=response.choices[0]["message"][
+ "content"
+ ], # response from litellm.completion()
+ expected="India", # expected output
+ input=question, # question passed to litellm.completion
+)
+
+print(result)
diff --git a/cookbook/benchmark/readme.md b/cookbook/benchmark/readme.md
new file mode 100644
index 000000000..afa59aa91
--- /dev/null
+++ b/cookbook/benchmark/readme.md
@@ -0,0 +1,181 @@
+
+ LLM-Bench
+
+
+
Benchmark LLMs response, cost and response time
+ LLM vs Cost per input + output token ($)
+
+
+
+ Bar Graph Excel Sheet here
+
+
+| Model | Provider | Cost per input + output token ($)|
+| --- | --- | --- |
+| openrouter/mistralai/mistral-7b-instruct | openrouter | 0.0 |
+| ollama/llama2 | ollama | 0.0 |
+| ollama/llama2:13b | ollama | 0.0 |
+| ollama/llama2:70b | ollama | 0.0 |
+| ollama/llama2-uncensored | ollama | 0.0 |
+| ollama/mistral | ollama | 0.0 |
+| ollama/codellama | ollama | 0.0 |
+| ollama/orca-mini | ollama | 0.0 |
+| ollama/vicuna | ollama | 0.0 |
+| perplexity/codellama-34b-instruct | perplexity | 0.0 |
+| perplexity/llama-2-13b-chat | perplexity | 0.0 |
+| perplexity/llama-2-70b-chat | perplexity | 0.0 |
+| perplexity/mistral-7b-instruct | perplexity | 0.0 |
+| perplexity/replit-code-v1.5-3b | perplexity | 0.0 |
+| text-bison | vertex_ai-text-models | 0.00000025 |
+| text-bison@001 | vertex_ai-text-models | 0.00000025 |
+| chat-bison | vertex_ai-chat-models | 0.00000025 |
+| chat-bison@001 | vertex_ai-chat-models | 0.00000025 |
+| chat-bison-32k | vertex_ai-chat-models | 0.00000025 |
+| code-bison | vertex_ai-code-text-models | 0.00000025 |
+| code-bison@001 | vertex_ai-code-text-models | 0.00000025 |
+| code-gecko@001 | vertex_ai-chat-models | 0.00000025 |
+| code-gecko@latest | vertex_ai-chat-models | 0.00000025 |
+| codechat-bison | vertex_ai-code-chat-models | 0.00000025 |
+| codechat-bison@001 | vertex_ai-code-chat-models | 0.00000025 |
+| codechat-bison-32k | vertex_ai-code-chat-models | 0.00000025 |
+| palm/chat-bison | palm | 0.00000025 |
+| palm/chat-bison-001 | palm | 0.00000025 |
+| palm/text-bison | palm | 0.00000025 |
+| palm/text-bison-001 | palm | 0.00000025 |
+| palm/text-bison-safety-off | palm | 0.00000025 |
+| palm/text-bison-safety-recitation-off | palm | 0.00000025 |
+| anyscale/meta-llama/Llama-2-7b-chat-hf | anyscale | 0.0000003 |
+| anyscale/mistralai/Mistral-7B-Instruct-v0.1 | anyscale | 0.0000003 |
+| openrouter/meta-llama/llama-2-13b-chat | openrouter | 0.0000004 |
+| openrouter/nousresearch/nous-hermes-llama2-13b | openrouter | 0.0000004 |
+| deepinfra/meta-llama/Llama-2-7b-chat-hf | deepinfra | 0.0000004 |
+| deepinfra/mistralai/Mistral-7B-Instruct-v0.1 | deepinfra | 0.0000004 |
+| anyscale/meta-llama/Llama-2-13b-chat-hf | anyscale | 0.0000005 |
+| amazon.titan-text-lite-v1 | bedrock | 0.0000007 |
+| deepinfra/meta-llama/Llama-2-13b-chat-hf | deepinfra | 0.0000007 |
+| text-babbage-001 | text-completion-openai | 0.0000008 |
+| text-ada-001 | text-completion-openai | 0.0000008 |
+| babbage-002 | text-completion-openai | 0.0000008 |
+| openrouter/google/palm-2-chat-bison | openrouter | 0.000001 |
+| openrouter/google/palm-2-codechat-bison | openrouter | 0.000001 |
+| openrouter/meta-llama/codellama-34b-instruct | openrouter | 0.000001 |
+| deepinfra/codellama/CodeLlama-34b-Instruct-hf | deepinfra | 0.0000012 |
+| deepinfra/meta-llama/Llama-2-70b-chat-hf | deepinfra | 0.0000016499999999999999 |
+| deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1 | deepinfra | 0.0000016499999999999999 |
+| anyscale/meta-llama/Llama-2-70b-chat-hf | anyscale | 0.000002 |
+| anyscale/codellama/CodeLlama-34b-Instruct-hf | anyscale | 0.000002 |
+| gpt-3.5-turbo-1106 | openai | 0.000003 |
+| openrouter/meta-llama/llama-2-70b-chat | openrouter | 0.000003 |
+| amazon.titan-text-express-v1 | bedrock | 0.000003 |
+| gpt-3.5-turbo | openai | 0.0000035 |
+| gpt-3.5-turbo-0301 | openai | 0.0000035 |
+| gpt-3.5-turbo-0613 | openai | 0.0000035 |
+| gpt-3.5-turbo-instruct | text-completion-openai | 0.0000035 |
+| openrouter/openai/gpt-3.5-turbo | openrouter | 0.0000035 |
+| cohere.command-text-v14 | bedrock | 0.0000035 |
+| gpt-3.5-turbo-0613 | openai | 0.0000035 |
+| claude-instant-1 | anthropic | 0.00000714 |
+| claude-instant-1.2 | anthropic | 0.00000714 |
+| openrouter/anthropic/claude-instant-v1 | openrouter | 0.00000714 |
+| anthropic.claude-instant-v1 | bedrock | 0.00000714 |
+| openrouter/mancer/weaver | openrouter | 0.00001125 |
+| j2-mid | ai21 | 0.00002 |
+| ai21.j2-mid-v1 | bedrock | 0.000025 |
+| openrouter/jondurbin/airoboros-l2-70b-2.1 | openrouter | 0.00002775 |
+| command-nightly | cohere | 0.00003 |
+| command | cohere | 0.00003 |
+| command-light | cohere | 0.00003 |
+| command-medium-beta | cohere | 0.00003 |
+| command-xlarge-beta | cohere | 0.00003 |
+| command-r-plus| cohere | 0.000018 |
+| j2-ultra | ai21 | 0.00003 |
+| ai21.j2-ultra-v1 | bedrock | 0.0000376 |
+| gpt-4-1106-preview | openai | 0.00004 |
+| gpt-4-vision-preview | openai | 0.00004 |
+| claude-2 | anthropic | 0.0000437 |
+| openrouter/anthropic/claude-2 | openrouter | 0.0000437 |
+| anthropic.claude-v1 | bedrock | 0.0000437 |
+| anthropic.claude-v2 | bedrock | 0.0000437 |
+| gpt-4 | openai | 0.00009 |
+| gpt-4-0314 | openai | 0.00009 |
+| gpt-4-0613 | openai | 0.00009 |
+| openrouter/openai/gpt-4 | openrouter | 0.00009 |
+| gpt-4-32k | openai | 0.00018 |
+| gpt-4-32k-0314 | openai | 0.00018 |
+| gpt-4-32k-0613 | openai | 0.00018 |
+
+
+
+## Setup:
+```
+git clone https://github.com/BerriAI/litellm
+```
+cd to `benchmark` dir
+```
+cd litellm/cookbook/benchmark
+```
+
+### Install Dependencies
+```
+pip install litellm click tqdm tabulate termcolor
+```
+
+### Configuration
+In `benchmark/benchmark.py` select your LLMs, LLM API Key and questions
+
+Supported LLMs: https://docs.litellm.ai/docs/providers
+
+```python
+# Define the list of models to benchmark
+models = ['gpt-3.5-turbo', 'togethercomputer/llama-2-70b-chat', 'claude-2']
+
+# Enter LLM API keys
+os.environ['OPENAI_API_KEY'] = ""
+os.environ['ANTHROPIC_API_KEY'] = ""
+os.environ['TOGETHERAI_API_KEY'] = ""
+
+# List of questions to benchmark (replace with your questions)
+questions = [
+ "When will BerriAI IPO?",
+ "When will LiteLLM hit $100M ARR?"
+]
+
+```
+
+## Run LLM-Bench
+```
+python3 benchmark.py
+```
+
+## Expected Output
+```
+Running question: When will BerriAI IPO? for model: claude-2: 100%|āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā| 3/3 [00:13<00:00, 4.41s/it]
+
+Benchmark Results for 'When will BerriAI IPO?':
++-----------------+----------------------------------------------------------------------------------+---------------------------+------------+
+| Model | Response | Response Time (seconds) | Cost ($) |
++=================+==================================================================================+===========================+============+
+| gpt-3.5-turbo | As an AI language model, I cannot provide up-to-date information or predict | 1.55 seconds | $0.000122 |
+| | future events. It is best to consult a reliable financial source or contact | | |
+| | BerriAI directly for information regarding their IPO plans. | | |
++-----------------+----------------------------------------------------------------------------------+---------------------------+------------+
+| togethercompute | I'm not able to provide information about future IPO plans or dates for BerriAI | 8.52 seconds | $0.000531 |
+| r/llama-2-70b-c | or any other company. IPO (Initial Public Offering) plans and timelines are | | |
+| hat | typically kept private by companies until they are ready to make a public | | |
+| | announcement. It's important to note that IPO plans can change and are subject | | |
+| | to various factors, such as market conditions, financial performance, and | | |
+| | regulatory approvals. Therefore, it's difficult to predict with certainty when | | |
+| | BerriAI or any other company will go public. If you're interested in staying | | |
+| | up-to-date with BerriAI's latest news and developments, you may want to follow | | |
+| | their official social media accounts, subscribe to their newsletter, or visit | | |
+| | their website periodically for updates. | | |
++-----------------+----------------------------------------------------------------------------------+---------------------------+------------+
+| claude-2 | I do not have any information about when or if BerriAI will have an initial | 3.17 seconds | $0.002084 |
+| | public offering (IPO). As an AI assistant created by Anthropic to be helpful, | | |
+| | harmless, and honest, I do not have insider knowledge about Anthropic's business | | |
+| | plans or strategies. | | |
++-----------------+----------------------------------------------------------------------------------+---------------------------+------------+
+```
+
+## Support
+**š¤ Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
diff --git a/cookbook/codellama-server/README.MD b/cookbook/codellama-server/README.MD
new file mode 100644
index 000000000..82a7e62f4
--- /dev/null
+++ b/cookbook/codellama-server/README.MD
@@ -0,0 +1,153 @@
+# CodeLlama Server: Streaming, Caching, Model Fallbacks (OpenAI + Anthropic), Prompt-tracking
+
+Works with: Anthropic, Huggingface, Cohere, TogetherAI, Azure, OpenAI, etc.
+
+[](https://pypi.org/project/litellm/)
+[](https://pypi.org/project/litellm/0.1.1/)
+
+
+[](https://railway.app/template/HuDPw-?referralCode=jch2ME)
+
+**LIVE DEMO** - https://litellm.ai/playground
+
+## What does CodeLlama Server do
+
+- Uses Together AI's CodeLlama to answer coding questions, with GPT-4 + Claude-2 as backups (you can easily switch this to any model from Huggingface, Replicate, Cohere, AI21, Azure, OpenAI, etc.)
+- Sets default system prompt for guardrails `system_prompt = "Only respond to questions about code. Say 'I don't know' to anything outside of that."`
+- Integrates with Promptlayer for model + prompt tracking
+- Example output
+
+
+
+- **Consistent Input/Output** Format
+ - Call all models using the OpenAI format - `completion(model, messages)`
+ - Text responses will always be available at `['choices'][0]['message']['content']`
+ - Stream responses will always be available at `['choices'][0]['delta']['content']`
+- **Error Handling** Using Model Fallbacks (if `CodeLlama` fails, try `GPT-4`) with cooldowns, and retries
+- **Prompt Logging** - Log successful completions to promptlayer for testing + iterating on your prompts in production! (Learn more: https://litellm.readthedocs.io/en/latest/advanced/
+
+ **Example: Logs sent to PromptLayer**
+
+
+
+
+- **Token Usage & Spend** - Track Input + Completion tokens used + Spend/model - https://docs.litellm.ai/docs/token_usage
+- **Caching** - Provides in-memory cache + GPT-Cache integration for more advanced usage - https://docs.litellm.ai/docs/caching/gpt_cache
+
+- **Streaming & Async Support** - Return generators to stream text responses - TEST IT š https://litellm.ai/
+
+## API Endpoints
+
+### `/chat/completions` (POST)
+
+This endpoint is used to generate chat completions for 50+ support LLM API Models. Use llama2, GPT-4, Claude2 etc
+
+#### Input
+
+This API endpoint accepts all inputs in raw JSON and expects the following inputs
+
+- `prompt` (string, required): The user's coding related question
+- Additional Optional parameters: `temperature`, `functions`, `function_call`, `top_p`, `n`, `stream`. See the full list of supported inputs here: https://litellm.readthedocs.io/en/latest/input/
+
+#### Example JSON body
+
+For claude-2
+
+```json
+{
+ "prompt": "write me a function to print hello world"
+}
+```
+
+### Making an API request to the Code-Gen Server
+
+```python
+import requests
+import json
+
+url = "localhost:4000/chat/completions"
+
+payload = json.dumps({
+ "prompt": "write me a function to print hello world"
+})
+headers = {
+ 'Content-Type': 'application/json'
+}
+
+response = requests.request("POST", url, headers=headers, data=payload)
+
+print(response.text)
+
+```
+
+### Output [Response Format]
+
+Responses from the server are given in the following format.
+All responses from the server are returned in the following format (for all LLM models). More info on output here: https://litellm.readthedocs.io/en/latest/output/
+
+```json
+{
+ "choices": [
+ {
+ "finish_reason": "stop",
+ "index": 0,
+ "message": {
+ "content": ".\n\n```\ndef print_hello_world():\n print(\"hello world\")\n",
+ "role": "assistant"
+ }
+ }
+ ],
+ "created": 1693279694.6474009,
+ "model": "togethercomputer/CodeLlama-34b-Instruct",
+ "usage": {
+ "completion_tokens": 14,
+ "prompt_tokens": 28,
+ "total_tokens": 42
+ }
+}
+```
+
+## Installation & Usage
+
+### Running Locally
+
+1. Clone liteLLM repository to your local machine:
+ ```
+ git clone https://github.com/BerriAI/litellm-CodeLlama-server
+ ```
+2. Install the required dependencies using pip
+ ```
+ pip install requirements.txt
+ ```
+3. Set your LLM API keys
+ ```
+ os.environ['OPENAI_API_KEY]` = "YOUR_API_KEY"
+ or
+ set OPENAI_API_KEY in your .env file
+ ```
+4. Run the server:
+ ```
+ python main.py
+ ```
+
+## Deploying
+
+1. Quick Start: Deploy on Railway
+
+ [](https://railway.app/template/HuDPw-?referralCode=jch2ME)
+
+2. `GCP`, `AWS`, `Azure`
+ This project includes a `Dockerfile` allowing you to build and deploy a Docker Project on your providers
+
+# Support / Talk with founders
+
+- [Our calendar š](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
+- [Community Discord š](https://discord.gg/wuPM9dRgDw)
+- Our emails āļø ishaan@berri.ai / krrish@berri.ai
+
+## Roadmap
+
+- [ ] Implement user-based rate-limiting
+- [ ] Spending controls per project - expose key creation endpoint
+- [ ] Need to store a keys db -> mapping created keys to their alias (i.e. project name)
+- [ ] Easily add new models as backups / as the entry-point (add this to the available model list)
diff --git a/cookbook/codellama-server/imgs/code-output.png b/cookbook/codellama-server/imgs/code-output.png
new file mode 100644
index 0000000000000000000000000000000000000000..67e298bd3be49e64f5cd199a6967e910468937d2
GIT binary patch
literal 238123
zcmZ_0cRbr)`#;{Qw$#>Esa3mH6s^5#Zxw3qT`@|88lhT6YscQT_ZBK@wDvAdj1XGX
z2#MOO_<4Ws-~IW1?sEU(A&+>S*SXGhp69xr*Errk(^et7b^q3tD_6+WR26|&u3RrY
z|JftDasG|{^X#?rUsu6E75OWb!wjoet~|J+rubCg7R@RvhIdUW*Oco2
z{mlQJ!U{lG2+IZ=pVf8=cI~EV>a$5F+xmNECM`%%aasMMPwcs0v6mAs@x$*ac
z{~2Zw-p;DDa?9Fu5=i3-l+54$}XOAYpRJf@rzqWIoV1`!*Z{uu=y52#!b3u
z!s`@2yL0F+8AVA+O{*tzOCjaC@T*>do|JOsyC~Ied;}@8DD$UYX7c(=cSnIJlA9^4
z^9jWAlb2<%;6Ws3XmNkHEIKXGa$hsQBqqYSEI=hhk+3
z5dkKt*u99L5Fxh9WLwdB01sJb?)&WbwZK;zxTVTdN>z)azsV_nTWoMGNB!~t0nGn?
ze2^rk&P^w|pMmbkfj$8~Qe!QRi^ZGTH+xQ1u4R3{7H@Qk-e0$~66I4SM1%|PGE74-
zEYS*|NNG^2YV1$^A9zI^&4rl`=OMJAhn7O>$|(>KjwP%&)XExml?F#}eMzz=x^yt{)sn|%`Y*u|jw=X5S7`;`
zbG%T6;xy{iHJ9p>#Jgbgp?0K~jua&T!%D#)>{lQr1#nV(-M!>T@B53DnZ}bHo*NC?
zGlJi_Xc0u
z1I{=p^t|qR_GTySf)&bVdqCy9>X5##3bqzZ7CW-2u#aimK;Qyd$XRuxR87gBN-wWH
zUY-PETyi|DC|Ns?FNhI*NKK^HaHyOGs1+AZ0lWOVYZOZR9Y?dTYEDXtx_ShLMSwqW
z)W1mP9{*-yGdowg7HzF)dO;KIrBsuZ1fHqm(nUtgcLs7dx}N6xNxp9suY^PkHi`VM
z?W6;5SIno`jO$z+e1))^t@=Z)r}6mX9V=$?O5lDd9XTBk|U?jVQuJP}4{buOD^dxzIn@q#4nTepok
zsAv)Q!N$BEA(i^uCr!UUM4;zPBCzldl^EB;`!C5C_}*U0kmxR17TE7In^%RqpCh#7
z(UBQq${54QEKhcVjPB}68%~KSnduE0yM5wb#-n$@Q5t@EW8@bM
z&$(qDe6FlbH*p87BUog&Kk@{iwvHH5PdH7<3&~(D#0hc9oA~)RPHWYu9I0sNTu`Z;
zw$;@2naK$cw}8TM$L2vgw1(bGqAqr5(5lkzOy~5SO+L?IrQy5i&U0knS<9UZ8_Inz
z%>1`DHJ@6g7R*e3Ug~s+tKsvk+`6wV=s0njYAQ9>(u&OW(-w03U_>Lh%(@ioHZI-;
z?3U_D$(NeOH5~aAaQ*!fpoAK_EPyI5Pwk`44n
zlqj|D_84tPKdJOmNW(2r517rbbbQ0yWXX)?V0-$1?8ZMI+Ybm0RIES-V5)HNds|IT
zy$X*~-{tc?8+l)F2L+}K)9Z@$n?)|DiDY6N$OTtYmZ(*jLx*I-SALSxU0@qYVeQW0
zE{hH=3&1*6>8zdE^KBPy74LkoR1g#gK3J*IdJ$GKt;K)4C{4EOGk5oj|4C6sbELQy
zX&Fm!qk%0Qw2|%PZ>MFYMpRxA`gZ)t_0gs{Q4hDRs-VvY!6UQYk|loIOus5h;==hLY~MS>1@Nsz^|&ZLKF%nlJHk3^abS!4>x;-qkaJh7%SF8F=e+
zXv8Okda+ZE`U2Sx$Xvf1og{$L*Nj;1cPvET8&flLgFTh^>JF04nmrH{ymkRjAKL;m
z>6)!+Buq8iSVF1H^5wL|C(L@V=tOSYB>XA&QbMN)NLSj-MxFBUbfmN+c2h4h*(Q>5
z^(S~oue1f~xe?7mz0fij^X=8b|g*I901?f{GCcZDB|R9h;K39T;SM!pgMmN5<&ybf!e|3%MD(hs#|i=@#y}
zkBXpiv>g`ho`iOh0NcoE
z$3YpWeELJT!9YY~k*NVsu5DXyn%BHhNbi=+1-)G%D3g2e?oWK#&sV{{TEX7ly`Fr=4iKv><1G(I-@uy+NiXrT7uYa#Hzdt
zCzlTKWxi+}gRKdVKBzV^;RULW)m(h-pc8o3S;z8o84dl7yVp&6XhB3Cpc4WcK(2
zXg8n^6Bv02jG6NJ+;N
zTLFl0+Rd;3lh!3e<3#bd=cF_WB+%1nL=qOP#c|u{czNE=Bzxcvh)_o;8}2}l_i&95
zturNBzqOj}1UN{=M_{?_O#8)`9~sQiE}nelFM_RvHCI!l%^
zI>c%Od(mNcYkDO~?!R-qHK$zrc#V&0Z>vYP^?
z%Nse>q-yq>efI%dvS?E38x~!+a(a4*fY-E3f)=Kmdh%`j9>#xDX7>N{?T_GwfL^)a
zS}y;^Y3PXffW2hD1ij10Hs8*hSQT_=&D5#d@)`FuG**owPxLDCHfHG&fqkDzZlm+z#Px5}ehvvEa{^n{Ho@-5M7Tm1g!u(y
zGk$WfhE%@Ud*zScpF6f8^Ik2od!I?DPjV<+x3n+_xO>(p$VUi))N|z;vU#{g$d~fMQz5pSb8+d2ftZhA+Yeb
z)4xKvq-u3*^IiuFBp`ypo#91PHIpe*49fx3KM)y-RqAg{0Piw_m`-LmWR8fil{PHU
z2?@ns4dxjITLpFvweOd(tzcS$lCIyAFi)q?<|i%l2)0D@nZbQ)=@3Ut1L!e7@PUAH
z<*4E=ZS6~!;7QRX-<^m%NG7m!QU-qD;ZC*A6Sq$ZkN`X2fBiy|90
zUbP0+7>`F#j{8;AtVO!YCKS1!?L@KFGY{?S4aZ64NmRTB*{Vqg{%c|ZL;QxQ#SKWd
z(eX}AmD>lwuQswv2K49DA@8YFSS9dB`N%V$*}r^PwJVVqvudO3BSM-qZ5II`ZQ-2|
zs9iaz0&3&AOQ&Z
z8|5*t^M>C&-fxCwLG4(5x^Ho9=Zyr;k0uCbzd2Cg?)TH|Bu$>&F}@5Duh3t)
zmsdpj@s(vEVilC-*DXl%UXlvFsRu3HS=gadKs%yB;9i(YkVyUpYqthRS$w#2Dp6z;
z_cBMZZHO;v1ioQ_tJroN5WnN$uN;{96)0g58`=|T8P!U0w1ZZLUa5})h-mz0gp*5T
zl`H8H6~@JvHq>tC5z?c{Ep!}xcUlJ>c+0w?D(-P>N`s&+K>iU$6c;4&trEmid}_B|
zReI^$X8y_ddQ-qtesi#Ii|sVU92ch05HB7^XL~61#$6Id&Y9v)oKD(oqVdT}{y&{+
z1AlP!*O>+7IP%Y9i$C96LikDVPn_r#;U071>aD3N23(06}+Taqra@hPjTh%Xj*UnTcw9Nu%v
zjNRC~(#`YN#u^cREd)=q&{w*W*--`9Yjupi>miPbBY_w<=Li9XV)C^N2f>u%uqUc$
zRkz-zDBErZdQ$;iMqYl7XL?boHVmt@Fz;!#XBw|k{GX1?WbpEBClhxzxa5v|Q&h=$
zhS`;@ehr;2qtkDg*mZ#ahf$DAk(2#J%jC0ax|2}DFFt)Q#Om~A{BIOlOx%mvA3V7n
zc`oxV7JZX25irfT!UBcGgS=p`%5Nzp-K)H1c}j3iKxh&sK(;G|yZf82T2EX(xY4+R
z)R{PJ%Xt9JN!ia`TX7xus`|m*K`KQ2jULK0E8@9iTpfoQEz4gP@HZu`0)Ss~y*l)6
z`6Kzy2<6*d4^r4>*roK+1ix-4Fy#9K$F#>BZdvoc*BDdg!C#j&*dclSO>8VhVPPi@
zrJL(VdNsjjP$`BbwF=|l_mrP%^JIK!F0F>{*OIDr4E=ch7eNBDuflhzZgSn4Yn^ZU
zs!*Xw(r01z{$1_gsjk~pv*X`5;b8YNel4PS(`nKj{*ta`>>6N6*e)UYVu)qp*K_g4(SFtw;0f4BoYmLoM
zN;*uDcq>sq{?uMeN=ZZ5zO!pbJjFDO#*xU=RK)T-&0Vce7dtS3aUKT2{j
zS-ig_G(pR!Kv$Rv+MrCaKAT}_JJ}Uv5R!ONgVW%v_#VZK;2=YpYqpH))=;Ke-~^_(nO2J$h&2iPiy|{KXR)
ztZQPOd^>@@=Il7duZXbnq+TbrfIX(Q?_8l_ARUTSVXv4;TS=we6xQ;M$%u8XRNe%o
z%mI%zPon;So%CktlwRa*6I*|Ad2#7C3HY%@YMNGhN7cE?bituds%V%;b)fG`U+>?U
zQ-hH9*SD_n%D0!J0%=Fe#0?5C9?qieVNR{yX*76~zA_Nzo~5v~5TdF4_s;KEBI#z_rhWd%oY)+xP9vs`Sw)^
zSW-~3W`lrLxK{T5!o)iDp%4}02lw!qq~F^ZNeCH@g)-viDz-p41O6B`2k5^W`
z!n0)_@5Ie0+0(boGloUa_zD&3xlHe{jxs(Slp1SbQNty?4=$hoHlr{bvtc?g@a<=~
z!gWY?r021E*!VJ<4qBUk0u$5TD~ucR0ITCi9($-c!WP&W1%K{;PFlUoyeF0
ze#VmpP2Q?wBq%!hlezuV`9(L!q*p&|-Yu4$1Uh9liQ#9#IpVw?^fjaTwrO$vr7=69
z=>^5w0&MAG>ZD
zx4orq!6+U=*S}p1Q64;?WLYK56JN~9pLp5F$eVX;XW4VBCkLlc+pu<0-RWA4?7nW4
zAB(3(N9ytw*T=0UhoA
znls85LM6-WRg%Jy6>`V7)~W~b*&bOy@0;(Stj7F_Xh9r8G2kFjH89yh571H^9hZLY
zW`WF2w2gp?@B*2mTkGE>(pY?Y*DB#rpWG(MsM7fvOzsh>)YLy3L@Lkq&Q}28XHp
zH07GnRen16>GLB-E0PmAWg+C9F~iTw`~Y;UzG9Z-V^5a85f%FsQ{&)k#^Q)IwR91W
zj+>@PXjZFDz1dyOH@C!Y4d;vC(Ecm$&PEyjJ%Np|x|ro=?=PXwFXd8#e086b`Z3zv
zt{Ydcsm>|fEAVJ)u$*fZgtemeh_MDeL@Ox`HLRa?{QGdS&dP8_^5m!8^UPJH)c2DYF{RebG)1go2NqT#oU-dYP(fa#&qMCa0!){k+OxK
zqnRZ?2bRp3(^&JH>rB^}-jqRTy;`WtfpJn_WnlYgi&kF}xp!N#VX^zZmVVgc%W_ST
z>7GbN=X7~sGokhUDYGgGx6cx}NDdhrtWHl?#>vr_rgLR_vcFiP%hw|uQF_c3uCVpv
z?@U3Y3i&|c04?T}Ur@7HCoOjW`(w4odq>Bf^D;So11#!s9r
znh*GL-5`40CYf#wD3=RtipR^{Oa6;B1M
z3f1U(eS))yBNRfN;om@bwz9
zgxOJfbTe(TSW?wYOC+tYwAqz4r=c~T;adE2YDF4%o@_X2{D?oUZM`JBb>8P;Cfd_0(sUGMR3reNc%t~BLE8>E
z$~adUvyMw>YUKk(Xl++PW?Dz4bIo|%p=%_#vY%f~IHw07A@gyS6SGOSQS^wnFh~8K
z@H#=XdUr;1KAtTQN{3Xq4wp)jVU^Z9sE|fH;>M*sRJLf&qri9`9>bt$>9LnX((N?^
zG-{Vrw673cyPY198m6{EYdsPBrWHbwFy_XLk<~B>;~v28R#nCzcO8sl*vs|8cVTi)
zI(#Mu%M;HUMhb8?-I1=-yQ(Rg=i*4bFjZ?-QpY}qO>l^GJSnE9uFZ=KGX{0&XLks<
zcL*lYgJV;YdL7i#@m3iHPuPzupxb1(2VX31Rny)Gh>|}4b^dCUW>!=3R(0lFF-N;uGzJiT=rdty5f7>#lJiy;?9)^QcEB9WT3kF*v~yb
zFjqXv3@^GL>qu}3Cs?4OyUdY
zx-JIeUMubWkuQ!L87eTCqNrn&u3ImfuHd;cel7T5woaCkWEE?ZNK)B_IMegzEJUSC7J7RXN}3wI*^LHkpjWv13%$)>TXn+~Pt&fMdNkE`K%X>o
zq#~nAj(V2(Mp?rLlxg)iJGp)`CfC6Q*r&e1L=Mer{pHY=zG`;1(~^bB|11D3^v?p8
zcuMR!L=YO#e86TLYvGpPKqtiX|1b3mRNbjyuMrE2l;(4mzT&lmxjEexl329AyoEg2(
zgXl<=St-skbfPZqLzARl(-U
zdi5TEDaV^h2#MOU8hRJ>9rDdmFE+HxvGpsrI<(VHH+?O(&uIcNMuIikWXKj|{4@@G
zOJAnNU`pOOc|rvBf}9vF@We|4*t(iBqA`^S~z9Nf2oqWA=qZ^ln*K5AlKV+E0H-#)hfsskIiPpsk{
z4YcYq7;p8Z1z1a&d>Tt+WB6z+5axY?=1b0}NgR53|6OhI_gh%i7r3(I3eoo*&!KC*
zX(s7B5_&TF^pfgk0raW2cvs1~s?&b=15M{XxU4E+0R#VL}
zpU)x@@yHsw@i8Ft7e410O7E`Qrx_Z`T+~Qt7hKpMb5GAyqLI->@jZnaH?{u)NkNs7
z!?An$m$_EbXzQ1Rq!+ukoNg1{vLdbz9y4W7uwq&k(a-XS9Q_LCTjWa98{k04=3^eMh%p0AGH&$}}q^@Et+`I~1!^EQc6o-+;
zD%zp4%q*djwZ#hdGD{DNZPf!6Ej28))?Ly=)PspYw#lZI3W^b3RnHun-JCZW%q{D0
z4sOtUYZx_}ZnY5p9iGWq9TJs4iJ;9lZkL;yfmPJ#5-)Egpk;zrMSLqpLN%Z%<~9Wl
zaY`;D>%3QMNx!1++CFGT^MAL8H7SOb<6FPL!kyo{?DdyDdruhrD1LCm?Ry-gSrdl7
z3##SA66som_gaPexZ%tAsd#jN$8!d8F@BK^zua8JEt~G}CVbX0oT38j&j~Oi@Zx=e
z(I>e^*Ra#yKhdZ?_W9!RQtYB$%QWROU<{$Lif>2?>KLP(=ooGNk2S7)D`MuYfu;u0
ztVV7M*SyrN>*_<*#LqtK-w6}SG~X1Lo%my?PwM-5LDhxXk~!~&+7~8@zWGXcB3P~a
zHwKj%ro~er0n*oCuW`3go4ezhqy!XQ7<`wbQH@Z1sk#>`Tu9P5$zVp)5&j0j3r3XL
zsUZDX1R2@#!yNQT6dRzj9DVxQi$(FB)TgI!Y5sOS)i#8%@TI|zWH-O_(!9U2erssd
zlBfDDB#-ecoaO^r)KvI4L%$B1#8Df>3+=Kt)OJ${WCiSMV*XWvw$3O&-J{Q@`tXTI
z@8VuctqE(Y9()r+ZP7Q!QOZF*C7P4({y}8=Ij|?`XoaKCp9mZKY~)u!+u)0u3_SJs
zv9#}kTZqrZ@@IYz8XhYXG-ass?f+iuXaidjUNDy81O#)uCSZXnmly`_mbYNe{!;JI
zU%ZWRrn+II!?WR(-_7Nx^_bZ8Eo`rTHOVk%`~Wwxxo;g^PCnN?3=&N&wg|}|E-1e0
zkn71j3NtdovdKi4m@IX~;mkYPsYY9CnE~V#Af&MlB5&NQ2fmxc@H9++=EL*nz|8?d5Tb)LPtXeg1EpKfy-uV_7HvOR(T&{17YDJG`jv=@!
zo7F=Ogenxm?lmA^t&5$`4%>ai|J6K&EOOTRBBS6lgZtiTSh
z`2O`x(|bLh_vwPSu)17>x#NPf0oy}ZOPRwFtZ?>W+}C(Ilog)!a>Q1vy+V~ynowqS(waD4z`(jQyTkudl%Qn<|=JFGQFWOHLcU7k-J=#zZ(GGR_NL+NBRCcp@~@|O8o
z_Sv81B7eU}!hd_c|6f7MtiM0rlvb7A^N2B&JznXlw}bQh4^W!?lJj#k>^UcRhUc_q
zC}$s9UXY9daDV4Hp`z|Gzy0p0FsMxd=;(l^>`?w;YY&`QxV83l8iuB-Vk{FVJ03T6
zf6n};oeYZrRq4{x(~q6VN&XG+{~uC(jKRCR5Xes@o#!$Bf39O^z$15wLAp4sycGra
zb=Hak42K^F|jMj
zHuv=RevkkjU-0C=0PMVOfH~k7vN+r6eNT?8g8vPdH^WC#0b28cag@!`YXO
zv#g`|m_WOCOuT&8|H~QkAPih`WUdI@Ob!%Z(?+t6{@d{7OV06@}V(>*D2-4>XR0nH^1I{(yplzxO#oYp)@_iM6)JS>xlpW-aIo^U1gT``kn
zx?eO5K{Mm);KM&q(H0Znl9L@P?w9qYR-)BN`wS4%hs8&W^7N|hw(P_
zC-OCwM-LZUlovBKE~*50L&A!gbb(==2iIeQ)<4LY7}5NfL$yfgcF-QS)t$MUcZ`B1
zC0r#Y*$SjKVXZzMFT=3y(KB~?`
z^ThY0-0YbFe5C&cMnHm%wx4f}MIh#f3Vy{l;vFVsJIZC9%G~*70O%4)#5{YgiBXjw
zxiQK{w`J1B*|;ypm8(Shml7pvOyw@_lIEU&F4m;VahC25aKy7=4UQMRaD^mSTkUEQ
z=BEgnC8!;oT@UTBG#%8Zv|Bud%+hpj5mO7Tug}b!Vl1U{P5Sdt7SFTKGK@4Zg>1c2
zAzSBy>9+Kb7>-|hhv&fq_TR1aGLyrFB6gm*>Vnkqz_{sRV;to|sDP+?gk5%p!ENgK
zL)xD}mP+p^i*DMXpP}!aEdweC?!wTly3)@@Ab9|YTWZ;py}rWev^{k&=o>C;R?5A;#ZQxd0!}MfWO3*4xcn`ev_+P4GSBO?=4P}t#Bf~)O%Og
z63&{2V>M5dnfxX>5}MV$9;?4TSTbid#&Cfp^vCn|_FTCwTkcV#&g00`r3yEg@)lUJ
zFXmv2=`f)d#xO+9Y|l+?XpX%DLzhD_A8ixqe9!Hw(Nq}8y`jhz>pnkP+A;`bY$;lV$0p}5bsBP#k7$w;lAE+i}W-?vGH
zW#Tf{e)h+w7l@V%G2kVT4x}EbNNqJ-23SM-@V!p#Y}sW!^8eGp-B11L6uL3;xSn@l
z$~dcoB~Gv3*{6wW`%1?aFd>+rqYzobe`>
zPMCde9o9UFd0!>HOEDUJm*aQVerub@E9;{{Y=frv`^6@PN~dm-{c>l-s8q&Yibj(IoNCqx#ggoE8jCTLo
z<5t#m#JXODy&e+=>Mg1Flj-M!hsP_A=YLQiYpbp|cZ)oIH6F>>md-FsRDL{9%4D
z{F*t3$RKWtD_vsgcPAeMTj8_5m>Lc^e=8MgE9@@+#UkQA9>5@34p1t5H|^gj2dcI~_g-(6$nhzXdDMOncv9afX-KY5~
ztmQRWO6tTvJmru_--~Zk?D-92#@9%8)}u?~yT~?OKdjQKRS=sWEOHHPao2%dwWRGP
z5QvE*ZOBseABX9-rh-<42N1f~87eVa6GT(<(OfdouUA?W)iOK_Jl@7OC}fH?g+G*7GBR{tm^H;NHmSSf`lc%wb)Am8AgfksM
zSvs`7^k=KFu4H=aylIdM{LzeF^bjXL}yMuE6R{6k!kS&nm3(QA0anylR)#O|H!0cS7kkc((4vnsLgmuBzLn<
zG1Py{bf-OzAF_T>h;9MvL66ELQO&1*MMuaeNt*j}-6xI6nrtJzo{hQ^?<_MzNuMfS
zxQ`=K$Ip1|;wBlyb2$_ZT_A__XS+o`ywu-ta=yg*&rZi_Wxl}heYLJd
zb5vrCQMpYt`UG}v#wgbFy5_frj}Tnj%NzB&ZxJu@%YFhK_4Z3OG6h6R7sK>4O;eFm
z)m@X7G0>Y_i0-V2Vj0I}@?BoX4Lyn(c;&F{HYw!jMNtW(5+}SxMMSg_nUkl2(mJLn
zJv&!Q;C{n3tSOhM3N#@TdX#h&1eknAArYjJ8H7hj*pB!)2HcEsDNc98JUjFFo_2(x
zA4j={b>ID(?)VV0!&)GM&p}o;+sM85CvDZM=>RX*NUiF`4^17G-2
zd_c=^8Ce+2dQF3BVi-Q1JD7Z4+u_r0!#jW$wD8aBO1Kg?UdSX_CgJ5|vvQJ={blg^
z`PnW#Zk=17;4ihUE9tQLY`ix0vNKgkL^zYz!{xo54?p%L*=p@;HIcBmOn&Zt-7u?8
zr!N1|cRE>rSaRx$gBCKN&LdghkzSF8`2JzCsZPS~|1C`^($g_p7x1C2`
ztYKSy<{0nJCAWE)F^VH{PA4GE|9T({pzFM!rH8`%XHeap+pJ>N?}<5wSetsBIipha
zW%RN9elV!2xwRgtW)Cs-Y*c!s@ma)K&Y#@9+_jO-0vzo?P@rfKhMp9fB`TPtIpy2*
zoC~q@8a%GJk76OtjL-{m#a8a)_^dmwErW!+8Pl|+vk^#T?pRed{=UKSJXwg1ZUx8g
zMn;_uW_P{0xTPo)y>{q*u#$;Qn#MnClXz1;4m)ZD$W-FJhqw46m
zmeBilp+10GGz}_MN`$Yzxl&(M1F9X$Kihd0ZJ
zB!n9VNbd617jVdR50|z18ep
zDy*Xue_{skJj=O_wLSCmd}#nl(mDxmao(qUp>(wQRs}WAMG^lz)5}nZ_7eOI$o;@f
zm>D|it-ZdUiW>6!t^{4lb>9mU*FN;HYjJibrtF#QMjEtW5rqxD{5D-)Q;k%5CwZW*P{2Nv0N~
zWio>MYx)V)1HleIo$0zAjx`5?ut&|=6?pKeuX9hdH#F^AAl5Gk4A#KB`aXp1oBO1d
zN%Y6FQ*^j8Q{8-jz^C*W10H#}30(!LAm6q5luCHiE~`%@B2FemP^>v9_EU$5lhXyC=zDho!V3R81t
z!oToz>8h;Kpjy-eUy%a`d$v^Z(P_wnYJmOzom>gtV~@r<udq+4^qlwJ!#~k~CS09I?s9RT3Encl@t+;!|6*d5g!RS4
z`A%-iZ|&v&)J}~~Fw`md-5W9iiYGjG?KY3mg*H$wxanp#x^u>>3sw9wMqz%*nr%<}
zv+J`ej#LN!_!f8DJRJn7E#cS2Q}w1P`G`P5%!Y4*67Le*+c%P1*V{bN2sisvA3<~K
z@@oCuxr}7{G$7{09ixQylUpIpez>8=5(>tqx`~4rXs6n4ZI4UI@!Ud!#fd^@>%p!q
z7Q$?y$9%Ln{7i58xaYKrAMWvm8=K&dTSLhrS0@cAig)RwOY}}~Z<5Bbl!54G`U?TqA%fl~Du$tO)2#+)Mbn!5dUj*e3D`sbj!rM%OdHVi
zn8fFqiuwZfBJjM$uyO#rHN-x&p?opSJXylzoSux=HjB?D`5k@FLi|2HgMHW06KL)+
zM@=0M)7s;w$r<#kj*s}Ur3xzkQD-~Mv(E(l2h${$0nPrq6p*8F#vas~%;$pHUD36i
z!r}dmfW;(m7(CmUK!UNaB9gk&0owKvZ~qYMyk#*7@P
z{Kl@ku0-p>9CAXdE=DirCL!^t!BX-M*yra_{yX=|>}|t`>1bmGM8ZTXpKD&Hs_|By
zbTmtA1&kekgfun&rqf2N>nf{Fg$l=_4ygy!FF|}P^{WZggxz>!0xBnR#d|ddNU(hL
zDb97he;BatA){Sa#hqksFT1rLA@hr%MW`jR$TDOLY1se=Rm
z>DujWiy@gTG_s|xgiSq=5DP4VA^nE*!kyKFo*gM1U4P?!sfE`i6w=L<#N^ka{Xg2c
zMUcG6(3yF@k68RsVThxgz`>Zi2qM17*O~y!BrHQs)95vJn3kL@=B)Ug%k0#;^2?MV
zX)eyviOnKjC(-;V+rQ;+a(YL&Wb2FcJ1xd9f+2$w(APE(NDwJ`Ol@?A8b-~|x{Wf1
zi-C^BC{rvfBDcOL9e=76r!PS7zrLR;(V6tzgd&{-GUgxSlx)D7|D@RV
z#0@{6q|po#Y`|4AW5)6)7>*1~t%Sj=WTO^il<_xm4-Ah0tP0>hy90(U!o{^U(nN7^
zur|1Pr>}G`>68|xRq^&daL@MBrzA(c0W8pwmq)6Wiw!Nd{>U9OIO8^6Kptbw+@8W*
zI60U<%!p6&>;5i~wOF$mg!j9*yRe*9u!qCMd!1ejiz#mVwClY)=hkHV?Nb}Lk9fED
ze96O7(20*8>%+$sna`MyC*rY&Rrf{kKbN`Td;jAHi#Vb-4!-0$uf>O<&glpFfYqho
z+#{?Em_R*0jCyFy$T!DQY{*aBk?^H8#`GqyL~Yvid5d{nDeiEnZ8}#XsOs|{H*q(&
z=-6WKVcfuZ^~Cw`7+=fm0NAOfW)^?LGKDnXE)qMaW?+jEJ`OVV^LdZw`7R+VRWq0m
z0FM<>ql^z@z$5w;tv$YK%)sp9qake2EW#3sJ$V0gLFS)$PZWTDHs{e+&w4P|_PBgq
zrRkyh>B4W}apdgH3`wU<@8u55PtS2Jo*ETFg06-BV-GH4Cs&CgwlRw5H6)xM@%sb+`M=FLMjp-T-RNrc
zI7mTk7hZFoD*!q7e0i!cN_SbP%EuARTj^(*sO9RwBx(mQ(}tfkv4W=(8C?e!4Q|xe
z5vuB$G_qI38M`u(npbWoAr6bOj4&_ZZZLJ3W!cY%Z$s!C5NQUVePxyx^#^E~&S?RWq2gFIPT
zYrbQSIp&yimAG(sr%|5{dwL5gybPv%EZ|MNPXFxk##}WISZR^5JM%O%)NlDzk|>Xd
z)9Xe}=)LSI{;aydOZOqYV0xmN&D=OQ^cKqdeO_>nAKa?9NH5}eMD7R!IoszZTISun
z!9XgXm%r1r7~LieJ)bg<{?&w@mMh)Mz*-rF?^^d_s_ojWe*^J-ZVZ~P?~{vn*~Exa
zUE}Y%aJBLX%jRuYxTrl|-ub69D{>_=!1l08s1Pmk7Cc~TZPr5P%WI70Dc?|M0jdNh
zXVYN4rzSGZDgCqjs()MJN*?XIm_`##KrC3E`#FO~b)Wr$%lEHh3q0I#GZ0jnaE&$g
zxtFuqU|rb)NM#Rp?3%$_1A+{J^hH4SL;TQv^TH|=WYWhHI-*d^Es#~dFmAnBI
ze5t%Z{g8B{Y;d;Bi9~Js`D)5{D#(wG5wf|^H+G5CQ8k@zV+7lNVi~&1RP4VcCyoA3
zxp&-`L)a^4eiM|{{)|cZ5B$9UJO(a5QFQ4E1a{Il*FXNUjbBloXiBde!FSJM%)A_(
zgSD~u8(heI#*NiUs^x8dzL`7No?jr3J?9|;LIFm>k&v(C0AbbVLA#(Hy^5$((g_3l@fl*t}e
zqd+#b~dtA?wQXPIzx)J2#<5oGQ)gIkmyjQ;oMfVnG~uquwNMkFl>upTGrP_c0
z|4OuS{&bPsxb&%}gADn=DkJOQod$R9w5h;yBch$9v75t>Uzz`&eE`aWp@feLbAe7$
zdQ-fy{L#W0Y(wqq>+;#nVmv=a2S$b-n#S=RSzoq3@lm)krCcw;UGOlE%ZiqduRPQe
zTd?jKthWcOeYsr*z2~g8t>L5+RZy3{ST=H)_t}Fm^~vh2fUa(sWA0R{nkVd%
z;EU{XL@wSp?&%+h^>0!By{aHmzs%i516G9#Y=YkT
zE!qPmUa?ylG>wvox^g2I2N)E=9`3AP%i*^5J$4zrE|E*Aa2Z)oRsG{yvEM5X6eAlu
z7UK3##a?Q*g3Y^P%Nsy{i*$L^(ZvY54cA;OAikZPv%;d*#bWVnlkHYmPdp^i@c9z@
z1UlC+Kd%l&LL9QqRmu=agAH5;CfU;Y^c~g}%}Opv54UQ*Dmve+%VGk!?@Bsj;$`K~
zDf~>-OxQ?WnAdiIlGMM4kpHQdo$2QGlLAr_@~dMR
zZT_=v<~{yO&h1Y`~)yZ+Y;>XoDAwqLZaR
ziJ0cs;rF?`UV{*Zn)EF
zeNAEO;JwRRuu}iqVV-G=EcD
zZFA05&jZhMyRmi1qd52bW>7<|fLM?FT2L4w7r7p7`A*%=H4h<1G!Yldp0bwgGPaXf
zQK`w8AX%Xq9#!z8HmNf!H;|>ze@gs&L)L$uOY6B-lIPYPN#u0xlrrm>*wSTl+OF^q
z_Qap+M-kmg73(w6jf4JoC9?;&t#(?CerVoLT@s`g@IvOpfBuZE66L>!#coGE^jWW#
z%^95|wXZU2?&g2~{EHA{^9G$entmy#ExY;?m{53ZQ@0pBru{{ho40l`8hbnXH6jq50gP!?M(kdAPd#_yo!-9%rbFWmdT~e@cRQE^&R?<9
zTld%TRwsk$o{Py!{h;~$4tHUUn~T4%s)ox5)-qY%4sLA{kj5ra1SNP7oHP89rt&mv%YRUv{&SN&&e@zaG+MtvGh&N}P*e^>+9Kj3O_zONcm=_%
zq2Y1yv;>wZ5I~gGRiTBJ@ri|aw?G+a$y@p|n=U7(d%cb(+T8Hf{t_Af0BZ~Sy0Trh
z>(Np@j>fVK(25@%nfj2li(_g^Tg@>9yyB(vbbsQ!s*faTVI8E6q;c-kwaCyHEyu0=
zXm=M6jtW9
zRa-Xf@yyx~7Y_KdP|D#azIjsk;nA)?6^g*G&!mrR1VUspfsn?pDlb(yY;;pojxl+C
zpPsOubFr-2j0CSkva|hwe$xs}m0#w82jNbDPQsDQgduoCe&`Ng$;N>`U>l0DFCO4@
zQdxeZuT&%k?de9iL%6zP5V?beDfj+M4DfxsijC49CYo833Q6;4VKimEt`}DKZPLf3
zmp%6d&o5izvGWP1S*){$Ni<1Q7;M5zqDgiQyDnNi-tpnmKS4VeqN6_4Dq=
zp#BnnCnMN1%9+vNO~qU~jeiXY{PVOT&!2BS+-Y=nl421p-$?MVmVp^HZ@Y{kMAJ9c
z*PYo?jPB^wA%{=1I4sszd!afWLS=%WBnP5NfU`+5Kk|qZ`IUK~IYf+VPQOj~qRty{
zWwzg$GXMJh$g<$AYzG-s5(8N2`$Tfte}PW*M?&CP=85JHl}jkolR{?9C9vm#U@Cbq
z+{DTv1Oc~xo_)>W*|#4R8c^fAIDQqOK#vf^!P%gGpdmoXO;__wU%lh>jO3$g0>k$~
z5mDzr&c-WJclDw$p&O@}2cI`I$Ve?zN);A|LM$;=61KlWB#|z=clj9qATk2J(Ef9+
zYogwsnCypEw2q$IIpQJ}zMfoh0{uvtx;*_c$P@R%hfscG9q0m^ZHNsOJh0~WN~<}!
z6=HLk>;;~UK}y5g7<>x>y0j0Nh=y06OXNhmW~}v5b15Cqt1Yy$C^oWvR!sxZI)gwL
zMuQKJ!D7s^QiNI*jzXNv3N@iDzGI{KaCg17pJzt1*Oy)F|9G7Mql@~OJr@!v=9$=e
z*}7K#E!u$-nHfK||2-iTWa{CpHfWz#JT=5ul
zn)D@YN7Qor;4y~@PL&=l>)MoqCJ(o|2RZ`TR-@ZDC6T2eCla!M=4=nnwQdi?R)l2I
zOo7s}_IUH-{HB<5^##B+xg18&m$&$#IPAYqlMnpp`GH2=UDd03ZsQ_Fh@B-7nw@Up7?icCcny7p_ogDqyk0@LzcX$10qHowNR
zV#s~l77raSxY7&sruzlr>sB$c`Hxl!TYQJn0RC)^CC1cS_)6d*Cc
zwf~zJjH9eBOIjw+V`NVFI_9j@YogvXWv7!l+eu|W#3#I&F+hgooC=xSY%I1t{(S8L
z<>I5T{}Qqf9)7#z9prnI-}SfIR-dC^)>Tg`-qc(ETHmlbTjhQ+s~FWU4V3C)3i~Y2
zK9qv$B4ra)diP1GAE^*(g)D!-gt!mtW>s@b|ND8m`#V~&zVZ<#ut8o#WV}%
z^^PJLxRI`05w6?>*Ek5w(Tfvl;jG-K8<$3_=$YfznRrcsN2Hvc$ZBj-ZG;Dv7Kw=j
z%<#lPqQx8P%B0o9vr=0r@pc+oq>H&PU;WdU`M=+JM+;9F|NF)Nb2*v0ADz5IjMe`(w;X?C
zg>_fhfwLA)BE2TdMoM;v)-q-%0Huqw{oWO@XU6UwnSAKS$7p~8tV$2~Ip&G!`ulOg
zR~ft0{rq7R*9@lfxKb|{)0V+7#|Q7D#jWs|s00QlVC~SO@*bTBc;oGe
zz5Xu#)rwTI%@8p`8
zO*=F%e{tBD0!XBfFMG3_iL6SaE7%i*Y;bIPV8V|7+Fs#m9CYnwpcx*M?R;sY1QXUD
z-;c;Wk`ctCDcO772NVhvcCJVVY1R{o!m?XqyGt
z^Mb!`vfm`awo5Ipb_%QCZ1a5a$71JkN++Bl+UMKj_96dUjX9TdQ16x*yI()`;znnt
zA2B)_uVmo^%Ti$Nzch_iPG|mMRpfuP+P7DpuY9!4`Cl&pg?-Kfc7kvN2fLZl*CF*k
zaaxy8v5b)B<1=+rJ*bvxLW0T=Y+plNzNe+9^F3K*83vQPz4d2m`QoS6d4~}VD-yE0
z={#P)=c@_mNng=K-!n65-Clt<(FOSA2(p%xWi5>3ifm{wWe(n%pUew>*heN0;tc7
zEzb9fBea%f^{)#Ej^m11QbJ@F
zefYCm+ziA&9BUsjYM8U7m;mB$Z(@Z0cE%BBb$bAA%vCkcBBt5;y|ukvimp7ExGq1~nz^*M~C=#%2m2lM1s*rXNcW`>S6Xwd`{%hA1A?qhiF^&k&8Z
zfu4|Wlb-)CkWm^J++x&WaCx0m`~D?!!Bl`e=5TrrISj8<%0hJ94?e}8c(~PX8jrHVSsj7(##pz
zx_r%OAv4SGPE_41F5FE$A5^y7VesSLzco=U7;OeeFsV$qdb2_T(j%dmjjpO}D{hd9
z51LAp+kL|osG01H2UXJ@I6mShAt(~W_IeQsn^gE|r8ESQvL
ziS^flelNH71)2xT&o`21mo)g8f+fMTI;}~4h<(l=i+Hzi8R+CK@632l;nOp)98P)6
z7Y>
z`b8d_TCCzoDsx|ZUp;rb)B)h?e>WdD1+N<&FIQUgOYeLWI)qd`3`v2fb3z6kMneFZ
zOr4pX?d#_0Yr_vbG)N-3FX{dZwU&s~HCYkFLN7fTRf~0g?YC;!jr2kldesB@&lF(P
z-e|G!V@QLI%QM}jFPBv8HK95oxTtxHa`D9AMBf-r_23qcI
z>n(5M%;)ZaDI8Ju^5
zXJgT5-;A4Gx@?J5mhBKV5SI4CN(5<~M)^<#%?1~KcA@zZsGbZIwpNyVyIUe^+WO>dAw`nI_{3rutOW>9}7
zF1?+o9EGueyn`V{pLAXaHgVPf$FA5DFvwy{{m;3Z3nuBlKRVOcaU5N~uAK@I6*w9j
z=o*o%qh#UyWk{3D89|5QtsVpyn%(~f2+wG%4-0XEkN~u$)RcG?GQ
zy`y=K8%4`>T>Zv-Cn!=+jgtjiOjD&Bw&>h^Tn^SMa>EO@yMty+AEH!E=6$|>@xh59
zojxqw=mtP^|Gu{i8$(-P1cD)B%QbPVdR<$rz0QG)_=cxSomqY9?*>NfTa{Dif7d_%
zu1+4Y0M;up%?k1IsVu=Z-a(kx1s5VsPyax;1+;iW@Pk+1C^sfIIOJYE~E&1om{6bh*`3u`|Xvc&JdYbF;RVd>i3LUxSP4kmL@bOf
zz%q6awa9XHPFdkt$l$t_NA`~7XJyPvSGUB+jJ?nD=E_I~^;It{TX2GIq)=ml)r&+C|1<E=2f{OVnht#+T3sCG$39AdC=lOSA%bPV?>LkQKIn=gI3HO&
z0l47Pbjlsg8lJ9VQd#`weWGzT+jtdv5>U*&zRwKp?>pJHFo1`c$r(m5g#O&n2Y;S9
zzQapvWnykW1x3K?>a~I5!d#zd!y#MuVOqM1lkIEqD=~ulG^wPM4U4BqT=xdDJm6tM
zmAQ995{<(L#9LZ4P=~g~&Fs*+^1h3Kp69o1;X$8YGPl$tqC;^DfIdH
zSkC-#6Tya!Vl{xK*jo;$R~K;A6)Hm)Xu4}J##pwi+wOM`A*(c?g{?7mhCmC`wSY1J
zqkb*|(cJn!Ym@&M1wV3}Yb8sEr=`z6&7d-Jq*-Jv^$9;uTVRbAf9G(mq_L#|1*Iu0
zG>}q9yV^C>QT?3lS7@8Sx6#1REms{Y+5~6jPqKgmEea`S>z2!25N5AfljL8D;7_W$uEO4%1d5>YBLNx3Db55abm9iv8D{F#hRPxzQFQk_4$^
z18HI)`@PA=k#29?_ZX;5!@|=pv;GvLySUHILqT4YC$WSE4}aL^ok5?w%2iaOTb5^w
z0|FDR;dl`#BiR9Ew_A3N4k9FIz7|TQ5FJ)(`Bom*=G;-0>;#`m2M**I3pR=A#z-gQ
z;Juc*vW=I_3PfWCke!FieOZP_erc3*xH>Tn(qu-^l5ig)m(0_5{3kr5p`Ez$oKzfc
za?Wuq;z6!AwD|SF{v~(*HZ1UydP&V}O07Gqg@m0&jfXc{6`{y8BJMRjHI2pT;6rPn$LwY*Xrp(&bzI=~w<{Vk+^YBoz
z6HcnF9P5SNvHzvo?X16MMcV=>wL9p{My7sK=YW%}FZTii0`8|w1}~Yd-ZKQ7kP8KK
z^iSCgJ;Eex3L{$vp{(|DyC1U}E;&9)2KZBXS~O5Fu(@lqfdVWqYlpf8Uo2mb{_Exc
zKOMNEiK;%eq*pFG;*_ybw>2C$=m+RWJROq@n|^#t+5q|63;MqC5FHj=NV|+A{Dw^YfIz#AT6d
z;~7)_{`CN9RhLaNZOPCh-v2>>jIh4IEk>Jgc`c(`?Ks_e^8<*`VkZm5g!y@Z~HQWXsHw7-2+V
z%S3iQ#1|c@pfy$F5RE=)+sZ*TZ8S_{KD|7voqf=L(B>_(&lxaNu)j4wpXoB1t3;>@
zXmj6ht`4UV30<*Q3cJUof6i`&1waP`OY7f>?7Um;H&CJYK}Y0P9DQc|CiDrvwijLo
z&TM!E7?mO0Zg1)7y3`$@h^jFkeL~xAQ!vFOOAsAbaYls??F{
zrb*-Tw^yt6r`M!m)nnyDHsJ<{4%y=#+4h1YPKAa6Jtxn9PD&&noh~@X=|OhZc>50b
z$kcBWwpFw^p|4Kzx|vGdCKTUuG9k}VQ{}sH?^Vsa+eaUTu&Lo<-ZvEuYusG+J(C@h
z@FX7xuU$8p<+5Jy#NsppP-bu!ANt~ahgjR55p-vfoWS$&^DXiVDYWflPsUu~~?%v&SN-4{{xm_r<)}oig^GrOM7V!ntGKm%`3Utq?xZ@4XOLy
zR2HQH(8)KD5G60HZAEG8{ATm&`Bj?8bF>F7^Ayb6+&w;*WMWmz!_XyRyI5Aw!|ahI
z_dT2Yge}yFJW+&H=^a{b9zzZ`q(JX?oNR8LJfsZIAM029tkV5sL;U&>I3L*R8U|y`
z4}WdbQncU)%96^Y|6QVf<3Lms-`)$NiLARWv3$lS#a=MJ&N+%sT&2puTx78^{C%rl
z(?jU&&AjT&fv<5%awS)>Coh?y7<>ybrhTFo7no5eHUxH&Y#azT!H+r3084eEhGdDh
z0hqcf4HWQ*U6T=92R=ubQf1TG-Q0trTWTf-yvXqN-Y}6Pp6ZY%+fYJTDbkb?TOEWg
zuXP&MG$Dpxy-ao*Y7afa>;l_cL^i{z2jS-2CqJXxWD8i1tbW2gRd!O7hm_g<)3*+N
z@-jBZ#M^(_Lr)G@%)M>H4wov!+b+iuqr+CCF`(m*AjjdVj*}qLl6bfrChFi4ExElN
z*f^6%p=ZlUEUO#uI2`u=jJSOY@WY(^&;S*LH`{D(ZwMe)cBIiOOB|@_gRrdwBBey=fdd?L*tJwA7o{o>U|9g0;
z3@Y%ey><>k_aQqJIXpF?^&6E?miy|R4U!+Som4-wVY?aOFBvyZUv-xSS`Z^z`xn;m
zu>^S>I!mn*#xsi8k0-c=d{>M6LB<3|XblqqJ3X{4xg=zSpvjK^Hr{!eoxNQ-LzsEv1P8*ot~T4pVN
z$?D;ztZMhp4ke9|X68>e=5w7Y@&e`pEuUP0jBB`rt#Mb7K;826lOgvtQ5o0BXZjELol@;NgzHt_mmmTEG745#+A9E=s_gKpyxHIu
zMV69roM5%lqHVlOUL_axSFbRo^l+SNq-{sv5;FbZvn}$@gVZKYO;l)23HiEGuYO^8
zS$T)^>#(xvt1ayTD8!`7MZgJQ5Kf-(wGoo|pD}J{&5z935dgJ}xV4;7S%Q$570$~5
zHL7jKS5qV7u0JieMVU(Sxf_RyZK`(rJNqjaE;4G?>2tl52xyi9R-mw_E5=Q6K=B|Y
zJL+|8?gLlec(h>pE*7QPL{!x>-n^cLKXpGR*}LCEd)XJE3@z8t2KRK%j^+f;3SG00
z-}u{x`a{@GBL3`WWX4G*tr#4gO*T=XD;1+ibk9kyV0H`!9dmMQ+xNsB{gl4#&<}1j
z&Q}4ujzH1_?{zd#BVky}59DJ0Rm;tDth~dliiY|s-)idl
zmXYPQ{VDfroC6K-4P1;=7T8An!cG-SAyjs=Mg<@*A+Uw910^TAaSwV8=-UMoyTS>Z
z;rhZNM(h(_`@8>ekB|E|ex9vvuZL#Zn;K7%nG@>M1*IZvMYE1CLy(oC=acfzf#SjV=wM_0On4~{E$`i%Q`a=-$)Mw0}5x|&d}7oiB;(xIwPMB4-O
zs?XXNz6gZZPA0*9kK=$J9+YGrmH7_=R$vs+M3u9ksuVl{c@~=^1F{pTa0^tm$5!)K
z#H%xx?>m|Kh=mfv7|J~N*lr=~BHxpWH!4=O`jhQX$e_9mKXQVb0~%1L@`RT;g^&=c
z<6VHhh0Uj*=M+ZvRvSQI)R=OE**Bwr6q=6rTMQ^$g83voJI=Nqic;vsl<+xaweDMmonX~
zpHju{MFd*B=jDH0zxurn5YL$cbN8+TZ}=qg&g$FC1>F?
z(Zeby3o6d){;`B;rS-?Uu#qp~Z5^af0c7R
zLtgXJ522$(yA#E4r$BwnSf90KQYc)nY*qbsD2|_`RE?toE5ZZ@bFn_FN5)&)p&e^&
zRSwsWig737u=cwS`P61I@Cf8ctt(36*=9I5ES0&PSjy7MP-UX@WoZ)(
ze;?j=aDfa{Clgf<$z^%uvi0Cg6cQx74P{3T7NCSyQl?r0s&^TmP^4|CJNK50d&lSN
z*6a+|LaTcPIt~bAuG&^{QX8UqZA*#7N)855Y!BNiPPme;{ZE$jq5~Ln$~Rn`P-OX#*QLsI(^>ICH0ywMX;2Jeq&I`R!eTQ@OKDyHxQ-Og)-mHBr3y_(luW!0r4scTFu0PR9Irc`bXVWhkE^gL3E9er=3f
z&f_vEQ4-M`U`=(@b=b*Yv_3(=1iY$*oE)b-Jmp~KE5u`~HgDatALarv7;
z=|qnTau6d{l#Zo8+13{Poag~z|g
zkqxMO2H1`hVn;*9`yaWuT3x*$n2qRd%x5#0HS2C}co#2M>a2HrTh*Do_5;*M_T6Hs
zGN=|R3=+7Mju*$}9?h%>iQY!VU=BiXdRx0&0Mqo6j+J`=q&U{MV_zNI!`>})91nIl
zj-CcSiX^q|09?9Z7k1yF3lSbzK-KI$2pK)NOTeZ3t_2?&=aEg!)ThJJkKn@WGa+Wb
zS(tyA=ly0`M(l7oX2rdpiw6sYFflHS1r`h1=Nj+8V1oxwmJ+>W_kyEf9mh*}@s%R+
zdGU>*t&X7@dyp|^VRc8ZYLT8Ys7K-0r@U!x#qAy3#@YBLqQAYMuT{6)F!y+1B=f&}
zW-jvNYoesG<@LO)>fTfEn8{CZ5A+2~Z_ZVLh}worIlbU0aJXDl0MPwn?^=wp(-MZQH-V6oj^t?ix1-iCBF?sK%%q
zcdFdWlPjqom-ZxqE|T*bcb>}TTq@-6wRFizcQShN_;H6(LIH7~AYw`x&$?GE0E=@I
zjUs{EH*e%4X8PFsfIRO30P=;
zeg;i)$Rcmucc@kZ%K-W8l^oLwDf{|+LUrNla4YVxlHLitN|30$l)_9Kj+xt7k+B
zo9dlBD)FGc39GmVigV&1?~=FPMHw(=mS+i7x_f$9y`x^F
zgMci5j}L*st+rB26_*6rKsl8~{3u~PUFru!xQ}7`9tsm`uYo#Q($5;JsxGjNZ?CdS
z$qrVx^M?3$hOD&}fy^d|t;aiq)>-TFETYwW$`d$Cm)C3Dtt3()g=m*|X^9CdA8$c5
zQg6f&aVi+yT3h3p>f)#4HEZjJp{r>b-9cdh=MM!DjxfuI*`l;u1Z(eP&ZKiXB;|G^9n)(
z>Ri5S7b831eRsgeFP&CUvN*ssbc(+zn_tN<@N!FnHm5rE^wrsDK{vF2fAWqm!)1@O
zuLwzW-Lk(YAtd3ra#J4bpXdfG;+Uw4f^5bubLjCPTcQHcFPVqcnf2CYeoe59WT`d~
z^c5t_00Gc?WB7raOQ>7#uZ}&WxfnS&U}Q!90odCt2MdpS8Mc-nM(K>)a?%y1yo_RT
zwu&H7heGq0YEJ&b^S2r&JnnIs+)0mf+l@Axe@Bu%wAv$Z8?D1~pylBaUw|IFiYb;!
zZBQ$x24Uya0GCTT`Lr4(9`g9s$R6mHd<&Mj)Tso5?@eFwa@qUZ;rNAi^GCe+NZ6w>
zNrJw2kg=ZP6SYzx_2^Mk_JCD=`Ke3gQL!hFb+)dGF8wC3d`)>r$BY##VFK%AA8F#Ho$0+v2O-uMA)dvWvB&n?}@n3CHBMF(RFLapi
zMBI3BR7CT>soTFd}
zTR(hbHM(JD>v!(W@kDd{V6pogc3$zOkfnD1_M_30kEdC@Xs|RyMw12aM35M$LNVb=
zAm_`y8hVAE=h*yrfv0cisa)W0WfND-EVMSxavY+cxnY$R#lTYVnfAHrO15`cA~xHz
zA?vj3x|A3ghwX{QFx%^i@BSh`<6gW3`x+Xy8gK@(
zV!?&o3f&5xJt(Tmf9Lz{>+v95(R*xFwzY}dXMQ=Dp5930LX>wuq0#Eg+cOVUQOtx#
zwj~geke%kM#vhsQoOy7{_>nF1uP@OmMo|-L@Gi4Ff_-p$pP{BbFowj#joLC-
z*KYr*YmLNLvRa&Ei?^!fzf&pO<#)_XrYJo3tCbQpY0%kL^u-&PsI!4090H07Q6H_f
z$BW>%PxUd~pZN$iSi6*}GQ>ivjQ`mei=YOyEJXMB3E
zret?qf%boX`B9`toLg!tYOO(4y
z)VUJQJOjz)z;7*o<
zD$y?dp{WLIX5HLVvQ2TSqXzKl(!bp!nm>QbLK68tT~;BXV8M%l310
zjDJhA;sv>C{=bm%WMLW3lL(`5C7uO8H7GWyP%--Eg%;YTa>Hhw7=foV_-LB0+k2%9
zZv;;UkCw&Z*GF4$t;^D4&yu6P+G!d!C7y*!*VD#4*8h_B^Onr{Bye<90F!I6sO+ss
z=j&F*tv07(-<-QVI(qk~SGo8hmhYA~u4;wF;^gg?KI=@2mSj1r42ZZm|MP0nsXD6u-`J}+?d!(U?%kPu{UpLIi8pmZ
z!}!Y8`r7C5(_ev@!MlGJ$HencA|p~d_yDy&nL
ztXTcF&=e*KJmo@fzbEB$ubWTl-RQ|>hF`b*a^Z@*{Z$n89p9}d|T9PpyF{DmcJnEPCsXh^^GgHqjXjzIF-e5P8l+
z*!v1t#hde?-mWsPw?Xih;D!*A-^hcRxW$Wx8qN8cWqlwOf2VG_#d-AVmwnw?U^
zBt)Q#I$5EiIh*>078x&EILUo7AwvW;nt}{B8e#2LXR+=lYq>`@2xo_znjMklPPS^$
z^Vss#k;iF%TQP);t{BL=N_{o7nQ|xGE%rmMtYIjzvJu_>s`3oKQz2uvvUEM+bc-Lv
z0aHNCrn5T#G@t9y#Z&>BPO+20^}+(v3HMx$TRZuYIcF{t(K_RjA(b_hn5#FPV~dLG
zBMBJdeycJ)bMdRoax^3AASF!b^9UdEd1^r(^N40^
zN3uKt@y_wO$u64AV(sz$n}}ib*-y%;7^6c*Lw!D}2#t|50=J@c0?C)F*M4x`|77=u
zE-z>-Lg&f1Te4R@Xg;6QReW*EGmRPX*T|3iiNhe(R9cZFOPg2lBnew-gfqt-uGeYz
zds@TlXf9v`5)<1cyhn;`s
zt-}|TcL(PVM2*|sEdM%@e=gtEx;|tgdv0+qE^bL-ET8PaePpqx4EJ1ItSoQ0n%*yo
z5PWnYleXmJU1k~+TwB@MUnXbIGYRM7F3Ip{-i6+Mrod!b`9|h->X>x6+biRZf$+p%
zroEvhE0;`;_Jj3r{e`W;9^U6g()Wp^_Sb)K(IDMbg`P|2zUZm`*DWKAKUaYYO2Ld#
z38L}1yknFd!El@H{*R)zZKErxugz)%zKdP?$_iawctBhE)$V4mA4Ku}>BQ$xKdQ){
zk&&*yDWw#9(Lpohlf&G|o4|I#H(AlFT+RZX`PaWP@7AALr!5vBUkwjCt3z{3;F?`X
zP5fm1RZh;irBiiV>UVkydg#YcUVB`DQ=IwtkN1{B-Ip#~0RaL4PI*!%Hz|UY*88ToMVPxe$}z
zP0Qy@f=|o-ra0HRAKp}t2BJc7pW!(aUMi+h=1>g*r<(6m!x|?&bj#B^q!>Y_;644
z^hZX|H_A0YSJAmOZr+JueVSqC5)EQ3RU&@luU(1Ix7!0QA|E94zNlENKhgEWulx$K
zj`Y{3M|eUFPMd%>*z0Jq4hc93!ntb8`QSa)`Kx=`$~9Y3o)leW?uC}WD@F~jhwwm2
z+X&|oxt%`V*{THJ_VD7kMd=t(u0-icw43*I7VZB8iGz?~1i|$#D!KNr9m47=dLz2_
z{3$rY1B69l*FJb)dokv1ph;n#hLOQ_Fr>PZ;slRzvomMtU;1GNN5`wMz}mSHW?VOz
zfHl8{mGDM~pVn}=96M*F$o8Cm?Byd36VNHFsli%CSNs*Sv2Q|n=zMT5$LJLt|Ch82
zG#`HHb>u{-CvwmT>85+w+(&NYQP3WWjHI$)&UEKe5PT^!%Hd6QqKsp%-ff7VuPm
zh8J8_uKP9IqkF-Gh;wxqn>4z1xk*(=mmSd)aE&_q;clc`PW}hc&t@Od4M-ix0)Q%*
zLf)6pm$(eoU$y*IrlCCUtb5IHnD&&_Lgzysuw&-OYEM_l$7P6`YaA=raoxDUd$ELICegdY1`VISS=ui|$eEebV=
zMJD4F7$ZkuDOF>g4iN+ivX(jI>%*HxmPMMk1!<&N{30er_?U+2MI6D&a)Cp1uj?Zb
zUdxx}{F&TAkzW#2l7zkA=&-A^=PR$w)>jVg?x`&g-XE~|=`j@OV3z-mDC5_f1lE4e
zc{Eqp`>QH{-qD>rhnZaI0sc;wh>vmW*H%+F=l3l)PiTkb;n0ppMa^VMbEwyE{P+zc
z57Z;6N`y|VW1Kv^`B(F7=~O2%>HO8xdZ0@Uk!=%aun1OKuS08f4b*gzV#l$$sX2;)
z8rWiovYGHYf3=*%e-o+a3*uZ8IeVSn1a~uEE@CIhze6juY(W`!zk|rt&+IP4rJ4QA
z?n%JToK;c$Qgma1*^w5LUK*X*>$t5dnIU2RXR0I!(u1DyzO0!MTDzs`i_QI
zc2^HVo!j9YA|oiKf_I#W{R~&%^WitN7K(?9Z#f+?P8VJI8*b>&WqV%M(fAIWH?OKE
zZzJ4jlt2|}C0$zAD;!KeHs}{`segF5&JwQ-b2#6(bRc6W8v89mU-fpqUc#1hsVG8H;Xko{k#p>gUglD)9Zw|X@u8agXk#8Y6OU93GW
z$;YD48DF%Vz%+*26C}W4R{^U8H7E&&@ox#_BTRR{tY*`U&UYd`khif
z34j)UnCsUzThP_);U)61Kt_=$`t)d^uzGYO;_O)u_6XKAi^X58UoJe?H7Wfo{jma+
zZv5^|BnjmjboJWPWP_V$wp`X}-|5;lmGdelE0b3pKdVQ|aAX&$elOp8EU8)Z_bK@H
zqY=KuHrlY^8rg^TD}1??k6(0R4&Sz3PTnZdF_}_Zyy-2qN)!jX2PM?T?2H7oPjE7n
zGq7a(jfED`Bb@EXoS=?9=v-C)94nv0v_uav^JA<`qSA6CMx!z|JV2y2AVFeY2K4>Y
z$k$V=KW}}i>E)Vk(Ayf_4(Z)|p{%p`d|mXSkds0P#@P4K^GBCb$Qm5Uv$EuKpA`)j
zn?&5Aj+nNSI4@tWd>*s#Bxy9<-0uJ3>#L)p?AE?#NC5%qZV;44=^7eDkQgO&NKqOD
zh7=gOK|lcs8OdSj?vPYadgv0QTe`p7=RD{Ao^#f_-oIF^#e(79`@Z&d{c0b`IYc<{
z)`KOO=PDL0`|#1{GCm;J`6t#>TJ^$2l+uPRwl0UKTn
zMq|wXnt21{2ydYr4-4=>rGqV)jnz<`Xmd9=j0h*9SAdn_kp8YyG{RO}2sAC4xe*Ri
zZ+YICHswnm@-1XLoUyVfr;M~q>vL&u+Jl7X@SS^yjwrVMo
zSJe5@A@C{xvajQwNzX{|GfRY^Ecg+Y)V2F!1h>0(_{75q+rAjXUjwv*k?Pj$SpmhM
z0NTsJ`0No7hCao%^p_3;F=CqCdD3Gw!^8bs8_1kH!%c7Vijast7X0uEU3`Hlsp9q|#wQMQg
z-K)p*wB&0F=@}hoe_VpDwMyR~!;hj9hLDdHQj~}_YYdKl3aPnX&?0LicXiem(Oh9R
zY0M1ecGJ(iW4e$MRt!St*-hlOhD@({1xNkE3=pV!rRDG;C(Pl~h+Y^UH4>Ld
zw|o~d)xdu&mKLxf*Ze<2Ns2h`^@-Blc}KtoNXp
z*PtcAoN4G8j&@GX2wCT8;vz?JAFy3`+_NU{3Ad_($d2z=D2Wj3{3!RB0B$Jn67(y#
zQ{!Xfpdcf4@vOp12p|;ReNh|-G4E!XD}~JERk$qo-X3@r**50vL%gRqEgE3kkG*Ss
zPPE;A9Xh@~9mz*sMp`=CeHjI3`W>pcB?|jE;bq88o|<*_#ws8f-vAVc5N7TB1I=B&
z@hao4NIduncSo>0U^g6rG11_XLeKfUg+t(C
zbydtAM5hJYp2ZeB(aX_3EF%gY@^^~zoDI)?io&*sSqq<3d?~-^p78!rA;-Q&cT$cE
zv~eF3y`*tiF3Sm%BxXEP3=EYyhU^FzD;?~0pI^|KI)B0KUwuxqKPC^&Gc*w3t5%4V3^zhQ($~?(ZUV7Z-%7w%!Qku_0A=Et@r4)zv^v^c`EWDO7V{
z6S|FOf98$v|4gh-(kyGb2Pd$*^J5wbuY(zB>e`sRF1LCkNwi6djfuD38O&4i(K@Ka
z8&LRF9ff-4&K3ia#>!Um->|jlMl*fAxLNjf6;{%)XnoHu;un9as;75MVhJe}-WUuN
z0eP)0|D;({Bo|?S(xvp?zXKnyWriY#5P^68twyGkJcE$33lDMSS$z(u1&dyE2qUSe
z2<<^Q`;dX^&mc~ZS-3$hg4N0xNq@^5*f90@o=EJ{ZEPN%Ho`VKIkeUl`__VIiHjC0
z5uewr70=)NWtJa*S_pYo4Oav^2D%=OL)#{d}BLpl%aj~W%BWr!)jdY(r=w}
z{juo+^z{sdSkGAIW~(U5zKMoPpA6Dr-Dwb<;>deZHe1
zZsV#SUrU?KZ86odD{H)y2_V3OLx(T0OF~S9mOceFQ}{A%io&|j
z0oY~Ksl2%m5hT4uhf`vdUJ>pNA`qD&rV
z_fOgWs}sI+e@~u83aup4RrLfeOE3KiP)c6t#=YOct&xlXAePCym$l@&+j1}bkWw`X
z^RF%fpzuvns%EALR4j(BFni@`Ms%$E=U%TWPgmvYnstzjK~tWR&MYk(OsmPTnq0$E
z$IgEP3WZ;vbYY>rP&iqj_S50VInQXDh4HHBvtykTu-C|`I+8ESIO)_9Vi`jB@S55)A9d{%`UC9n4#
zu$KHXJ-C0p6=Hi%q!V5~O$BweZ$q5|Jy0ls_>LgreS4btK+gS#4o3E7=8sm0s{mG7
zZWVF-u)9h7uIkkbr4qUWf~yyVpo{1S{XEomSxioW@*A}hdmmpbR1^qZ8%=zE$nUAn
zG~qvZBAHp1Q{?yNP
zW^5)dSUQ44!G~lk>a0>+%S`g!9=N_4Mnd4W5%JtI1Rs%IQvpp2)tyc5^Wx$?*KSfL
zjgXN2nCj%W_%EfYkLWC;Md_s9o)8}B?|lr|Sbk`+{X**xuxIE^D4{d^?r}L*mcAV3
z1CL#Xj&O8rtk-j&vSJ1Jn#1;IH@*V)aZ3>Z%6mPE$H`0uE5H8gA#lKv$K%}FYm09@TwvKg?r?(;&+zjK7;pi$kZ^mE!>GPasg1Zut`2H
z$3?UkHND}jA&8-!2D#C>#Gb#g~bS#QPQoKS!-t;O4*nUh~&s1L7xUOeZ5cNK+SL=f_
zE;V9ZrHG$~zl!CzFHg~AiTlw5?5SH9ybefyzLg>IBag3gpH_b3P2e%u>wH|&N2aJz*mhQ^6b8_E8S+f?d6AaF~OE-dAIUvzdn#w)P1{XVDFz4i#A
z=x2L%RdfZ4w*E!5NO!8P&+cz-w`yJ6ah5MK25eNs_vSZ{0m$(^y>SVov5(mwV5b6%tj2=Uay!1bIWm1Eypy~w{81*n
zmn`Kg%_Wplh?E22z{(_#tq49;t~v~ aBNnkbKRml^pJN$Sv|g?lBgZJ!NV_
zvz809rcI`trAbyYxNED&lN;?tZ&s&axx%>45DlMFc<(`|&ey%!^@5!!;Ba0nEu+gb
zw(s+gCMz7i*YUAuQD3zk{6}-ZL^MrcQV+whe`^8SuP6n#1rFvFZEUY9p$!AR4qLl~
zhh*n8t|Ct4@aEMg2aQ-nb-$Ss>I)qL1jb&Uk#HT09p2RJ3fuF}tI?VA;Z6Gbe}086
z9s4)l%13#cd2{tn(~wp(Z-PKhO}5b0FkNZ2J2+CoMcIFMfTNs>l{Ar#x>YRz&UxN|
zg_bQUQHA!+haxs?nXcb$4`EzpiRA5O$s#+WKgc(jvNK~UaWP#(yx&NDJ8-wo{Fjgo
zx2_!@MW>ACa^w`6Nr5oEc2m=#!QV&i(2@bVMt2ELTn5hb0lyIR~Ue5ncR==JNo`O>3ffZTnFGzPt)8`{3ns|MSI
zjnJD3-a-kG)f}e^GU8(!
z?49OI!&7y&Vwql^HOJ|0iu)TpTl5GLJ{dIG1Vx~&n(NcinoDM_thFJh?`S1%m&50|
zS(PL3Q`lns*s#L=%w&Hz`2v_`+;$f!nq#mrOdbxdx4c-pwSJeE6n}uljnkA*zBlvf
zFL>0l+E1HV>Nnf%^;g1YLy)5v194*#`ewTK5OLgX^oW>J;UP@o>1zRWVoi7Izk%d`
zE(RYRKy&tIC*>#V2-hy}$+7hwXEWa;#woqfw=0Yf$5jVq@RDH&p>`K>b>c_lSeV@6
z&^X7uhbqu8vaugxtmGfJ>|)Ly_3ck_pnGsJN1c((gz~ggp~^)vZk+HR`BJsaZ{954
zUzEC>WXfNv!9yp1XSC(E_%~+xH+1>WFG7J>qB^s-;0o2eM2;Hor(w=g#{RP-^fEB5
zcG`*HTfrvkI!uV&%W;wRdTY%_wd(oW8G*~@(;V^;ej1rtZVeTv;9X$jS~tnqWb3}`R9lEL(XU_fWa84
z&lXOnjMYTnWV2j7(*mcL-?*m5=HROHrAj>xNBKKFX+tSg#&!xgJIB8jF15Rd*bwu#
z?H8ujgSv62)Z*Vmpv`h}=P2pRmE8n=4U3P)k(3Trh`BiP?
z&ljRG+W+N$+g$(NV|7@Ct!-dhZ+`nx3kQC!6H?tQsA
zo1d}h2wG)NmHG%3#>X)6DW)&E^?Ed*Ou3}LXBWt}@i!OyrcJfEQblj#geP2p>fVIFv#d^V{@i!THqViqWqhK+yv8qr5
zPzBMrKz5BNYA4mRoa^(VEq-;HZxJlGR=ieBr}-=Q6T6mwmHhuKJ^!vio`jqVaEfpf
zGRSsp%Lp^`ss>e*?*?*8X+r~2-O;7u@|7MbVB^A>FtsZ!2FwQrE<0#7xYe{v_M(NL
zx-#6AT}3UHh&Lvk!+=7J+N}QZyrCAFRs)FwRmcb80g)}0QYFXfvCsVmIhiK;-0!w*
zKh4~5k90Oz*G{)`ctOK
z8b_Lnc1u(IbfvW|=Zd9vyCU6r0US*AcFtUu--*&_cQ&=rY+c+aMOeC
zq~2zqJ|6WfW=It;|6lLQUmO)9z4(Q`rt%?uJuDo*g3E18zDEJhG3Z!9xG%lZqfgDK
z@Vo1N8A%pk^Y|(!=>Y)Z0k+&vnT!pxKfB7!NM|(=KsT*7@{}XlP-D8o^iXL;QFgih
zrxA0vDo%5^o0{j|Re>9inDEd&>77La|Hp0&=qiO!LTkOB^6JcX0T#D+w765Bal?OC
z;YpP`UyqUp6u(7RzlQHnYzL(jMv&4%;MO>;lyyXd#PLxA<9BK^NP~$)dSsk2uu;Z??4RMci@XhdJTfn|~
z^^KC|=ArHDzc}9ibps;q2|)u$uTO5M#~7_TD;}X!erG=0le8gY*SG`BoC`L^#ftO_
zaP6e7XPh=yY||k(&yem+&KExl%Z!k<^TzgCzX)%!CO1qez&&qY`E)f)IjQ-_ZAju{
zs=wN1Ea-skNWT}3vDiRhfg4z?YDo8bf7vs0Mn`qP)vUebCj*{Y^y+(;lwiBGU(?J`1Kop@@WVhe=`p2
zA(2_hm*qNNbuPLrh9~#gjWF>w*Vc0;J}SvQgxQ}8AGjiCN`b$b@JjNgqU
ziGn4^@pFX_qEIHr2L%~1|JS=4Ob9{;3^TJw;a71=+3#icZ%bqijfObEKdS4pe_Hx9
z7Le@58~~foCxB|=xXEwE-PI{Juh#BeyEfUO<$QCP>8_DS+rHPke|VHW9=`DgQvJKx
zF$P6Ke*bRmc$=Y_7Z%h*t7&3UKLZCdbd|)6@|zm@lvE736sv940`Qr?I<{UN6XfT2
z)lYMsDK0WBXZJa-%-j}hJm{CM^x18$b)dPJp)_y1I&P_
z?)AyMzyI;Y{EgpG^*p@{6YeS2A77y|@6AZ&_4`@hli3l;JuFP`Oh4ld;71g=*fhOq
zfT&e>{P+13r4&Gq()Q9eNVr?#>^x>}zsv@^k;ESa$}JXJ$=}wBioMJrc8R0uX`t#8
zI5ttN$Z+F?{EW7rV3X)#{DjL9jsyd$WWtFx4q_ScxSxwi25l)f4%`-_E(=E7)st*^
zavM4hRBEWlet|$Nl}3|c?Y{G)mxZ%6aE5q8cGc$XCU2J^ASo-QvLkT2Zf7G
z``=tWFoPR9Hg4B!Rc2nE^(COdoqsiFj~G&AaDWyGTrbkJe%%4AoIn~gVCMSq
z>FoRKR|B5i4=VEIjtU$v<}X*o9gAi>=6y5zZK9>txCgw)Dql!{I_>0^n&920Mf1Z;
zTd&V|UPuV81eW(P%%7@`?_^$Wy`DT<&6IaPq8L}p^lYd2^zLdu&Of~_b|UsWah%W0
zQbjA^U!05)pOdhi)dU#CZeUq<*6zXJn5UB;3``^|N0_F>X-hA*D&y>KFb1X>us2;)
z52Ta5B=!K^$AtYXd>pP-|1*@K!vAuKMc-)*aMq>_^%#P#&j$SEj+Q3D|Nj{MbJ(BX
zV?|CWtWwDZHqZfJi`4wH=fR`fitNY+pzodtrPQFsF3Z5eSaG`3+h^;^o@HIwJ*00S
zLY0h3sxf0!vO35T?v1_8m9)c+o-N&wz(Q-9F)6F?PLoa=jOSlv*%45Q0_A@bjZvNH
z&9=q-q;(~!=NdA%ksbar%z=fTMv7(LT(mZ~&WFR0#q)ci;xCn@P5iHpCJg{7z*x3p
z3Z>>^p**CYt%XnPB
z{MWnfS-hZP`M3ODbhcN*l+bau2gFfoiWg5F0T(VH8?5sqdt{8rO4Te
zn+xpZFe=h6IvHg*dwwHwSyBY~K)1%sBrq*NrNg9cbFtG{8YKaXMM{k-21P8BpiK|6
zXso1nn<^960J&(YBVE%Me^I6(V{p=CfU$9F>&^$^o9v*lp|EZ}An$`8Ame
z#L0jK0j%8>K`_2R&~t0-8j(X$cCq?5zrMju6~mGr@&mkap|9p
ziNqn#e>nM7MtK9TvA@iH)SN{We+7T-jEK+P7kt>fQ`A&uB1om{l4lHsRkYe?g1
zXLaoMR!p&!o1!vqK1KZCXc``-H?9FRt6-sQ5P;ueCA-dJKdG%|9)+vMPwzvnGrOpz
z#gUDas?V$yj$d=Y#%THnis`^imb|brO@X8nRtg~@RF5c7)DvIb=Wni0DxKagF?4+c
zm!3>JwMJX(unLncm5Zn{Aq4h?|Kh88lin#6F6xdgLMQp@l4
zXU-mH^8peV#aj!RAtB}9(}yK}GGp~n$dQb#bf4G{EU0Ts<5q>{aXU@R;Bqu8Iu%!y
z_n(g`1?%850vjXAbF9$I`+~=57)zP!(cw@pQ!|gk8kodU5sg&j^N#(|xF+s%&CMyn5pt_U2dzd`F@8xu_fQ2+|urN;EBWWU+
z)%$jG+f^ek+=>L~6*J*uMw5T67RG`M#`r@BWp_Qjp?<82a(2m`x+U+k7Obg)o1r+r
z!_nsKWFpj96#ik_pPT@qhZlSq9RL9!HLjQWtq3CO^o7yoD_hAu)0^}0O1qN!S2l$9
zudV(uZ?>^RS#dDTP44?>0ScO$3uA*?tqun!HW=0dFg8}z5}~Ne7(uVVV`rJfP)&L5
zY4REMnNW^|LD%b1bljg>GeYZ@f;o5um%Rx&c4FNxWR4IS)6Z`uy
zr)m2uHau4SZCWBIMU2L1si8EW%x2n=xjr9nt(o&i#GZ!AFW-CeP2EH#xM*$C%AW*;
zAP;Q1lsES(
zjM*Dic_{(t!z+Qz1FvEv{dRfP=bQxOkW)bE`#+BKNrRT(AA0Mfb;Ck`p6c`*dVkY2
zNof8f{_5#|{gG#`WEo&q-_&%FNZ0)skXX9?`rdG^bMgM`*G)qPYov$enb(K;lUFD6
zEXC|Nm=~noOup%Fdg{0*jnb0O-t9+P5mAHmYUcfX##X}aGgbH*ip|NNbd@x-I`B7=cJIbFnKut!25B>3HB&nb5TOGbP9
z>I;ezc~_j({)yG8VValyYZP5ll+Y2u^~~=9hKGYfyYea*AL8O1jQVC^#e7s3HM{cXh0s$L@-MQ&3;Q3!=G$k
zpbG8bgSd$(^_zl0Ga^v}U=P{zdJCGY1=qxrs@mVy8g+(M
zngO;6_x#LkIQ4Jm=W58Jy*#^nK5#mgV?pZxZ{19gmHyA4bQ9f5&P96-g1S)gIZw-N
z%PhZg?K^kqr&%Oihg4|bzdtfzL$SBiC2%i?aD@;@N)%g$z$p&c=47#hvp65R8v`&A;tCg|@_Jml@%W
z37{urI>f?Zas4h#LW@;e*S(tVs@+`;qNJ$TTDKJaa*Dlxoe?3@v(!)wdZ^KKT9Mq$
zHzjX94;ZGf^K}g$qOJjRV^Vx46{>(pD6#x5@?nwXnx`)i*a6$;agv^ISzwun`hnOH
zcjz%IN91Ie3-`5vR1(~~sl!6Qe-?3~fSm`@G_@(%-kX?l=1(EYDYO21+3}p9fzj{u
zr=zJ%3b?)0ciYa+m8LACd<;|Qqo)%S`hE4gX?eRMAwiHOk%{FI@yddbB&WlA;^Q}8
z;md}9m|G#$M+TKB1Dbdk;E#F!U@-MXQr}dii=HJ@}A#7Cs&;$t^7(W
zmXG`u**k80p#`vzgfC+~p^e#8@smd{BU?lZa-y$C?jH&`$SLxY)fauH3NMwDcdw7X
zC~zabCvSRC(zxr}=46ofm~Tt8Q6{mJ|MePC
zNu7H?)PZs8JVtDUJAAbr-yweo%nAL@jk_&AKtbB9m!AWM6oCX%6rXO=WyZWbFxhZ}
z@y$}goC9P)%IvF%3Xp8p2I$Q=%maqUNJKP9tV4ohVY+b7fz3ABHv*Ck>Wh%jQi*!h
ztOVE#>Xk*X^+3=z_KjaJc(e$3#b@30@tb(INOcz9h9j}>favH*P0xpu6Ih*riNf_x
z0+vudN&iQba)$EWT6UcKJ17n07Q~Pc^ubPpjsC&RC-3TA*X_lZ9~(=Ye5R)f>%MyZ
zv9x4t^M@bLSjNfD9CfIrZUVBvL>zLSNAa#yU$Mfu6w}##X~R$A1?k@%(Y2P}{;B!u7stT#Sv$cGPU7LS;R8>&x
zi2R?NqZ1EU_Xfj@2VRN6=T_SGLG2CvN+rBUw@|O)`xQu5SJ
zt%7Pg88w$YvgpP@Tc%$JNI*%u^jxSB%&ghfiaYsHF21D(Hb3>22_PkSCxhXGeqE
zl=Jlv^sAZvez
zh9Ik4r1fbMd9}vk%RZz}55A$bd@r?FQSJQ|epDjn|B11UjnFI&?A5w0J8G?rYoFVR8UAJ@>zZ=riOkNN-s|8{l;
zA3d2cgYiNc&lj<(?5HREhQ3QcH(x6K%0NfAuKs5W+d&0_`dp`BZvY)y
zMAZ%+T8VTW9is+9!WeL_ZLMcW&?6=ow01$GMCiq5@`Ch8)NvyY{OQIJ{AqT%mFZRK
zi~4Wu=mX0x2>fGF*)1ahi{gALU0A1!!Q0%cd(ZeGXx;uZYFQ3+8b?{Rfg7DJY*tMT
zdV+^x%F2OvRz+owz4M6&>_Ym0x{Y4_2ezcHgW=tWDn!49qEwVmr|R&YH}(5o9gTB8
z3Hd_9KmqHA^_LRiFcP8z2$d6U7c>~a2z+Kwq@4gn4hOsNS#j@aTteVWsYR(*HlQjT
z6TOEM+gZByX~GcEBdq#gPp_db-)@~MC%g9lEL?{{juh~je#yY3|CpJ`enfL*IZa3INUN^Jz{e5Uu~d)_7-=rHZKzQCE(cp9LWil8Lx1d$L%Y=jq$sVp!e
z@!!c7ukm@U;j{mNah%@U@u0NVKZEu7>RJ3+m9+r1;c1+L7l>7iepKa`q)A%?=v6==
zC5Q@PsZIxId+;E#<22F<0wLQ0J8RBUHu+EAHa8+{1Dup|6Fs8>yKd`#q=bmJ8D|xp
zJwGM98kG997tAd|Y7^;K2=XLe`y~ooxhON@TjIbN4sv?_e$uYAIh`vgHqT-BDMIFm
zJm$&6m-9e9kR#-brV41DaCp1&AamX4R;_R5z7%!z)}}u%S#MRcwawhxntpeRQngz3
zF(nNH-5fKSyl1mDxxiO-`6piEt|n8jR-#NVE!+=&h>L%6G}Vc5z$e@IP6ok17gZf(
z733X%U$7K99fC2IFAQYyP9@==EV4UNuGiG$jT`+fz#&wWAzpb;hCAhay*}4sPPo3w
zmkm0UE&Y$3<*zsVJ>e*%I;zA(H=;sYaZ?hQ%cvPp)Y#S@xg9uR-k{*UEPC*EEnh7)
z@95gUwEz*GkBL~185%5eH*ayEefv1?S*EGMX>?#(wrc||SZH3eMqpPQRmZC6iYsdv
zWTAL3R*=S5KOoh+mh^3+U|kvAa$jVyuFLI9kLFESe+4HV>_YnzLE9$LQh$j*`XSIE
z+*6qaKB^n@loS`ERT|a`biN+jShnC=H!{PVY1XOBQrR)?74qg`ZZIdL+
za#(oaUL>pl&)g&k?`Z)UGDY3p!b0Qft#d{sypONBg`#e?z7FLUb|=sa9aE6+91hAV
zzW2-6PKTd}N;w=WBQ-L%-^qtUVS|i;g54$pm5~D|xqmxRUxd~ur&-2~mz(7%-_9l_
zVsMChb?)#E;MU7s6BLGgqa~pbxPk`d+@iy~Ah!t4j!rd-zcO7nEdYU&+h4fyG#4AZzS$4(Pn%uKT#Ua^g>It6nv|V@
z*~NO(jq=T{Y%mGrZf|YtmBYzAXNct;qtKIL&X@M+uwXB(JGMN)6V>LTryp1utYqlJ
zX^OsQ9g$bS4&laS-2XKPJSxH`)*C-RIFV~DzIo-CDQgDHYkBdl2;BoUgsS<&e5
zFBqr|IRV(G09kg;^@QKWb|nA_PkaL*M_U$@HV4O_VpH*%A%2`YDVt!|Hz0n^x=;U5
zsl&>%c1;_XSiK8L#>vbCp^WoAVYxx`%jhMBmQQr<30dgJ=YdX;_J&W_`Z!S(hDHnmDQC!quS{6&zFLllf+
zRrO~~T8Y!fcXYG+i+|E-NflT(XJ2jvt_rxwC$FYILErc(RM!yO9H3SKrl728#*0F`|!3O5=cuBZlm)J4SuT}9EMx=SWlTyhlSLR(Cm3+&DlG<*jj$(
zS_tt1o=V$&c}t?~&!@ET@E>6{?PhENavIS4S0U`a*a$Zutcz2SI6^I8YUWTQ^j6N6
z-b+T?(k+3Dbq5@%IA#doC4p6JQ`Zn7D8oL@wdby){^P;dRLqwhyjYV2mJ%sdMV?9t
z757RWnFrOphq}D*z4nDvn+*Pn3A{UR<9mBWc%=_Ym+hqE!J;GE4PQ>N46xm3&kJ4K
zd@XnI@dNH-Iqcvi1D5k2C+mHc6G<1?PKA3bZG+)wMpg5%Hva^>^zwu)}P6&>VI
zwd`Ei@E`6fE}e@`_l2`ZabLr9z14_tP85c(4R=z!+S|LsH|~~h%-~NOaQA2i*_K$Y
zYi#@JxAw5P@?|xc#EoJ|&{r+X?&l}HrfCZB_$kgzyrepl)b*l>_
zxZyg{Z;S;7fp`7R=0=rFhA1}`iAo6cmkOdBEAa2h=_93X(gJMfFFY4;6LeCG@DDdv
zI3K-yeviICFF~bRYtB=O;^>i0Ah(pQJF|jlA|V+DlvbncaVnPhkSxwTD<+Nb2wbe;
z9U=Zeu1U}0+g^LaD=_*Gb|tMNt1&eG%+%G3-^8SE{K{ds`J_KQD=y$hc;lVAjYA07
zI^VG#?*+HLbGWyI&j;AlsAxgP8qYhU>6k-|ClBfqAh0F`u6AxyMc56#?-U%3RDZ^9
zeD9uJ=w08Q6r~oDkIZq0>ozuel4k8|H7$@%v(4b5?F+;jjDJ{)B|tCB
z7*_q73#gTaFcIA&bR{@R7?jG{y(}A?t~At$4S-gxggC}A4g3QE2v|k~IBoA@HV8vD
zO-lN{(Ag^$2~7DkV=PPx@cLb#Dahc$-^8fD2G2SM5hBKp+00D=e8^PO#1=W_pTw=7
z?xe$XP*(!lO@Edzg9dtv%%(v#vE&6KWPM~iGd8qviINOMtUIm}Tpcv;b&({2qBb1=
zC0mFecs%D&vAv)`drLVs5*&J$K4KleYK3W}0JzO$}hjYq*M)JGVY*W8zKlj2+m!
zafqB4L9)G7k;6c;8a&|{A{xq~7x3y=Z9(+AyFj?Bs5AiVc-!Emx8;A-D2iIK474?q
zY_KC;GhFJy!b~v-hbvHnc$A-7!W24X%~IsP21dqKB)J1t59K!fe2{CbnsiCyj#Mox
zpeI}B+0Hdb|k{>mB~(LM9Vuw&waI2GmY
z7c_l|!X>;t8F;9%^D?0zD)Ot8tgG+bsf@46DC%hq7d-!PNh^B&U;gXZYv;WaXp3Lm
z!5Kk3G>nVV4&0W}^9~H(phO99v=p}0F*EVU|NP?4?#i{!?$5`8o^`@yq@Ipp^u8b^
zircs}sBRq<<;$8HPe@)SLi=L1i$qF0UJc