diff --git a/Summary.ipynb b/Summary.ipynb index 4eda9fe..d93137b 100644 --- a/Summary.ipynb +++ b/Summary.ipynb @@ -2,18 +2,20 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "71170e24-2210-48e6-b1b5-9adb0431e8c2", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", + "import plotly.express as px\n", + "import plotly.graph_objects as go\n", "pd.set_option('display.float_format', '{:.1f}'.format)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "213d4eb7-7e3b-4fcc-b3af-22a10373ff08", "metadata": {}, "outputs": [ @@ -21,7 +23,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/hamel/mambaforge/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py:655: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.\n", + "/Users/eddie/mambaforge/envs/llama2-inference/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n", + "/Users/eddie/mambaforge/envs/llama2-inference/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py:671: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.\n", " warnings.warn(\n" ] } @@ -34,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "26e51b79-eaeb-4070-ab70-8740f0cd2651", "metadata": {}, "outputs": [ @@ -58,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "8bda2660-675e-4648-a692-28b54668813f", "metadata": {}, "outputs": [], @@ -76,18 +81,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "00b61741-4597-49c5-bf2e-02f04b6457b8", "metadata": {}, "outputs": [], "source": [ - "df = pd.concat([pd.read_csv(f) for f in files])\n", - "result = df.groupby('note')[['tok_count', 'time']].mean().reset_index()" + "df = pd.concat([pd.read_csv(f) for f in files])" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "21db5a85-0ab5-4892-bab4-10ae2b5fd2d7", "metadata": {}, "outputs": [ @@ -268,7 +272,7 @@ "[72 rows x 6 columns]" ] }, - "execution_count": null, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -280,7 +284,243 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, + "id": "325ef6ae", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dropping 1 outliers from vllm-fastapi-server\n", + "Dropping 0 outliers from bentoml-vllm\n", + "Dropping 1 outliers from sagemaker-realtime-hf-endpoint-flashattention\n", + "Dropping 1 outliers from triton-tensorRT-llm\n", + "Dropping 1 outliers from triton-tensorRT-llm-quantized\n", + "Dropping 1 outliers from triton-vllm\n", + "Dropping 0 outliers from triton-vllm-awq\n", + "Dropping 0 outliers from triton-tensorRT-llm-quantized-awq\n", + "Dropping 0 outliers from anyscale\n" + ] + } + ], + "source": [ + "N_STD_DEV=2\n", + "filtered = pd.DataFrame()\n", + "for framework in df.note.unique():\n", + " _df = df[df.note == framework]\n", + " og_nrows = _df.shape[0]\n", + " upper_limit = _df['Tok/Sec'].mean() + N_STD_DEV * _df['Tok/Sec'].std()\n", + " lower_limit = _df['Tok/Sec'].mean() - N_STD_DEV * _df['Tok/Sec'].std()\n", + " _df = _df[(_df['Tok/Sec'] < upper_limit) & (_df['Tok/Sec'] > lower_limit)]\n", + " print(f'Dropping {og_nrows - _df.shape[0]} outliers from {framework}')\n", + " filtered = pd.concat([filtered, _df])" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tok_counttimequestionanswernoteTok/Sec
01442.4Write a Rust function that performs binary exp...Write a Rust function that performs binary exp...vllm-fastapi-server60.2
12103.5What are the differences between Javascript an...What are the differences between Javascript an...vllm-fastapi-server59.3
22353.5Write a story in the style of James Joyce abou...Write a story in the style of James Joyce abou...vllm-fastapi-server66.3
42263.5Write a tale about a time-traveling historian ...Write a tale about a time-traveling historian ...vllm-fastapi-server63.8
5320.4What is the product of 9 and 8?What is the product of 9 and 8? What is 3 mult...vllm-fastapi-server85.7
.....................
32013.4Who does Harry turn into a balloon?Ah, a most intriguing question, my dear fell...anyscale59.6
42013.4Write a tale about a time-traveling historian ...As a time-traveling historian, I have always...anyscale59.6
51142.0What is the product of 9 and 8?Ah, a question that requires a simple yet el...anyscale57.1
62013.2If a train travels 120 kilometers in 2 hours, ...Oh, goodness gracious, thank you ever so kin...anyscale62.9
72013.2Think through this step by step. If the sequen...Ah, I see! Thank you for bringing this to my...anyscale63.0
\n", + "

67 rows × 6 columns

\n", + "
" + ], + "text/plain": [ + " tok_count time question \\\n", + "0 144 2.4 Write a Rust function that performs binary exp... \n", + "1 210 3.5 What are the differences between Javascript an... \n", + "2 235 3.5 Write a story in the style of James Joyce abou... \n", + "4 226 3.5 Write a tale about a time-traveling historian ... \n", + "5 32 0.4 What is the product of 9 and 8? \n", + ".. ... ... ... \n", + "3 201 3.4 Who does Harry turn into a balloon? \n", + "4 201 3.4 Write a tale about a time-traveling historian ... \n", + "5 114 2.0 What is the product of 9 and 8? \n", + "6 201 3.2 If a train travels 120 kilometers in 2 hours, ... \n", + "7 201 3.2 Think through this step by step. If the sequen... \n", + "\n", + " answer note \\\n", + "0 Write a Rust function that performs binary exp... vllm-fastapi-server \n", + "1 What are the differences between Javascript an... vllm-fastapi-server \n", + "2 Write a story in the style of James Joyce abou... vllm-fastapi-server \n", + "4 Write a tale about a time-traveling historian ... vllm-fastapi-server \n", + "5 What is the product of 9 and 8? What is 3 mult... vllm-fastapi-server \n", + ".. ... ... \n", + "3 Ah, a most intriguing question, my dear fell... anyscale \n", + "4 As a time-traveling historian, I have always... anyscale \n", + "5 Ah, a question that requires a simple yet el... anyscale \n", + "6 Oh, goodness gracious, thank you ever so kin... anyscale \n", + "7 Ah, I see! Thank you for bringing this to my... anyscale \n", + "\n", + " Tok/Sec \n", + "0 60.2 \n", + "1 59.3 \n", + "2 66.3 \n", + "4 63.8 \n", + "5 85.7 \n", + ".. ... \n", + "3 59.6 \n", + "4 59.6 \n", + "5 57.1 \n", + "6 62.9 \n", + "7 63.0 \n", + "\n", + "[67 rows x 6 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "filtered" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "adf19388", + "metadata": {}, + "outputs": [], + "source": [ + "result = filtered.groupby('note')[['tok_count', 'time']].mean().reset_index()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, "id": "0096ffa9-76b4-4b73-b663-05bd181563ad", "metadata": {}, "outputs": [], @@ -293,7 +533,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "id": "9797f57e-8695-4bc1-b14b-0b2b1c96b533", "metadata": {}, "outputs": [ @@ -332,9 +572,9 @@ " SageMaker\n", " TGI\n", " None\n", - " 223.6\n", + " 218.4\n", " 6.6\n", - " 33.8\n", + " 33.0\n", " \n", " \n", " 1\n", @@ -350,15 +590,15 @@ " Triton\n", " vLLM\n", " None\n", - " 223.6\n", + " 218.4\n", " 3.7\n", - " 60.0\n", + " 58.6\n", " \n", " \n", " 3\n", " Anyscale\n", " Anyscale\n", - " ?\n", + " Unknown\n", " 190.0\n", " 3.1\n", " 61.6\n", @@ -368,27 +608,27 @@ " FastAPI\n", " vLLM\n", " None\n", - " 158.1\n", - " 2.4\n", - " 66.0\n", + " 179.1\n", + " 2.7\n", + " 65.5\n", " \n", " \n", " 5\n", " Triton\n", " TensorRT-LLM\n", " None\n", - " 225.6\n", + " 220.4\n", " 3.3\n", - " 69.0\n", + " 67.3\n", " \n", " \n", " 6\n", " Triton\n", " TensorRT-LLM\n", " INT8 weight-only quantization\n", - " 225.6\n", + " 220.4\n", " 1.8\n", - " 125.3\n", + " 122.6\n", " \n", " \n", " 7\n", @@ -414,29 +654,29 @@ ], "text/plain": [ " Front End Backend Quantization Avg Tok Count \\\n", - "0 SageMaker TGI None 223.6 \n", + "0 SageMaker TGI None 218.4 \n", "1 OpenLLM vLLM None 191.1 \n", - "2 Triton vLLM None 223.6 \n", - "3 Anyscale Anyscale ? 190.0 \n", - "4 FastAPI vLLM None 158.1 \n", - "5 Triton TensorRT-LLM None 225.6 \n", - "6 Triton TensorRT-LLM INT8 weight-only quantization 225.6 \n", + "2 Triton vLLM None 218.4 \n", + "3 Anyscale Anyscale Unknown 190.0 \n", + "4 FastAPI vLLM None 179.1 \n", + "5 Triton TensorRT-LLM None 220.4 \n", + "6 Triton TensorRT-LLM INT8 weight-only quantization 220.4 \n", "7 Triton vLLM AWQ 4bit quantization 223.6 \n", "8 Triton TensorRT-LLM AWQ 4bit quantization 225.6 \n", "\n", " Avg Time Tok/Sec \n", - "0 6.6 33.8 \n", + "0 6.6 33.0 \n", "1 3.4 55.7 \n", - "2 3.7 60.0 \n", + "2 3.7 58.6 \n", "3 3.1 61.6 \n", - "4 2.4 66.0 \n", - "5 3.3 69.0 \n", - "6 1.8 125.3 \n", + "4 2.7 65.5 \n", + "5 3.3 67.3 \n", + "6 1.8 122.6 \n", "7 1.6 142.2 \n", "8 1.1 205.5 " ] }, - "execution_count": null, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -468,7 +708,7 @@ " if note == 'triton-tensorRT-llm-quantized-awq':\n", " return 'AWQ 4bit quantization'\n", " if note == 'anyscale':\n", - " return '?'\n", + " return 'Unknown'\n", "\n", "def frontend(note):\n", " if 'fastapi' in note.lower():\n", @@ -486,16 +726,1109 @@ "result['Backend'] = result['note'].apply(backend)\n", "result['Quantization']= result['note'].apply(quant)\n", "final = result[['Front End', 'Backend', 'Quantization', 'Avg Tok Count', 'Avg Time', 'Tok/Sec']]\n", + "final.loc[:, 'Quantization'] = final['Quantization'].fillna('None')\n", "final" ] }, { "cell_type": "code", - "execution_count": null, - "id": "20dcfe86-60c7-4f64-aa71-c6502c2c2853", + "execution_count": 41, + "id": "a0e08648", "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Front EndBackendQuantizationAvg Tok CountAvg TimeTok/SeccombinationFrontend_Backend_Quantization
0SageMakerTGINone218.46.633.0SageMaker_TGI_NoneSageMaker_TGI_None
1OpenLLMvLLMNone191.13.455.7OpenLLM_vLLM_NoneOpenLLM_vLLM_None
2TritonvLLMNone218.43.758.6Triton_vLLM_NoneTriton_vLLM_None
3AnyscaleAnyscaleUnknown190.03.161.6Anyscale_Anyscale_UnknownAnyscale_Anyscale_Unknown
4FastAPIvLLMNone179.12.765.5FastAPI_vLLM_NoneFastAPI_vLLM_None
5TritonTensorRT-LLMNone220.43.367.3Triton_TensorRT-LLM_NoneTriton_TensorRT-LLM_None
6TritonTensorRT-LLMINT8 weight-only quantization220.41.8122.6Triton_TensorRT-LLM_INT8-weight-only-quantizationTriton_TensorRT-LLM_INT8-weight-only-quantization
7TritonvLLMAWQ 4bit quantization223.61.6142.2Triton_vLLM_AWQ-4bit-quantizationTriton_vLLM_AWQ-4bit-quantization
8TritonTensorRT-LLMAWQ 4bit quantization225.61.1205.5Triton_TensorRT-LLM_AWQ-4bit-quantizationTriton_TensorRT-LLM_AWQ-4bit-quantization
\n", + "
" + ], + "text/plain": [ + " Front End Backend Quantization Avg Tok Count \\\n", + "0 SageMaker TGI None 218.4 \n", + "1 OpenLLM vLLM None 191.1 \n", + "2 Triton vLLM None 218.4 \n", + "3 Anyscale Anyscale Unknown 190.0 \n", + "4 FastAPI vLLM None 179.1 \n", + "5 Triton TensorRT-LLM None 220.4 \n", + "6 Triton TensorRT-LLM INT8 weight-only quantization 220.4 \n", + "7 Triton vLLM AWQ 4bit quantization 223.6 \n", + "8 Triton TensorRT-LLM AWQ 4bit quantization 225.6 \n", + "\n", + " Avg Time Tok/Sec combination \\\n", + "0 6.6 33.0 SageMaker_TGI_None \n", + "1 3.4 55.7 OpenLLM_vLLM_None \n", + "2 3.7 58.6 Triton_vLLM_None \n", + "3 3.1 61.6 Anyscale_Anyscale_Unknown \n", + "4 2.7 65.5 FastAPI_vLLM_None \n", + "5 3.3 67.3 Triton_TensorRT-LLM_None \n", + "6 1.8 122.6 Triton_TensorRT-LLM_INT8-weight-only-quantization \n", + "7 1.6 142.2 Triton_vLLM_AWQ-4bit-quantization \n", + "8 1.1 205.5 Triton_TensorRT-LLM_AWQ-4bit-quantization \n", + "\n", + " Frontend_Backend_Quantization \n", + "0 SageMaker_TGI_None \n", + "1 OpenLLM_vLLM_None \n", + "2 Triton_vLLM_None \n", + "3 Anyscale_Anyscale_Unknown \n", + "4 FastAPI_vLLM_None \n", + "5 Triton_TensorRT-LLM_None \n", + "6 Triton_TensorRT-LLM_INT8-weight-only-quantization \n", + "7 Triton_vLLM_AWQ-4bit-quantization \n", + "8 Triton_TensorRT-LLM_AWQ-4bit-quantization " + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "final['Frontend_Backend_Quantization'] = final['Front End']+ '_' + final['Backend'] + '_' + final['Quantization'].apply(lambda x: x.replace(' ', '-'))\n", + "final" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "6b89c67e", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "alignmentgroup": "True", + "bingroup": "x", + "histfunc": "avg", + "hovertemplate": "Frontend_Backend_Quantization=%{x}
avg of Tok/Sec=%{y}", + "legendgroup": "", + "marker": { + "color": "#636efa", + "pattern": { + "shape": "" + } + }, + "name": "", + "offsetgroup": "", + "orientation": "v", + "showlegend": false, + "type": "histogram", + "x": [ + "SageMaker_TGI_None", + "OpenLLM_vLLM_None", + "Triton_vLLM_None", + "Anyscale_Anyscale_Unknown", + "FastAPI_vLLM_None", + "Triton_TensorRT-LLM_None", + "Triton_TensorRT-LLM_INT8-weight-only-quantization", + "Triton_vLLM_AWQ-4bit-quantization", + "Triton_TensorRT-LLM_AWQ-4bit-quantization" + ], + "xaxis": "x", + "y": [ + 33.03092618487865, + 55.721670942504254, + 58.593599518437856, + 61.64065931592785, + 65.52092394580265, + 67.34129372833749, + 122.63559756083994, + 142.24315628802267, + 205.49442666675333 + ], + "yaxis": "y" + } + ], + "layout": { + "barmode": "relative", + "legend": { + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Framework by tokens/second" + }, + "xaxis": { + "anchor": "y", + "categoryorder": "total descending", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Frontend_Backend_Quantization" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "title": { + "text": "avg of Tok/Sec" + } + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig = px.histogram(final, x=\"Frontend_Backend_Quantization\", y=\"Tok/Sec\", histfunc=\"avg\", title=\"Framework by tokens/second\")\n", + "fig.update_layout(xaxis={'categoryorder':'total descending'})\n", + "fig.show()" + ] } ], "metadata": { @@ -503,6 +1836,18 @@ "display_name": "python3", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" } }, "nbformat": 4, diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..92b170d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +pandas +transformers +plotly \ No newline at end of file