From 10ee48e09a4f8d0cb869060d076103af0fcc9ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Mon, 8 Apr 2024 18:01:59 -0300 Subject: [PATCH 1/5] OpenAI compatible servers benchmark based from the anyscale and exllama benchmarks. --- openai/Dockerfile | 28 ++++++++++++++++++++++++++ openai/README.md | 16 +++++++++++++++ openai/bench.py | 17 ++++++++++++++++ openai/client.py | 44 +++++++++++++++++++++++++++++++++++++++++ openai/requirements.txt | 22 +++++++++++++++++++++ 5 files changed, 127 insertions(+) create mode 100644 openai/Dockerfile create mode 100644 openai/README.md create mode 100644 openai/bench.py create mode 100644 openai/client.py create mode 100644 openai/requirements.txt diff --git a/openai/Dockerfile b/openai/Dockerfile new file mode 100644 index 0000000..d4f9f5c --- /dev/null +++ b/openai/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 + +# From the parent directory (main directory of this repo) run: +# +# docker build --build-arg USERID=$(id -u) -t localhost/openai-bench openai +# +# docker run --rm -it --name openai-bench -e OPENAI_API_KEY='secret' \ +# -v"$PWD":/home/user/llama-inference localhost/openai-bench \ +# --baseurl https://api.openai.com:443/v1 --model gpt-3.5-turbo \ +# --filename gpt-3.5-turbo.bench.csv --note "gpt-3.5-turbo" +# +# The container isn't self contained but designed to be used with the repo. + +# Look for newer tags here https://hub.docker.com/_/python: +FROM docker.io/python:3.12-bookworm +ARG USERID=1000 +COPY requirements.txt /tmp +RUN pip3 install --no-cache-dir -r /tmp/requirements.txt +RUN adduser --disabled-password --gecos '' -u $USERID user +USER user +WORKDIR /home/user/llama-inference/openai/ +ENTRYPOINT ["python3", "bench.py"] +CMD [ \ + "--baseurl", "https://api.openai.com:443/v1", \ + "--model", "gpt-3.5-turbo", \ + "--filename", "/dev/stdout", \ + "--note", "gpt-3.5-turbo" \ +] diff --git a/openai/README.md b/openai/README.md new file mode 100644 index 0000000..14797ba --- /dev/null +++ b/openai/README.md @@ -0,0 +1,16 @@ +# OpenAI Chat API compatible benchmark + +Benchmark for servers compatible with the [OpenAI Chat Completions API](https://platform.openai.com/docs/guides/text-generation/chat-completions-api). + +You can follow the instructions in the [Dockerfile](Dockerfile) to build a container to run the benchmark. + +Or alternatively on your own machine: +```sh +cd openai +pip3 -r requirements.txt +export OPENAI_API_KEY='secret' #obtain from https://platform.openai.com/api-keys +python3 client.py #to test a single run +python3 bench.py --help #get usage help +python3 bench.py --baseurl https://api.openai.com:443/v1 --model gpt-3.5-turbo \ + --filename gpt-3.5-turbo.bench.csv --note "gpt-3.5-turbo" +``` diff --git a/openai/bench.py b/openai/bench.py new file mode 100644 index 0000000..18a92a7 --- /dev/null +++ b/openai/bench.py @@ -0,0 +1,17 @@ +from client import chat,args +import sys +sys.path.append('../common/') +from questions import questions +import pandas as pd + +if __name__ == '__main__': + counter = 1 + responses = [] + for q in questions: + response = chat(q) + if counter >= 2: # allow for a warmup + responses.append(response) + counter +=1 + + df = pd.DataFrame(responses) + df.to_csv(args.filename, index=False) diff --git a/openai/client.py b/openai/client.py new file mode 100644 index 0000000..f1b243e --- /dev/null +++ b/openai/client.py @@ -0,0 +1,44 @@ +from openai import OpenAI +import argparse +import time + +#TODO: should use a parent parser without --filename here. +parser = argparse.ArgumentParser(description='Run LLM inference requests and save to a CSV.') +parser.add_argument('--filename', type=str, default='/dev/stdout', + help='Path to the output CSV file (stdout by default).') +parser.add_argument('--note', type=str, + help='Note to add to the rows of the file (--model by default).') +parser.add_argument('--model', type=str, default='gpt-3.5-turbo', + help='Model to use (gpt-3.5-turbo by default).') +parser.add_argument('--baseurl', type=str, default='https://api.openai.com:443/v1', + help='Endpoint base URL (https://api.openai.com:443/v1 by default).') +args = parser.parse_args() +if ('note' not in args): + args.note = args.model + +client = OpenAI(base_url=args.baseurl) + +def chat(prompt:str): + start = time.perf_counter() + result = client.chat.completions.create( + model=args.model, + max_tokens=200, + messages=[ + {"role": "system", "content": "You are a very verbose and helpful assistant"}, + {"role": "user", "content": prompt} + ] + ) + request_time = time.perf_counter() - start + return {'tok_count': result.usage.completion_tokens, + 'time': request_time, + 'question': prompt, + 'answer': result.choices[0].message.content, + 'note': args.note} + +if __name__ == '__main__': + prompt = "San Francisco is a city in" + result = chat(prompt) + tokPerSec = result['tok_count']/result['time'] + print(f"User: {prompt}\n" + f"Chatbot in {result['time']}s with {result['tok_count']} tokens ({tokPerSec} t/s):\n" + f"{result['answer']}") diff --git a/openai/requirements.txt b/openai/requirements.txt new file mode 100644 index 0000000..dd36072 --- /dev/null +++ b/openai/requirements.txt @@ -0,0 +1,22 @@ +annotated-types==0.6.0 +anyio==4.3.0 +certifi==2024.2.2 +distro==1.9.0 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +idna==3.6 +numpy==1.26.4 +openai==1.16.2 +pandas==2.2.1 +pydantic==2.6.4 +pydantic_core==2.16.3 +python-dateutil==2.9.0.post0 +pytz==2024.1 +setuptools==69.1.1 +six==1.16.0 +sniffio==1.3.1 +tqdm==4.66.2 +typing_extensions==4.11.0 +tzdata==2024.1 +wheel==0.43.0 From d6f273b5720e2e2161d79e3ae0b3f03cbadcb8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Mon, 8 Apr 2024 18:33:11 -0300 Subject: [PATCH 2/5] Apply Megalinter suggestions. --- openai/bench.py | 14 +++++---- openai/client.py | 77 ++++++++++++++++++++++++++++++++---------------- 2 files changed, 59 insertions(+), 32 deletions(-) diff --git a/openai/bench.py b/openai/bench.py index 18a92a7..2bfa7c3 100644 --- a/openai/bench.py +++ b/openai/bench.py @@ -1,17 +1,19 @@ -from client import chat,args import sys -sys.path.append('../common/') -from questions import questions import pandas as pd -if __name__ == '__main__': +from client import args, chat + +sys.path.append("../common/") +from questions import questions + +if __name__ == "__main__": counter = 1 responses = [] for q in questions: response = chat(q) - if counter >= 2: # allow for a warmup + if counter >= 2: # allow for a warmup responses.append(response) - counter +=1 + counter += 1 df = pd.DataFrame(responses) df.to_csv(args.filename, index=False) diff --git a/openai/client.py b/openai/client.py index f1b243e..9934b70 100644 --- a/openai/client.py +++ b/openai/client.py @@ -1,44 +1,69 @@ -from openai import OpenAI import argparse import time -#TODO: should use a parent parser without --filename here. -parser = argparse.ArgumentParser(description='Run LLM inference requests and save to a CSV.') -parser.add_argument('--filename', type=str, default='/dev/stdout', - help='Path to the output CSV file (stdout by default).') -parser.add_argument('--note', type=str, - help='Note to add to the rows of the file (--model by default).') -parser.add_argument('--model', type=str, default='gpt-3.5-turbo', - help='Model to use (gpt-3.5-turbo by default).') -parser.add_argument('--baseurl', type=str, default='https://api.openai.com:443/v1', - help='Endpoint base URL (https://api.openai.com:443/v1 by default).') +from openai import OpenAI + +# TODO: should use a parent parser without --filename here. +parser = argparse.ArgumentParser( + description="Run LLM inference requests and save to a CSV." +) +parser.add_argument( + "--filename", + type=str, + default="/dev/stdout", + help="Path to the output CSV file (stdout by default).", +) +parser.add_argument( + "--note", type=str, help="Note to add to the rows of the file (--model by default)." +) +parser.add_argument( + "--model", + type=str, + default="gpt-3.5-turbo", + help="Model to use (gpt-3.5-turbo by default).", +) +parser.add_argument( + "--baseurl", + type=str, + default="https://api.openai.com:443/v1", + help="Endpoint base URL (https://api.openai.com:443/v1 by default).", +) args = parser.parse_args() -if ('note' not in args): +if "note" not in args: args.note = args.model client = OpenAI(base_url=args.baseurl) -def chat(prompt:str): + +def chat(prompt: str): start = time.perf_counter() result = client.chat.completions.create( model=args.model, max_tokens=200, messages=[ - {"role": "system", "content": "You are a very verbose and helpful assistant"}, - {"role": "user", "content": prompt} - ] + { + "role": "system", + "content": "You are a very verbose and helpful assistant", + }, + {"role": "user", "content": prompt}, + ], ) request_time = time.perf_counter() - start - return {'tok_count': result.usage.completion_tokens, - 'time': request_time, - 'question': prompt, - 'answer': result.choices[0].message.content, - 'note': args.note} + return { + "tok_count": result.usage.completion_tokens, + "time": request_time, + "question": prompt, + "answer": result.choices[0].message.content, + "note": args.note, + } + -if __name__ == '__main__': +if __name__ == "__main__": prompt = "San Francisco is a city in" result = chat(prompt) - tokPerSec = result['tok_count']/result['time'] - print(f"User: {prompt}\n" - f"Chatbot in {result['time']}s with {result['tok_count']} tokens ({tokPerSec} t/s):\n" - f"{result['answer']}") + tokPerSec = result["tok_count"] / result["time"] + print( + f"User: {prompt}\n" + f"Chatbot in {result['time']}s with {result['tok_count']} tokens ({tokPerSec} t/s):\n" + f"{result['answer']}" + ) From 397658992de3a29ed98d160382a5299d23b8918d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Mon, 8 Apr 2024 18:39:48 -0300 Subject: [PATCH 3/5] Megalinter was unhappy still... --- openai/bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openai/bench.py b/openai/bench.py index 2bfa7c3..8a7f552 100644 --- a/openai/bench.py +++ b/openai/bench.py @@ -1,6 +1,6 @@ import sys -import pandas as pd +import pandas as pd from client import args, chat sys.path.append("../common/") From e2d075cae0d1d424320b7e82ce2afcf81f2faa03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Mon, 8 Apr 2024 19:05:26 -0300 Subject: [PATCH 4/5] Ignore some warnings from Hadolint and Trivy. --- openai/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openai/Dockerfile b/openai/Dockerfile index d4f9f5c..66b1eaf 100644 --- a/openai/Dockerfile +++ b/openai/Dockerfile @@ -16,6 +16,7 @@ FROM docker.io/python:3.12-bookworm ARG USERID=1000 COPY requirements.txt /tmp RUN pip3 install --no-cache-dir -r /tmp/requirements.txt +# hadolint ignore=DL3059 RUN adduser --disabled-password --gecos '' -u $USERID user USER user WORKDIR /home/user/llama-inference/openai/ @@ -26,3 +27,4 @@ CMD [ \ "--filename", "/dev/stdout", \ "--note", "gpt-3.5-turbo" \ ] +#trivy:ignore:ds026 From 43b7b8dc46969a011598ea92145962968f361c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Thu, 6 Jun 2024 16:33:46 -0300 Subject: [PATCH 5/5] Update some Megalinter reported security issues in pip requirements. --- openai/requirements.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openai/requirements.txt b/openai/requirements.txt index dd36072..0b47719 100644 --- a/openai/requirements.txt +++ b/openai/requirements.txt @@ -2,10 +2,13 @@ annotated-types==0.6.0 anyio==4.3.0 certifi==2024.2.2 distro==1.9.0 +exceptiongroup==1.2.1 +gpg==1.15.1 h11==0.14.0 httpcore==1.0.5 httpx==0.27.0 -idna==3.6 +idna==3.7 +libcomps==0.1.18 numpy==1.26.4 openai==1.16.2 pandas==2.2.1 @@ -13,10 +16,9 @@ pydantic==2.6.4 pydantic_core==2.16.3 python-dateutil==2.9.0.post0 pytz==2024.1 -setuptools==69.1.1 +rpm==4.16.1.3 six==1.16.0 sniffio==1.3.1 -tqdm==4.66.2 +tqdm==4.66.4 typing_extensions==4.11.0 tzdata==2024.1 -wheel==0.43.0