From c6d2ea51a2b2cfabb86ff04f10f60424df5e26a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Wed, 31 Jan 2024 15:05:51 -0300 Subject: [PATCH 01/10] Add PowerInfer benchmark. --- powerinfer/Dockerfile | 53 +++++++++++++++++++++++++++++++++++++++++++ powerinfer/README.md | 19 ++++++++++++++++ powerinfer/bench.py | 17 ++++++++++++++ powerinfer/client.py | 21 +++++++++++++++++ 4 files changed, 110 insertions(+) create mode 100644 powerinfer/Dockerfile create mode 100644 powerinfer/README.md create mode 100755 powerinfer/bench.py create mode 100755 powerinfer/client.py diff --git a/powerinfer/Dockerfile b/powerinfer/Dockerfile new file mode 100644 index 0000000..5d68019 --- /dev/null +++ b/powerinfer/Dockerfile @@ -0,0 +1,53 @@ +# From the parent directory (main directory of this repo) run: +# +# docker build --build-arg USERID=$(id -u) -t local/powerinfer-bench powerinfer +# +# mkdir $HOME/.cache/{huggingface,PowerInfer} +# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z \ +# -v$HOME/.cache/PowerInfer/:/home/user/.cache/PowerInfer/:Z \ +# local/powerinfer-bench \ +# sh -c 'cd /home/user/.cache/PowerInfer/ && \ +# huggingface-cli download --resume-download --local-dir ReluLLaMA-7B \ +# --local-dir-use-symlinks False PowerInfer/ReluLLaMA-7B-PowerInfer-GGUF' +# +# docker run --rm -it --gpus all -v$HOME/.cache/PowerInfer/:/home/user/.cache/PowerInfer/:Z \ +# local/powerinfer-bench \ +# build/bin/main --log-disable -m \ +# /home/user/.cache/PowerInfer/ReluLLaMA-7B/llama-7b-relu.powerinfer.gguf \ +# -p "Once upon a time" +# +# docker run --rm -it --name powerinfer-bench --gpus all \ +# -v$HOME/.cache/PowerInfer/:/home/user/.cache/PowerInfer/:Z \ +# -v$(pwd):/home/user/llama-inference/:Z \ +# local/powerinfer-bench \ +# build/bin/server -v -m \ +# /home/user/.cache/PowerInfer/ReluLLaMA-7B/llama-7b-relu.powerinfer.gguf +# +# In another terminal: +# +# docker exec -it powerinfer-bench curl --request POST --url http://localhost:8080/completion \ +# --header "Content-Type: application/json" --data \ +# '{"prompt": "Building a website can be done in 10 simple steps:", "n_predict": 128}' +# +# docker exec -it powerinfer-bench sh -c 'cd /home/user/llama-inference/powerinfer && \ +# python3 bench.py' +# +# If using Podman with CDI substitute +# --gpus all +# for +# --device nvidia.com/gpu=all --security-opt=label=disable + +# Select an available version from +# https://gitlab.com/nvidia/container-images/cuda/blob/master/doc/supported-tags.md: +FROM nvidia/cuda:12.3.1-devel-rockylinux9 +ARG USERID=1000 +RUN yum install -y python3-pip cmake libcudnn8 git && yum clean all && rm -rf /var/cache/yum/* +RUN git clone https://github.com/SJTU-IPADS/PowerInfer +WORKDIR PowerInfer +RUN pip install --no-cache-dir -r requirements.txt +RUN cmake -S . -B build -DLLAMA_CUBLAS=ON +RUN cmake --build build --config Release -j $(nproc) +RUN pip install --no-cache-dir pandas #for the benchmark. +RUN adduser -u $USERID user +USER user + diff --git a/powerinfer/README.md b/powerinfer/README.md new file mode 100644 index 0000000..a72ad06 --- /dev/null +++ b/powerinfer/README.md @@ -0,0 +1,19 @@ +[PowerInfer](https://github.com/SJTU-IPADS/PowerInfer) + +Note that the model loses some inference quality in exchange for speed as shown in https://huggingface.co/SparseLLM/ReluLLaMA-7B. + +You can compile PowerInfer following their instructions and then run the server: + +```bash +build/bin/server -v -m ReluLLaMA-7B/llama-7b-relu.powerinfer.gguf +``` + +And in another terminal run: +```bash +python3 bench.py +``` + +The results will be in the bench-powerinfer.csv file. + +Or alternatively you can follow the instructions in the [Dockerfile](Dockerfile) to build a container to run the server and the benchmark inside it. + diff --git a/powerinfer/bench.py b/powerinfer/bench.py new file mode 100755 index 0000000..9c17be5 --- /dev/null +++ b/powerinfer/bench.py @@ -0,0 +1,17 @@ +import pandas as pd +import sys +sys.path.append('../common/') +from client import chat +from questions import questions + +if __name__ == '__main__': + counter = 1 + responses = [] + for q in questions: + response = chat(q) + if counter >= 2: # allow for a warmup + responses.append(response) + counter +=1 + + df = pd.DataFrame(responses) + df.to_csv('bench-powerinfer.csv', index=False) diff --git a/powerinfer/client.py b/powerinfer/client.py new file mode 100755 index 0000000..a932c8f --- /dev/null +++ b/powerinfer/client.py @@ -0,0 +1,21 @@ +import os, requests, time, json + +api_base = "http://localhost:8080" + +def chat(prompt:str): + payload = {"prompt": f"{prompt}", "n_predict": 200} + headers = {"Content-Type": "application/json"} + start = time.perf_counter() + response = requests.post(f"{api_base}/completion", headers = headers, json = payload) + response = response.json() + request_time = time.perf_counter() - start + + return {'tok_count': response['tokens_predicted'], + 'time': request_time, + 'question': prompt, + 'answer': response['content'], + 'note': 'PowerInfer ' + response['model']} + +if __name__ == '__main__': + prompt = "San Francisco is a city in" + print(f"User: {prompt}\nPowerInfer: {chat(prompt)['answer']}") From 232658f9a8ba6060e0c10b5fd0dc8128064550d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Wed, 31 Jan 2024 15:21:54 -0300 Subject: [PATCH 02/10] Make MarkdownLint happy. --- powerinfer/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/powerinfer/README.md b/powerinfer/README.md index a72ad06..13bae21 100644 --- a/powerinfer/README.md +++ b/powerinfer/README.md @@ -1,4 +1,6 @@ -[PowerInfer](https://github.com/SJTU-IPADS/PowerInfer) +#PowerInfer benchmark + +Benchmark for [PowerInfer](https://github.com/SJTU-IPADS/PowerInfer). Note that the model loses some inference quality in exchange for speed as shown in https://huggingface.co/SparseLLM/ReluLLaMA-7B. From ade534eacc6e0c753a5ec1e0b6bfe87b7804a0f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Wed, 31 Jan 2024 15:29:55 -0300 Subject: [PATCH 03/10] Make Kics happier. --- powerinfer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powerinfer/Dockerfile b/powerinfer/Dockerfile index 5d68019..5caad21 100644 --- a/powerinfer/Dockerfile +++ b/powerinfer/Dockerfile @@ -43,7 +43,7 @@ FROM nvidia/cuda:12.3.1-devel-rockylinux9 ARG USERID=1000 RUN yum install -y python3-pip cmake libcudnn8 git && yum clean all && rm -rf /var/cache/yum/* RUN git clone https://github.com/SJTU-IPADS/PowerInfer -WORKDIR PowerInfer +WORKDIR /PowerInfer RUN pip install --no-cache-dir -r requirements.txt RUN cmake -S . -B build -DLLAMA_CUBLAS=ON RUN cmake --build build --config Release -j $(nproc) From f81e384cfda266737204d28161492fbb2f369b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Wed, 31 Jan 2024 15:32:47 -0300 Subject: [PATCH 04/10] Make Hadolint happier. --- powerinfer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powerinfer/Dockerfile b/powerinfer/Dockerfile index 5caad21..a1de8fe 100644 --- a/powerinfer/Dockerfile +++ b/powerinfer/Dockerfile @@ -46,7 +46,7 @@ RUN git clone https://github.com/SJTU-IPADS/PowerInfer WORKDIR /PowerInfer RUN pip install --no-cache-dir -r requirements.txt RUN cmake -S . -B build -DLLAMA_CUBLAS=ON -RUN cmake --build build --config Release -j $(nproc) +RUN cmake --build build --config Release -j "$(nproc)" RUN pip install --no-cache-dir pandas #for the benchmark. RUN adduser -u $USERID user USER user From fe8ff94fa8f1c1052096fa7154e354ddb2dad8c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Wed, 31 Jan 2024 15:52:36 -0300 Subject: [PATCH 05/10] Megalinter formatted the README.md and it's right! --- powerinfer/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/powerinfer/README.md b/powerinfer/README.md index 13bae21..48d6c27 100644 --- a/powerinfer/README.md +++ b/powerinfer/README.md @@ -1,8 +1,8 @@ -#PowerInfer benchmark +# PowerInfer benchmark Benchmark for [PowerInfer](https://github.com/SJTU-IPADS/PowerInfer). -Note that the model loses some inference quality in exchange for speed as shown in https://huggingface.co/SparseLLM/ReluLLaMA-7B. +Note that the model loses some inference quality in exchange for speed as shown in . You can compile PowerInfer following their instructions and then run the server: From b784326064ec81ea6653c84de1308fe13b302b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Wed, 31 Jan 2024 15:59:22 -0300 Subject: [PATCH 06/10] Cosmetic changes in powerinfer/client.py. --- powerinfer/client.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/powerinfer/client.py b/powerinfer/client.py index a932c8f..d888150 100755 --- a/powerinfer/client.py +++ b/powerinfer/client.py @@ -1,21 +1,26 @@ -import os, requests, time, json +import time +import requests api_base = "http://localhost:8080" -def chat(prompt:str): + +def chat(prompt: str): payload = {"prompt": f"{prompt}", "n_predict": 200} headers = {"Content-Type": "application/json"} start = time.perf_counter() - response = requests.post(f"{api_base}/completion", headers = headers, json = payload) + response = requests.post(f"{api_base}/completion", headers=headers, json=payload) response = response.json() request_time = time.perf_counter() - start - return {'tok_count': response['tokens_predicted'], - 'time': request_time, - 'question': prompt, - 'answer': response['content'], - 'note': 'PowerInfer ' + response['model']} + return { + "tok_count": response["tokens_predicted"], + "time": request_time, + "question": prompt, + "answer": response["content"], + "note": "PowerInfer " + response["model"], + } + -if __name__ == '__main__': +if __name__ == "__main__": prompt = "San Francisco is a city in" print(f"User: {prompt}\nPowerInfer: {chat(prompt)['answer']}") From 4fd9dbe5baf3f64adbf06c6bbf4540c9be2ef0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Wed, 27 Mar 2024 12:27:46 -0300 Subject: [PATCH 07/10] Make Megalinter happier. --- powerinfer/bench.py | 14 ++++++++------ powerinfer/client.py | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/powerinfer/bench.py b/powerinfer/bench.py index 9c17be5..bde0417 100755 --- a/powerinfer/bench.py +++ b/powerinfer/bench.py @@ -1,17 +1,19 @@ -import pandas as pd import sys -sys.path.append('../common/') + +import pandas as pd + +sys.path.append("../common/") from client import chat from questions import questions -if __name__ == '__main__': +if __name__ == "__main__": counter = 1 responses = [] for q in questions: response = chat(q) - if counter >= 2: # allow for a warmup + if counter >= 2: # allow for a warmup responses.append(response) - counter +=1 + counter += 1 df = pd.DataFrame(responses) - df.to_csv('bench-powerinfer.csv', index=False) + df.to_csv("bench-powerinfer.csv", index=False) \ No newline at end of file diff --git a/powerinfer/client.py b/powerinfer/client.py index d888150..9475177 100755 --- a/powerinfer/client.py +++ b/powerinfer/client.py @@ -1,4 +1,5 @@ import time + import requests api_base = "http://localhost:8080" @@ -23,4 +24,4 @@ def chat(prompt: str): if __name__ == "__main__": prompt = "San Francisco is a city in" - print(f"User: {prompt}\nPowerInfer: {chat(prompt)['answer']}") + print(f"User: {prompt}\nPowerInfer: {chat(prompt)['answer']}") \ No newline at end of file From 5ad8c94cb3cdbcbeb5036e767b9f7d7a0f5d22e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Thu, 4 Apr 2024 15:43:14 -0300 Subject: [PATCH 08/10] Newer image including CUDNN9, fix for dnf. --- powerinfer/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/powerinfer/Dockerfile b/powerinfer/Dockerfile index a1de8fe..ead0a5f 100644 --- a/powerinfer/Dockerfile +++ b/powerinfer/Dockerfile @@ -39,15 +39,15 @@ # Select an available version from # https://gitlab.com/nvidia/container-images/cuda/blob/master/doc/supported-tags.md: -FROM nvidia/cuda:12.3.1-devel-rockylinux9 -ARG USERID=1000 -RUN yum install -y python3-pip cmake libcudnn8 git && yum clean all && rm -rf /var/cache/yum/* +FROM nvcr.io/nvidia/cuda:12.3.2-cudnn9-devel-rockylinux9 +RUN dnf install -y python3-pip cmake git && dnf clean all && rm -rf /var/cache/dnf/* RUN git clone https://github.com/SJTU-IPADS/PowerInfer WORKDIR /PowerInfer RUN pip install --no-cache-dir -r requirements.txt RUN cmake -S . -B build -DLLAMA_CUBLAS=ON RUN cmake --build build --config Release -j "$(nproc)" RUN pip install --no-cache-dir pandas #for the benchmark. +ARG USERID=1000 RUN adduser -u $USERID user USER user From a8de8084f3f495a55d93c57039e368276c3816ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Thu, 4 Apr 2024 16:50:27 -0300 Subject: [PATCH 09/10] CUDNN9 doesn't work yet for PowerInfer, downgrading to CUDNN8. --- powerinfer/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/powerinfer/Dockerfile b/powerinfer/Dockerfile index ead0a5f..932173d 100644 --- a/powerinfer/Dockerfile +++ b/powerinfer/Dockerfile @@ -39,7 +39,9 @@ # Select an available version from # https://gitlab.com/nvidia/container-images/cuda/blob/master/doc/supported-tags.md: -FROM nvcr.io/nvidia/cuda:12.3.2-cudnn9-devel-rockylinux9 +# 2024-04-04 CUDNN9 not supported yet: +#FROM nvcr.io/nvidia/cuda:12.3.2-cudnn9-devel-rockylinux9 +FROM nvcr.io/nvidia/cuda:12.2.2-cudnn8-devel-rockylinux9 RUN dnf install -y python3-pip cmake git && dnf clean all && rm -rf /var/cache/dnf/* RUN git clone https://github.com/SJTU-IPADS/PowerInfer WORKDIR /PowerInfer From f3a29dbdbe50025eb71ca236f06e4ae2b19d9fbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Thu, 6 Jun 2024 15:44:29 -0300 Subject: [PATCH 10/10] Megalinter nitpicking... --- powerinfer/bench.py | 2 +- powerinfer/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) mode change 100755 => 100644 powerinfer/bench.py mode change 100755 => 100644 powerinfer/client.py diff --git a/powerinfer/bench.py b/powerinfer/bench.py old mode 100755 new mode 100644 index bde0417..3afd3d0 --- a/powerinfer/bench.py +++ b/powerinfer/bench.py @@ -16,4 +16,4 @@ counter += 1 df = pd.DataFrame(responses) - df.to_csv("bench-powerinfer.csv", index=False) \ No newline at end of file + df.to_csv("bench-powerinfer.csv", index=False) diff --git a/powerinfer/client.py b/powerinfer/client.py old mode 100755 new mode 100644 index 9475177..ea4b8ba --- a/powerinfer/client.py +++ b/powerinfer/client.py @@ -24,4 +24,4 @@ def chat(prompt: str): if __name__ == "__main__": prompt = "San Francisco is a city in" - print(f"User: {prompt}\nPowerInfer: {chat(prompt)['answer']}") \ No newline at end of file + print(f"User: {prompt}\nPowerInfer: {chat(prompt)['answer']}")