Add PowerInfer benchmark.

powerinfer/Dockerfile

@@ -0,0 +1,55 @@
+# From the parent directory (main directory of this repo) run:
+#
+# docker build --build-arg USERID=$(id -u) -t local/powerinfer-bench powerinfer
+#
+# mkdir $HOME/.cache/{huggingface,PowerInfer}
+# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z \
+#   -v$HOME/.cache/PowerInfer/:/home/user/.cache/PowerInfer/:Z \
+#   local/powerinfer-bench \
+#   sh -c 'cd /home/user/.cache/PowerInfer/ && \
+#   huggingface-cli download --resume-download --local-dir ReluLLaMA-7B \
+#   --local-dir-use-symlinks False PowerInfer/ReluLLaMA-7B-PowerInfer-GGUF'
+#
+# docker run --rm -it --gpus all -v$HOME/.cache/PowerInfer/:/home/user/.cache/PowerInfer/:Z \
+#   local/powerinfer-bench \
+#   build/bin/main --log-disable -m \
+#   /home/user/.cache/PowerInfer/ReluLLaMA-7B/llama-7b-relu.powerinfer.gguf \
+#   -p "Once upon a time"
+#
+# docker run --rm -it --name powerinfer-bench --gpus all \
+#   -v$HOME/.cache/PowerInfer/:/home/user/.cache/PowerInfer/:Z \
+#   -v$(pwd):/home/user/llama-inference/:Z \
+#   local/powerinfer-bench \
+#   build/bin/server -v -m \
+#   /home/user/.cache/PowerInfer/ReluLLaMA-7B/llama-7b-relu.powerinfer.gguf
+#
+# In another terminal:
+#
+# docker exec -it powerinfer-bench curl --request POST --url http://localhost:8080/completion \
+#   --header "Content-Type: application/json" --data \
+#   '{"prompt": "Building a website can be done in 10 simple steps:", "n_predict": 128}'
+#
+# docker exec -it powerinfer-bench sh -c 'cd /home/user/llama-inference/powerinfer && \
+#   python3 bench.py'
+#
+# If using Podman with CDI substitute
+#   --gpus all
+# for
+#   --device nvidia.com/gpu=all --security-opt=label=disable
+
+# Select an available version from
+# https://gitlab.com/nvidia/container-images/cuda/blob/master/doc/supported-tags.md:
+# 2024-04-04 CUDNN9 not supported yet:
+#FROM nvcr.io/nvidia/cuda:12.3.2-cudnn9-devel-rockylinux9
+FROM nvcr.io/nvidia/cuda:12.2.2-cudnn8-devel-rockylinux9
+RUN dnf install -y python3-pip cmake git && dnf clean all && rm -rf /var/cache/dnf/*
+RUN git clone https://github.com/SJTU-IPADS/PowerInfer
+WORKDIR /PowerInfer
+RUN pip install --no-cache-dir -r requirements.txt
+RUN cmake -S . -B build -DLLAMA_CUBLAS=ON
+RUN cmake --build build --config Release -j "$(nproc)"
+RUN pip install --no-cache-dir pandas #for the benchmark.
+ARG USERID=1000
+RUN adduser -u $USERID user
+USER user
+

powerinfer/README.md

@@ -0,0 +1,21 @@
+# PowerInfer benchmark
+
+Benchmark for [PowerInfer](https://github.com/SJTU-IPADS/PowerInfer).
+
+Note that the model loses some inference quality in exchange for speed as shown in <https://huggingface.co/SparseLLM/ReluLLaMA-7B>.
+
+You can compile PowerInfer following their instructions and then run the server:
+
+```bash
+build/bin/server -v -m ReluLLaMA-7B/llama-7b-relu.powerinfer.gguf
+```
+
+And in another terminal run:
+```bash
+python3 bench.py
+```
+
+The results will be in the bench-powerinfer.csv file.
+
+Or alternatively you can follow the instructions in the [Dockerfile](Dockerfile) to build a container to run the server and the benchmark inside it.
+

powerinfer/bench.py

@@ -0,0 +1,19 @@
+import sys
+
+import pandas as pd
+
+sys.path.append("../common/")
+from client import chat
+from questions import questions
+
+if __name__ == "__main__":
+    counter = 1
+    responses = []
+    for q in questions:
+        response = chat(q)
+        if counter >= 2:  # allow for a warmup
+            responses.append(response)
+        counter += 1
+
+    df = pd.DataFrame(responses)
+    df.to_csv("bench-powerinfer.csv", index=False)

powerinfer/client.py

@@ -0,0 +1,27 @@
+import time
+
+import requests
+
+api_base = "http://localhost:8080"
+
+
+def chat(prompt: str):
+    payload = {"prompt": f"{prompt}", "n_predict": 200}
+    headers = {"Content-Type": "application/json"}
+    start = time.perf_counter()
+    response = requests.post(f"{api_base}/completion", headers=headers, json=payload)
+    response = response.json()
+    request_time = time.perf_counter() - start
+
+    return {
+        "tok_count": response["tokens_predicted"],
+        "time": request_time,
+        "question": prompt,
+        "answer": response["content"],
+        "note": "PowerInfer " + response["model"],
+    }
+
+
+if __name__ == "__main__":
+    prompt = "San Francisco is a city in"
+    print(f"User: {prompt}\nPowerInfer: {chat(prompt)['answer']}")