From 6149be5237cd6abbf575e220cfebfb4388ccea7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Tue, 23 Jan 2024 14:18:20 -0300 Subject: [PATCH 1/3] Add mlc/Dockerfile with instructions inside it. Requires small changes to mlc.py which will be sent in a separate branch+PR. --- mlc/Dockerfile | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 mlc/Dockerfile diff --git a/mlc/Dockerfile b/mlc/Dockerfile new file mode 100644 index 0000000..fc7c49b --- /dev/null +++ b/mlc/Dockerfile @@ -0,0 +1,59 @@ +# From the parent directory (main directory of this repo) run: +# +# docker build --build-arg USERID=$(id -u) -t local/mlc-bench mlc +# +# If not already using and having a $HOME/.cache/huggingface/ then: +# +# mkdir $HOME/.cache/huggingface/ +# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z local/hf-bench \ +# huggingface-cli login +# Answer n to: Add token as git credential? (Y/n) n +# +# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z local/hf-bench \ +# huggingface-cli download meta-llama/Llama-2-7b-hf +# +# mkdir $HOME/.cache/mlc_chat/ +# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z \ +# -v$HOME/.cache/mlc_chat/:/home/user/.cache/mlc_chat/:Z -v$(pwd):/home/user/llama-inference \ +# --gpus all local/mlc-bench \ +# sh -c 'cd /home/user/llama-inference/mlc && mlc_chat convert_weight --quantization q4f16_1 \ +# -o Llama-2-7b-hf-q4f16_1 ~/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-hf/snapshots/*' +# +# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z \ +# -v$HOME/.cache/mlc_chat/:/home/user/.cache/mlc_chat/:Z -v$(pwd):/home/user/llama-inference \ +# --gpus all local/mlc-bench \ +# sh -c 'cd /home/user/llama-inference/mlc && mlc_chat gen_config --quantization q4f16_1 \ +# --conv-template LM -o Llama-2-7b-hf-q4f16_1 \ +# ~/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-hf/snapshots/*' +# +# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z \ +# -v$HOME/.cache/mlc_chat/:/home/user/.cache/mlc_chat/:Z -v$(pwd):/home/user/llama-inference \ +# --gpus all local/mlc-bench \ +# sh -c 'cd /home/user/llama-inference/mlc && python3 mlc.py' +# +# If using Podman with CDI substitute +# --gpus all +# for +# --device nvidia.com/gpu=all --security-opt=label=disable + +# Select an available version from +# https://gitlab.com/nvidia/container-images/cuda/blob/master/doc/supported-tags.md: +FROM nvidia/cuda:12.2.2-cudnn8-devel-rockylinux9 + +# requests isn't declared as a dependency by mlc-chat: +RUN yum install -y python3-pip python3-requests && yum clean all && rm -rf /var/cache/yum/* + +# Don't know why libcuda.so.1 isn't on another place or configured in ld.so.conf.d, +# /usr/local/lib64/ doesn't work neither; also PyTorch doesn't find libcupti.so.12 by default: +RUN ln -s /usr/local/cuda-12.2/compat/libcuda.so.1 /usr/lib64/ && \ + ln -s /usr/local/cuda-12/lib64/libcupti.so.12 /usr/lib64/ + +# Select a matching version from https://llm.mlc.ai/docs/install/mlc_llm.html#install-mlc-packages, +# also torch isn't declared as dependency by mlc-chat and it's required for convert_weight command: +RUN pip install --no-cache-dir --pre -f https://mlc.ai/wheels mlc-chat-nightly-cu122 \ + mlc-ai-nightly-cu122 transformers torch pandas + +ARG USERID=1000 +RUN adduser -u $USERID user +USER user + From 8d9afaa409b6a067ed3f165f130896afd9dd4f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Tue, 23 Jan 2024 14:24:46 -0300 Subject: [PATCH 2/3] mlc.py: can work with the base model instead of -chat and adjust questions import path to run from it's folder instead of a subfolder. --- mlc/mlc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mlc/mlc.py b/mlc/mlc.py index 2dc2628..147c752 100644 --- a/mlc/mlc.py +++ b/mlc/mlc.py @@ -3,16 +3,16 @@ from transformers import AutoTokenizer import time import sys -sys.path.append('../../common/') +sys.path.append('../common/') from questions import questions import pandas as pd -tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") +tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf") cfg = ChatConfig(max_gen_len=200) -cm = ChatModule(model="Llama-2-7b-chat-hf-q4f16_1", chat_config=cfg) +cm = ChatModule(model="Llama-2-7b-hf-q4f16_1", chat_config=cfg) def tok_count(prompt:str): inputs = tokenizer(prompt) @@ -27,7 +27,7 @@ def predict(prompt:str): 'time': request_time, 'question': prompt, 'answer': output, - 'note': 'mlc chat 7b q4f16_1'} + 'note': 'mlc 7b q4f16_1'} if __name__ == '__main__': counter = 1 From 13b7e8dc2bf966ab906f2257ed2d4af9a1475105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Tue, 2 Apr 2024 19:38:05 -0300 Subject: [PATCH 3/3] Upstream renamed from mlc_chat to mlc_llm, explicit registry for newer Podman, cleanup fixed for dnf, remove workarounds now that they were fixed by upstream. --- mlc/Dockerfile | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/mlc/Dockerfile b/mlc/Dockerfile index fc7c49b..3514e45 100644 --- a/mlc/Dockerfile +++ b/mlc/Dockerfile @@ -5,29 +5,29 @@ # If not already using and having a $HOME/.cache/huggingface/ then: # # mkdir $HOME/.cache/huggingface/ -# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z local/hf-bench \ +# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z local/mlc-bench \ # huggingface-cli login # Answer n to: Add token as git credential? (Y/n) n # -# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z local/hf-bench \ +# docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z local/mlc-bench \ # huggingface-cli download meta-llama/Llama-2-7b-hf # -# mkdir $HOME/.cache/mlc_chat/ +# mkdir $HOME/.cache/mlc_llm/ # docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z \ -# -v$HOME/.cache/mlc_chat/:/home/user/.cache/mlc_chat/:Z -v$(pwd):/home/user/llama-inference \ +# -v$HOME/.cache/mlc_llm/:/home/user/.cache/mlc_llm/:Z -v$(pwd):/home/user/llama-inference \ # --gpus all local/mlc-bench \ -# sh -c 'cd /home/user/llama-inference/mlc && mlc_chat convert_weight --quantization q4f16_1 \ +# sh -c 'cd /home/user/llama-inference/mlc && mlc_llm convert_weight --quantization q4f16_1 \ # -o Llama-2-7b-hf-q4f16_1 ~/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-hf/snapshots/*' # # docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z \ -# -v$HOME/.cache/mlc_chat/:/home/user/.cache/mlc_chat/:Z -v$(pwd):/home/user/llama-inference \ +# -v$HOME/.cache/mlc_llm/:/home/user/.cache/mlc_llm/:Z -v$(pwd):/home/user/llama-inference \ # --gpus all local/mlc-bench \ -# sh -c 'cd /home/user/llama-inference/mlc && mlc_chat gen_config --quantization q4f16_1 \ +# sh -c 'cd /home/user/llama-inference/mlc && mlc_llm gen_config --quantization q4f16_1 \ # --conv-template LM -o Llama-2-7b-hf-q4f16_1 \ # ~/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-hf/snapshots/*' # # docker run --rm -it -v$HOME/.cache/huggingface/:/home/user/.cache/huggingface/:Z \ -# -v$HOME/.cache/mlc_chat/:/home/user/.cache/mlc_chat/:Z -v$(pwd):/home/user/llama-inference \ +# -v$HOME/.cache/mlc_llm/:/home/user/.cache/mlc_llm/:Z -v$(pwd):/home/user/llama-inference \ # --gpus all local/mlc-bench \ # sh -c 'cd /home/user/llama-inference/mlc && python3 mlc.py' # @@ -38,20 +38,19 @@ # Select an available version from # https://gitlab.com/nvidia/container-images/cuda/blob/master/doc/supported-tags.md: -FROM nvidia/cuda:12.2.2-cudnn8-devel-rockylinux9 +# 2024-04-02 CUDNN9 not supported yet: +#FROM nvcr.io/nvidia/cuda:12.3.2-cudnn9-devel-rockylinux9 +FROM nvcr.io/nvidia/cuda:12.2.2-cudnn8-devel-rockylinux9 -# requests isn't declared as a dependency by mlc-chat: -RUN yum install -y python3-pip python3-requests && yum clean all && rm -rf /var/cache/yum/* +RUN dnf install -y python3-pip && dnf clean all && rm -rf /var/cache/dnf/* # Don't know why libcuda.so.1 isn't on another place or configured in ld.so.conf.d, -# /usr/local/lib64/ doesn't work neither; also PyTorch doesn't find libcupti.so.12 by default: -RUN ln -s /usr/local/cuda-12.2/compat/libcuda.so.1 /usr/lib64/ && \ - ln -s /usr/local/cuda-12/lib64/libcupti.so.12 /usr/lib64/ +# /usr/local/lib64/ doesn't work neither: +RUN ln -s /usr/local/cuda-12.*/compat/libcuda.so.1 /usr/lib64/ -# Select a matching version from https://llm.mlc.ai/docs/install/mlc_llm.html#install-mlc-packages, -# also torch isn't declared as dependency by mlc-chat and it's required for convert_weight command: -RUN pip install --no-cache-dir --pre -f https://mlc.ai/wheels mlc-chat-nightly-cu122 \ - mlc-ai-nightly-cu122 transformers torch pandas +# Select a matching version from https://llm.mlc.ai/docs/install/mlc_llm.html#install-mlc-packages: +RUN pip install --no-cache-dir --pre -f https://mlc.ai/wheels mlc-llm-nightly-cu122 \ + mlc-ai-nightly-cu122 transformers pandas ARG USERID=1000 RUN adduser -u $USERID user