diff --git a/.env.example b/.env.example index 0c71bf0..bbac78f 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,52 @@ # 9Router API credentials — copy this file to .env and fill in your values API_BASE_URL=http://127.0.0.1:20128/v1 API_KEY=your-9router-api-key-here + +# Optional overrides (defaults live in src/distill/config.py) +# TEACHER_MODEL=cx/gpt-5.5-xhigh +# JUDGE_MODEL=cx/gpt-5.5-high +# STUDENT_MODEL_ID=D:/models/qwen15-1.5b + +# ── Training ─────────────────────────────────────────────────────────────── +# MAX_SEQ_LENGTH is the TRAINING cap only. v0.5 trained at 512; raising it +# raises VRAM use and OOMs bf16 LoRA on a 6 GB card. It does NOT affect +# evaluation — use EVAL_MAX_SEQ_LENGTH for that. +# MAX_SEQ_LENGTH=512 +# NUM_EPOCHS=3 + +# These two ship with defaults that do NOT work on a 6 GB card, and v0.5 was +# trained with both flipped. Set them before training on comparable hardware. +# LOAD_IN_4BIT defaults to true, but bitsandbytes 4-bit is broken in this +# environment (Python 3.14 + torch nightly), so training falls back to LoRA on +# the full bf16 base — which only fits with gradient checkpointing on. +# LOAD_IN_4BIT=false +# GRADIENT_CHECKPOINTING=true + +# LORA_R=16 +# LORA_ALPHA=32 +# LEARNING_RATE=2e-4 +# EARLY_STOPPING_PATIENCE=3 # evals without improvement before stopping +# TRAIN_ON_COMPLETIONS_ONLY=true # mask prompt tokens out of the loss + +# ── Dataset ──────────────────────────────────────────────────────────────── +# SPLIT_SEED=42 # fixes the stratified train/validation/test draw + +# ── Evaluation (python -m distill.evaluate) ──────────────────────────────── +# EVAL_MAX_SEQ_LENGTH is the perplexity truncation cap. It decides how much of +# the held-out set is scored at all, so a perplexity is only comparable against +# another measured at the same cap. Default 2048 scores 100% of the current +# test split; 1024 scores 91.9% and 512 scores 70.4%. +# EVAL_MAX_SEQ_LENGTH=2048 +# EVAL_SEED=42 # seeds sampled answer generation (ROUGE-L reproducibility) +# EVAL_MAX_SAMPLES=200 # cap on held-out samples evaluated +# EVAL_MAX_NEW_TOKENS=512 # generation length per answer +# EVAL_TEMPERATURE=0.7 +# EVAL_TOP_P=0.9 + +# GGUF export tooling (machine-specific paths) +# LLAMACPP_BIN=D:/tools/llama-cpp-b10107 +# LLAMACPP_SRC=D:/tools/llama.cpp-src + +# Inference service (services/api) +# MODEL_PATH=checkpoints/gguf/distill-gpt55-v0.5-Q4_K_M.gguf +# CORS_ALLOW_ORIGINS=http://localhost:3000 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f4edf35 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,26 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + - package-ecosystem: pip + directory: "/services/api" + schedule: + interval: weekly + - package-ecosystem: npm + directory: "/services/web" + schedule: + interval: weekly + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + - package-ecosystem: docker + directory: "/services/api" + schedule: + interval: weekly + - package-ecosystem: docker + directory: "/services/web" + schedule: + interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ccfc552 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: ci + +on: + push: + branches: [master] + pull_request: + +jobs: + python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install test deps + run: | + pip install pytest ruff openai fastapi httpx uvicorn prometheus-client pydantic + - name: Lint (ruff) + run: ruff check src/ tests/ services/api/ + - name: Core package tests + run: python -m pytest tests/ -q + - name: API service tests + working-directory: services/api + run: python -m pytest tests/ -q + + web: + runs-on: ubuntu-latest + defaults: + run: + working-directory: services/web + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: services/web/pnpm-lock.yaml + - run: pnpm install --frozen-lockfile + - name: Verify generated client is in sync with the contract + run: | + pnpm run generate-client + git diff --exit-code src/api/schema.d.ts + - run: pnpm test + - run: pnpm build diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..e7088c4 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,30 @@ +name: docker-publish + +on: + push: + branches: [master] + +jobs: + publish: + runs-on: ubuntu-latest + strategy: + matrix: + service: [api, web] + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: docker/build-push-action@v6 + with: + context: services/${{ matrix.service }} + push: true + build-args: | + REVISION=${{ github.sha }} + tags: | + nguyenson1710/distill-gpt55-${{ matrix.service }}:latest + nguyenson1710/distill-gpt55-${{ matrix.service }}:${{ github.sha }} + cache-from: type=gha,scope=${{ matrix.service }} + cache-to: type=gha,scope=${{ matrix.service }},mode=max diff --git a/.gitignore b/.gitignore index 3b8aaa5..5863b00 100644 --- a/.gitignore +++ b/.gitignore @@ -4,19 +4,29 @@ __pycache__/ *.egg-info/ dist/ build/ +.pytest_cache/ +.ruff_cache/ -# Models & checkpoints (quá nặng) +# Models & checkpoints (too heavy for git) checkpoints/ models/ *.bin *.safetensors *.pt *.pth +*.gguf -# Dataset raw +# Dataset data/raw/ data/processed/ +# Logs +logs/ + +# Node +node_modules/ +services/web/dist/ + # IDE .idea/ .vscode/ @@ -26,7 +36,34 @@ data/processed/ .DS_Store Thumbs.db -# Environment +# Environment & secrets .env +.env.* +!.env.example venv/ .venv/ + +# Assistant tooling & private dirs (never commit) +.claude/ +.codex/ +.commandcode/ +.claude-private/ +.private/ +AGENTS.md +CLAUDE.md + +# Private notes / drafts (root-level; plans/ is this repo's tracked workflow) +/PLAN*.md +/plan*.md +/NOTES*.md +/notes*.md +/TODO*.md +/todo*.md +/DRAFT*.md +/draft*.md +*.private.md +*.local.md + +# Test + tooling artifacts +.coverage +.playwright-mcp/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..29e43c9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,50 @@ +# Changelog + +All notable changes to this project are documented here. +Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [Unreleased] + +### Added +- `src/distill` package: resilient teacher client (retryable/fatal error + classification, backoff + jitter, output validation), resumable atomic + generation, dataset quality pipeline (mojibake/dedup screening, stratified + train/validation/test splits), training with validation + early stopping, + bf16 merge, evaluation suite (held-out PPL, ROUGE-L vs teacher, optional + LLM-as-judge), GGUF export wrapper. +- `services/api`: OpenAI-compatible FastAPI inference service over llama.cpp + (streaming SSE, rate limiting, /healthz /readyz /metrics), tests, Dockerfile. +- `services/web`: Vite + React chat UI with client generated from the committed + OpenAPI contract, streaming rendering, tests, Dockerfile (nginx). +- `docker-compose.yml`, GitHub Actions CI + Docker Hub publish, dependabot, + pyproject, unit test suites, MIT license. + +### Changed +- Training samples now use the exact Qwen2.5 chat template with + `<|im_start|>`/`<|im_end|>` special tokens (v0.4 trained on a plain-text + approximation, mismatching every standard inference path). +- Adapter merge now applies LoRA onto the bf16 base instead of the 4-bit + dequantized base. + +### Fixed +- Transient teacher API errors (quota/connection) no longer recorded as + permanent failures — the v0.4 run lost 134/530 prompts to this. +- Vietnamese teacher outputs no longer mojibake-corrupted (UTF-8 handling + + replacement-character validation at the client). + +### Removed +- All 13 legacy root-level scripts (`gen_batch.py`, `train_student.py`, + `format_dataset.py`, `evaluate*.py`, `chat.py`, root `config.py`, ...) — + superseded by the tested `src/distill` package (`python -m distill.`). + +## [0.4.0] - 2026-07-25 + +### Added +- Honest held-out evaluation: 357 train / 38 stratified test samples, + perplexity 6.93, ROUGE-L vs teacher; per-category breakdown. + +## [0.3.0] - 2026-07-24 + +### Added +- First end-to-end pipeline: 300-sample generation, QLoRA training on RTX 3060 + 6GB, merge, perplexity 4.70 (in-sample — superseded by v0.4's honest eval). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5df46aa --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Nguyen Tien Son + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index d9f23c4..94994b3 100644 --- a/README.md +++ b/README.md @@ -1,96 +1,189 @@ -# Distill GPT-5.5-xhigh → Qwen2.5-1.5B +# distill-gpt55 -**Knowledge Distillation** từ GPT-5.5-xhigh (9Router API) sang Qwen2.5-1.5B-Instruct, chạy local trên RTX 3060 6GB VRAM. +> Chưng cất GPT-5.5-xhigh qua 9Router thành Qwen2.5-1.5B-Instruct, huấn luyện cục bộ trên RTX 3060 6GB, rồi phục vụ qua API tương thích OpenAI và web chat streaming. -## Kết quả +![Giao diện chat streaming của model đã distill](docs/assets/chat-streaming.gif) -| Metric | Before | After | -|--------|--------|-------| -| Training Loss | 1.43 | **1.14** ↓20% | -| Token Accuracy | 66.0% | **72.6%** | -| Perplexity | — | **4.70** (Excellent) | +Ảnh chụp thực tế: student 1.5B trả lời qua SSE từ llama.cpp chạy CPU, khoảng 1.4 token/giây; video được tăng tốc. + +## Mục lục + +- [Dự án này làm gì?](#dự-án-này-làm-gì) +- [Kết quả v0.5](#kết-quả-v05) +- [Kiến trúc](#kiến-trúc) +- [Chạy nhanh bằng Docker](#chạy-nhanh-bằng-docker) +- [Dùng web chat và lịch sử hội thoại](#dùng-web-chat-và-lịch-sử-hội-thoại) +- [Pipeline huấn luyện](#pipeline-huấn-luyện) +- [Kiểm thử](#kiểm-thử) +- [Tài liệu chi tiết](#tài-liệu-chi-tiết) + +## Dự án này làm gì? + +`distill-gpt55` có hai phần tách biệt: + +| Phần | Mục đích | Khi nào cần chạy | +|---|---|---| +| **Offline training** | Sinh teacher outputs, lọc/split dataset, fine-tune LoRA, đánh giá, export GGUF | Khi tái tạo hoặc cải thiện model | +| **Online serving** | Phục vụ GGUF bằng FastAPI + llama.cpp và chat UI React | Khi muốn dùng model | + +Bạn **không cần** 9Router, GPU, hay môi trường training để chạy bản GGUF đã export. Serving hiện chạy CPU trong API container. + +## Kết quả v0.5 + +| Metric | v0.4 | **v0.5 hiện tại** | +|---|---:|---:| +| Held-out perplexity, cap 2048 | Chưa đo ở cap này | **5.23** | +| Held-out perplexity, cap 512 | 6.93 | **5.38** (giảm 22.4%) | +| Validation loss tốt nhất | Không có validation split | **1.409** | +| Dataset train / validation / test | 357 / 0 / 38 | **426 / 51 / 51** | +| Teacher outputs | 396 / 530 | **530 sinh, 528 giữ lại** | +| Chat template | Plain-text gần đúng | Qwen `<|im_start|>` chính xác | + +Perplexity luôn đi kèm truncation cap. Test split có median 525 token: cap 512 chỉ chấm khoảng 70% token, còn cap 2048 chấm 100%. Chỉ so sánh các số đo ở **cùng cap**. + +![Perplexity và ROUGE-L theo category](docs/assets/evaluation-by-category.png) + +Tái tạo headline: `python -m distill.evaluate --label v0.5`. Báo cáo đầy đủ: [`plans/reports/evaluation-v0.5.md`](plans/reports/evaluation-v0.5.md). ## Kiến trúc +```text +OFFLINE — RTX 3060 6GB ONLINE — docker compose +prompts.json (530) ┌────────────┐ REST / SSE ┌──────────────┐ + → generate_dataset (9Router) │ web │ ───────────────▶│ api │ + → dataset (quality gate + split) │ React/Vite │ │ FastAPI │ + → train (bf16 LoRA + validation) │ nginx │◀─────────────────│ llama.cpp CPU│ + → merge → evaluate → export_gguf └────────────┘ └──────┬───────┘ + GGUF /models (RO) ``` -GPT-5.5-xhigh ──9Router API──▶ teacher_outputs.json ──▶ QLoRA Train ──▶ Merged Model - (teacher) (300 samples, 933K tokens) (Qwen2.5-1.5B) + +Chi tiết thành phần, contract và data flow: [`docs/system-architecture.md`](docs/system-architecture.md). + +## Chạy nhanh bằng Docker + +### Điều kiện cần + +1. Docker Desktop với Compose v2. +2. File GGUF tại `checkpoints/gguf/distill-gpt55-v0.5-Q4_K_M.gguf`. + +### Khởi động + +```bash +docker compose up --build ``` -- **Teacher:** `cx/gpt-5.5-xhigh` (9Router, 2048 max tokens) -- **Student:** `Qwen2.5-1.5B-Instruct` (Alibaba, 4-bit NF4) -- **GPU:** RTX 3060 Laptop 6GB VRAM -- **Framework:** PyTorch + Transformers + PEFT (LoRA) + BitsAndBytes +| Dịch vụ | URL | Ghi chú | +|---|---|---| +| Web chat | http://localhost:3000 | Chỉ khởi động sau khi API ready | +| API | http://localhost:8000 | OpenAI-compatible | +| Readiness | http://localhost:8000/readyz | `503` trong lúc GGUF đang load | +| Liveness | http://localhost:8000/healthz | Process còn sống | -## Quick Start +Xác minh API sau khi model load: ```bash -# Test connection -python test_connection.py +curl http://localhost:8000/readyz +curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" \ + -d '{"messages":[{"role":"user","content":"What is 2+2?"}]}' +``` -# Generate data from teacher (resumable, ~14s/prompt) -python gen_batch.py +Nếu `/readyz` vẫn trả `503`, xem `docker compose logs api`; đừng kết luận service hỏng chỉ vì model đang cold-start. Hướng dẫn triển khai và rollback: [`docs/deployment-guide.md`](docs/deployment-guide.md). -# Format for training -python format_dataset.py +## Dùng web chat và lịch sử hội thoại -# Train (3 epochs, ~8 min for 200 samples) -python train_student.py +Giao diện là một chat view với streaming SSE, markdown đã sanitize cho assistant output, tùy chọn generation và history local-first. -# Evaluate -python evaluate.py -python test_model.py +### Luồng sử dụng -# Chat -python chat.py -``` +1. Mở http://localhost:3000 và chờ badge hiện **model ready**. +2. Nhập câu hỏi; `Enter` gửi, `Shift+Enter` xuống dòng. +3. Dùng **New chat** hoặc sidebar để tạo/chuyển/xóa hội thoại. +4. Khi đang sinh, dùng **Stop** để abort request. Điều hướng history bị khóa trong thời gian này để token không ghi nhầm vào chat khác. + +### Lịch sử được lưu ở đâu? + +History chỉ lưu trong `localStorage` của **browser profile hiện tại**, dưới key `distill-gpt55.chat-history.v1`. -## Hyperparameters +| Hành vi | Thực tế | +|---|---| +| Giới hạn | 30 conversations gần nhất; 100 completed messages/conversation | +| Tiêu đề | Tạo từ user prompt đầu tiên, gọn tối đa 48 ký tự | +| Dữ liệu không lưu | Assistant response rỗng, lỗi hoặc chưa hoàn tất | +| Khi storage lỗi/đầy | Chat hiện tại vẫn dùng được trong RAM; persistence bị bỏ qua | +| Sync / account / export / recovery | **Không có** | +| Khi xóa site data hoặc đổi browser/profile | History biến mất khỏi browser đó | -| Param | Value | Note | -|-------|-------|------| -| LoRA r | 16 | | -| LoRA alpha | 32 | | -| Learning rate | 2e-4 | | -| Batch | 1 + grad_accum 8 | vừa 6GB VRAM | -| Max seq len | 512 | | -| Epochs | 3 | | -| Quant | NF4, float16 | bfloat16 không hỗ trợ RTX 3060 | -| Optimizer | AdamW 8-bit | | +History trong UI được gửi lại làm context ở lượt kế tiếp, cùng với system prompt. Đây **không** phải bộ nhớ dài hạn: API mặc định có cửa sổ context 4096 token và hiện web không token-truncate history trước request. Giữ conversation ngắn nếu câu trả lời bắt đầu lỗi context hoặc kém liên quan. -## Project Structure +Chi tiết UX, accessibility và quy ước UI: [`docs/design-guidelines.md`](docs/design-guidelines.md). Hướng dẫn frontend: [`services/web/README.md`](services/web/README.md). +## Pipeline huấn luyện + +> Cần Python environment phù hợp, 9Router cho generation/judge và RTX 3060 6GB cho training theo cấu hình v0.5. + +```bash +pip install -e .[train,dev] +set PYTHONPATH=src + +python -m distill.download_student +python -m distill.generate_dataset +python -m distill.dataset +python -m distill.train +python -m distill.merge +python -m distill.evaluate --label v0.5 +python -m distill.export_gguf +python -m distill.chat ``` -distill-gpt55/ -├── config.py # Central config -├── gen_batch.py # API data generation (resumable) -├── format_dataset.py # Convert to chat template -├── train_student.py # QLoRA training -├── evaluate.py # Perplexity evaluation -├── test_model.py # Quick inference test -├── chat.py # Interactive chat -├── test_connection.py # Verify 9Router API -├── data/ -│ ├── prompts.json # 530 prompts (10 categories) -│ ├── raw/ # Teacher outputs -│ └── processed/ # Training dataset -├── checkpoints/ -│ ├── adapter/ # LoRA weights -│ └── merged/ # Final model (1.5GB) -└── docs/ # Project documentation + +Copy `.env.example` thành `.env` rồi điền API key trước khi gọi teacher. Với máy tương tự, v0.5 dùng `LOAD_IN_4BIT=false`, `GRADIENT_CHECKPOINTING=true`, `MAX_SEQ_LENGTH=512`: bitsandbytes 4-bit hiện lỗi trong Python 3.14 + torch nightly. Không commit `.env`. + +## Kiểm thử + +```bash +python -m pytest tests/ -q # training pipeline +cd services/api && python -m pytest tests/ -q # API, fake runtime, không cần model/GPU +cd services/web && pnpm test && pnpm build # UI tests + typecheck + production build +ruff check src/ tests/ services/api/ +``` + +API contract chuẩn là [`docs/openapi.yaml`](docs/openapi.yaml). Sau khi sửa API, regenerate frontend types rồi kiểm tra diff: + +```bash +cd services/web +pnpm run generate-client +pnpm build ``` -## Pipeline Stages +## Tài liệu chi tiết + +| Tài liệu | Nội dung | +|---|---| +| [`docs/project-overview-pdr.md`](docs/project-overview-pdr.md) | Requirements, metric, constraint và risk sản phẩm | +| [`docs/system-architecture.md`](docs/system-architecture.md) | Kiến trúc offline/online và chat data flow | +| [`docs/deployment-guide.md`](docs/deployment-guide.md) | Docker, local run, smoke test, resource và rollback | +| [`docs/code-standards.md`](docs/code-standards.md) | Quy ước code, test, contract và training constraints | +| [`docs/design-guidelines.md`](docs/design-guidelines.md) | Design tokens, responsive/accessibility, state và history UX | +| [`docs/project-roadmap.md`](docs/project-roadmap.md) | Những phần đã hoàn thành, giới hạn đã biết và roadmap | +| [`services/api/README.md`](services/api/README.md) | API endpoints, config và runbook | +| [`services/web/README.md`](services/web/README.md) | Web setup, history behaviour và frontend troubleshooting | + +## Repo layout + +```text +src/distill/ Training pipeline package +services/api/ FastAPI + llama.cpp inference service +services/web/ React chat UI +data/ Prompts, teacher outputs, processed splits +checkpoints/ Adapter, merged model, GGUF artifacts (gitignored) +docs/ Architecture, deployment, standards, roadmap, OpenAPI +plans/ ClaudeKit plans và evaluation reports +``` -1. **Generate:** API calls GPT-5.5-xhigh, saves after each prompt -2. **Format:** Convert to Qwen `<|im_start|>` chat template -3. **Train:** QLoRA 4-bit + LoRA rank 16, 3 epochs -4. **Merge:** Combine adapter with base model -5. **Evaluate:** Perplexity + qualitative tests +## Known constraints -## Constraints +- **6GB VRAM:** v0.5 dùng bf16 LoRA + gradient checkpointing; sequence length training bị giới hạn 512. +- **Python 3.14:** cần torch nightly CUDA; load model CPU-first rồi mới đưa sang GPU để tránh crash đã biết. +- **9Router:** chỉ cần cho sinh dataset/judge, không cần khi serving. +- **Local deployment:** API serialize generation vì llama.cpp context không thread-safe; không phải multi-user scale-out service. -- **6GB VRAM** → mandatory QLoRA 4-bit -- **Python 3.14** → requires nightly PyTorch CUDA 12.8 build -- **Windows symlink limitation** → model cache in `D:/models/` -- **API:** 9Router localhost:20128 must be running +Xem các giới hạn, lý do và hướng xử lý tại [`docs/project-roadmap.md`](docs/project-roadmap.md). \ No newline at end of file diff --git a/chat.py b/chat.py deleted file mode 100644 index 91e1538..0000000 --- a/chat.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Interactive chat with the distilled model for evaluation.""" -import sys -import io -import os - -sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') - -import torch -from transformers import AutoModelForCausalLM, AutoTokenizer - -import config - - -def load_model(model_path=config.MERGED_MODEL_DIR): - """Load merged student model.""" - print(f"Loading model from {model_path}...") - model = AutoModelForCausalLM.from_pretrained( - model_path, - torch_dtype=torch.float16, - device_map="auto", - trust_remote_code=True, - ) - tokenizer = AutoTokenizer.from_pretrained( - model_path, - trust_remote_code=True, - ) - if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - print(f"Loaded! VRAM: {torch.cuda.max_memory_allocated()/1024**3:.2f} GB") - return model, tokenizer - - -def generate(model, tokenizer, prompt, max_tokens=512): - """Generate response from model.""" - messages = [ - {"role": "system", "content": "You are a helpful, knowledgeable assistant."}, - {"role": "user", "content": prompt}, - ] - text = tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True, - ) - inputs = tokenizer(text, return_tensors="pt").to(model.device) - - with torch.no_grad(): - outputs = model.generate( - **inputs, - max_new_tokens=max_tokens, - temperature=0.7, - top_p=0.9, - do_sample=True, - pad_token_id=tokenizer.pad_token_id, - ) - - response = tokenizer.decode(outputs[0][len(inputs.input_ids[0]):], skip_special_tokens=True) - return response - - -def chat(): - """Interactive chat loop.""" - model, tokenizer = load_model() - - print("\n" + "=" * 60) - print("Distilled Student Model Chat") - print("Commands: /exit, /clear, /batch") - print("=" * 60) - - while True: - try: - user_input = input("\nYou: ").strip() - except (EOFError, KeyboardInterrupt): - print("\nGoodbye!") - break - - if not user_input: - continue - - if user_input == "/exit": - print("Goodbye!") - break - - if user_input == "/clear": - os.system("cls" if sys.platform == "win32" else "clear") - continue - - if user_input == "/batch": - run_test_set(model, tokenizer) - continue - - print("Student: ", end="", flush=True) - response = generate(model, tokenizer, user_input) - print(response) - print("-" * 40) - - -def run_test_set(model, tokenizer): - """Run a small test set for evaluation.""" - test_prompts = [ - "Write a Python function to calculate Fibonacci numbers.", - "Explain what black holes are in simple terms.", - "What is the difference between AI and machine learning?", - "Giải thích cách nấu phở bò truyền thống.", - "If I flip a coin 3 times, what is the probability of getting exactly 2 heads?", - ] - for prompt in test_prompts: - print(f"\n[Q] {prompt}") - print(f"[A] ", end="", flush=True) - resp = generate(model, tokenizer, prompt, max_tokens=256) - print(resp[:300]) - print("-" * 40) - - -if __name__ == "__main__": - chat() diff --git a/config.py b/config.py deleted file mode 100644 index 363b6fb..0000000 --- a/config.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Central configuration for GPT-5.5 Distill project. Loads secrets from .env.""" - -import os -from pathlib import Path - -# ── Load .env file ───────────────────────────────────── -def _load_dotenv(): - env_path = Path(__file__).parent / ".env" - if env_path.exists(): - with open(env_path) as f: - for line in f: - line = line.strip() - if line and not line.startswith("#") and "=" in line: - key, val = line.split("=", 1) - if key.strip() not in os.environ: - os.environ[key.strip()] = val.strip() - -_load_dotenv() - -# ── Paths ────────────────────────────────────────────── -PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) -DATA_DIR = os.path.join(PROJECT_DIR, "data") -RAW_DIR = os.path.join(DATA_DIR, "raw") -PROCESSED_DIR = os.path.join(DATA_DIR, "processed") -CHECKPOINT_DIR = os.path.join(PROJECT_DIR, "checkpoints") -MERGED_DIR = os.path.join(PROJECT_DIR, "merged_model") - -# ── 9Router API ───────────────────────────────────────── -API_BASE_URL = os.environ.get("API_BASE_URL", "http://127.0.0.1:20128/v1") -API_KEY = os.environ.get("API_KEY", "") - -# ── Teacher (distill source) ──────────────────────────── -TEACHER_MODEL = "cx/gpt-5.5-xhigh" -TEACHER_MAX_TOKENS = 2048 -TEACHER_TEMPERATURE = 0.7 - -# ── Student (distill target) ──────────────────────────── -STUDENT_MODEL_ID = "D:/models/qwen15-1.5b" -STUDENT_LOCAL_DIR = os.path.join(CHECKPOINT_DIR, "student_base") - -# ── Dataset generation ────────────────────────────────── -PROMPTS_FILE = os.path.join(DATA_DIR, "prompts.json") -TEACHER_OUTPUT_FILE = os.path.join(RAW_DIR, "teacher_outputs.json") -PROCESSED_DATASET_FILE = os.path.join(PROCESSED_DIR, "dataset_train.json") -TEST_SPLIT_RATIO = 0.1 -REQUEST_DELAY = 1.5 # seconds between API calls -CHECKPOINT_EVERY = 10 # save partial after N prompts -MAX_RETRIES = 3 - -# ── LoRA / QLoRA Training ─────────────────────────────── -LORA_R = 16 -LORA_ALPHA = 32 -LORA_DROPOUT = 0.05 -LORA_TARGET_MODULES = ["q_proj", "k_proj", "v_proj", "o_proj"] - -BATCH_SIZE = 1 -GRADIENT_ACCUMULATION_STEPS = 8 -LEARNING_RATE = 2e-4 -NUM_EPOCHS = 3 -MAX_SEQ_LENGTH = 512 -SAVE_STEPS = 100 -LOGGING_STEPS = 10 -FP16 = True -WARMUP_RATIO = 0.03 -GRADIENT_CHECKPOINTING = False - -# 4-bit quantization -LOAD_IN_4BIT = True -BNB_4BIT_QUANT_TYPE = "nf4" -BNB_4BIT_COMPUTE_DTYPE = "float16" -BNB_4BIT_DOUBLE_QUANT = True - -# ── Adapter save paths ────────────────────────────────── -ADAPTER_DIR = os.path.join(CHECKPOINT_DIR, "adapter") -MERGED_MODEL_DIR = os.path.join(CHECKPOINT_DIR, "merged") - -# Ensure all directories exist on import -for _d in [DATA_DIR, RAW_DIR, PROCESSED_DIR, CHECKPOINT_DIR, MERGED_DIR]: - os.makedirs(_d, exist_ok=True) diff --git a/data/prompts.json b/data/prompts.json index caddd62..8e1dc1f 100644 --- a/data/prompts.json +++ b/data/prompts.json @@ -528,5 +528,45 @@ {"id": 527, "category": "coding", "instruction": "Write a React custom hook for debouncing a value.", "expected_length": "short"}, {"id": 528, "category": "reasoning", "instruction": "What is the difference between induction and deduction? Give real-world examples.", "expected_length": "medium"}, {"id": 529, "category": "reasoning", "instruction": "A train leaves Station A at 8 AM traveling at 60 km/h. Another train leaves Station B at 9 AM traveling at 80 km/h toward Station A. If stations are 300 km apart, when do they meet?", "expected_length": "medium"}, - {"id": 530, "category": "creative", "instruction": "Write a bedtime story about the Moon learning to share the night sky with the stars.", "expected_length": "medium"} + {"id": 530, "category": "creative", "instruction": "Write a bedtime story about the Moon learning to share the night sky with the stars.", "expected_length": "medium"}, + {"id": 531, "category": "creative", "instruction": "Write a sci-fi flash fiction about a linguist who decodes an alien message that rewrites every human word.", "expected_length": "medium"}, + {"id": 532, "category": "creative", "instruction": "Write a humorous monologue from a coffee mug that has survived the office dishwasher for ten years.", "expected_length": "medium"}, + {"id": 533, "category": "creative", "instruction": "Compose song lyrics (one verse and one chorus) about a road trip with no destination, capturing freedom and uncertainty.", "expected_length": "medium"}, + {"id": 534, "category": "creative", "instruction": "Write the opening scene of a horror screenplay set in an abandoned radio station where broadcasts are still playing.", "expected_length": "medium"}, + {"id": 535, "category": "creative", "instruction": "Invent a fictional sport played in zero gravity and write a play-by-play commentary of the championship final.", "expected_length": "medium"}, + {"id": 536, "category": "creative", "instruction": "Write a humorous letter of resignation from a dragon who has decided treasure-guarding is no longer fulfilling.", "expected_length": "medium"}, + {"id": 537, "category": "creative", "instruction": "Describe a bustling alien marketplace using only sounds, smells, and textures without ever naming any object.", "expected_length": "medium"}, + {"id": 538, "category": "creative", "instruction": "Write a children's story about a cloud that is afraid of heights and must learn to float among the others.", "expected_length": "medium"}, + {"id": 539, "category": "creative", "instruction": "Compose a drinking song of four verses for a guild of dwarf stargazers who map constellations through ale-soaked lenses.", "expected_length": "medium"}, + {"id": 540, "category": "creative", "instruction": "Write a short mystery where the detective is a houseplant that solves its owner's disappearance through observation.", "expected_length": "medium"}, + {"id": 541, "category": "creative", "instruction": "Create a worldbuilding document for a floating city powered by captured lightning, including its governance, cuisine, and main export.", "expected_length": "medium"}, + {"id": 542, "category": "creative", "instruction": "Write a tender goodbye letter from an old lighthouse to the ships it will no longer guide.", "expected_length": "medium"}, + {"id": 543, "category": "creative", "instruction": "Compose a rap battle of eight bars each between the Sun and a black hole debating who is more powerful.", "expected_length": "medium"}, + {"id": 544, "category": "creative", "instruction": "Write a comic strip script of four panels about a robot learning sarcasm and getting it wrong every time.", "expected_length": "medium"}, + {"id": 545, "category": "creative", "instruction": "Describe the first morning after humanity collectively decides to stop sleeping, told from the perspective of a sunrise.", "expected_length": "medium"}, + {"id": 546, "category": "vietnamese", "instruction": "Giải thích cách mạng xã hội ảnh hưởng đến tâm lý của thanh niên Việt Nam hiện nay.", "expected_length": "medium"}, + {"id": 547, "category": "vietnamese", "instruction": "Viết hướng dẫn chi tiết cách đăng ký và sử dụng ví điện tử MoMo cho người mới bắt đầu.", "expected_length": "medium"}, + {"id": 548, "category": "vietnamese", "instruction": "Phân tích sự phát triển của ngành công nghệ thông tin tại Việt Nam trong mười năm qua.", "expected_length": "medium"}, + {"id": 549, "category": "vietnamese", "instruction": "Giải thích văn hóa chạy sô ngày cưới ở nông thôn Việt Nam và lý do nó đang dần thay đổi.", "expected_length": "medium"}, + {"id": 550, "category": "vietnamese", "instruction": "Viết bài giới thiệu về các địa điểm du lịch sinh thái ít người biết đến ở miền Tây Nam Bộ.", "expected_length": "medium"}, + {"id": 551, "category": "vietnamese", "instruction": "Phân tích truyện ngắn Vợ nhặt của Kim Lân về tình yêu và lòng thương người trong hoàn cảnh khó khăn.", "expected_length": "medium"}, + {"id": 552, "category": "vietnamese", "instruction": "Giải thích hệ thống giao thông công cộng ở Hà Nội và đề xuất ba cách cải thiện.", "expected_length": "medium"}, + {"id": 553, "category": "vietnamese", "instruction": "Viết hướng dẫn cách chọn vải và bảo quản áo dài truyền thống sao cho bền đẹp.", "expected_length": "medium"}, + {"id": 554, "category": "vietnamese", "instruction": "Phân tích tác động của đại dịch COVID-19 đến nền kinh tế Việt Nam và quá trình phục hồi.", "expected_length": "medium"}, + {"id": 555, "category": "vietnamese", "instruction": "Giải thích vai trò của phụ nữ Việt Nam trong xã hội hiện đại và những thách thức còn lại.", "expected_length": "medium"}, + {"id": 556, "category": "vietnamese", "instruction": "Viết bài về ẩm thực đường phố Việt Nam và lý do nó nổi tiếng quốc tế.", "expected_length": "medium"}, + {"id": 557, "category": "vietnamese", "instruction": "Phân tích các yếu tố làm nên bản sắc văn hóa Việt Nam riêng biệt so với các nước Đông Nam Á.", "expected_length": "medium"}, + {"id": 558, "category": "vietnamese", "instruction": "Giải thích cấu trúc và ý nghĩa của một câu ca dao hoặc tục ngữ Việt Nam do bạn tự chọn.", "expected_length": "medium"}, + {"id": 559, "category": "vietnamese", "instruction": "Viết hướng dẫn cách trồng và chăm sóc hoa lan cho người mới bắt đầu ở khí hậu nhiệt đới.", "expected_length": "medium"}, + {"id": 560, "category": "vietnamese", "instruction": "Phân tích bài thơ Tràng giang của Huy Cận về nỗi buồn sầu và cảnh vật thiên nhiên.", "expected_length": "medium"}, + {"id": 561, "category": "reasoning", "instruction": "You are in a room with two doors guarded by two people; one always tells the truth and one always lies. One door leads to freedom. You may ask one guard one question. What do you ask and why does it work?", "expected_length": "medium"}, + {"id": 562, "category": "reasoning", "instruction": "A farmer must cross a river with a fox, a chicken, and a bag of grain. The boat holds the farmer and one item. If left alone, the fox eats the chicken and the chicken eats the grain. Explain the safe crossing step by step.", "expected_length": "medium"}, + {"id": 563, "category": "reasoning", "instruction": "Explain the sunk cost fallacy and give a real example of how it affects business decisions.", "expected_length": "medium"}, + {"id": 564, "category": "reasoning", "instruction": "Three friends pay thirty dollars for a hotel room, ten each. The room was twenty-five, so the bellboy returns five but keeps two, giving one to each friend. They each paid nine, twenty-seven total, plus two is twenty-nine. Where is the missing dollar? Explain the flaw.", "expected_length": "medium"}, + {"id": 565, "category": "reasoning", "instruction": "You have twelve balls, one of a different weight. Using a balance scale only three times, explain how you find the odd ball and whether it is heavier or lighter.", "expected_length": "medium"}, + {"id": 566, "category": "reasoning", "instruction": "Explain the concept of expected value and apply it to decide whether buying a lottery ticket is a rational choice.", "expected_length": "medium"}, + {"id": 567, "category": "reasoning", "instruction": "A man pushes his car to a hotel and discovers he is bankrupt. Explain what game he is playing and why that makes him bankrupt.", "expected_length": "medium"}, + {"id": 568, "category": "reasoning", "instruction": "Five pirates divide one hundred gold coins. The captain proposes a split; if half or more disagree he walks the plank and the next proposes. Pirates are rational, greedy, and value survival above all. What split does the captain propose and why?", "expected_length": "medium"}, + {"id": 569, "category": "reasoning", "instruction": "Explain the difference between necessary and sufficient conditions, giving one everyday example of each.", "expected_length": "medium"}, + {"id": 570, "category": "reasoning", "instruction": "A doctor gives you three pills and tells you to take one every half hour. How long until you have taken all three? Explain the reasoning step by step.", "expected_length": "medium"} ] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e62628c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +services: + api: + build: + context: ./services/api + image: nguyenson1710/distill-gpt55-api:latest + environment: + MODEL_PATH: /models/distill-gpt55-v0.5-Q4_K_M.gguf + CORS_ALLOW_ORIGINS: http://localhost:3000 + volumes: + - ./checkpoints/gguf:/models:ro + ports: + - "8000:8000" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8000/readyz"] + interval: 15s + timeout: 5s + start_period: 90s + retries: 5 + networks: [distill] + + web: + build: + context: ./services/web + args: + VITE_API_BASE_URL: http://localhost:8000 + image: nguyenson1710/distill-gpt55-web:latest + depends_on: + api: + condition: service_healthy + ports: + - "3000:3000" + networks: [distill] + +networks: + distill: diff --git a/docs/assets/chat-desktop.png b/docs/assets/chat-desktop.png new file mode 100644 index 0000000..34ac00a Binary files /dev/null and b/docs/assets/chat-desktop.png differ diff --git a/docs/assets/chat-streaming.gif b/docs/assets/chat-streaming.gif new file mode 100644 index 0000000..b9df272 Binary files /dev/null and b/docs/assets/chat-streaming.gif differ diff --git a/docs/assets/evaluation-by-category.png b/docs/assets/evaluation-by-category.png new file mode 100644 index 0000000..4c387b6 Binary files /dev/null and b/docs/assets/evaluation-by-category.png differ diff --git a/docs/code-standards.md b/docs/code-standards.md index a5e2278..4184391 100644 --- a/docs/code-standards.md +++ b/docs/code-standards.md @@ -1,47 +1,67 @@ # Code Standards -## File Naming -- Python scripts: **snake_case** for descriptive names (`gen_batch.py`, `train_student.py`, `format_dataset.py`) -- Config: `config.py` at project root -- Docs: Markdown in `docs/` dir +## Layout -## Code Structure -- **≤ 200 LOC** per file — split when exceeding (CK rule) -- **kebab-case** for non-Python files per CK convention -- **No dead code** — remove test/temp scripts after use -- **Imports at top**, grouped: stdlib → third-party → local +- Training pipeline lives in the **`src/distill` package** (`python -m distill.`); + root-level scripts are legacy and being retired. +- Services are standalone under `services//` — the api service must not + import from `src/distill` (keeps torch out of its Docker image). +- Python: **snake_case** modules; ≤ 200 LOC per file — split when exceeding. +- Non-Python files: **kebab-case**; docs as Markdown in `docs/`. +- Imports at top, grouped: stdlib → third-party → local. ## Configuration -- All settings in `config.py` — no hardcoded values in scripts -- Directory creation handled automatically on `import config` -- Paths relative to `PROJECT_DIR`, computed with `os.path.join` -## Error Handling -- API calls: retry 3× with exponential backoff -- Training: let crash for visibility, fix root cause -- File I/O: explicit encoding `utf-8` -- UTF-8 output wrapper for Windows console: `TextIOWrapper` +- Training tunables in `src/distill/config.py`, all env-overridable; secrets + only via `.env` (gitignored) — never hardcoded, never printed. +- Service config in `services/api/app/config.py`, env-only (container-friendly). + +## Error handling + +- Teacher API calls: classify retryable vs fatal, exponential backoff + jitter, + validate outputs (empty/short/U+FFFD rejected) — see `distill/teacher_client.py`. +- Dataset writes are atomic (temp file + rename), resumable across runs. +- File I/O: explicit `encoding="utf-8"`; console via `distill/logging_utils.py`. +- Training: let it crash for visibility; fix root cause (see phase-03 incident log). + +## Testing & lint + +- `python -m pytest tests/ -q` (package) and `services/api: python -m pytest tests/ -q` + (fake runtime — no GPU/model needed); web: `pnpm test` + `pnpm build`. +- `ruff check src/ tests/ services/api/` must be clean (CI gate). +- API contract `docs/openapi.yaml` is generated from the FastAPI app; the web + client is generated from it (`pnpm run generate-client`) — CI fails on drift. + +## Git workflow -## Git Workflow - **Conventional commits:** `feat:`, `fix:`, `docs:`, `test:`, `chore:` -- **No AI refs** in commit messages (except Co-authored-by trailer) -- **Focused commits:** one logical unit per commit -- **No secrets:** API keys belong in env vars, not code (current: config.py, to be migrated) +- **No AI refs** in commit messages; sole author Nguyen Tien Son. +- Focused commits; no secrets, no private notes, no model artifacts. + +## Model training (RTX 3060 6GB — hard-won constraints) -## Model Training -- **4-bit NF4** quantization mandatory for 6GB VRAM -- **float16** compute dtype (RTX 3060 không hỗ trợ bfloat16 amp) -- **No gradient checkpointing** on RTX 3060 (causes OOM with bitsandbytes) -- **adamw_8bit** optimizer for memory efficiency -- **SFTTrainer** from TRL with `processing_class=` (new API) +- **bf16 end-to-end** — the base checkpoint is bf16 and fp16 conversion at load + crashes the current torch nightly on Windows (see phase-03 incident log). +- **Model loads go CPU-first, then `.to("cuda:0")`** — safetensors + direct-to-GPU is broken against the current torch nightly. +- **LoRA on full bf16** (r16, q/k/v/o) with gradient checkpointing; + the QLoRA 4-bit path is kept behind `LOAD_IN_4BIT` but bnb on-the-fly + quantization crashes in this environment. +- **MAX_SEQ_LENGTH 512** — vocab-sized logits (152K) OOM 6GB at 1024. +- **adamw_8bit** optimizer; batch 1 × grad-accum 8; validation split with + early stopping and best-checkpoint restore. ## Dataset -- **200+ samples** minimum for meaningful distillation -- **10 categories:** coding, reasoning, math, creative, science, ml_ai, vietnamese, business, health, philosophy -- **Qwen chat template:** `<|im_start|>system/user/assistant<|im_end|>` -- **Save after every generation** to prevent data loss + +- 530 prompts / 10 categories; every sample validated at generation time. +- **Exact Qwen2.5 chat template** with `<|im_start|>`/`<|im_end|>` special + tokens (a plain-text approximation trains a format no inference stack uses). +- Quality gate before splitting: mojibake, min-length, duplicate instructions. +- **Stratified 80/10/10 train/validation/test**, seed 42, splits disjoint by id. ## Evaluation -- **Perplexity** on held-out set as primary metric -- **Qualitative test** with diverse prompts (code, reasoning, general knowledge) -- **Loss tracking** per epoch for convergence monitoring + +- Perplexity on the held-out test split (never in-sample), per category. +- ROUGE-L + token-F1 vs teacher reference; optional LLM-as-judge via 9Router. +- Reports versioned under `plans/reports/evaluation-