From fd1919056ef1b9b3a3254569852fb95b667c4f32 Mon Sep 17 00:00:00 2001 From: Kauna <16511995+klei22@users.noreply.github.com> Date: Sun, 21 Jun 2026 20:28:12 -0700 Subject: [PATCH 1/4] Add FLORES char-BPE validation BPB demo --- data/char_bpe_exploration/.gitignore | 3 + data/char_bpe_exploration/README.md | 97 ++++++++ .../scripts/run_char_bpe_exploration.py | 218 ++++++++++++++++++ data/char_bpe_exploration_train/.gitignore | 2 + demos/char_bpe_flores_validation_bpb_demo.sh | 119 ++++++++++ 5 files changed, 439 insertions(+) create mode 100644 data/char_bpe_exploration/.gitignore create mode 100644 data/char_bpe_exploration/README.md create mode 100755 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py create mode 100644 data/char_bpe_exploration_train/.gitignore create mode 100755 demos/char_bpe_flores_validation_bpb_demo.sh diff --git a/data/char_bpe_exploration/.gitignore b/data/char_bpe_exploration/.gitignore new file mode 100644 index 0000000000..645be261a5 --- /dev/null +++ b/data/char_bpe_exploration/.gitignore @@ -0,0 +1,3 @@ +/texts/ +/runs/ +/results/ diff --git a/data/char_bpe_exploration/README.md b/data/char_bpe_exploration/README.md new file mode 100644 index 0000000000..b4608dc1f4 --- /dev/null +++ b/data/char_bpe_exploration/README.md @@ -0,0 +1,97 @@ +# FLORES-200 Char-BPE Exploration + +This folder contains a reproducible pipeline for comparing nanoGPT's `char_bpe` +tokenizer across selected FLORES-200 languages and stepped vocabulary sizes. + +The pipeline does three things: + +1. Downloads/loads FLORES-200 via Hugging Face `datasets`. +2. Writes one UTF-8 `.txt` file per requested language in `texts/`. +3. Runs `data/template/prepare.py --method char_bpe` at each requested vocabulary + size, then writes comparable metrics to `results/summary.csv` and + `results/summary.json`. By default, 90% of each text becomes `train.bin` + and 10% becomes `val.bin` so downstream validation-loss comparisons work. + +## Languages + +| Label | FLORES-200 code | +| --- | --- | +| kiswahili | `swh_Latn` | +| bahasa_indonesian | `ind_Latn` | +| korean | `kor_Hang` | +| korean_nfd | `kor_Hang` converted with `data/hangul/hangul_nfc_to_nfd.py` | +| english | `eng_Latn` | +| chinese | `zho_Hans` | +| japanese | `jpn_Jpan` | +| arabic | `arb_Arab` | +| spanish | `spa_Latn` | +| german | `deu_Latn` | +| russian | `rus_Cyrl` | +| thai | `tha_Thai` | +| filipino | `tgl_Latn` | +| hindi | `hin_Deva` | +| finnish | `fin_Latn` | +| italian | `ita_Latn` | + +## Quick start + +From the repository root: + +```bash +python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py +``` + +Optional examples: + +```bash +# Use a smaller sweep while testing the pipeline. +python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py \ + --vocab-sizes 384,512 + +# Refresh language text files from Hugging Face before tokenizing. +python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py \ + --refresh-texts + +# Use the full text as train.bin when you only need tokenizer compression metrics. +python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py \ + --percentage-train 1.0 +``` + +If `datasets` is not installed, install it first: + +```bash +python3 -m pip install datasets numpy tqdm sentencepiece tiktoken +``` + +## Validation-loss and bits-per-byte demo + +To train the same small model across language/vocab-size pairs and compare +validation loss plus bits per byte, run: + +```bash +bash demos/char_bpe_flores_validation_bpb_demo.sh +``` + +The demo writes a comparison table to +`out/char_bpe_flores_validation_bpb/summary.csv`. You can override `LANGUAGES`, +`VOCAB_SIZES`, `MAX_ITERS`, `DEVICE`, and other shell variables before running +the script. + +## Outputs + +Generated artifacts are intentionally ignored by Git: + +- `texts/*.txt`: per-language FLORES text files. +- `runs//vocab_/`: nanoGPT `char_bpe` outputs (`meta.pkl`, + `train.bin`, `val.bin`, `char_bpe_vocab.json`, token counts, and metrics). +- `results/summary.csv` and `results/summary.json`: comparison table. + +Key metrics: + +- `bytes_per_token`: lower means the tokenizer compresses that language into + fewer tokens for the same UTF-8 byte length. +- `chars_per_token`: lower/higher should be interpreted by script and Unicode + normalization; compare Korean NFC vs NFD carefully because NFD increases the + number of Unicode code points. +- `unk_byte_fallback_tokens`: count of raw-byte fallback tokens used by the + trained tokenizer. diff --git a/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py b/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py new file mode 100755 index 0000000000..3ecfba0a6b --- /dev/null +++ b/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Build FLORES-200 language text files and run stepped char-BPE sweeps.""" + +from __future__ import annotations + +import argparse +import csv +import json +import pickle +import shutil +import subprocess +import sys +from pathlib import Path + +LANGUAGES = [ + ("kiswahili", "swh_Latn"), + ("bahasa_indonesian", "ind_Latn"), + ("korean", "kor_Hang"), + ("english", "eng_Latn"), + ("chinese", "zho_Hans"), + ("japanese", "jpn_Jpan"), + ("arabic", "arb_Arab"), + ("spanish", "spa_Latn"), + ("german", "deu_Latn"), + ("russian", "rus_Cyrl"), + ("thai", "tha_Thai"), + ("filipino", "tgl_Latn"), + ("hindi", "hin_Deva"), + ("finnish", "fin_Latn"), + ("italian", "ita_Latn"), +] + +DEFAULT_VOCAB_SIZES = [384, 512, 768, 1024, 1536, 2048, 4096] + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def parse_vocab_sizes(value: str) -> list[int]: + sizes = [int(part.strip()) for part in value.split(",") if part.strip()] + bad = [size for size in sizes if size <= 256] + if bad: + raise argparse.ArgumentTypeError( + f"char_bpe vocab sizes must be > 256 because IDs 0..255 are byte fallback tokens: {bad}" + ) + return sizes + + +def load_flores_split(code: str, split: str) -> list[str]: + try: + from datasets import load_dataset + except ImportError as exc: + raise SystemExit( + "Missing dependency: install Hugging Face datasets with " + "`python3 -m pip install datasets` and rerun." + ) from exc + + dataset = load_dataset("facebook/flores", code, split=split) + if "sentence" not in dataset.column_names: + raise RuntimeError(f"Unexpected facebook/flores columns for {code}: {dataset.column_names}") + return [str(row["sentence"]) for row in dataset] + + +def write_language_texts(text_dir: Path, refresh: bool) -> dict[str, Path]: + text_dir.mkdir(parents=True, exist_ok=True) + outputs: dict[str, Path] = {} + + for label, code in LANGUAGES: + out_path = text_dir / f"{label}.txt" + if refresh or not out_path.exists(): + sentences: list[str] = [] + for split in ("dev", "devtest"): + sentences.extend(load_flores_split(code, split)) + out_path.write_text("\n".join(sentences) + "\n", encoding="utf-8") + outputs[label] = out_path + + korean_nfd = text_dir / "korean_nfd.txt" + if refresh or not korean_nfd.exists(): + converter = repo_root() / "data" / "hangul" / "hangul_nfc_to_nfd.py" + subprocess.run( + [sys.executable, str(converter), str(outputs["korean"]), str(korean_nfd)], + check=True, + cwd=repo_root(), + ) + outputs["korean_nfd"] = korean_nfd + return outputs + + +def count_bin_tokens(path: Path, vocab_size: int) -> int: + dtype_bytes = 4 if vocab_size > 65535 else 2 + return path.stat().st_size // dtype_bytes + + +def run_prepare( + label: str, + text_path: Path, + vocab_size: int, + run_dir: Path, + percentage_train: float, +) -> dict[str, object]: + run_dir.mkdir(parents=True, exist_ok=True) + for artifact in ( + "train.bin", + "val.bin", + "meta.pkl", + "char_bpe_vocab.json", + "char_bpe_token_counts.json", + "metrics.json", + ): + (run_dir / artifact).unlink(missing_ok=True) + prepare = repo_root() / "data" / "template" / "prepare.py" + cmd = [ + sys.executable, + str(prepare), + "--train_input", + str(text_path), + "--method", + "char_bpe", + "--vocab_size", + str(vocab_size), + "--percentage_train", + str(percentage_train), + "--track_token_counts", + "--train_output", + "train.bin", + "--val_output", + "val.bin", + ] + subprocess.run(cmd, check=True, cwd=run_dir) + + meta_path = run_dir / "meta.pkl" + with meta_path.open("rb") as f: + meta = pickle.load(f) + + train_tokens = count_bin_tokens(run_dir / "train.bin", meta["vocab_size"]) + val_path = run_dir / "val.bin" + val_tokens = count_bin_tokens(val_path, meta["vocab_size"]) if val_path.exists() else 0 + text = text_path.read_text(encoding="utf-8") + split_index = int(len(text) * percentage_train) + train_text = text[:split_index] + val_text = text[split_index:] if percentage_train < 1.0 else "" + byte_fallback_tokens = sum( + count for token_id, count in meta.get("token_counts", {}).items() if int(token_id) < 256 + ) + + metrics = { + "language": label, + "requested_vocab_size": vocab_size, + "actual_vocab_size": meta["vocab_size"], + "utf8_bytes": len(text.encode("utf-8")), + "unicode_codepoints": len(text), + "train_utf8_bytes": len(train_text.encode("utf-8")), + "val_utf8_bytes": len(val_text.encode("utf-8")), + "train_unicode_codepoints": len(train_text), + "val_unicode_codepoints": len(val_text), + "train_tokens": train_tokens, + "val_tokens": val_tokens, + "bytes_per_token": round(len(train_text.encode("utf-8")) / train_tokens, 6) if train_tokens else 0, + "val_bytes_per_token": round(len(val_text.encode("utf-8")) / val_tokens, 6) if val_tokens else 0, + "chars_per_token": round(len(train_text) / train_tokens, 6) if train_tokens else 0, + "val_chars_per_token": round(len(val_text) / val_tokens, 6) if val_tokens else 0, + "unk_byte_fallback_tokens": byte_fallback_tokens, + } + (run_dir / "metrics.json").write_text(json.dumps(metrics, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return metrics + + +def write_summary(rows: list[dict[str, object]], results_dir: Path) -> None: + results_dir.mkdir(parents=True, exist_ok=True) + json_path = results_dir / "summary.json" + csv_path = results_dir / "summary.csv" + json_path.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + if rows: + with csv_path.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + base = repo_root() / "data" / "char_bpe_exploration" + parser.add_argument("--base-dir", type=Path, default=base) + parser.add_argument("--vocab-sizes", type=parse_vocab_sizes, default=DEFAULT_VOCAB_SIZES) + parser.add_argument("--refresh-texts", action="store_true") + parser.add_argument("--clean-runs", action="store_true") + parser.add_argument( + "--percentage-train", + type=float, + default=0.9, + help="Fraction of each language text used for train.bin; the remainder becomes val.bin.", + ) + args = parser.parse_args() + + if not 0 < args.percentage_train <= 1: + raise SystemExit("--percentage-train must be in the interval (0, 1].") + + if args.clean_runs: + shutil.rmtree(args.base_dir / "runs", ignore_errors=True) + shutil.rmtree(args.base_dir / "results", ignore_errors=True) + + texts = write_language_texts(args.base_dir / "texts", args.refresh_texts) + ordered_labels = [label for label, _ in LANGUAGES] + ["korean_nfd"] + + rows: list[dict[str, object]] = [] + for label in ordered_labels: + for vocab_size in args.vocab_sizes: + run_dir = args.base_dir / "runs" / label / f"vocab_{vocab_size}" + print(f"[char_bpe_exploration] {label} vocab={vocab_size}", flush=True) + rows.append(run_prepare(label, texts[label], vocab_size, run_dir, args.percentage_train)) + write_summary(rows, args.base_dir / "results") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data/char_bpe_exploration_train/.gitignore b/data/char_bpe_exploration_train/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/data/char_bpe_exploration_train/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/demos/char_bpe_flores_validation_bpb_demo.sh b/demos/char_bpe_flores_validation_bpb_demo.sh new file mode 100755 index 0000000000..45577cc672 --- /dev/null +++ b/demos/char_bpe_flores_validation_bpb_demo.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# char_bpe_flores_validation_bpb_demo.sh +# +# Demonstrates how to compare FLORES-200 char-BPE tokenizers by training the +# same small model on each language/vocab-size pair and reporting validation +# loss plus bits per byte (BPB). BPB normalizes token-level loss by the number +# of UTF-8 bytes represented per validation token: +# +# bits_per_byte = validation_loss_nats / (ln(2) * val_bytes_per_token) +# +# The default settings are intentionally small so the script is practical as a +# demo. Override LANGUAGES, VOCAB_SIZES, MAX_ITERS, etc. for larger sweeps. + +set -euo pipefail + +BASE_DIR="data/char_bpe_exploration" +DATASET_ROOT="data/char_bpe_exploration_train" +OUT_ROOT="out/char_bpe_flores_validation_bpb" +SUMMARY_CSV="${OUT_ROOT}/summary.csv" + +LANGUAGES=${LANGUAGES:-"kiswahili bahasa_indonesian korean korean_nfd english chinese japanese arabic spanish german russian thai filipino hindi finnish italian"} +VOCAB_SIZES=${VOCAB_SIZES:-"384 512"} +MAX_ITERS=${MAX_ITERS:-200} +EVAL_INTERVAL=${EVAL_INTERVAL:-50} +EVAL_ITERS=${EVAL_ITERS:-20} +BATCH_SIZE=${BATCH_SIZE:-8} +BLOCK_SIZE=${BLOCK_SIZE:-64} +N_LAYER=${N_LAYER:-2} +N_HEAD=${N_HEAD:-2} +N_EMBD=${N_EMBD:-128} +DEVICE=${DEVICE:-"cpu"} +DTYPE=${DTYPE:-"float32"} +PERCENTAGE_TRAIN=${PERCENTAGE_TRAIN:-"0.9"} + +mkdir -p "${DATASET_ROOT}" "${OUT_ROOT}" + +VOCAB_CSV=$(python3 - <<'PY' "${VOCAB_SIZES}" +import sys +print(",".join(sys.argv[1].split())) +PY +) + +echo "=== Step 1: Build FLORES text files and char-BPE train/val bins ===" +python3 "${BASE_DIR}/scripts/run_char_bpe_exploration.py" \ + --vocab-sizes "${VOCAB_CSV}" \ + --percentage-train "${PERCENTAGE_TRAIN}" + +echo "language,vocab_size,dataset,validation_loss,bits_per_byte,out_dir" > "${SUMMARY_CSV}" + +for language in ${LANGUAGES}; do + for vocab_size in ${VOCAB_SIZES}; do + run_dir="${BASE_DIR}/runs/${language}/vocab_${vocab_size}" + dataset="char_bpe_exploration_train/${language}_vocab_${vocab_size}" + dataset_dir="data/${dataset}" + out_dir="${OUT_ROOT}/${language}/vocab_${vocab_size}" + + if [ ! -f "${run_dir}/train.bin" ] || [ ! -f "${run_dir}/val.bin" ] || [ ! -f "${run_dir}/meta.pkl" ]; then + echo "Missing tokenized artifacts in ${run_dir}" >&2 + exit 1 + fi + + mkdir -p "${dataset_dir}" "${out_dir}" + cp "${run_dir}/train.bin" "${dataset_dir}/train.bin" + cp "${run_dir}/val.bin" "${dataset_dir}/val.bin" + cp "${run_dir}/meta.pkl" "${dataset_dir}/meta.pkl" + + echo "=== Step 2: Train ${language} vocab=${vocab_size} ===" + python3 train.py \ + --dataset "${dataset}" \ + --out_dir "${out_dir}" \ + --block_size "${BLOCK_SIZE}" \ + --batch_size "${BATCH_SIZE}" \ + --n_layer "${N_LAYER}" \ + --n_head "${N_HEAD}" \ + --n_embd "${N_EMBD}" \ + --max_iters "${MAX_ITERS}" \ + --eval_interval "${EVAL_INTERVAL}" \ + --eval_iters "${EVAL_ITERS}" \ + --learning_rate 6e-4 \ + --weight_decay 0.1 \ + --device "${DEVICE}" \ + --dtype "${DTYPE}" \ + --no-compile + + echo "=== Step 3: Compute BPB for ${language} vocab=${vocab_size} ===" + python3 - <<'PY' "${language}" "${vocab_size}" "${dataset}" "${run_dir}/metrics.json" "${out_dir}" "${SUMMARY_CSV}" +import csv +import json +import math +import sys +from pathlib import Path + +language, vocab_size, dataset, metrics_path, out_dir, summary_csv = sys.argv[1:] +best_path = Path(out_dir) / "best_val_loss_and_iter.txt" +if not best_path.exists(): + raise SystemExit(f"Missing validation-loss file: {best_path}") +validation_loss = float(best_path.read_text(encoding="utf-8").splitlines()[0].split(",")[0]) +metrics = json.loads(Path(metrics_path).read_text(encoding="utf-8")) +val_bytes_per_token = float(metrics["val_bytes_per_token"]) +if val_bytes_per_token <= 0: + raise SystemExit(f"val_bytes_per_token must be > 0 in {metrics_path}") +bits_per_byte = validation_loss / (math.log(2) * val_bytes_per_token) +row = { + "language": language, + "vocab_size": vocab_size, + "dataset": dataset, + "validation_loss": f"{validation_loss:.6f}", + "bits_per_byte": f"{bits_per_byte:.6f}", + "out_dir": out_dir, +} +with Path(summary_csv).open("a", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=row.keys()) + writer.writerow(row) +print(json.dumps(row, indent=2)) +PY + done +done + +echo "Comparison complete: ${SUMMARY_CSV}" From e1bacbdbbb4284c43dcb2fa1923a773b8b0a12cf Mon Sep 17 00:00:00 2001 From: Kauna <16511995+klei22@users.noreply.github.com> Date: Sun, 21 Jun 2026 21:31:33 -0700 Subject: [PATCH 2/4] Use restructured FLORES parquet source --- data/char_bpe_exploration/README.md | 12 ++-- .../scripts/run_char_bpe_exploration.py | 56 ++++++++++++------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/data/char_bpe_exploration/README.md b/data/char_bpe_exploration/README.md index b4608dc1f4..c0651ff161 100644 --- a/data/char_bpe_exploration/README.md +++ b/data/char_bpe_exploration/README.md @@ -5,7 +5,7 @@ tokenizer across selected FLORES-200 languages and stepped vocabulary sizes. The pipeline does three things: -1. Downloads/loads FLORES-200 via Hugging Face `datasets`. +1. Downloads the `muhammadravi251001/restructured-flores200` parquet files from Hugging Face using the same helper pattern as `data/flores200-res/get_dataset.sh`. 2. Writes one UTF-8 `.txt` file per requested language in `texts/`. 3. Runs `data/template/prepare.py --method char_bpe` at each requested vocabulary size, then writes comparable metrics to `results/summary.csv` and @@ -52,15 +52,19 @@ python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py \ python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py \ --refresh-texts +# Override the restructured FLORES-200 parquet folder URL if needed. +python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py \ + --source-url https://huggingface.co/datasets/muhammadravi251001/restructured-flores200/tree/main/data + # Use the full text as train.bin when you only need tokenizer compression metrics. python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py \ --percentage-train 1.0 ``` -If `datasets` is not installed, install it first: +Install the parquet extraction dependencies first: ```bash -python3 -m pip install datasets numpy tqdm sentencepiece tiktoken +python3 -m pip install pandas pyarrow requests beautifulsoup4 tqdm numpy sentencepiece tiktoken ``` ## Validation-loss and bits-per-byte demo @@ -81,7 +85,7 @@ the script. Generated artifacts are intentionally ignored by Git: -- `texts/*.txt`: per-language FLORES text files. +- `texts/*.txt`: per-language FLORES text files emitted from `text_` parquet columns. - `runs//vocab_/`: nanoGPT `char_bpe` outputs (`meta.pkl`, `train.bin`, `val.bin`, `char_bpe_vocab.json`, token counts, and metrics). - `results/summary.csv` and `results/summary.json`: comparison table. diff --git a/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py b/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py index 3ecfba0a6b..1face1305a 100755 --- a/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py +++ b/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Build FLORES-200 language text files and run stepped char-BPE sweeps.""" +"""Build restructured FLORES-200 text files and run stepped char-BPE sweeps.""" from __future__ import annotations @@ -12,6 +12,8 @@ import sys from pathlib import Path +SOURCE_URL = "https://huggingface.co/datasets/muhammadravi251001/restructured-flores200/tree/main/data" + LANGUAGES = [ ("kiswahili", "swh_Latn"), ("bahasa_indonesian", "ind_Latn"), @@ -47,32 +49,41 @@ def parse_vocab_sizes(value: str) -> list[int]: return sizes -def load_flores_split(code: str, split: str) -> list[str]: - try: - from datasets import load_dataset - except ImportError as exc: - raise SystemExit( - "Missing dependency: install Hugging Face datasets with " - "`python3 -m pip install datasets` and rerun." - ) from exc - - dataset = load_dataset("facebook/flores", code, split=split) - if "sentence" not in dataset.column_names: - raise RuntimeError(f"Unexpected facebook/flores columns for {code}: {dataset.column_names}") - return [str(row["sentence"]) for row in dataset] +def emit_restructured_flores_text(code: str, output_path: Path, source_url: str) -> None: + """Emit one text_ column from the restructured FLORES-200 parquet files. + + This mirrors data/flores200-res/get_dataset.sh, which uses the shared + data/template/utils/get_parquet_dataset.py helper against the + muhammadravi251001/restructured-flores200 Hugging Face parquet folder. + """ + helper = repo_root() / "data" / "template" / "utils" / "get_parquet_dataset.py" + output_path.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + sys.executable, + str(helper), + "--url", + source_url, + "--include_keys", + f"text_{code}", + "--value_prefix", + "\n", + "--output_text_file", + str(output_path), + ], + check=True, + cwd=output_path.parent, + ) -def write_language_texts(text_dir: Path, refresh: bool) -> dict[str, Path]: +def write_language_texts(text_dir: Path, refresh: bool, source_url: str) -> dict[str, Path]: text_dir.mkdir(parents=True, exist_ok=True) outputs: dict[str, Path] = {} for label, code in LANGUAGES: out_path = text_dir / f"{label}.txt" if refresh or not out_path.exists(): - sentences: list[str] = [] - for split in ("dev", "devtest"): - sentences.extend(load_flores_split(code, split)) - out_path.write_text("\n".join(sentences) + "\n", encoding="utf-8") + emit_restructured_flores_text(code, out_path, source_url) outputs[label] = out_path korean_nfd = text_dir / "korean_nfd.txt" @@ -185,6 +196,11 @@ def main() -> int: parser.add_argument("--vocab-sizes", type=parse_vocab_sizes, default=DEFAULT_VOCAB_SIZES) parser.add_argument("--refresh-texts", action="store_true") parser.add_argument("--clean-runs", action="store_true") + parser.add_argument( + "--source-url", + default=SOURCE_URL, + help="Hugging Face parquet folder URL for muhammadravi251001/restructured-flores200.", + ) parser.add_argument( "--percentage-train", type=float, @@ -200,7 +216,7 @@ def main() -> int: shutil.rmtree(args.base_dir / "runs", ignore_errors=True) shutil.rmtree(args.base_dir / "results", ignore_errors=True) - texts = write_language_texts(args.base_dir / "texts", args.refresh_texts) + texts = write_language_texts(args.base_dir / "texts", args.refresh_texts, args.source_url) ordered_labels = [label for label, _ in LANGUAGES] + ["korean_nfd"] rows: list[dict[str, object]] = [] From 24259506d8f4056ce90349d01a2861c2902faf6b Mon Sep 17 00:00:00 2001 From: Kauna <16511995+klei22@users.noreply.github.com> Date: Sun, 21 Jun 2026 22:10:19 -0700 Subject: [PATCH 3/4] Add Plotly report for FLORES BPB runs --- data/char_bpe_exploration/README.md | 18 +- .../scripts/render_bpb_report.py | 311 ++++++++++++++++++ .../scripts/run_char_bpe_exploration.py | 2 +- demos/char_bpe_flores_validation_bpb_demo.sh | 10 +- 4 files changed, 337 insertions(+), 4 deletions(-) create mode 100755 data/char_bpe_exploration/scripts/render_bpb_report.py diff --git a/data/char_bpe_exploration/README.md b/data/char_bpe_exploration/README.md index c0651ff161..acb2ec4e2e 100644 --- a/data/char_bpe_exploration/README.md +++ b/data/char_bpe_exploration/README.md @@ -46,7 +46,7 @@ Optional examples: ```bash # Use a smaller sweep while testing the pipeline. python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py \ - --vocab-sizes 384,512 + --vocab-sizes 320,384,512,768,1024 # Refresh language text files from Hugging Face before tokenizing. python3 data/char_bpe_exploration/scripts/run_char_bpe_exploration.py \ @@ -79,7 +79,21 @@ bash demos/char_bpe_flores_validation_bpb_demo.sh The demo writes a comparison table to `out/char_bpe_flores_validation_bpb/summary.csv`. You can override `LANGUAGES`, `VOCAB_SIZES`, `MAX_ITERS`, `DEVICE`, and other shell variables before running -the script. +the script. The demo also renders a standalone Plotly report at +`out/char_bpe_flores_validation_bpb/report.html`. + +## Plotly report + +If you already have `out/char_bpe_flores_validation_bpb/summary.csv`, regenerate +the HTML report without retraining: + +```bash +python3 data/char_bpe_exploration/scripts/render_bpb_report.py +``` + +The report includes line charts, heatmaps, best-observed BPB rankings, and +scatter plots connecting validation loss, BPB, validation bytes/token, and +byte-fallback counts. ## Outputs diff --git a/data/char_bpe_exploration/scripts/render_bpb_report.py b/data/char_bpe_exploration/scripts/render_bpb_report.py new file mode 100755 index 0000000000..37ddb5bc4f --- /dev/null +++ b/data/char_bpe_exploration/scripts/render_bpb_report.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +"""Render a Plotly HTML report for FLORES char-BPE validation/BPB runs.""" + +from __future__ import annotations + +import argparse +import csv +import html +import json +import math +from collections import defaultdict +from pathlib import Path +from typing import Any + +DEFAULT_TRAINING_SUMMARY = Path("out/char_bpe_flores_validation_bpb/summary.csv") +DEFAULT_TOKENIZATION_SUMMARY = Path("data/char_bpe_exploration/results/summary.csv") +DEFAULT_OUTPUT = Path("out/char_bpe_flores_validation_bpb/report.html") + + +def read_csv_rows(path: Path) -> list[dict[str, str]]: + if not path.exists(): + raise SystemExit(f"Missing CSV file: {path}") + with path.open(newline="", encoding="utf-8") as f: + return list(csv.DictReader(f)) + + +def as_float(row: dict[str, str], key: str) -> float: + value = row.get(key, "") + if value == "": + return float("nan") + return float(value) + + +def as_int(row: dict[str, str], key: str) -> int: + value = row.get(key, "") + if value == "": + return 0 + return int(float(value)) + + +def finite_or_none(value: float) -> float | None: + return value if math.isfinite(value) else None + + +def sorted_vocab_sizes(rows: list[dict[str, str]]) -> list[int]: + return sorted({as_int(row, "vocab_size") for row in rows if row.get("vocab_size")}) + + +def sorted_languages(rows: list[dict[str, str]]) -> list[str]: + return sorted({row["language"] for row in rows if row.get("language")}) + + +def line_traces(rows: list[dict[str, str]], metric: str) -> list[dict[str, Any]]: + grouped: dict[str, list[dict[str, str]]] = defaultdict(list) + for row in rows: + grouped[row["language"]].append(row) + + traces = [] + for language in sorted(grouped): + lang_rows = sorted(grouped[language], key=lambda row: as_int(row, "vocab_size")) + traces.append( + { + "type": "scatter", + "mode": "lines+markers", + "name": language, + "x": [as_int(row, "vocab_size") for row in lang_rows], + "y": [finite_or_none(as_float(row, metric)) for row in lang_rows], + "hovertemplate": f"{language}
vocab=%{{x}}
{metric}=%{{y:.5f}}", + } + ) + return traces + + +def heatmap_trace(rows: list[dict[str, str]], metric: str) -> list[dict[str, Any]]: + vocabs = sorted_vocab_sizes(rows) + languages = sorted_languages(rows) + lookup = {(row["language"], as_int(row, "vocab_size")): as_float(row, metric) for row in rows} + z = [[finite_or_none(lookup.get((language, vocab), float("nan"))) for vocab in vocabs] for language in languages] + return [ + { + "type": "heatmap", + "x": vocabs, + "y": languages, + "z": z, + "colorscale": "Viridis", + "hovertemplate": "language=%{y}
vocab=%{x}
value=%{z:.5f}", + } + ] + + +def best_bar_trace(rows: list[dict[str, str]], metric: str) -> list[dict[str, Any]]: + best_rows = [] + for language in sorted_languages(rows): + candidates = [row for row in rows if row.get("language") == language and math.isfinite(as_float(row, metric))] + if candidates: + best_rows.append(min(candidates, key=lambda row: as_float(row, metric))) + best_rows.sort(key=lambda row: as_float(row, metric)) + return [ + { + "type": "bar", + "orientation": "h", + "x": [as_float(row, metric) for row in best_rows], + "y": [row["language"] for row in best_rows], + "customdata": [as_int(row, "vocab_size") for row in best_rows], + "hovertemplate": "language=%{y}
best " + metric + "=%{x:.5f}
vocab=%{customdata}", + } + ] + + +def scatter_loss_vs_bpb(rows: list[dict[str, str]]) -> list[dict[str, Any]]: + return [ + { + "type": "scatter", + "mode": "markers", + "x": [as_float(row, "validation_loss") for row in rows], + "y": [as_float(row, "bits_per_byte") for row in rows], + "text": [row["language"] for row in rows], + "customdata": [as_int(row, "vocab_size") for row in rows], + "marker": { + "size": 10, + "color": [as_int(row, "vocab_size") for row in rows], + "colorscale": "Turbo", + "showscale": True, + "colorbar": {"title": "vocab"}, + }, + "hovertemplate": "language=%{text}
vocab=%{customdata}
val_loss=%{x:.5f}
BPB=%{y:.5f}", + } + ] + + +def tokenization_lookup(tokenization_rows: list[dict[str, str]]) -> dict[tuple[str, int], dict[str, str]]: + lookup: dict[tuple[str, int], dict[str, str]] = {} + for row in tokenization_rows: + language = row.get("language") + vocab = row.get("requested_vocab_size") or row.get("vocab_size") + if language and vocab: + lookup[(language, int(float(vocab)))] = row + return lookup + + +def enrich_training_rows( + training_rows: list[dict[str, str]], + tokenization_rows: list[dict[str, str]], +) -> list[dict[str, str]]: + lookup = tokenization_lookup(tokenization_rows) + enriched = [] + for row in training_rows: + merged = dict(row) + token_row = lookup.get((row.get("language", ""), as_int(row, "vocab_size")), {}) + for key in ("val_bytes_per_token", "val_chars_per_token", "val_tokens", "unk_byte_fallback_tokens", "actual_vocab_size"): + if key in token_row: + merged[key] = token_row[key] + enriched.append(merged) + return enriched + + +def metric_scatter(rows: list[dict[str, str]], x_metric: str, y_metric: str) -> list[dict[str, Any]]: + usable = [row for row in rows if row.get(x_metric) not in (None, "") and row.get(y_metric) not in (None, "")] + return [ + { + "type": "scatter", + "mode": "markers", + "x": [as_float(row, x_metric) for row in usable], + "y": [as_float(row, y_metric) for row in usable], + "text": [row["language"] for row in usable], + "customdata": [as_int(row, "vocab_size") for row in usable], + "marker": {"size": 10, "color": [as_float(row, "bits_per_byte") for row in usable], "colorscale": "Viridis", "showscale": True}, + "hovertemplate": "language=%{text}
vocab=%{customdata}
" + x_metric + "=%{x:.5f}
" + y_metric + "=%{y:.5f}", + } + ] + + +def make_plots(rows: list[dict[str, str]]) -> list[dict[str, Any]]: + return [ + { + "id": "validation-loss-lines", + "title": "Validation loss by char-BPE vocabulary size", + "description": "Lower is better. Use this to see how each language responds to larger char-BPE vocabularies.", + "data": line_traces(rows, "validation_loss"), + "layout": {"xaxis": {"title": "Requested vocabulary size"}, "yaxis": {"title": "Validation loss (nats/token)"}}, + }, + { + "id": "bpb-lines", + "title": "Bits per byte by char-BPE vocabulary size", + "description": "Lower is better. BPB normalizes validation loss by UTF-8 bytes per validation token.", + "data": line_traces(rows, "bits_per_byte"), + "layout": {"xaxis": {"title": "Requested vocabulary size"}, "yaxis": {"title": "Bits per byte"}}, + }, + { + "id": "validation-loss-heatmap", + "title": "Validation-loss heatmap", + "description": "A compact view of all language/vocab combinations; darker/lighter changes expose language-specific scaling trends.", + "data": heatmap_trace(rows, "validation_loss"), + "layout": {"xaxis": {"title": "Requested vocabulary size"}, "yaxis": {"title": "Language"}}, + }, + { + "id": "bpb-heatmap", + "title": "Bits-per-byte heatmap", + "description": "Highlights which vocabulary sizes give the best compression-normalized validation behavior per language.", + "data": heatmap_trace(rows, "bits_per_byte"), + "layout": {"xaxis": {"title": "Requested vocabulary size"}, "yaxis": {"title": "Language"}}, + }, + { + "id": "best-bpb-bar", + "title": "Best observed BPB per language", + "description": "Ranks languages by their best observed BPB and shows which vocab size produced it.", + "data": best_bar_trace(rows, "bits_per_byte"), + "layout": {"xaxis": {"title": "Best bits per byte"}, "yaxis": {"title": "Language", "automargin": True}}, + }, + { + "id": "loss-vs-bpb", + "title": "Validation loss vs bits per byte", + "description": "Separates pure token-level modeling loss from byte-normalized efficiency; color indicates vocabulary size.", + "data": scatter_loss_vs_bpb(rows), + "layout": {"xaxis": {"title": "Validation loss (nats/token)"}, "yaxis": {"title": "Bits per byte"}}, + }, + { + "id": "val-bytes-token-vs-bpb", + "title": "Validation bytes/token vs BPB", + "description": "Shows whether BPB changes are driven by better byte coverage, better modeling, or both.", + "data": metric_scatter(rows, "val_bytes_per_token", "bits_per_byte"), + "layout": {"xaxis": {"title": "Validation bytes per token"}, "yaxis": {"title": "Bits per byte"}}, + }, + { + "id": "byte-fallback-vs-bpb", + "title": "Byte-fallback count vs BPB", + "description": "Useful for spotting languages/vocab sizes where byte fallback may still dominate tokenization behavior.", + "data": metric_scatter(rows, "unk_byte_fallback_tokens", "bits_per_byte"), + "layout": {"xaxis": {"title": "Byte-fallback tokens"}, "yaxis": {"title": "Bits per byte"}}, + }, + ] + + +def render_html(rows: list[dict[str, str]], plots: list[dict[str, Any]]) -> str: + rows_json = json.dumps(rows, ensure_ascii=False) + plots_json = json.dumps(plots, ensure_ascii=False) + return f""" + + + + + FLORES char-BPE validation and BPB report + + + + +

FLORES char-BPE validation and bits-per-byte report

+

This report compares validation loss and BPB for char-BPE vocab sweeps. BPB is computed as validation_loss_nats / (ln(2) * val_bytes_per_token).

+

Total completed runs: {len(rows)}

+
+

Raw completed-run table

+
+ + + +""" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--training-summary", type=Path, default=DEFAULT_TRAINING_SUMMARY) + parser.add_argument("--tokenization-summary", type=Path, default=DEFAULT_TOKENIZATION_SUMMARY) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + args = parser.parse_args() + + training_rows = read_csv_rows(args.training_summary) + tokenization_rows = read_csv_rows(args.tokenization_summary) if args.tokenization_summary.exists() else [] + rows = enrich_training_rows(training_rows, tokenization_rows) + plots = make_plots(rows) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(render_html(rows, plots), encoding="utf-8") + print(f"Wrote Plotly report to {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py b/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py index 1face1305a..c4cf979635 100755 --- a/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py +++ b/data/char_bpe_exploration/scripts/run_char_bpe_exploration.py @@ -32,7 +32,7 @@ ("italian", "ita_Latn"), ] -DEFAULT_VOCAB_SIZES = [384, 512, 768, 1024, 1536, 2048, 4096] +DEFAULT_VOCAB_SIZES = [320, 384, 512, 640, 768, 1024, 1280, 1536, 2048, 3072, 4096, 6144, 8192] def repo_root() -> Path: diff --git a/demos/char_bpe_flores_validation_bpb_demo.sh b/demos/char_bpe_flores_validation_bpb_demo.sh index 45577cc672..499f894263 100755 --- a/demos/char_bpe_flores_validation_bpb_demo.sh +++ b/demos/char_bpe_flores_validation_bpb_demo.sh @@ -17,9 +17,10 @@ BASE_DIR="data/char_bpe_exploration" DATASET_ROOT="data/char_bpe_exploration_train" OUT_ROOT="out/char_bpe_flores_validation_bpb" SUMMARY_CSV="${OUT_ROOT}/summary.csv" +REPORT_HTML="${OUT_ROOT}/report.html" LANGUAGES=${LANGUAGES:-"kiswahili bahasa_indonesian korean korean_nfd english chinese japanese arabic spanish german russian thai filipino hindi finnish italian"} -VOCAB_SIZES=${VOCAB_SIZES:-"384 512"} +VOCAB_SIZES=${VOCAB_SIZES:-"320 384 512 640 768 1024 1280 1536 2048 3072 4096"} MAX_ITERS=${MAX_ITERS:-200} EVAL_INTERVAL=${EVAL_INTERVAL:-50} EVAL_ITERS=${EVAL_ITERS:-20} @@ -116,4 +117,11 @@ PY done done +echo "=== Step 4: Render Plotly HTML report ===" +python3 data/char_bpe_exploration/scripts/render_bpb_report.py \ + --training-summary "${SUMMARY_CSV}" \ + --tokenization-summary "${BASE_DIR}/results/summary.csv" \ + --output "${REPORT_HTML}" + echo "Comparison complete: ${SUMMARY_CSV}" +echo "Plotly report: ${REPORT_HTML}" From 6fa471d05ff874c63b7542c241c62652bbb895bf Mon Sep 17 00:00:00 2001 From: klei22 Date: Mon, 22 Jun 2026 08:55:00 -0700 Subject: [PATCH 4/4] Update training settings for flores exploration --- demos/char_bpe_flores_validation_bpb_demo.sh | 31 +++++++++++++------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/demos/char_bpe_flores_validation_bpb_demo.sh b/demos/char_bpe_flores_validation_bpb_demo.sh index 499f894263..251b4a9d0f 100755 --- a/demos/char_bpe_flores_validation_bpb_demo.sh +++ b/demos/char_bpe_flores_validation_bpb_demo.sh @@ -21,16 +21,16 @@ REPORT_HTML="${OUT_ROOT}/report.html" LANGUAGES=${LANGUAGES:-"kiswahili bahasa_indonesian korean korean_nfd english chinese japanese arabic spanish german russian thai filipino hindi finnish italian"} VOCAB_SIZES=${VOCAB_SIZES:-"320 384 512 640 768 1024 1280 1536 2048 3072 4096"} -MAX_ITERS=${MAX_ITERS:-200} +MAX_ITERS=${MAX_ITERS:-3000} EVAL_INTERVAL=${EVAL_INTERVAL:-50} -EVAL_ITERS=${EVAL_ITERS:-20} +EVAL_ITERS=${EVAL_ITERS:-50} BATCH_SIZE=${BATCH_SIZE:-8} -BLOCK_SIZE=${BLOCK_SIZE:-64} -N_LAYER=${N_LAYER:-2} -N_HEAD=${N_HEAD:-2} +BLOCK_SIZE=${BLOCK_SIZE:-256} +N_LAYER=${N_LAYER:-6} +N_HEAD=${N_HEAD:-3} N_EMBD=${N_EMBD:-128} -DEVICE=${DEVICE:-"cpu"} -DTYPE=${DTYPE:-"float32"} +DEVICE=${DEVICE:-"cuda:0"} +DTYPE=${DTYPE:-"bfloat16"} PERCENTAGE_TRAIN=${PERCENTAGE_TRAIN:-"0.9"} mkdir -p "${DATASET_ROOT}" "${OUT_ROOT}" @@ -74,14 +74,25 @@ for language in ${LANGUAGES}; do --n_layer "${N_LAYER}" \ --n_head "${N_HEAD}" \ --n_embd "${N_EMBD}" \ + --use_rotary_embeddings \ + --no-use_abs_pos_embeddings \ + --attention_variant infinite \ + --use_concat_heads \ + --n_qk_head_dim 120 \ + --n_v_head_dim 120 \ + --use_qk_norm \ + --use_qk_norm_scale \ + --activation_variant squared_relu \ + --use_peri_ln \ --max_iters "${MAX_ITERS}" \ --eval_interval "${EVAL_INTERVAL}" \ --eval_iters "${EVAL_ITERS}" \ - --learning_rate 6e-4 \ - --weight_decay 0.1 \ + --learning_rate 0.001 \ + --weight_decay 0.0 \ + --optimizer muon \ --device "${DEVICE}" \ --dtype "${DTYPE}" \ - --no-compile + --compile echo "=== Step 3: Compute BPB for ${language} vocab=${vocab_size} ===" python3 - <<'PY' "${language}" "${vocab_size}" "${dataset}" "${run_dir}/metrics.json" "${out_dir}" "${SUMMARY_CSV}"