log_words.py

import os
import json
from transformers import AutoTokenizer
from config import LOG_DIR, BASIL_MODEL_NAME

tokenizer = AutoTokenizer.from_pretrained(BASIL_MODEL_NAME)

# Get all today's log filenames
log_fnames = [f for f in os.listdir(LOG_DIR) if f.startswith("log_2025") and f.endswith(".jsonl")]

for fname in log_fnames:
    total_words = 0
    total_tokens = 0
    full_text = ""

    with open(os.path.join(LOG_DIR, fname), "r", encoding="utf-8") as f:
        for line in f:
            entry = json.loads(line)
            if entry["speaker"] in ["Tutor", "Sophie", "Basil"]:
                full_text += f"{entry['speaker']}: {entry['text']}\n"

    total_words = len(full_text.split())
    total_tokens = len(tokenizer(full_text)["input_ids"])

    print(f"{fname}: {total_words:,} words, {total_tokens:,} tokens")