Distillation v1

distillation/configs/infer.yaml

@@ -0,0 +1,38 @@
+# Inference & evaluation config for Wikontic distillation.
+# Paths are resolved relative to the directory where infer.py lives.
+#
+# Workflow:
+#   - Use --split val  (default) during development: comparing checkpoints,
+#     tuning whitelist settings, etc.
+#   - Use --split test ONLY once for the final unbiased evaluation after
+#     model & hyperparameters are frozen.
+
+# Model paths
+base_model: "HuggingFaceTB/SmolLM2-1.7B-Instruct"
+adapter_path: "./checkpoints/wikontic-v4"
+
+# Data (same chat format used for training)
+val_path: "./data/val.jsonl"
+test_path: "./data/test.jsonl"
+output_dir: "./inference_outputs"
+
+# Generation
+temperature: 0.0
+do_sample: false
+max_new_tokens: 2048
+batch_size: 1
+
+# Quantization (must match training settings)
+load_in_4bit: true
+bnb_4bit_quant_type: "nf4"
+bnb_4bit_compute_dtype: "bfloat16"
+
+# Whitelist filtering
+# Extract allowed relations/types from training data and filter predictions
+use_whitelist: true
+whitelist_soft_mapping: false   # hard drop — reject any unknown relation
+whitelist_min_similarity: 0.6   # minimum similarity threshold for soft mapping
+
+# Comparison report
+# Path to training data for distribution comparison and whitelist extraction
+train_path: "./data/train.jsonl"

distillation/configs/train.yaml

@@ -0,0 +1,51 @@
+# Training config for Wikontic distillation — v4
+# (stronger LoRA, Focal Loss, no curriculum, no hand-tuned weights)
+# Paths are resolved relative to the directory where train.py lives.
+
+train_path: "./data/train.jsonl"
+val_path: "./data/val.jsonl"
+test_path: "./data/test.jsonl"
+system_prompt_path: "../src/wikontic/utils/prompts/triplet_extraction/propmt_1_types_qualifiers.txt"
+output_dir: "./checkpoints/wikontic-v4"
+
+# Model
+base_model: "HuggingFaceTB/SmolLM2-1.7B-Instruct"
+
+# Quantization (QLoRA)
+load_in_4bit: true
+bnb_4bit_quant_type: "nf4"
+bnb_4bit_compute_dtype: "bfloat16"
+
+# LoRA
+lora_r: 32                        # increased from 16
+lora_alpha: 64
+lora_dropout: 0.05
+target_modules:
+  - "q_proj"
+  - "v_proj"
+  - "k_proj"
+  - "o_proj"
+  - "gate_proj"
+  - "up_proj"
+  - "down_proj"
+
+# Training hyperparameters
+num_train_epochs: 5
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 1
+gradient_accumulation_steps: 4
+learning_rate: 2.0e-4
+warmup_steps: 100
+logging_steps: 10
+save_steps: 1000
+eval_strategy: "steps"
+eval_steps: 1000
+bf16: true
+dataloader_num_workers: 0
+save_total_limit: 2
+report_to: "none"
+
+# ---------------------------------------------------------------------------
+# Focal Loss gamma (set to 0.0 for standard CE, 1.0-2.0 for focal)
+# ---------------------------------------------------------------------------
+focal_gamma: 2.0