diff --git a/config/constract_llm/dataset/cleanse/config.json b/config/constract_llm/dataset/cleanse/config.json index f0f7b346..3d2447b9 100644 --- a/config/constract_llm/dataset/cleanse/config.json +++ b/config/constract_llm/dataset/cleanse/config.json @@ -1,13 +1,13 @@ { - "input_name_or_path": "imdb", - "output_dir": "./data/misc/json", - "text_fields": ["text"], - "do_deduplicate": true, - "do_rm_duplicated_by_minhash": true, - "minhash_threshold": 0.95, - "do_rm_time_schedule": true, - "rm_time_schedule_threshold": 3, - "do_rm_only_numeric": true, - "do_rm_include_url_text": true, - "do_rm_include_email_text": true + "input_name_or_path": "imdb", + "output_dir": "data/misc/json", + "text_fields": ["text"], + "do_deduplicate": true, + "do_rm_duplicated_by_minhash": true, + "minhash_threshold": 0.95, + "do_rm_time_schedule": true, + "rm_time_schedule_threshold": 3, + "do_rm_only_numeric": true, + "do_rm_include_url_text": true, + "do_rm_include_email_text": true } diff --git a/config/constract_llm/dataset/preprocess/config.json b/config/constract_llm/dataset/preprocess/config.json index 32614ba3..2f50923d 100644 --- a/config/constract_llm/dataset/preprocess/config.json +++ b/config/constract_llm/dataset/preprocess/config.json @@ -1,5 +1,5 @@ { - "input_name_or_path": "imdb", - "output_dir": "./data/misc/json", - "text_fields": ["text", "label"] + "input_name_or_path": "imdb", + "output_dir": "data/misc/json", + "text_fields": ["text", "label"] } diff --git a/config/constract_llm/dataset/split/config.json b/config/constract_llm/dataset/split/config.json index d2388aa3..eff94b0e 100644 --- a/config/constract_llm/dataset/split/config.json +++ b/config/constract_llm/dataset/split/config.json @@ -1,9 +1,9 @@ { - "dataset_name_or_path": "imdb", - "output_dir": "./data/misc/json", - "test_size": 0.1, - "val_size": 0.1, - "split_mode": "random", - "random_seed": 42, - "stratify_key": "label" + "dataset_name_or_path": "imdb", + "output_dir": "data/misc/json", + "test_size": 0.1, + "val_size": 0.1, + "split_mode": "random", + "random_seed": 42, + "stratify_key": "label" } diff --git a/config/constract_llm/eval/analysis/example.json b/config/constract_llm/eval/analysis/example.json index 474d6854..358f2481 100644 --- a/config/constract_llm/eval/analysis/example.json +++ b/config/constract_llm/eval/analysis/example.json @@ -1,18 +1,18 @@ { - "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-0.5B", - "model_revision": null, - "output_dir": "data/outputs/eval/analysis/Sentence-ModernBERT-JP-0.5B", - "language": "ja", - "num_examples": 2000, - "num_attention_examples": 32, - "attention_max_tokens": 128, - "batch_size": 16, - "max_seq_length": 512, - "seed": 42, - "dtype": "bfloat16", - "with_attention": true, - "with_layer_sts": true, - "with_kill_test": true, - "dump_layer_embeddings": false, - "sts_max_pairs": 1500 + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-0.5B", + "model_revision": null, + "output_dir": "data/outputs/eval/analysis/Sentence-ModernBERT-JP-0.5B", + "language": "ja", + "num_examples": 2000, + "num_attention_examples": 32, + "attention_max_tokens": 128, + "batch_size": 16, + "max_seq_length": 512, + "seed": 42, + "dtype": "bfloat16", + "with_attention": true, + "with_layer_sts": true, + "with_kill_test": true, + "dump_layer_embeddings": false, + "sts_max_pairs": 1500 } diff --git a/config/constract_llm/eval/bench_sbert/sentence-llama-bi-3b.json b/config/constract_llm/eval/bench_sbert/sentence-llama-bi-3b.json deleted file mode 100644 index 14792285..00000000 --- a/config/constract_llm/eval/bench_sbert/sentence-llama-bi-3b.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B", - "output_dir": "data/outputs/bench_sbert/llama-bi-3b", - "dataset_name_or_path": "fujiki/wiki40b_ja", - "dataset_config_name": "default", - "dataset_split": "train", - "text_fields": [ - "text" - ], - "streaming": true, - "batch_sizes": [ - 16, - 32, - 64, - 128, - 256 - ], - "seq_lengths": [ - 128, - 256, - 512 - ], - "device": "cuda", - "dtype": "bfloat16", - "normalize_embeddings": true, - "max_use_samples": 100000, - "monitor_interval_ms": 100, - "warmup_fraction": 0.0, - "show_progress": true, - "trust_remote_code": true, - "attn_implementation": "flash_attention_2", - "model_extra_kwargs": {}, - "tokenizer_extra_kwargs": {}, - "common_extra_kwargs": {} -} diff --git a/config/constract_llm/eval/bench_sbert/sentence-llama-bi.json b/config/constract_llm/eval/bench_sbert/sentence-llama-bi.json deleted file mode 100644 index 4148cd46..00000000 --- a/config/constract_llm/eval/bench_sbert/sentence-llama-bi.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-0.5B", - "output_dir": "data/outputs/bench_sbert/llama-bi", - "dataset_name_or_path": "fujiki/wiki40b_ja", - "dataset_config_name": "default", - "dataset_split": "train", - "text_fields": ["text"], - "streaming": true, - "batch_sizes": [16, 32, 64, 128, 256], - "seq_lengths": [128, 256, 512], - "device": "cuda", - "dtype": "bfloat16", - "normalize_embeddings": true, - "max_use_samples": 100000, - "monitor_interval_ms": 100, - "warmup_fraction": 0.0, - "show_progress": true, - "trust_remote_code": true, - "attn_implementation": "flash_attention_2", - "model_extra_kwargs": {}, - "tokenizer_extra_kwargs": {}, - "common_extra_kwargs": {} -} diff --git a/config/constract_llm/eval/bench_sbert/sentence-modernbert-3b.json b/config/constract_llm/eval/bench_sbert/sentence-modernbert-3b.json deleted file mode 100644 index bf6bf655..00000000 --- a/config/constract_llm/eval/bench_sbert/sentence-modernbert-3b.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-3B", - "output_dir": "data/outputs/bench_sbert/modernbert-3b", - "dataset_name_or_path": "fujiki/wiki40b_ja", - "dataset_config_name": "default", - "dataset_split": "train", - "text_fields": [ - "text" - ], - "streaming": true, - "batch_sizes": [ - 16, - 32, - 64, - 128, - 256 - ], - "seq_lengths": [ - 128, - 256, - 512 - ], - "device": "cuda", - "dtype": "bfloat16", - "normalize_embeddings": true, - "max_use_samples": 100000, - "monitor_interval_ms": 100, - "warmup_fraction": 0.0, - "show_progress": true, - "trust_remote_code": true, - "attn_implementation": "flash_attention_2", - "model_extra_kwargs": {}, - "tokenizer_extra_kwargs": {}, - "common_extra_kwargs": {} -} diff --git a/config/constract_llm/eval/bench_sbert/sentence-modernbert.json b/config/constract_llm/eval/bench_sbert/sentence-modernbert.json deleted file mode 100644 index 94f4c347..00000000 --- a/config/constract_llm/eval/bench_sbert/sentence-modernbert.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-0.5B", - "output_dir": "data/outputs/bench_sbert/modernbert", - "dataset_name_or_path": "fujiki/wiki40b_ja", - "dataset_config_name": "default", - "dataset_split": "train", - "text_fields": ["text"], - "streaming": true, - "batch_sizes": [16, 32, 64, 128, 256], - "seq_lengths": [128, 256, 512], - "device": "cuda", - "dtype": "bfloat16", - "normalize_embeddings": true, - "max_use_samples": 100000, - "monitor_interval_ms": 100, - "warmup_fraction": 0.0, - "show_progress": true, - "trust_remote_code": true, - "attn_implementation": "flash_attention_2", - "model_extra_kwargs": {}, - "tokenizer_extra_kwargs": {}, - "common_extra_kwargs": {} -} diff --git a/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-3b.json b/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-3b.json deleted file mode 100644 index bc182947..00000000 --- a/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-3b.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-3B", - "output_dir": "data/outputs/bench_sbert/sarashina-bi-3b", - "dataset_name_or_path": "fujiki/wiki40b_ja", - "dataset_config_name": "default", - "dataset_split": "train", - "text_fields": [ - "text" - ], - "streaming": true, - "batch_sizes": [ - 16, - 32, - 64, - 128, - 256 - ], - "seq_lengths": [ - 128, - 256, - 512 - ], - "device": "cuda", - "dtype": "bfloat16", - "normalize_embeddings": true, - "max_use_samples": 100000, - "monitor_interval_ms": 100, - "warmup_fraction": 0.0, - "show_progress": true, - "trust_remote_code": true, - "attn_implementation": "flash_attention_2", - "model_extra_kwargs": {}, - "tokenizer_extra_kwargs": {}, - "common_extra_kwargs": {} -} diff --git a/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi.json b/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi.json deleted file mode 100644 index 8d090f7f..00000000 --- a/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-0.5B", - "output_dir": "data/outputs/bench_sbert/sarashina-bi", - "dataset_name_or_path": "fujiki/wiki40b_ja", - "dataset_config_name": "default", - "dataset_split": "train", - "text_fields": ["text"], - "streaming": true, - "batch_sizes": [16, 32, 64, 128, 256], - "seq_lengths": [128, 256, 512], - "device": "cuda", - "dtype": "bfloat16", - "normalize_embeddings": true, - "max_use_samples": 100000, - "monitor_interval_ms": 100, - "warmup_fraction": 0.0, - "show_progress": true, - "trust_remote_code": true, - "attn_implementation": "flash_attention_2", - "model_extra_kwargs": {}, - "tokenizer_extra_kwargs": {}, - "common_extra_kwargs": {} -} diff --git a/config/constract_llm/eval/bench_sbert/sentence-instructlm-bi-en-0.5b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-InstructLM-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-instructlm-bi-en-0.5b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-InstructLM-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/bench_sbert/sentence-llama-bi-en-0.5b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-llama-bi-en-0.5b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-1B/bs1024.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-1B/bs1024.json new file mode 100644 index 00000000..5a50fd7b --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-1B/bs1024.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "data/outputs/bench_sbert/sentence-llama-bi-en-1b-bs1024", + "dataset_name_or_path": "google/wiki40b", + "dataset_config_name": "en", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-llama-bi-en-1b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-1B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-llama-bi-en-1b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-1B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-3B/bs2048.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-3B/bs2048.json new file mode 100644 index 00000000..31a231e8 --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-3B/bs2048.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-3B-bs2048", + "output_dir": "data/outputs/bench_sbert/sentence-llama-bi-en-3b-bs2048", + "dataset_name_or_path": "google/wiki40b", + "dataset_config_name": "en", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-llama-bi-en-3b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-3B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-llama-bi-en-3b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-EN-3B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-0.5B/default.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-0.5B/default.json new file mode 100644 index 00000000..436d81de --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-0.5B/default.json @@ -0,0 +1,23 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-0.5B", + "output_dir": "data/outputs/bench_sbert/llama-bi", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": ["text"], + "streaming": true, + "batch_sizes": [16, 32, 64, 128, 256], + "seq_lengths": [128, 256, 512], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-1B/bs1024.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-1B/bs1024.json new file mode 100644 index 00000000..1af5138d --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-1B/bs1024.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-1B-bs1024", + "output_dir": "data/outputs/bench_sbert/llama-bi-1b", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-llama-bi-1b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-1B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-llama-bi-1b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-1B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-3B/bs2048.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-3B/bs2048.json new file mode 100644 index 00000000..b16435f0 --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-3B/bs2048.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B-bs2048", + "output_dir": "data/outputs/bench_sbert/llama-bi-3b", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-3B/default.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-3B/default.json new file mode 100644 index 00000000..e2f9e81e --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Llama-Bi-JP-3B/default.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B", + "output_dir": "data/outputs/bench_sbert/llama-bi-3b", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-mistral-bi-en-0.5b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Mistral-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-mistral-bi-en-0.5b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Mistral-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/bench_sbert/sentence-modernbert-en-0.5b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-modernbert-en-0.5b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-0.5B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-1B/bs1024.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-1B/bs1024.json new file mode 100644 index 00000000..457ed315 --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-1B/bs1024.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "data/outputs/bench_sbert/sentence-modernbert-en-1b-bs1024", + "dataset_name_or_path": "google/wiki40b", + "dataset_config_name": "en", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-modernbert-en-1b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-1B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-modernbert-en-1b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-1B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-3B/bs2048.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-3B/bs2048.json new file mode 100644 index 00000000..216fe23d --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-3B/bs2048.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-3B-bs2048", + "output_dir": "data/outputs/bench_sbert/sentence-modernbert-en-3b-bs2048", + "dataset_name_or_path": "google/wiki40b", + "dataset_config_name": "en", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-modernbert-en-3b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-3B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-modernbert-en-3b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-EN-3B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-0.5B/default.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-0.5B/default.json new file mode 100644 index 00000000..6a6e64cc --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-0.5B/default.json @@ -0,0 +1,23 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-0.5B", + "output_dir": "data/outputs/bench_sbert/modernbert", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": ["text"], + "streaming": true, + "batch_sizes": [16, 32, 64, 128, 256], + "seq_lengths": [128, 256, 512], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-1B/bs1024.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-1B/bs1024.json new file mode 100644 index 00000000..b45e9b12 --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-1B/bs1024.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-1B-bs1024", + "output_dir": "data/outputs/bench_sbert/modernbert-1b", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-modernbert-1b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-1B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-modernbert-1b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-1B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-3B/bs2048.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-3B/bs2048.json new file mode 100644 index 00000000..99f9dadd --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-3B/bs2048.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-3B-bs2048", + "output_dir": "data/outputs/bench_sbert/modernbert-3b", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-3B/default.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-3B/default.json new file mode 100644 index 00000000..cc52897f --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-ModernBERT-JP-3B/default.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-3B", + "output_dir": "data/outputs/bench_sbert/modernbert-3b", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-qwen2.5-bi-en-0.5b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Qwen2.5-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-qwen2.5-bi-en-0.5b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Qwen2.5-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/bench_sbert/sentence-qwen2.5-scratch-bi-en-0.5b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-qwen2.5-scratch-bi-en-0.5b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-0.5B/default.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-0.5B/default.json new file mode 100644 index 00000000..9f7555d0 --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-0.5B/default.json @@ -0,0 +1,23 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-0.5B", + "output_dir": "data/outputs/bench_sbert/sarashina-bi", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": ["text"], + "streaming": true, + "batch_sizes": [16, 32, 64, 128, 256], + "seq_lengths": [128, 256, 512], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-en-0.5b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-en-0.5b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-1B/bs1024.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-1B/bs1024.json new file mode 100644 index 00000000..86a5e52b --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-1B/bs1024.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "data/outputs/bench_sbert/sentence-sarashina-bi-en-1b-bs1024", + "dataset_name_or_path": "google/wiki40b", + "dataset_config_name": "en", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-en-1b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-1B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-en-1b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-1B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-3B/bs2048.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-3B/bs2048.json new file mode 100644 index 00000000..6554a6cc --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-3B/bs2048.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-3B-bs2048", + "output_dir": "data/outputs/bench_sbert/sentence-sarashina-bi-en-3b-bs2048", + "dataset_name_or_path": "google/wiki40b", + "dataset_config_name": "en", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-en-3b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-3B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-en-3b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-EN-3B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-1B/bs1024.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-1B/bs1024.json new file mode 100644 index 00000000..7271030d --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-1B/bs1024.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-1B-bs1024", + "output_dir": "data/outputs/bench_sbert/sarashina-bi-1b", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-1b.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-1B/default.json similarity index 100% rename from config/constract_llm/eval/bench_sbert/sentence-sarashina-bi-1b.json rename to config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-1B/default.json diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-3B/bs2048.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-3B/bs2048.json new file mode 100644 index 00000000..db56ad23 --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-3B/bs2048.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-3B-bs2048", + "output_dir": "data/outputs/bench_sbert/sarashina-bi-3b", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-3B/default.json b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-3B/default.json new file mode 100644 index 00000000..513c5d38 --- /dev/null +++ b/config/constract_llm/eval/ft/bench_sbert/Sentence-Sarashina-Bi-JP-3B/default.json @@ -0,0 +1,35 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-3B", + "output_dir": "data/outputs/bench_sbert/sarashina-bi-3b", + "dataset_name_or_path": "fujiki/wiki40b_ja", + "dataset_config_name": "default", + "dataset_split": "train", + "text_fields": [ + "text" + ], + "streaming": true, + "batch_sizes": [ + 16, + 32, + 64, + 128, + 256 + ], + "seq_lengths": [ + 128, + 256, + 512 + ], + "device": "cuda", + "dtype": "bfloat16", + "normalize_embeddings": true, + "max_use_samples": 100000, + "monitor_interval_ms": 100, + "warmup_fraction": 0.0, + "show_progress": true, + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "common_extra_kwargs": {} +} diff --git a/config/constract_llm/eval/isotropic/Sentence-InstructLM-Bi-EN-0.5B.json b/config/constract_llm/eval/ft/isotropic/Sentence-InstructLM-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-InstructLM-Bi-EN-0.5B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-InstructLM-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-EN-0.5B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Llama-Bi-EN-0.5B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-1B/bs1024.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-1B/bs1024.json new file mode 100644 index 00000000..2e92b9b0 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-1B/bs1024.json @@ -0,0 +1,15 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "data/outputs/eval/isotropic", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "wiki_name": "google/wiki40b", + "wiki_lang": "en", + "positive_pair_dataset_name": "sentence-transformers/all-nli", + "positive_pair_dataset_config_name": "triplet", + "positive_pair_dataset_split": "dev", + "positive_pair_sentence1_column": "anchor", + "positive_pair_sentence2_column": "positive" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-EN-1B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-1B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Llama-Bi-EN-1B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-1B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-3B/bs2048.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-3B/bs2048.json new file mode 100644 index 00000000..0538c723 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-3B/bs2048.json @@ -0,0 +1,15 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-3B-bs2048", + "output_dir": "data/outputs/eval/isotropic", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "wiki_name": "google/wiki40b", + "wiki_lang": "en", + "positive_pair_dataset_name": "sentence-transformers/all-nli", + "positive_pair_dataset_config_name": "triplet", + "positive_pair_dataset_split": "dev", + "positive_pair_sentence1_column": "anchor", + "positive_pair_sentence2_column": "positive" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-EN-3B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-3B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Llama-Bi-EN-3B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-EN-3B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-0.5B/default.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-0.5B/default.json new file mode 100644 index 00000000..e27aa0aa --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-0.5B/default.json @@ -0,0 +1,13 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-0.5B", + "output_dir": "data/outputs/eval/isotropic/Sentence-Llama-Bi-0.5B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-1B/bs1024.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-1B/bs1024.json new file mode 100644 index 00000000..ae4621df --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-1B/bs1024.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-1B-bs1024", + "output_dir": "data/outputs/eval/isotropic/Sentence-Llama-Bi-1B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-1B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-1B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Llama-Bi-1B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-1B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-3B/bs2048.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-3B/bs2048.json new file mode 100644 index 00000000..7a13b14b --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-3B/bs2048.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B-bs2048", + "output_dir": "data/outputs/eval/isotropic/Sentence-Llama-Bi-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-3B/default.json b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-3B/default.json new file mode 100644 index 00000000..0519c66b --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Llama-Bi-JP-3B/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B", + "output_dir": "data/outputs/eval/isotropic/Sentence-Llama-Bi-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-Mistral-Bi-EN-0.5B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Mistral-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Mistral-Bi-EN-0.5B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Mistral-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/isotropic/Sentence-ModernBERT-EN-0.5B.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-ModernBERT-EN-0.5B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-0.5B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-1B/bs1024.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-1B/bs1024.json new file mode 100644 index 00000000..88c1dd7a --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-1B/bs1024.json @@ -0,0 +1,15 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "data/outputs/eval/isotropic", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "wiki_name": "google/wiki40b", + "wiki_lang": "en", + "positive_pair_dataset_name": "sentence-transformers/all-nli", + "positive_pair_dataset_config_name": "triplet", + "positive_pair_dataset_split": "dev", + "positive_pair_sentence1_column": "anchor", + "positive_pair_sentence2_column": "positive" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-ModernBERT-EN-1B.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-1B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-ModernBERT-EN-1B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-1B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-3B/bs2048.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-3B/bs2048.json new file mode 100644 index 00000000..010f4e47 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-3B/bs2048.json @@ -0,0 +1,15 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-3B-bs2048", + "output_dir": "data/outputs/eval/isotropic", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "wiki_name": "google/wiki40b", + "wiki_lang": "en", + "positive_pair_dataset_name": "sentence-transformers/all-nli", + "positive_pair_dataset_config_name": "triplet", + "positive_pair_dataset_split": "dev", + "positive_pair_sentence1_column": "anchor", + "positive_pair_sentence2_column": "positive" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-ModernBERT-EN-3B.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-3B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-ModernBERT-EN-3B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-EN-3B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-0.5B/default.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-0.5B/default.json new file mode 100644 index 00000000..2b089f16 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-0.5B/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-0.5B", + "output_dir": "data/outputs/eval/isotropic/Sentence-Modernbert-0.5B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-1B/bs1024.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-1B/bs1024.json new file mode 100644 index 00000000..5b889cd3 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-1B/bs1024.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-1B-bs1024", + "output_dir": "data/outputs/eval/isotropic/Sentence-ModernBERT-1B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-ModernBERT-1B.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-1B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-ModernBERT-1B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-1B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-3B/bs2048.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-3B/bs2048.json new file mode 100644 index 00000000..23062e36 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-3B/bs2048.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-3B-bs2048", + "output_dir": "data/outputs/eval/isotropic/Sentence-ModernBERT-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-3B/default.json b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-3B/default.json new file mode 100644 index 00000000..edbb3b79 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-ModernBERT-JP-3B/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-3B", + "output_dir": "data/outputs/eval/isotropic/Sentence-ModernBERT-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-Qwen2.5-Bi-EN-0.5B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Qwen2.5-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Qwen2.5-Bi-EN-0.5B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Qwen2.5-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/isotropic/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-0.5B/default.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-0.5B/default.json new file mode 100644 index 00000000..7f971878 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-0.5B/default.json @@ -0,0 +1,13 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-0.5B", + "output_dir": "data/outputs/eval/isotropic/Sentence-Sarashina-Bi-0.5B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-EN-0.5B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-0.5B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-EN-0.5B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-0.5B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-1B/bs1024.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-1B/bs1024.json new file mode 100644 index 00000000..659d8b91 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-1B/bs1024.json @@ -0,0 +1,15 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "data/outputs/eval/isotropic", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "wiki_name": "google/wiki40b", + "wiki_lang": "en", + "positive_pair_dataset_name": "sentence-transformers/all-nli", + "positive_pair_dataset_config_name": "triplet", + "positive_pair_dataset_split": "dev", + "positive_pair_sentence1_column": "anchor", + "positive_pair_sentence2_column": "positive" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-EN-1B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-1B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-EN-1B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-1B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-3B/bs2048.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-3B/bs2048.json new file mode 100644 index 00000000..9505271f --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-3B/bs2048.json @@ -0,0 +1,15 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-3B-bs2048", + "output_dir": "data/outputs/eval/isotropic", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "wiki_name": "google/wiki40b", + "wiki_lang": "en", + "positive_pair_dataset_name": "sentence-transformers/all-nli", + "positive_pair_dataset_config_name": "triplet", + "positive_pair_dataset_split": "dev", + "positive_pair_sentence1_column": "anchor", + "positive_pair_sentence2_column": "positive" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-EN-3B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-3B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-EN-3B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-EN-3B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-1B/bs1024.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-1B/bs1024.json new file mode 100644 index 00000000..e31941e6 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-1B/bs1024.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-1B-bs1024", + "output_dir": "data/outputs/eval/isotropic/Sentence-Sarashina-Bi-1B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-1B.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-1B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-1B.json rename to config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-1B/default.json diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-3B/bs2048.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-3B/bs2048.json new file mode 100644 index 00000000..933ab28c --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-3B/bs2048.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-3B-bs2048", + "output_dir": "data/outputs/eval/isotropic/Sentence-Sarashina-Bi-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-3B/default.json b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-3B/default.json new file mode 100644 index 00000000..f79e8958 --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/Sentence-Sarashina-Bi-JP-3B/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-3B", + "output_dir": "data/outputs/eval/isotropic/Sentence-Sarashina-Bi-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/ft/isotropic/example.json b/config/constract_llm/eval/ft/isotropic/example.json new file mode 100644 index 00000000..f1b8aa8f --- /dev/null +++ b/config/constract_llm/eval/ft/isotropic/example.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "cl-nagoya/ruri-v3-30m", + "output_dir": "data/outputs/eval/isotropic", + "num_examples": 100, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/mteb_full/sentence-instructlm-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-instructlm-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-instructlm-bi-en-0.5b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/mteb/sentence-instructlm-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-instructlm-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-InstructLM-Bi-EN-0.5B/subset.json diff --git a/config/constract_llm/eval/mteb_full/sentence-llama-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-llama-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-0.5b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/mteb/sentence-llama-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-llama-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-0.5B/subset.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full-bs1024.json new file mode 100644 index 00000000..85597c21 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full-bs1024.json @@ -0,0 +1,63 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "ArguAna", + "ArXivHierarchicalClusteringP2P", + "ArXivHierarchicalClusteringS2S", + "AskUbuntuDupQuestions", + "BIOSSES", + "Banking77Classification", + "BiorxivClusteringP2P.v2", + "CQADupstackGamingRetrieval", + "CQADupstackUnixRetrieval", + "ClimateFEVERHardNegatives", + "FEVERHardNegatives", + "FiQA2018", + "HotpotQAHardNegatives", + "ImdbClassification", + "MTOPDomainClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "MindSmallReranking", + "SCIDOCS", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STSBenchmark", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "TRECCOVID", + "Touche2020Retrieval.v3", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering.v2", + "AmazonCounterfactualClassification", + "STS17", + "STS22.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full/sentence-llama-bi-en-1b.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-llama-bi-en-1b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/clf-clust-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/clf-clust-bs1024.json new file mode 100644 index 00000000..f9a00591 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/clf-clust-bs1024.json @@ -0,0 +1,42 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "AmazonCounterfactualClassification", + "Banking77Classification", + "ImdbClassification", + "MTOPDomainClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "ArXivHierarchicalClusteringP2P", + "ArXivHierarchicalClusteringS2S", + "BiorxivClusteringP2P.v2", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "TwentyNewsgroupsClustering.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-arguana-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-arguana-bs1024.json new file mode 100644 index 00000000..f1f0effe --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-arguana-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "ArguAna" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-climatefeverhardnega-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-climatefeverhardnega-bs1024.json new file mode 100644 index 00000000..61d5baa3 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-climatefeverhardnega-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "ClimateFEVERHardNegatives" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret-bs1024.json new file mode 100644 index 00000000..3215dfe9 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "CQADupstackGamingRetrieval" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri-bs1024.json new file mode 100644 index 00000000..31438b8c --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "CQADupstackUnixRetrieval" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-feverhardnegatives-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-feverhardnegatives-bs1024.json new file mode 100644 index 00000000..061b2881 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-feverhardnegatives-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "FEVERHardNegatives" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-fiqa2018-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-fiqa2018-bs1024.json new file mode 100644 index 00000000..94f2c264 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-fiqa2018-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "FiQA2018" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative-bs1024.json new file mode 100644 index 00000000..e053362f --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "HotpotQAHardNegatives" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-scidocs-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-scidocs-bs1024.json new file mode 100644 index 00000000..1ffd45a6 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-scidocs-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "SCIDOCS" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-touche2020retrieval-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-touche2020retrieval-bs1024.json new file mode 100644 index 00000000..944ab6fd --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-touche2020retrieval-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "Touche2020Retrieval.v3" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-treccovid-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-treccovid-bs1024.json new file mode 100644 index 00000000..b1ca925c --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-treccovid-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "TRECCOVID" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/sts-rerank-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/sts-rerank-bs1024.json new file mode 100644 index 00000000..3f9aa653 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/sts-rerank-bs1024.json @@ -0,0 +1,37 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "BIOSSES", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STS17", + "STS22.v2", + "STSBenchmark", + "AskUbuntuDupQuestions", + "MindSmallReranking" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-llama-bi-en-1b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/subset-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/subset-bs1024.json new file mode 100644 index 00000000..75fec986 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/subset-bs1024.json @@ -0,0 +1,29 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "data/outputs/eval/mteb/sentence-llama-bi-en-1b-bs1024", + "tasks": [ + "SciFact", + "STSBenchmark", + "Banking77Classification.v2", + "AskUbuntuDupQuestions", + "StackExchangeClustering.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb/sentence-llama-bi-en-1b.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-llama-bi-en-1b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-1B/subset.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-3B/subset-bs2048.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-3B/subset-bs2048.json new file mode 100644 index 00000000..82089c00 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-3B/subset-bs2048.json @@ -0,0 +1,29 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-3B-bs2048", + "output_dir": "data/outputs/eval/mteb/sentence-llama-bi-en-3b-bs2048", + "tasks": [ + "SciFact", + "STSBenchmark", + "Banking77Classification.v2", + "AskUbuntuDupQuestions", + "StackExchangeClustering.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb/sentence-llama-bi-en-3b.json b/config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-3B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-llama-bi-en-3b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Llama-Bi-EN-3B/subset.json diff --git a/config/constract_llm/eval/mteb_full/sentence-mistral-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-mistral-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-mistral-bi-en-0.5b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/mteb/sentence-mistral-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-mistral-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Mistral-Bi-EN-0.5B/subset.json diff --git a/config/constract_llm/eval/mteb_full/sentence-modernbert-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-modernbert-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-0.5b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/mteb/sentence-modernbert-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-modernbert-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-0.5B/subset.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full-bs1024.json new file mode 100644 index 00000000..6f373eb3 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full-bs1024.json @@ -0,0 +1,63 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "ArguAna", + "ArXivHierarchicalClusteringP2P", + "ArXivHierarchicalClusteringS2S", + "AskUbuntuDupQuestions", + "BIOSSES", + "Banking77Classification", + "BiorxivClusteringP2P.v2", + "CQADupstackGamingRetrieval", + "CQADupstackUnixRetrieval", + "ClimateFEVERHardNegatives", + "FEVERHardNegatives", + "FiQA2018", + "HotpotQAHardNegatives", + "ImdbClassification", + "MTOPDomainClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "MindSmallReranking", + "SCIDOCS", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STSBenchmark", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "TRECCOVID", + "Touche2020Retrieval.v3", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering.v2", + "AmazonCounterfactualClassification", + "STS17", + "STS22.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full/sentence-modernbert-en-1b.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-modernbert-en-1b.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/clf-clust-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/clf-clust-bs1024.json new file mode 100644 index 00000000..d5c12f1e --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/clf-clust-bs1024.json @@ -0,0 +1,42 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "AmazonCounterfactualClassification", + "Banking77Classification", + "ImdbClassification", + "MTOPDomainClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "ArXivHierarchicalClusteringP2P", + "ArXivHierarchicalClusteringS2S", + "BiorxivClusteringP2P.v2", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "TwentyNewsgroupsClustering.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-arguana-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-arguana-bs1024.json new file mode 100644 index 00000000..d8e66d7f --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-arguana-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "ArguAna" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-climatefeverhardnega-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-climatefeverhardnega-bs1024.json new file mode 100644 index 00000000..4a6f6fc5 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-climatefeverhardnega-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "ClimateFEVERHardNegatives" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackgamingret-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackgamingret-bs1024.json new file mode 100644 index 00000000..1fbe9252 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackgamingret-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "CQADupstackGamingRetrieval" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackunixretri-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackunixretri-bs1024.json new file mode 100644 index 00000000..76488d71 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackunixretri-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "CQADupstackUnixRetrieval" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-feverhardnegatives-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-feverhardnegatives-bs1024.json new file mode 100644 index 00000000..d7ec600c --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-feverhardnegatives-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "FEVERHardNegatives" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-fiqa2018-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-fiqa2018-bs1024.json new file mode 100644 index 00000000..268ee21a --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-fiqa2018-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "FiQA2018" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-hotpotqahardnegative-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-hotpotqahardnegative-bs1024.json new file mode 100644 index 00000000..9120d81b --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-hotpotqahardnegative-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "HotpotQAHardNegatives" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-scidocs-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-scidocs-bs1024.json new file mode 100644 index 00000000..568d453e --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-scidocs-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "SCIDOCS" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-touche2020retrieval-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-touche2020retrieval-bs1024.json new file mode 100644 index 00000000..d08a229d --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-touche2020retrieval-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "Touche2020Retrieval.v3" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-treccovid-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-treccovid-bs1024.json new file mode 100644 index 00000000..78e937ed --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-treccovid-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "TRECCOVID" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/sts-rerank-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/sts-rerank-bs1024.json new file mode 100644 index 00000000..1be0d73b --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/sts-rerank-bs1024.json @@ -0,0 +1,37 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "BIOSSES", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STS17", + "STS22.v2", + "STSBenchmark", + "AskUbuntuDupQuestions", + "MindSmallReranking" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-modernbert-en-1b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/subset-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/subset-bs1024.json new file mode 100644 index 00000000..4b611aca --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/subset-bs1024.json @@ -0,0 +1,29 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "data/outputs/eval/mteb/sentence-modernbert-en-1b-bs1024", + "tasks": [ + "SciFact", + "STSBenchmark", + "Banking77Classification.v2", + "AskUbuntuDupQuestions", + "StackExchangeClustering.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb/sentence-modernbert-en-1b.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-modernbert-en-1b.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-1B/subset.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-3B/subset-bs2048.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-3B/subset-bs2048.json new file mode 100644 index 00000000..31905417 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-3B/subset-bs2048.json @@ -0,0 +1,29 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-3B-bs2048", + "output_dir": "data/outputs/eval/mteb/sentence-modernbert-en-3b-bs2048", + "tasks": [ + "SciFact", + "STSBenchmark", + "Banking77Classification.v2", + "AskUbuntuDupQuestions", + "StackExchangeClustering.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb/sentence-modernbert-en-3b.json b/config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-3B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-modernbert-en-3b.json rename to config/constract_llm/eval/ft/mteb/Sentence-ModernBERT-EN-3B/subset.json diff --git a/config/constract_llm/eval/mteb_full/sentence-qwen2.5-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-qwen2.5-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-bi-en-0.5b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/mteb/sentence-qwen2.5-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-qwen2.5-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Bi-EN-0.5B/subset.json diff --git a/config/constract_llm/eval/mteb_full/sentence-qwen2.5-scratch-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-qwen2.5-scratch-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-qwen2.5-scratch-bi-en-0.5b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/mteb/sentence-qwen2.5-scratch-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-qwen2.5-scratch-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/subset.json diff --git a/config/constract_llm/eval/mteb_full/sentence-sarashina-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-sarashina-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-0.5b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/mteb/sentence-sarashina-bi-en-0.5b.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-sarashina-bi-en-0.5b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-0.5B/subset.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full-bs1024.json new file mode 100644 index 00000000..2154fec9 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full-bs1024.json @@ -0,0 +1,63 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "ArguAna", + "ArXivHierarchicalClusteringP2P", + "ArXivHierarchicalClusteringS2S", + "AskUbuntuDupQuestions", + "BIOSSES", + "Banking77Classification", + "BiorxivClusteringP2P.v2", + "CQADupstackGamingRetrieval", + "CQADupstackUnixRetrieval", + "ClimateFEVERHardNegatives", + "FEVERHardNegatives", + "FiQA2018", + "HotpotQAHardNegatives", + "ImdbClassification", + "MTOPDomainClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "MindSmallReranking", + "SCIDOCS", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STSBenchmark", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "TRECCOVID", + "Touche2020Retrieval.v3", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "TwentyNewsgroupsClustering.v2", + "AmazonCounterfactualClassification", + "STS17", + "STS22.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full/sentence-sarashina-bi-en-1b.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full.json similarity index 100% rename from config/constract_llm/eval/mteb_full/sentence-sarashina-bi-en-1b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/clf-clust-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/clf-clust-bs1024.json new file mode 100644 index 00000000..b82cd40b --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/clf-clust-bs1024.json @@ -0,0 +1,42 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "AmazonCounterfactualClassification", + "Banking77Classification", + "ImdbClassification", + "MTOPDomainClassification", + "MassiveIntentClassification", + "MassiveScenarioClassification", + "ToxicConversationsClassification", + "TweetSentimentExtractionClassification", + "ArXivHierarchicalClusteringP2P", + "ArXivHierarchicalClusteringS2S", + "BiorxivClusteringP2P.v2", + "MedrxivClusteringP2P.v2", + "MedrxivClusteringS2S.v2", + "StackExchangeClustering.v2", + "StackExchangeClusteringP2P.v2", + "TwentyNewsgroupsClustering.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--clf-clust.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/clf-clust.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--clf-clust.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/clf-clust.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-arguana-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-arguana-bs1024.json new file mode 100644 index 00000000..3e73518b --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-arguana-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "ArguAna" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-arguana.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-arguana.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-arguana.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-arguana.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-climatefeverhardnega-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-climatefeverhardnega-bs1024.json new file mode 100644 index 00000000..eb13ba0d --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-climatefeverhardnega-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "ClimateFEVERHardNegatives" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-climatefeverhardnega.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-climatefeverhardnega.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-climatefeverhardnega.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-climatefeverhardnega.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret-bs1024.json new file mode 100644 index 00000000..e4fcead3 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "CQADupstackGamingRetrieval" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-cqadupstackgamingret.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-cqadupstackgamingret.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackgamingret.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri-bs1024.json new file mode 100644 index 00000000..431db0fc --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "CQADupstackUnixRetrieval" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-cqadupstackunixretri.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-cqadupstackunixretri.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-cqadupstackunixretri.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-feverhardnegatives-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-feverhardnegatives-bs1024.json new file mode 100644 index 00000000..1ac75446 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-feverhardnegatives-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "FEVERHardNegatives" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-feverhardnegatives.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-feverhardnegatives.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-feverhardnegatives.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-feverhardnegatives.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-fiqa2018-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-fiqa2018-bs1024.json new file mode 100644 index 00000000..cb2346e9 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-fiqa2018-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "FiQA2018" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-fiqa2018.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-fiqa2018.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-fiqa2018.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-fiqa2018.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative-bs1024.json new file mode 100644 index 00000000..9991d451 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "HotpotQAHardNegatives" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-hotpotqahardnegative.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-hotpotqahardnegative.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-hotpotqahardnegative.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-scidocs-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-scidocs-bs1024.json new file mode 100644 index 00000000..f9d8252a --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-scidocs-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "SCIDOCS" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-scidocs.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-scidocs.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-scidocs.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-scidocs.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-touche2020retrieval-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-touche2020retrieval-bs1024.json new file mode 100644 index 00000000..bcb3f9d1 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-touche2020retrieval-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "Touche2020Retrieval.v3" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-touche2020retrieval..json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-touche2020retrieval.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-touche2020retrieval..json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-touche2020retrieval.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-treccovid-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-treccovid-bs1024.json new file mode 100644 index 00000000..a53e48ae --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-treccovid-bs1024.json @@ -0,0 +1,27 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "TRECCOVID" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-treccovid.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-treccovid.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--retr-treccovid.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/retr-treccovid.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/sts-rerank-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/sts-rerank-bs1024.json new file mode 100644 index 00000000..23317e17 --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/sts-rerank-bs1024.json @@ -0,0 +1,37 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "/cl/work16/tatsuki-o/mirei_devsplit/mteb_full/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "BIOSSES", + "SICK-R", + "STS12", + "STS13", + "STS14", + "STS15", + "STS17", + "STS22.v2", + "STSBenchmark", + "AskUbuntuDupQuestions", + "MindSmallReranking" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": { + "torch_dtype": "bfloat16" + }, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--sts-rerank.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/sts-rerank.json similarity index 100% rename from config/constract_llm/eval/mteb_full_chunks/sentence-sarashina-bi-en-1b--sts-rerank.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/full_chunks/sts-rerank.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/subset-bs1024.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/subset-bs1024.json new file mode 100644 index 00000000..e58cd37f --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/subset-bs1024.json @@ -0,0 +1,29 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "data/outputs/eval/mteb/sentence-sarashina-bi-en-1b-bs1024", + "tasks": [ + "SciFact", + "STSBenchmark", + "Banking77Classification.v2", + "AskUbuntuDupQuestions", + "StackExchangeClustering.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb/sentence-sarashina-bi-en-1b.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-sarashina-bi-en-1b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-1B/subset.json diff --git a/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-3B/subset-bs2048.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-3B/subset-bs2048.json new file mode 100644 index 00000000..8ad06dec --- /dev/null +++ b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-3B/subset-bs2048.json @@ -0,0 +1,29 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-3B-bs2048", + "output_dir": "data/outputs/eval/mteb/sentence-sarashina-bi-en-3b-bs2048", + "tasks": [ + "SciFact", + "STSBenchmark", + "Banking77Classification.v2", + "AskUbuntuDupQuestions", + "StackExchangeClustering.v2" + ], + "languages": [ + "eng" + ], + "eval_splits": null, + "exclusive_language_filter": false, + "overwrite_strategy": "only-missing", + "public_only": true, + "raise_error": true, + "num_proc": null, + "batch_size": 32, + "normalize_embeddings": true, + "show_progress_bar": true, + "device": "cuda", + "trust_remote_code": true, + "attn_implementation": "flash_attention_2", + "model_extra_kwargs": {}, + "tokenizer_extra_kwargs": {}, + "use_mteb_get_model": false +} diff --git a/config/constract_llm/eval/mteb/sentence-sarashina-bi-en-3b.json b/config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-3B/subset.json similarity index 100% rename from config/constract_llm/eval/mteb/sentence-sarashina-bi-en-3b.json rename to config/constract_llm/eval/ft/mteb/Sentence-Sarashina-Bi-EN-3B/subset.json diff --git a/config/constract_llm/eval/isotropic/Llama-0.5B.json b/config/constract_llm/eval/isotropic/Llama-0.5B.json deleted file mode 100644 index fd6976b6..00000000 --- a/config/constract_llm/eval/isotropic/Llama-0.5B.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-PT-stage2", - "output_dir": "data/outputs/eval/isotropic/Llama-0.5B", - "num_examples": 2000, - "batch_size": 2, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Llama-3B.json b/config/constract_llm/eval/isotropic/Llama-3B.json deleted file mode 100644 index e2989994..00000000 --- a/config/constract_llm/eval/isotropic/Llama-3B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2", - "output_dir": "data/outputs/eval/isotropic/Llama-3B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Llama-Bi-0.5B.json b/config/constract_llm/eval/isotropic/Llama-Bi-0.5B.json deleted file mode 100644 index 810157c1..00000000 --- a/config/constract_llm/eval/isotropic/Llama-Bi-0.5B.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-0.5B-PT-stage2", - "output_dir": "data/outputs/eval/isotropic/Llama-Bi-0.5B", - "num_examples": 2000, - "batch_size": 2, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Llama-Bi-3B.json b/config/constract_llm/eval/isotropic/Llama-Bi-3B.json deleted file mode 100644 index d30e6136..00000000 --- a/config/constract_llm/eval/isotropic/Llama-Bi-3B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-3B-PT-stage2", - "output_dir": "data/outputs/eval/isotropic/Llama-Bi-3B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/ModernBERT-0.5B.json b/config/constract_llm/eval/isotropic/ModernBERT-0.5B.json deleted file mode 100644 index c28bbb5c..00000000 --- a/config/constract_llm/eval/isotropic/ModernBERT-0.5B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-PT-stage2", - "output_dir": "data/outputs/eval/isotropic/Modernbert-0.5B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/ModernBERT-3B.json b/config/constract_llm/eval/isotropic/ModernBERT-3B.json deleted file mode 100644 index c09e6771..00000000 --- a/config/constract_llm/eval/isotropic/ModernBERT-3B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2", - "output_dir": "data/outputs/eval/isotropic/ModernBERT-3B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sarashina-0.5B.json b/config/constract_llm/eval/isotropic/Sarashina-0.5B.json deleted file mode 100644 index c73e89bd..00000000 --- a/config/constract_llm/eval/isotropic/Sarashina-0.5B.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "model_name_or_path": "sbintuitions/sarashina2.2-0.5b", - "output_dir": "data/outputs/eval/isotropic/Sarashina-0.5B", - "num_examples": 2000, - "batch_size": 2, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sarashina-3B.json b/config/constract_llm/eval/isotropic/Sarashina-3B.json deleted file mode 100644 index 4318ad53..00000000 --- a/config/constract_llm/eval/isotropic/Sarashina-3B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "sbintuitions/sarashina2.2-3b", - "output_dir": "data/outputs/eval/isotropic/Sarashina-3B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sarashina-Bi-0.5B.json b/config/constract_llm/eval/isotropic/Sarashina-Bi-0.5B.json deleted file mode 100644 index b94a2532..00000000 --- a/config/constract_llm/eval/isotropic/Sarashina-Bi-0.5B.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-0.5b", - "output_dir": "data/outputs/eval/isotropic/Sarashina-Bi-0.5B", - "num_examples": 2000, - "batch_size": 2, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sarashina-Bi-3B.json b/config/constract_llm/eval/isotropic/Sarashina-Bi-3B.json deleted file mode 100644 index 22776e8e..00000000 --- a/config/constract_llm/eval/isotropic/Sarashina-Bi-3B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-3b", - "output_dir": "data/outputs/eval/isotropic/Sarashina-Bi-3B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-0.5B.json b/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-0.5B.json deleted file mode 100644 index d6569d80..00000000 --- a/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-0.5B.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-0.5B", - "output_dir": "data/outputs/eval/isotropic/Sentence-Llama-Bi-0.5B", - "num_examples": 2000, - "batch_size": 2, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-3B.json b/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-3B.json deleted file mode 100644 index cc2eefd0..00000000 --- a/config/constract_llm/eval/isotropic/Sentence-Llama-Bi-3B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B", - "output_dir": "data/outputs/eval/isotropic/Sentence-Llama-Bi-3B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sentence-ModernBERT-0.5B.json b/config/constract_llm/eval/isotropic/Sentence-ModernBERT-0.5B.json deleted file mode 100644 index 2649a4fb..00000000 --- a/config/constract_llm/eval/isotropic/Sentence-ModernBERT-0.5B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-0.5B", - "output_dir": "data/outputs/eval/isotropic/Sentence-Modernbert-0.5B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sentence-ModernBERT-3B.json b/config/constract_llm/eval/isotropic/Sentence-ModernBERT-3B.json deleted file mode 100644 index 9d1de542..00000000 --- a/config/constract_llm/eval/isotropic/Sentence-ModernBERT-3B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-3B", - "output_dir": "data/outputs/eval/isotropic/Sentence-ModernBERT-3B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-0.5B.json b/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-0.5B.json deleted file mode 100644 index 6ca12087..00000000 --- a/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-0.5B.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-0.5B", - "output_dir": "data/outputs/eval/isotropic/Sentence-Sarashina-Bi-0.5B", - "num_examples": 2000, - "batch_size": 2, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-3B.json b/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-3B.json deleted file mode 100644 index f38e6f7e..00000000 --- a/config/constract_llm/eval/isotropic/Sentence-Sarashina-Bi-3B.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-3B", - "output_dir": "data/outputs/eval/isotropic/Sentence-Sarashina-Bi-3B", - "num_examples": 2000, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/isotropic/example.json b/config/constract_llm/eval/isotropic/example.json deleted file mode 100644 index 45d63fd0..00000000 --- a/config/constract_llm/eval/isotropic/example.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model_name_or_path": "cl-nagoya/ruri-v3-30m", - "output_dir": "data/outputs/eval/isotropic", - "num_examples": 100, - "dtype": "bfloat16", - "attn_implementation": "flash_attention_2", - "seed": 42, - "miracl_name": "miracl/miracl", - "miracl_lang": "ja", - "wiki_name": "wikimedia/wikipedia", - "wiki_lang": "20231101.ja" -} diff --git a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jcola.json b/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jcola.json deleted file mode 100644 index a075fad6..00000000 --- a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jcola.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-PT-stage2", - "task_name": "JCoLA", - "trust_remote_code": true, - "dataset_name": null, - "dataset_config_name": null, - "max_train_samples": null, - "max_eval_samples": null, - "run_name": "Llama-JP-0.5B-PT-stage2-jcola-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jcola-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jcola-test/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "num_train_epochs": 3, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 2e-5, - "per_device_train_batch_size": 32, - "per_device_eval_batch_size": 32, - "gradient_accumulation_steps": 1, - "eval_accumulation_steps": 1, - "max_seq_length": 512, - "force_add_bos_token": true, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "metric_for_best_model": "accuracy", - "logging_first_step": true, - "prediction_loss_only": false, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 2, - "load_best_model_at_end": true, - "greater_is_better": true, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" -} diff --git a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jcola.json b/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jcola.json deleted file mode 100644 index e56adebf..00000000 --- a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jcola.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2", - "task_name": "JCoLA", - "trust_remote_code": true, - "dataset_name": null, - "dataset_config_name": null, - "max_train_samples": null, - "max_eval_samples": null, - "run_name": "Llama-JP-3B-PT-stage2-jcola-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jcola-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jcola-test/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "num_train_epochs": 3, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-6, - "per_device_train_batch_size": 2, - "per_device_eval_batch_size": 2, - "gradient_accumulation_steps": 16, - "eval_accumulation_steps": 1, - "max_seq_length": 512, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "metric_for_best_model": "accuracy", - "logging_first_step": true, - "prediction_loss_only": false, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 2, - "load_best_model_at_end": true, - "greater_is_better": true, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" -} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jcola.json b/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jcola.json deleted file mode 100644 index cbc700e4..00000000 --- a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jcola.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-PT-stage2", - "task_name": "JCoLA", - "trust_remote_code": true, - "dataset_name": null, - "dataset_config_name": null, - "max_train_samples": null, - "max_eval_samples": null, - "run_name": "ModernBERT-JP-0.5B-PT-stage2-jcola-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jcola-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jcola-test/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "num_train_epochs": 3, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 2e-5, - "per_device_train_batch_size": 32, - "per_device_eval_batch_size": 32, - "gradient_accumulation_steps": 1, - "eval_accumulation_steps": 1, - "max_seq_length": 512, - "force_add_bos_token": true, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "metric_for_best_model": "accuracy", - "logging_first_step": true, - "prediction_loss_only": false, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 2, - "load_best_model_at_end": true, - "greater_is_better": true, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" -} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jcola.json b/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jcola.json deleted file mode 100644 index f7268ebc..00000000 --- a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jcola.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2", - "task_name": "JCoLA", - "trust_remote_code": true, - "dataset_name": null, - "dataset_config_name": null, - "max_train_samples": null, - "max_eval_samples": null, - "run_name": "ModernBERT-JP-3B-PT-stage2-jcola-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jcola-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jcola-test/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "num_train_epochs": 3, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-6, - "per_device_train_batch_size": 2, - "per_device_eval_batch_size": 2, - "gradient_accumulation_steps": 16, - "eval_accumulation_steps": 1, - "max_seq_length": 512, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "metric_for_best_model": "accuracy", - "logging_first_step": true, - "prediction_loss_only": false, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 2, - "load_best_model_at_end": true, - "greater_is_better": true, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" -} diff --git a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jcola.json b/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jcola.json deleted file mode 100644 index 23f97710..00000000 --- a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jcola.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "model_name_or_path": "sbintuitions/sarashina2.2-3b", - "task_name": "JCoLA", - "trust_remote_code": true, - "dataset_name": null, - "dataset_config_name": null, - "max_train_samples": null, - "max_eval_samples": null, - "run_name": "Sarashina-JP-3B-jcola-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jcola-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jcola-test/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "num_train_epochs": 3, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-6, - "per_device_train_batch_size": 2, - "per_device_eval_batch_size": 2, - "gradient_accumulation_steps": 16, - "eval_accumulation_steps": 1, - "max_seq_length": 512, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "metric_for_best_model": "accuracy", - "logging_first_step": true, - "prediction_loss_only": false, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 2, - "load_best_model_at_end": true, - "greater_is_better": true, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" -} diff --git a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_cola.json b/config/constract_llm/eval/pt/glue/InstructLM-EN-0.5B/cola.json similarity index 86% rename from config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_cola.json rename to config/constract_llm/eval/pt/glue/InstructLM-EN-0.5B/cola.json index 7d359571..d647ab80 100644 --- a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_cola.json +++ b/config/constract_llm/eval/pt/glue/InstructLM-EN-0.5B/cola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "InstructLM-EN-0.5B-cola-glue", - "output_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-cola-glue", - "logging_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-cola-glue", + "logging_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -46,6 +46,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_mnli.json b/config/constract_llm/eval/pt/glue/InstructLM-EN-0.5B/mnli.json similarity index 86% rename from config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_mnli.json rename to config/constract_llm/eval/pt/glue/InstructLM-EN-0.5B/mnli.json index 5c580357..fb56d5ae 100644 --- a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_mnli.json +++ b/config/constract_llm/eval/pt/glue/InstructLM-EN-0.5B/mnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "InstructLM-EN-0.5B-mnli-glue", - "output_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-mnli-glue", + "logging_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -46,6 +46,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_stsb.json b/config/constract_llm/eval/pt/glue/InstructLM-EN-0.5B/stsb.json similarity index 86% rename from config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_stsb.json rename to config/constract_llm/eval/pt/glue/InstructLM-EN-0.5B/stsb.json index 438eca14..43e48e09 100644 --- a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_stsb.json +++ b/config/constract_llm/eval/pt/glue/InstructLM-EN-0.5B/stsb.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "InstructLM-EN-0.5B-stsb-glue", - "output_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-stsb-glue", + "logging_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -46,6 +46,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_cola.json b/config/constract_llm/eval/pt/glue/Llama-EN-0.5B-PT-stage2/cola.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_cola.json rename to config/constract_llm/eval/pt/glue/Llama-EN-0.5B-PT-stage2/cola.json index 1c010a8f..36797a49 100644 --- a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_cola.json +++ b/config/constract_llm/eval/pt/glue/Llama-EN-0.5B-PT-stage2/cola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-0.5B-PT-stage2-cola-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-cola-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-cola-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_mnli.json b/config/constract_llm/eval/pt/glue/Llama-EN-0.5B-PT-stage2/mnli.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_mnli.json rename to config/constract_llm/eval/pt/glue/Llama-EN-0.5B-PT-stage2/mnli.json index d7415668..65211e62 100644 --- a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_mnli.json +++ b/config/constract_llm/eval/pt/glue/Llama-EN-0.5B-PT-stage2/mnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-0.5B-PT-stage2-mnli-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_stsb.json b/config/constract_llm/eval/pt/glue/Llama-EN-0.5B-PT-stage2/stsb.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_stsb.json rename to config/constract_llm/eval/pt/glue/Llama-EN-0.5B-PT-stage2/stsb.json index bb4a5052..d54f53b4 100644 --- a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_stsb.json +++ b/config/constract_llm/eval/pt/glue/Llama-EN-0.5B-PT-stage2/stsb.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-0.5B-PT-stage2-stsb-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/cola-bs1024.json b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/cola-bs1024.json new file mode 100644 index 00000000..3763d0c7 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/cola-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-1B-PT-stage2-bs1024", + "task_name": "cola", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "cola", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-1B-PT-stage2-bs1024-cola-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-cola-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-cola-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_cola.json b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/cola.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_cola.json rename to config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/cola.json index c3bee105..8e584fde 100644 --- a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_cola.json +++ b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/cola.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-1B-PT-stage2-cola-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-cola-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-cola-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/mnli-bs1024.json b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/mnli-bs1024.json new file mode 100644 index 00000000..28b737d4 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/mnli-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-1B-PT-stage2-bs1024", + "task_name": "mnli", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "mnli", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-1B-PT-stage2-bs1024-mnli-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-mnli-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_mnli.json b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/mnli.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_mnli.json rename to config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/mnli.json index 8485d548..a87590e1 100644 --- a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_mnli.json +++ b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/mnli.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-1B-PT-stage2-mnli-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/stsb-bs1024.json b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/stsb-bs1024.json new file mode 100644 index 00000000..eaa45419 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/stsb-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-1B-PT-stage2-bs1024", + "task_name": "stsb", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "stsb", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-1B-PT-stage2-bs1024-stsb-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-stsb-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "pearson", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_stsb.json b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/stsb.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_stsb.json rename to config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/stsb.json index 82d1faa6..ebd2baff 100644 --- a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_stsb.json +++ b/config/constract_llm/eval/pt/glue/Llama-EN-1B-PT-stage2/stsb.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-1B-PT-stage2-stsb-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/cola-bs2048.json b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/cola-bs2048.json new file mode 100644 index 00000000..ae230623 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/cola-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-3B-PT-stage2-bs2048", + "task_name": "cola", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "cola", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-3B-PT-stage2-bs2048-cola-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-cola-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-cola-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_cola.json b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/cola.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_cola.json rename to config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/cola.json index 683705bc..90ae4416 100644 --- a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_cola.json +++ b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/cola.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-3B-PT-stage2-cola-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-cola-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-cola-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/mnli-bs2048.json b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/mnli-bs2048.json new file mode 100644 index 00000000..ed2e03c6 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/mnli-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-3B-PT-stage2-bs2048", + "task_name": "mnli", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "mnli", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-3B-PT-stage2-bs2048-mnli-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-mnli-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_mnli.json b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/mnli.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_mnli.json rename to config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/mnli.json index d6a06f7e..d50bff27 100644 --- a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_mnli.json +++ b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/mnli.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-3B-PT-stage2-mnli-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/stsb-bs2048.json b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/stsb-bs2048.json new file mode 100644 index 00000000..51e7bb7f --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/stsb-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-3B-PT-stage2-bs2048", + "task_name": "stsb", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "stsb", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-3B-PT-stage2-bs2048-stsb-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-stsb-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "pearson", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_stsb.json b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/stsb.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_stsb.json rename to config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/stsb.json index c1bd0c3e..4fb4b448 100644 --- a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_stsb.json +++ b/config/constract_llm/eval/pt/glue/Llama-EN-3B-PT-stage2/stsb.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-3B-PT-stage2-stsb-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-JP-0.5B-PT-stage2/jcola.json b/config/constract_llm/eval/pt/glue/Llama-JP-0.5B-PT-stage2/jcola.json new file mode 100644 index 00000000..24b6f93b --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-JP-0.5B-PT-stage2/jcola.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-PT-stage2", + "task_name": "JCoLA", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-0.5B-PT-stage2-jcola-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jcola-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jcola-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 32, + "per_device_eval_batch_size": 32, + "gradient_accumulation_steps": 1, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jnli.json b/config/constract_llm/eval/pt/glue/Llama-JP-0.5B-PT-stage2/jnli.json similarity index 82% rename from config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jnli.json rename to config/constract_llm/eval/pt/glue/Llama-JP-0.5B-PT-stage2/jnli.json index 667a8d67..68d0a048 100644 --- a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jnli.json +++ b/config/constract_llm/eval/pt/glue/Llama-JP-0.5B-PT-stage2/jnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-0.5B-PT-stage2-jnli-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jnli-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jnli-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jnli-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jnli-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jsts.json b/config/constract_llm/eval/pt/glue/Llama-JP-0.5B-PT-stage2/jsts.json similarity index 82% rename from config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jsts.json rename to config/constract_llm/eval/pt/glue/Llama-JP-0.5B-PT-stage2/jsts.json index 743a99c6..004715e1 100644 --- a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jsts.json +++ b/config/constract_llm/eval/pt/glue/Llama-JP-0.5B-PT-stage2/jsts.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-0.5B-PT-stage2-jsts-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jsts-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jsts-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jsts-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jsts-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "pearson", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jcola-bs1024.json b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jcola-bs1024.json new file mode 100644 index 00000000..0d94c664 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jcola-bs1024.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-PT-stage2-bs1024", + "task_name": "JCoLA", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-1B-PT-stage2-bs1024-jcola-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jcola-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jcola-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jcola.json b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jcola.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jcola.json rename to config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jcola.json index 149f6fa1..0a71ebe0 100644 --- a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jcola.json +++ b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jcola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-1B-PT-stage2-jcola-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jcola-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jcola-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jcola-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jcola-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jnli-bs1024.json b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jnli-bs1024.json new file mode 100644 index 00000000..b2b2b756 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jnli-bs1024.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-PT-stage2-bs1024", + "task_name": "JNLI", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-1B-PT-stage2-bs1024-jnli-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jnli-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jnli-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jnli.json b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jnli.json similarity index 82% rename from config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jnli.json rename to config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jnli.json index a27cf8c3..4c48937a 100644 --- a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jnli.json +++ b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-1B-PT-stage2-jnli-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jnli-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jnli-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jnli-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jnli-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jsts-bs1024.json b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jsts-bs1024.json new file mode 100644 index 00000000..a49fcc67 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jsts-bs1024.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-PT-stage2-bs1024", + "task_name": "JSTS", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-1B-PT-stage2-bs1024-jsts-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jsts-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jsts-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "pearson", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jsts.json b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jsts.json similarity index 82% rename from config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jsts.json rename to config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jsts.json index 209a6ad0..4d82a8d1 100644 --- a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jsts.json +++ b/config/constract_llm/eval/pt/glue/Llama-JP-1B-PT-stage2/jsts.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-1B-PT-stage2-jsts-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jsts-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jsts-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jsts-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jsts-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "pearson", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jcola-bs2048.json b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jcola-bs2048.json new file mode 100644 index 00000000..7e5e77d5 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jcola-bs2048.json @@ -0,0 +1,50 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2-bs2048", + "task_name": "JCoLA", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-3B-PT-stage2-bs2048-jcola-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jcola-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jcola-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jcola.json b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jcola.json new file mode 100644 index 00000000..3884b81d --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jcola.json @@ -0,0 +1,50 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2", + "task_name": "JCoLA", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-3B-PT-stage2-jcola-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jcola-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jcola-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jnli-bs2048.json b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jnli-bs2048.json new file mode 100644 index 00000000..abf7b483 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jnli-bs2048.json @@ -0,0 +1,50 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2-bs2048", + "task_name": "JNLI", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-3B-PT-stage2-bs2048-jnli-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jnli-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jnli-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jnli.json b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jnli.json similarity index 81% rename from config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jnli.json rename to config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jnli.json index a53f79fe..bb3fd8a1 100644 --- a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jnli.json +++ b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-3B-PT-stage2-jnli-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jnli-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jnli-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jnli-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jnli-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -23,10 +23,10 @@ "gradient_accumulation_steps": 16, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -46,5 +46,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jsts-bs2048.json b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jsts-bs2048.json new file mode 100644 index 00000000..66aacfaf --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jsts-bs2048.json @@ -0,0 +1,50 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2-bs2048", + "task_name": "JSTS", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-3B-PT-stage2-bs2048-jsts-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jsts-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jsts-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "pearson", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jsts.json b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jsts.json similarity index 81% rename from config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jsts.json rename to config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jsts.json index 5f6ffb7c..e5215f38 100644 --- a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jsts.json +++ b/config/constract_llm/eval/pt/glue/Llama-JP-3B-PT-stage2/jsts.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-3B-PT-stage2-jsts-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jsts-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jsts-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jsts-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jsts-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -23,10 +23,10 @@ "gradient_accumulation_steps": 16, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "pearson", "logging_first_step": true, @@ -46,5 +46,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_cola.json b/config/constract_llm/eval/pt/glue/Mistral-EN-0.5B-PT-stage2/cola.json similarity index 85% rename from config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_cola.json rename to config/constract_llm/eval/pt/glue/Mistral-EN-0.5B-PT-stage2/cola.json index 32a5f85a..18fc5458 100644 --- a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_cola.json +++ b/config/constract_llm/eval/pt/glue/Mistral-EN-0.5B-PT-stage2/cola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Mistral-EN-0.5B-PT-stage2-cola-glue", - "output_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-cola-glue", - "logging_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-cola-glue", + "logging_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_mnli.json b/config/constract_llm/eval/pt/glue/Mistral-EN-0.5B-PT-stage2/mnli.json similarity index 85% rename from config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_mnli.json rename to config/constract_llm/eval/pt/glue/Mistral-EN-0.5B-PT-stage2/mnli.json index 5ad9a4d2..3f48a4d4 100644 --- a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_mnli.json +++ b/config/constract_llm/eval/pt/glue/Mistral-EN-0.5B-PT-stage2/mnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Mistral-EN-0.5B-PT-stage2-mnli-glue", - "output_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_stsb.json b/config/constract_llm/eval/pt/glue/Mistral-EN-0.5B-PT-stage2/stsb.json similarity index 85% rename from config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_stsb.json rename to config/constract_llm/eval/pt/glue/Mistral-EN-0.5B-PT-stage2/stsb.json index afd73e58..c061d562 100644 --- a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_stsb.json +++ b/config/constract_llm/eval/pt/glue/Mistral-EN-0.5B-PT-stage2/stsb.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Mistral-EN-0.5B-PT-stage2-stsb-glue", - "output_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_cola.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-0.5B-PT-stage2/cola.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_cola.json rename to config/constract_llm/eval/pt/glue/ModernBERT-EN-0.5B-PT-stage2/cola.json index f20d8c85..31919d78 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_cola.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-0.5B-PT-stage2/cola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-0.5B-PT-stage2-cola-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-cola-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-cola-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_mnli.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-0.5B-PT-stage2/mnli.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_mnli.json rename to config/constract_llm/eval/pt/glue/ModernBERT-EN-0.5B-PT-stage2/mnli.json index 02a6e36a..70a7a92a 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_mnli.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-0.5B-PT-stage2/mnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-0.5B-PT-stage2-mnli-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-mnli-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_stsb.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-0.5B-PT-stage2/stsb.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_stsb.json rename to config/constract_llm/eval/pt/glue/ModernBERT-EN-0.5B-PT-stage2/stsb.json index bf8c0dc7..c7ffab42 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_stsb.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-0.5B-PT-stage2/stsb.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-0.5B-PT-stage2-stsb-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-stsb-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/cola-bs1024.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/cola-bs1024.json new file mode 100644 index 00000000..324d67ba --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/cola-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage2-bs1024", + "task_name": "cola", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "cola", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-1B-PT-stage2-bs1024-cola-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-cola-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-cola-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_cola.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/cola.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_cola.json rename to config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/cola.json index 85f9e6f4..92e63904 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_cola.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/cola.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-1B-PT-stage2-cola-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-cola-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-cola-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/mnli-bs1024.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/mnli-bs1024.json new file mode 100644 index 00000000..464b0274 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/mnli-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage2-bs1024", + "task_name": "mnli", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "mnli", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-1B-PT-stage2-bs1024-mnli-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-mnli-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-mnli-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_mnli.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/mnli.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_mnli.json rename to config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/mnli.json index 41703ba0..4f53a7ed 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_mnli.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/mnli.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-1B-PT-stage2-mnli-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-mnli-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/stsb-bs1024.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/stsb-bs1024.json new file mode 100644 index 00000000..ebe9af3d --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/stsb-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage2-bs1024", + "task_name": "stsb", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "stsb", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-1B-PT-stage2-bs1024-stsb-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-stsb-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-stsb-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "pearson", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_stsb.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/stsb.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_stsb.json rename to config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/stsb.json index f16ddef5..9556355b 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_stsb.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-1B-PT-stage2/stsb.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-1B-PT-stage2-stsb-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-stsb-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/cola-bs2048.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/cola-bs2048.json new file mode 100644 index 00000000..f7ef937c --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/cola-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-3B-PT-stage2-bs2048", + "task_name": "cola", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "cola", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-3B-PT-stage2-bs2048-cola-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-cola-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-cola-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_cola.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/cola.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_cola.json rename to config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/cola.json index 56a201a1..526a9d32 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_cola.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/cola.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-3B-PT-stage2-cola-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-cola-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-cola-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/mnli-bs2048.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/mnli-bs2048.json new file mode 100644 index 00000000..7e750433 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/mnli-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-3B-PT-stage2-bs2048", + "task_name": "mnli", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "mnli", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-3B-PT-stage2-bs2048-mnli-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-mnli-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-mnli-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_mnli.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/mnli.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_mnli.json rename to config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/mnli.json index 53dc1c51..9f4b0383 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_mnli.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/mnli.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-3B-PT-stage2-mnli-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-mnli-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/stsb-bs2048.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/stsb-bs2048.json new file mode 100644 index 00000000..07516b9b --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/stsb-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-3B-PT-stage2-bs2048", + "task_name": "stsb", + "dataset_name": "nyu-mll/glue", + "dataset_config_name": "stsb", + "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c", + "trust_remote_code": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-3B-PT-stage2-bs2048-stsb-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-stsb-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-stsb-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "pearson", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_stsb.json b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/stsb.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_stsb.json rename to config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/stsb.json index 4cc6fbcf..62c4f45b 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_stsb.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-EN-3B-PT-stage2/stsb.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-3B-PT-stage2-stsb-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-stsb-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-JP-0.5B-PT-stage2/jcola.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-0.5B-PT-stage2/jcola.json new file mode 100644 index 00000000..5b0ae227 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-0.5B-PT-stage2/jcola.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-PT-stage2", + "task_name": "JCoLA", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-0.5B-PT-stage2-jcola-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jcola-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jcola-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 32, + "per_device_eval_batch_size": 32, + "gradient_accumulation_steps": 1, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jnli.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-0.5B-PT-stage2/jnli.json similarity index 81% rename from config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jnli.json rename to config/constract_llm/eval/pt/glue/ModernBERT-JP-0.5B-PT-stage2/jnli.json index c3e5d300..d3b587ea 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jnli.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-0.5B-PT-stage2/jnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-0.5B-PT-stage2-jnli-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jnli-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jnli-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jnli-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jnli-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jsts.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-0.5B-PT-stage2/jsts.json similarity index 81% rename from config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jsts.json rename to config/constract_llm/eval/pt/glue/ModernBERT-JP-0.5B-PT-stage2/jsts.json index 357d7617..d4b1fb97 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jsts.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-0.5B-PT-stage2/jsts.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-0.5B-PT-stage2-jsts-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jsts-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jsts-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jsts-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jsts-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "pearson", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jcola-bs1024.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jcola-bs1024.json new file mode 100644 index 00000000..82237914 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jcola-bs1024.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-PT-stage2-bs1024", + "task_name": "JCoLA", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-1B-PT-stage2-bs1024-jcola-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jcola-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jcola-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jcola.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jcola.json similarity index 84% rename from config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jcola.json rename to config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jcola.json index 50c5fc69..498ea209 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jcola.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jcola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-1B-PT-stage2-jcola-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jcola-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jcola-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jcola-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jcola-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jnli-bs1024.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jnli-bs1024.json new file mode 100644 index 00000000..e66326e9 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jnli-bs1024.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-PT-stage2-bs1024", + "task_name": "JNLI", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-1B-PT-stage2-bs1024-jnli-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jnli-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jnli-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jnli.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jnli.json similarity index 81% rename from config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jnli.json rename to config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jnli.json index b23a14f6..89cb7494 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jnli.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-1B-PT-stage2-jnli-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jnli-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jnli-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jnli-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jnli-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jsts-bs1024.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jsts-bs1024.json new file mode 100644 index 00000000..9203df1c --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jsts-bs1024.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-PT-stage2-bs1024", + "task_name": "JSTS", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-1B-PT-stage2-bs1024-jsts-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jsts-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jsts-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "force_add_bos_token": true, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "pearson", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jsts.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jsts.json similarity index 81% rename from config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jsts.json rename to config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jsts.json index 4a46667c..4460de58 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jsts.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-1B-PT-stage2/jsts.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-1B-PT-stage2-jsts-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jsts-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jsts-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jsts-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jsts-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "pearson", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jcola-bs2048.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jcola-bs2048.json new file mode 100644 index 00000000..e1cd415f --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jcola-bs2048.json @@ -0,0 +1,50 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2-bs2048", + "task_name": "JCoLA", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-3B-PT-stage2-bs2048-jcola-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jcola-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jcola-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jcola.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jcola.json new file mode 100644 index 00000000..3d5e3670 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jcola.json @@ -0,0 +1,50 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2", + "task_name": "JCoLA", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-3B-PT-stage2-jcola-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jcola-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jcola-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jnli-bs2048.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jnli-bs2048.json new file mode 100644 index 00000000..1a840726 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jnli-bs2048.json @@ -0,0 +1,50 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2-bs2048", + "task_name": "JNLI", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-3B-PT-stage2-bs2048-jnli-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jnli-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jnli-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jnli.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jnli.json similarity index 81% rename from config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jnli.json rename to config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jnli.json index 27fc4f7e..84518061 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jnli.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-3B-PT-stage2-jnli-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jnli-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jnli-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jnli-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jnli-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -23,10 +23,10 @@ "gradient_accumulation_steps": 16, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -46,5 +46,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jsts-bs2048.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jsts-bs2048.json new file mode 100644 index 00000000..a199e38c --- /dev/null +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jsts-bs2048.json @@ -0,0 +1,50 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2-bs2048", + "task_name": "JSTS", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-3B-PT-stage2-bs2048-jsts-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jsts-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jsts-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "pearson", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jsts.json b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jsts.json similarity index 81% rename from config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jsts.json rename to config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jsts.json index 086c3f8a..5bad09c7 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jsts.json +++ b/config/constract_llm/eval/pt/glue/ModernBERT-JP-3B-PT-stage2/jsts.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-3B-PT-stage2-jsts-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jsts-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jsts-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jsts-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jsts-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -23,10 +23,10 @@ "gradient_accumulation_steps": 16, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "pearson", "logging_first_step": true, @@ -46,5 +46,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_cola.json b/config/constract_llm/eval/pt/glue/Qwen2.5-EN-0.5B/cola.json similarity index 86% rename from config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_cola.json rename to config/constract_llm/eval/pt/glue/Qwen2.5-EN-0.5B/cola.json index 147327a4..58e9ba0d 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_cola.json +++ b/config/constract_llm/eval/pt/glue/Qwen2.5-EN-0.5B/cola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-EN-0.5B-cola-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-cola-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-cola-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -46,6 +46,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_mnli.json b/config/constract_llm/eval/pt/glue/Qwen2.5-EN-0.5B/mnli.json similarity index 86% rename from config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_mnli.json rename to config/constract_llm/eval/pt/glue/Qwen2.5-EN-0.5B/mnli.json index e85399a3..e9d9b6a4 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_mnli.json +++ b/config/constract_llm/eval/pt/glue/Qwen2.5-EN-0.5B/mnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-EN-0.5B-mnli-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -46,6 +46,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_stsb.json b/config/constract_llm/eval/pt/glue/Qwen2.5-EN-0.5B/stsb.json similarity index 86% rename from config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_stsb.json rename to config/constract_llm/eval/pt/glue/Qwen2.5-EN-0.5B/stsb.json index 4ed53898..85570e44 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_stsb.json +++ b/config/constract_llm/eval/pt/glue/Qwen2.5-EN-0.5B/stsb.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-EN-0.5B-stsb-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -46,6 +46,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_cola.json b/config/constract_llm/eval/pt/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2/cola.json similarity index 85% rename from config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_cola.json rename to config/constract_llm/eval/pt/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2/cola.json index 13622f35..4ea9fd60 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_cola.json +++ b/config/constract_llm/eval/pt/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2/cola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-Scratch-EN-0.5B-PT-stage2-cola-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-cola-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-cola-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_mnli.json b/config/constract_llm/eval/pt/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2/mnli.json similarity index 85% rename from config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_mnli.json rename to config/constract_llm/eval/pt/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2/mnli.json index cb8e96d2..fea1d8d1 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_mnli.json +++ b/config/constract_llm/eval/pt/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2/mnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-Scratch-EN-0.5B-PT-stage2-mnli-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_stsb.json b/config/constract_llm/eval/pt/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2/stsb.json similarity index 85% rename from config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_stsb.json rename to config/constract_llm/eval/pt/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2/stsb.json index 03c2093f..d4a11ec1 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_stsb.json +++ b/config/constract_llm/eval/pt/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2/stsb.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-Scratch-EN-0.5B-PT-stage2-stsb-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,6 +47,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "dataset_revision": "bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_cola.json b/config/constract_llm/eval/pt/glue/Sarashina-EN-0.5B/cola.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_cola.json rename to config/constract_llm/eval/pt/glue/Sarashina-EN-0.5B/cola.json index 9eb5ba85..7b499e35 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_cola.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-EN-0.5B/cola.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-0.5B-cola-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-cola-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-cola-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_mnli.json b/config/constract_llm/eval/pt/glue/Sarashina-EN-0.5B/mnli.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_mnli.json rename to config/constract_llm/eval/pt/glue/Sarashina-EN-0.5B/mnli.json index 854c13f4..cd38a5b2 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_mnli.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-EN-0.5B/mnli.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-0.5B-mnli-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_stsb.json b/config/constract_llm/eval/pt/glue/Sarashina-EN-0.5B/stsb.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_stsb.json rename to config/constract_llm/eval/pt/glue/Sarashina-EN-0.5B/stsb.json index a340f743..3feff776 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_stsb.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-EN-0.5B/stsb.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-0.5B-stsb-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_cola.json b/config/constract_llm/eval/pt/glue/Sarashina-EN-1B/cola.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_cola.json rename to config/constract_llm/eval/pt/glue/Sarashina-EN-1B/cola.json index 02a25159..418902a2 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_cola.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-EN-1B/cola.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-1B-cola-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-cola-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-1B-cola-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-1B-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_mnli.json b/config/constract_llm/eval/pt/glue/Sarashina-EN-1B/mnli.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_mnli.json rename to config/constract_llm/eval/pt/glue/Sarashina-EN-1B/mnli.json index 1a862a68..74981a85 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_mnli.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-EN-1B/mnli.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-1B-mnli-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-1B-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-1B-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_stsb.json b/config/constract_llm/eval/pt/glue/Sarashina-EN-1B/stsb.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_stsb.json rename to config/constract_llm/eval/pt/glue/Sarashina-EN-1B/stsb.json index d64b2e9b..3df783e6 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_stsb.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-EN-1B/stsb.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-1B-stsb-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-1B-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-1B-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_cola.json b/config/constract_llm/eval/pt/glue/Sarashina-EN-3B/cola.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_cola.json rename to config/constract_llm/eval/pt/glue/Sarashina-EN-3B/cola.json index 32f9c01e..b059279f 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_cola.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-EN-3B/cola.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-3B-cola-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-cola-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-cola-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-3B-cola-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-3B-cola-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_mnli.json b/config/constract_llm/eval/pt/glue/Sarashina-EN-3B/mnli.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_mnli.json rename to config/constract_llm/eval/pt/glue/Sarashina-EN-3B/mnli.json index 00c23d0c..c4936b63 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_mnli.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-EN-3B/mnli.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-3B-mnli-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-mnli-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-mnli-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-3B-mnli-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-3B-mnli-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_stsb.json b/config/constract_llm/eval/pt/glue/Sarashina-EN-3B/stsb.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_stsb.json rename to config/constract_llm/eval/pt/glue/Sarashina-EN-3B/stsb.json index 53312948..bd11d7e7 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_stsb.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-EN-3B/stsb.json @@ -8,8 +8,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-3B-stsb-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-stsb-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-stsb-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-3B-stsb-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-3B-stsb-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jcola.json b/config/constract_llm/eval/pt/glue/Sarashina-JP-0.5B/jcola.json similarity index 82% rename from config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jcola.json rename to config/constract_llm/eval/pt/glue/Sarashina-JP-0.5B/jcola.json index d8e952a4..fa8dc974 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jcola.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-JP-0.5B/jcola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-0.5B-jcola-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jcola-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jcola-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jcola-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jcola-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 0.00001, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 0.000001, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jnli.json b/config/constract_llm/eval/pt/glue/Sarashina-JP-0.5B/jnli.json similarity index 82% rename from config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jnli.json rename to config/constract_llm/eval/pt/glue/Sarashina-JP-0.5B/jnli.json index 95a877d7..3f16901d 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jnli.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-JP-0.5B/jnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-0.5B-jnli-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jnli-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jnli-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jnli-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jnli-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jsts.json b/config/constract_llm/eval/pt/glue/Sarashina-JP-0.5B/jsts.json similarity index 82% rename from config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jsts.json rename to config/constract_llm/eval/pt/glue/Sarashina-JP-0.5B/jsts.json index b7b6ccc6..a41c3a80 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jsts.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-JP-0.5B/jsts.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-0.5B-jsts-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jsts-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jsts-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jsts-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jsts-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "pearson", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jcola.json b/config/constract_llm/eval/pt/glue/Sarashina-JP-1B/jcola.json similarity index 85% rename from config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jcola.json rename to config/constract_llm/eval/pt/glue/Sarashina-JP-1B/jcola.json index e7407b2a..377c421e 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jcola.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-JP-1B/jcola.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-1B-jcola-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jcola-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jcola-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jcola-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jcola-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jnli.json b/config/constract_llm/eval/pt/glue/Sarashina-JP-1B/jnli.json similarity index 82% rename from config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jnli.json rename to config/constract_llm/eval/pt/glue/Sarashina-JP-1B/jnli.json index 18f4e38a..cc6711f9 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jnli.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-JP-1B/jnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-1B-jnli-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jnli-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jnli-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jnli-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jnli-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jsts.json b/config/constract_llm/eval/pt/glue/Sarashina-JP-1B/jsts.json similarity index 82% rename from config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jsts.json rename to config/constract_llm/eval/pt/glue/Sarashina-JP-1B/jsts.json index b0187145..9ff94d47 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jsts.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-JP-1B/jsts.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-1B-jsts-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jsts-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jsts-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jsts-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jsts-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -24,10 +24,10 @@ "eval_accumulation_steps": 1, "max_seq_length": 512, "force_add_bos_token": true, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "pearson", "logging_first_step": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/glue/Sarashina-JP-3B/jcola.json b/config/constract_llm/eval/pt/glue/Sarashina-JP-3B/jcola.json new file mode 100644 index 00000000..1932a0a9 --- /dev/null +++ b/config/constract_llm/eval/pt/glue/Sarashina-JP-3B/jcola.json @@ -0,0 +1,50 @@ +{ + "model_name_or_path": "sbintuitions/sarashina2.2-3b", + "task_name": "JCoLA", + "trust_remote_code": true, + "dataset_name": null, + "dataset_config_name": null, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Sarashina-JP-3B-jcola-test", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jcola-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jcola-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "load_best_model_at_end": true, + "greater_is_better": true, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jnli.json b/config/constract_llm/eval/pt/glue/Sarashina-JP-3B/jnli.json similarity index 82% rename from config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jnli.json rename to config/constract_llm/eval/pt/glue/Sarashina-JP-3B/jnli.json index 0a7cdf73..31011c21 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jnli.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-JP-3B/jnli.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-3B-jnli-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jnli-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jnli-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jnli-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jnli-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -23,10 +23,10 @@ "gradient_accumulation_steps": 16, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "accuracy", "logging_first_step": true, @@ -46,5 +46,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jsts.json b/config/constract_llm/eval/pt/glue/Sarashina-JP-3B/jsts.json similarity index 82% rename from config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jsts.json rename to config/constract_llm/eval/pt/glue/Sarashina-JP-3B/jsts.json index 3603ca07..28e90d59 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jsts.json +++ b/config/constract_llm/eval/pt/glue/Sarashina-JP-3B/jsts.json @@ -7,8 +7,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-3B-jsts-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jsts-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jsts-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jsts-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jsts-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -23,10 +23,10 @@ "gradient_accumulation_steps": 16, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "pearson", "logging_first_step": true, @@ -46,5 +46,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/isotropic/Llama-Bi-EN-0.5B.json b/config/constract_llm/eval/pt/isotropic/Llama-Bi-EN-0.5B-PT-stage2/default.json similarity index 99% rename from config/constract_llm/eval/isotropic/Llama-Bi-EN-0.5B.json rename to config/constract_llm/eval/pt/isotropic/Llama-Bi-EN-0.5B-PT-stage2/default.json index 375dac90..117ff04d 100644 --- a/config/constract_llm/eval/isotropic/Llama-Bi-EN-0.5B.json +++ b/config/constract_llm/eval/pt/isotropic/Llama-Bi-EN-0.5B-PT-stage2/default.json @@ -13,4 +13,4 @@ "positive_pair_sentence1_column": "anchor", "positive_pair_sentence2_column": "positive", "model_name_or_path": "iamtatsuki05/Llama-Bi-EN-0.5B-PT-stage2" -} \ No newline at end of file +} diff --git a/config/constract_llm/eval/pt/isotropic/Llama-Bi-EN-1B-PT-stage2/bs1024.json b/config/constract_llm/eval/pt/isotropic/Llama-Bi-EN-1B-PT-stage2/bs1024.json new file mode 100644 index 00000000..ea4e9aad --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Llama-Bi-EN-1B-PT-stage2/bs1024.json @@ -0,0 +1,16 @@ +{ + "output_dir": "data/outputs/eval/isotropic/Llama-Bi-EN-1B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "wiki_name": "google/wiki40b", + "wiki_lang": "en", + "positive_pair_dataset_name": "sentence-transformers/all-nli", + "positive_pair_dataset_config_name": "triplet", + "positive_pair_dataset_split": "dev", + "positive_pair_sentence1_column": "anchor", + "positive_pair_sentence2_column": "positive", + "model_name_or_path": "iamtatsuki05/Llama-Bi-EN-1B-PT-stage2-bs1024" +} diff --git a/config/constract_llm/eval/isotropic/Llama-Bi-EN-1B.json b/config/constract_llm/eval/pt/isotropic/Llama-Bi-EN-1B-PT-stage2/default.json similarity index 99% rename from config/constract_llm/eval/isotropic/Llama-Bi-EN-1B.json rename to config/constract_llm/eval/pt/isotropic/Llama-Bi-EN-1B-PT-stage2/default.json index 9307f27e..759f5fc0 100644 --- a/config/constract_llm/eval/isotropic/Llama-Bi-EN-1B.json +++ b/config/constract_llm/eval/pt/isotropic/Llama-Bi-EN-1B-PT-stage2/default.json @@ -13,4 +13,4 @@ "positive_pair_sentence1_column": "anchor", "positive_pair_sentence2_column": "positive", "model_name_or_path": "iamtatsuki05/Llama-Bi-EN-1B-PT-stage2" -} \ No newline at end of file +} diff --git a/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-0.5B-PT-stage2/default.json b/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-0.5B-PT-stage2/default.json new file mode 100644 index 00000000..7235d61f --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-0.5B-PT-stage2/default.json @@ -0,0 +1,13 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-0.5B-PT-stage2", + "output_dir": "data/outputs/eval/isotropic/Llama-Bi-0.5B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-1B-PT-stage2/bs1024.json b/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-1B-PT-stage2/bs1024.json new file mode 100644 index 00000000..0661cded --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-1B-PT-stage2/bs1024.json @@ -0,0 +1,13 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-1B-PT-stage2-bs1024", + "output_dir": "data/outputs/eval/isotropic/Llama-Bi-1B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Llama-Bi-1B.json b/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-1B-PT-stage2/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Llama-Bi-1B.json rename to config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-1B-PT-stage2/default.json diff --git a/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-3B-PT-stage2/bs2048.json b/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-3B-PT-stage2/bs2048.json new file mode 100644 index 00000000..1a8f3677 --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-3B-PT-stage2/bs2048.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-3B-PT-stage2-bs2048", + "output_dir": "data/outputs/eval/isotropic/Llama-Bi-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-3B-PT-stage2/default.json b/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-3B-PT-stage2/default.json new file mode 100644 index 00000000..33c42f1f --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Llama-Bi-JP-3B-PT-stage2/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-3B-PT-stage2", + "output_dir": "data/outputs/eval/isotropic/Llama-Bi-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/pt/isotropic/Llama-JP-0.5B-PT-stage2/default.json b/config/constract_llm/eval/pt/isotropic/Llama-JP-0.5B-PT-stage2/default.json new file mode 100644 index 00000000..4d847809 --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Llama-JP-0.5B-PT-stage2/default.json @@ -0,0 +1,13 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-PT-stage2", + "output_dir": "data/outputs/eval/isotropic/Llama-0.5B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/pt/isotropic/Llama-JP-1B-PT-stage2/bs1024.json b/config/constract_llm/eval/pt/isotropic/Llama-JP-1B-PT-stage2/bs1024.json new file mode 100644 index 00000000..0d2f807a --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Llama-JP-1B-PT-stage2/bs1024.json @@ -0,0 +1,13 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-PT-stage2-bs1024", + "output_dir": "data/outputs/eval/isotropic/Llama-1B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Llama-1B.json b/config/constract_llm/eval/pt/isotropic/Llama-JP-1B-PT-stage2/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Llama-1B.json rename to config/constract_llm/eval/pt/isotropic/Llama-JP-1B-PT-stage2/default.json diff --git a/config/constract_llm/eval/pt/isotropic/Llama-JP-3B-PT-stage2/bs2048.json b/config/constract_llm/eval/pt/isotropic/Llama-JP-3B-PT-stage2/bs2048.json new file mode 100644 index 00000000..c3fbeb8f --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Llama-JP-3B-PT-stage2/bs2048.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2-bs2048", + "output_dir": "data/outputs/eval/isotropic/Llama-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/pt/isotropic/Llama-JP-3B-PT-stage2/default.json b/config/constract_llm/eval/pt/isotropic/Llama-JP-3B-PT-stage2/default.json new file mode 100644 index 00000000..e6b165f9 --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Llama-JP-3B-PT-stage2/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2", + "output_dir": "data/outputs/eval/isotropic/Llama-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/ModernBERT-EN-0.5B.json b/config/constract_llm/eval/pt/isotropic/ModernBERT-EN-0.5B-PT-stage2/default.json similarity index 99% rename from config/constract_llm/eval/isotropic/ModernBERT-EN-0.5B.json rename to config/constract_llm/eval/pt/isotropic/ModernBERT-EN-0.5B-PT-stage2/default.json index 1b6d8302..9f2aac03 100644 --- a/config/constract_llm/eval/isotropic/ModernBERT-EN-0.5B.json +++ b/config/constract_llm/eval/pt/isotropic/ModernBERT-EN-0.5B-PT-stage2/default.json @@ -13,4 +13,4 @@ "positive_pair_sentence1_column": "anchor", "positive_pair_sentence2_column": "positive", "model_name_or_path": "iamtatsuki05/ModernBERT-EN-0.5B-PT-stage2" -} \ No newline at end of file +} diff --git a/config/constract_llm/eval/pt/isotropic/ModernBERT-EN-1B-PT-stage2/bs1024.json b/config/constract_llm/eval/pt/isotropic/ModernBERT-EN-1B-PT-stage2/bs1024.json new file mode 100644 index 00000000..cdc540f9 --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/ModernBERT-EN-1B-PT-stage2/bs1024.json @@ -0,0 +1,16 @@ +{ + "output_dir": "data/outputs/eval/isotropic/ModernBERT-EN-1B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "wiki_name": "google/wiki40b", + "wiki_lang": "en", + "positive_pair_dataset_name": "sentence-transformers/all-nli", + "positive_pair_dataset_config_name": "triplet", + "positive_pair_dataset_split": "dev", + "positive_pair_sentence1_column": "anchor", + "positive_pair_sentence2_column": "positive", + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage2-bs1024" +} diff --git a/config/constract_llm/eval/isotropic/ModernBERT-EN-1B.json b/config/constract_llm/eval/pt/isotropic/ModernBERT-EN-1B-PT-stage2/default.json similarity index 99% rename from config/constract_llm/eval/isotropic/ModernBERT-EN-1B.json rename to config/constract_llm/eval/pt/isotropic/ModernBERT-EN-1B-PT-stage2/default.json index 6705a985..d8934f09 100644 --- a/config/constract_llm/eval/isotropic/ModernBERT-EN-1B.json +++ b/config/constract_llm/eval/pt/isotropic/ModernBERT-EN-1B-PT-stage2/default.json @@ -13,4 +13,4 @@ "positive_pair_sentence1_column": "anchor", "positive_pair_sentence2_column": "positive", "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage2" -} \ No newline at end of file +} diff --git a/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-0.5B-PT-stage2/default.json b/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-0.5B-PT-stage2/default.json new file mode 100644 index 00000000..fabaa02a --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-0.5B-PT-stage2/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-PT-stage2", + "output_dir": "data/outputs/eval/isotropic/Modernbert-0.5B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-1B-PT-stage2/bs1024.json b/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-1B-PT-stage2/bs1024.json new file mode 100644 index 00000000..cf41b319 --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-1B-PT-stage2/bs1024.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-PT-stage2-bs1024", + "output_dir": "data/outputs/eval/isotropic/ModernBERT-1B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/ModernBERT-1B.json b/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-1B-PT-stage2/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/ModernBERT-1B.json rename to config/constract_llm/eval/pt/isotropic/ModernBERT-JP-1B-PT-stage2/default.json diff --git a/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-3B-PT-stage2/bs2048.json b/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-3B-PT-stage2/bs2048.json new file mode 100644 index 00000000..fcec1b06 --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-3B-PT-stage2/bs2048.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2-bs2048", + "output_dir": "data/outputs/eval/isotropic/ModernBERT-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-3B-PT-stage2/default.json b/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-3B-PT-stage2/default.json new file mode 100644 index 00000000..3c34656f --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/ModernBERT-JP-3B-PT-stage2/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2", + "output_dir": "data/outputs/eval/isotropic/ModernBERT-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/pt/isotropic/Sarashina-0.5B/default.json b/config/constract_llm/eval/pt/isotropic/Sarashina-0.5B/default.json new file mode 100644 index 00000000..bb14d7d2 --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Sarashina-0.5B/default.json @@ -0,0 +1,13 @@ +{ + "model_name_or_path": "sbintuitions/sarashina2.2-0.5b", + "output_dir": "data/outputs/eval/isotropic/Sarashina-0.5B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Sarashina-1B.json b/config/constract_llm/eval/pt/isotropic/Sarashina-1B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sarashina-1B.json rename to config/constract_llm/eval/pt/isotropic/Sarashina-1B/default.json diff --git a/config/constract_llm/eval/pt/isotropic/Sarashina-3B/default.json b/config/constract_llm/eval/pt/isotropic/Sarashina-3B/default.json new file mode 100644 index 00000000..1157767f --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Sarashina-3B/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "sbintuitions/sarashina2.2-3b", + "output_dir": "data/outputs/eval/isotropic/Sarashina-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-0.5B/default.json b/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-0.5B/default.json new file mode 100644 index 00000000..6e18f62e --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-0.5B/default.json @@ -0,0 +1,13 @@ +{ + "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-0.5b", + "output_dir": "data/outputs/eval/isotropic/Sarashina-Bi-0.5B", + "num_examples": 2000, + "batch_size": 2, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Sarashina-Bi-1B.json b/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-1B/default.json similarity index 100% rename from config/constract_llm/eval/isotropic/Sarashina-Bi-1B.json rename to config/constract_llm/eval/pt/isotropic/Sarashina-Bi-1B/default.json diff --git a/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-3B/default.json b/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-3B/default.json new file mode 100644 index 00000000..5e26f974 --- /dev/null +++ b/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-3B/default.json @@ -0,0 +1,12 @@ +{ + "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-3b", + "output_dir": "data/outputs/eval/isotropic/Sarashina-Bi-3B", + "num_examples": 2000, + "dtype": "bfloat16", + "attn_implementation": "flash_attention_2", + "seed": 42, + "miracl_name": "miracl/miracl", + "miracl_lang": "ja", + "wiki_name": "wikimedia/wikipedia", + "wiki_lang": "20231101.ja" +} diff --git a/config/constract_llm/eval/isotropic/Sarashina-Bi-EN-0.5B.json b/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-EN-0.5B/default.json similarity index 99% rename from config/constract_llm/eval/isotropic/Sarashina-Bi-EN-0.5B.json rename to config/constract_llm/eval/pt/isotropic/Sarashina-Bi-EN-0.5B/default.json index 33a40eba..a5474661 100644 --- a/config/constract_llm/eval/isotropic/Sarashina-Bi-EN-0.5B.json +++ b/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-EN-0.5B/default.json @@ -13,4 +13,4 @@ "positive_pair_sentence1_column": "anchor", "positive_pair_sentence2_column": "positive", "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-0.5b" -} \ No newline at end of file +} diff --git a/config/constract_llm/eval/isotropic/Sarashina-Bi-EN-1B.json b/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-EN-1B/default.json similarity index 99% rename from config/constract_llm/eval/isotropic/Sarashina-Bi-EN-1B.json rename to config/constract_llm/eval/pt/isotropic/Sarashina-Bi-EN-1B/default.json index dbb86ae7..75ab6aac 100644 --- a/config/constract_llm/eval/isotropic/Sarashina-Bi-EN-1B.json +++ b/config/constract_llm/eval/pt/isotropic/Sarashina-Bi-EN-1B/default.json @@ -13,4 +13,4 @@ "positive_pair_sentence1_column": "anchor", "positive_pair_sentence2_column": "positive", "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-1b" -} \ No newline at end of file +} diff --git a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_csqa.json b/config/constract_llm/eval/pt/qa/InstructLM-EN-0.5B/csqa.json similarity index 86% rename from config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_csqa.json rename to config/constract_llm/eval/pt/qa/InstructLM-EN-0.5B/csqa.json index 78fa0c92..b728fc5d 100644 --- a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_csqa.json +++ b/config/constract_llm/eval/pt/qa/InstructLM-EN-0.5B/csqa.json @@ -9,8 +9,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "InstructLM-EN-0.5B-csqa-glue", - "output_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-csqa-glue", + "logging_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_squad.json b/config/constract_llm/eval/pt/qa/InstructLM-EN-0.5B/squad.json similarity index 86% rename from config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_squad.json rename to config/constract_llm/eval/pt/qa/InstructLM-EN-0.5B/squad.json index 0edbf1f4..34d57c1c 100644 --- a/config/constract_llm/eval/pt/InstructLM-EN-0.5B/InstructLM-EN-0.5B_squad.json +++ b/config/constract_llm/eval/pt/qa/InstructLM-EN-0.5B/squad.json @@ -12,8 +12,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "InstructLM-EN-0.5B-squad-glue", - "output_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-squad-glue", - "logging_dir": "./data/outputs/eval/glue/InstructLM-EN-0.5B-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-squad-glue", + "logging_dir": "data/outputs/eval/glue/InstructLM-EN-0.5B-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_csqa.json b/config/constract_llm/eval/pt/qa/Llama-EN-0.5B-PT-stage2/csqa.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_csqa.json rename to config/constract_llm/eval/pt/qa/Llama-EN-0.5B-PT-stage2/csqa.json index 253cd30e..9271766a 100644 --- a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_csqa.json +++ b/config/constract_llm/eval/pt/qa/Llama-EN-0.5B-PT-stage2/csqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-0.5B-PT-stage2-csqa-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_squad.json b/config/constract_llm/eval/pt/qa/Llama-EN-0.5B-PT-stage2/squad.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_squad.json rename to config/constract_llm/eval/pt/qa/Llama-EN-0.5B-PT-stage2/squad.json index 4ec91cc3..d133d5f0 100644 --- a/config/constract_llm/eval/pt/Llama-EN-0.5B-PT-stage2/Llama-EN-0.5B-PT-stage2_squad.json +++ b/config/constract_llm/eval/pt/qa/Llama-EN-0.5B-PT-stage2/squad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-0.5B-PT-stage2-squad-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-squad-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-squad-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-0.5B-PT-stage2-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/csqa-bs1024.json b/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/csqa-bs1024.json new file mode 100644 index 00000000..e4e2320b --- /dev/null +++ b/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/csqa-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-1B-PT-stage2-bs1024", + "task_name": "csqa", + "trust_remote_code": true, + "dataset_name": "tau/commonsense_qa", + "dataset_config_name": null, + "dataset_revision": "94630fe30dad47192a8546eb75f094926d47e155", + "max_seq_length": 128, + "force_add_bos_token": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-1B-PT-stage2-bs1024-csqa-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-csqa-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_csqa.json b/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/csqa.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_csqa.json rename to config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/csqa.json index 3128cb4f..ed0da79e 100644 --- a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_csqa.json +++ b/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/csqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-1B-PT-stage2-csqa-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/squad-bs1024.json b/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/squad-bs1024.json new file mode 100644 index 00000000..18094de9 --- /dev/null +++ b/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/squad-bs1024.json @@ -0,0 +1,55 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-1B-PT-stage2-bs1024", + "task_name": "squad", + "trust_remote_code": true, + "dataset_name": "rajpurkar/squad", + "dataset_config_name": null, + "dataset_revision": "7b6d24c440a36b6815f21b70d25016731768db1f", + "max_seq_length": 384, + "force_add_bos_token": true, + "doc_stride": 128, + "n_best_size": 20, + "max_answer_length": 30, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-1B-PT-stage2-bs1024-squad-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-squad-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-bs1024-squad-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "f1", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_squad.json b/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/squad.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_squad.json rename to config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/squad.json index ee57253b..f48172e1 100644 --- a/config/constract_llm/eval/pt/Llama-EN-1B-PT-stage2/Llama-EN-1B-PT-stage2_squad.json +++ b/config/constract_llm/eval/pt/qa/Llama-EN-1B-PT-stage2/squad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-1B-PT-stage2-squad-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-squad-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-1B-PT-stage2-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-squad-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-1B-PT-stage2-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/csqa-bs2048.json b/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/csqa-bs2048.json new file mode 100644 index 00000000..58c15e7f --- /dev/null +++ b/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/csqa-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-3B-PT-stage2-bs2048", + "task_name": "csqa", + "trust_remote_code": true, + "dataset_name": "tau/commonsense_qa", + "dataset_config_name": null, + "dataset_revision": "94630fe30dad47192a8546eb75f094926d47e155", + "max_seq_length": 128, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-3B-PT-stage2-bs2048-csqa-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-csqa-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_csqa.json b/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/csqa.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_csqa.json rename to config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/csqa.json index 85123b7c..dc219ae2 100644 --- a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_csqa.json +++ b/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/csqa.json @@ -9,8 +9,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-3B-PT-stage2-csqa-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/squad-bs2048.json b/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/squad-bs2048.json new file mode 100644 index 00000000..21b3fb81 --- /dev/null +++ b/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/squad-bs2048.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-3B-PT-stage2-bs2048", + "task_name": "squad", + "trust_remote_code": true, + "dataset_name": "rajpurkar/squad", + "dataset_config_name": null, + "dataset_revision": "7b6d24c440a36b6815f21b70d25016731768db1f", + "max_seq_length": 384, + "doc_stride": 128, + "n_best_size": 20, + "max_answer_length": 30, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-EN-3B-PT-stage2-bs2048-squad-glue", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-squad-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-bs2048-squad-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "f1", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_squad.json b/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/squad.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_squad.json rename to config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/squad.json index 84db6ab6..e5b20e32 100644 --- a/config/constract_llm/eval/pt/Llama-EN-3B-PT-stage2/Llama-EN-3B-PT-stage2_squad.json +++ b/config/constract_llm/eval/pt/qa/Llama-EN-3B-PT-stage2/squad.json @@ -12,8 +12,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-EN-3B-PT-stage2-squad-glue", - "output_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-squad-glue", - "logging_dir": "./data/outputs/eval/glue/Llama-EN-3B-PT-stage2-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-squad-glue", + "logging_dir": "data/outputs/eval/glue/Llama-EN-3B-PT-stage2-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jcsqa.json b/config/constract_llm/eval/pt/qa/Llama-JP-0.5B-PT-stage2/jcsqa.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jcsqa.json rename to config/constract_llm/eval/pt/qa/Llama-JP-0.5B-PT-stage2/jcsqa.json index a97b7eca..b49de16f 100644 --- a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jcsqa.json +++ b/config/constract_llm/eval/pt/qa/Llama-JP-0.5B-PT-stage2/jcsqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-0.5B-PT-stage2-jcsqa-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jcsqa-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jcsqa-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jcsqa-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jsquad.json b/config/constract_llm/eval/pt/qa/Llama-JP-0.5B-PT-stage2/jsquad.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jsquad.json rename to config/constract_llm/eval/pt/qa/Llama-JP-0.5B-PT-stage2/jsquad.json index 93a780ad..44f8b2c8 100644 --- a/config/constract_llm/eval/pt/Llama-JP-0.5B-PT-stage2/Llama-JP-0.5B-PT-stage2_jsquad.json +++ b/config/constract_llm/eval/pt/qa/Llama-JP-0.5B-PT-stage2/jsquad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-0.5B-PT-stage2-jsquad-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jsquad-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jsquad-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-0.5B-PT-stage2-jsquad-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jcsqa-bs1024.json b/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jcsqa-bs1024.json new file mode 100644 index 00000000..c0881176 --- /dev/null +++ b/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jcsqa-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-PT-stage2-bs1024", + "task_name": "JCSQA", + "trust_remote_code": true, + "dataset_name": "shunk031/JGLUE", + "dataset_config_name": "JCommonsenseQA", + "dataset_revision": "41cc99f1c01d41b1ab13435ae97b7076c2199f4c", + "max_seq_length": 128, + "force_add_bos_token": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-1B-PT-stage2-bs1024-jcsqa-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jcsqa-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jcsqa.json b/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jcsqa.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jcsqa.json rename to config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jcsqa.json index 4c6840a5..5cb3994d 100644 --- a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jcsqa.json +++ b/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jcsqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-1B-PT-stage2-jcsqa-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jcsqa-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jcsqa-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jcsqa-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jsquad-bs1024.json b/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jsquad-bs1024.json new file mode 100644 index 00000000..91544d46 --- /dev/null +++ b/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jsquad-bs1024.json @@ -0,0 +1,55 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-PT-stage2-bs1024", + "task_name": "JSQuAD", + "trust_remote_code": true, + "dataset_name": "shunk031/JGLUE", + "dataset_config_name": "JSQuAD", + "dataset_revision": "41cc99f1c01d41b1ab13435ae97b7076c2199f4c", + "max_seq_length": 384, + "force_add_bos_token": true, + "doc_stride": 128, + "n_best_size": 20, + "max_answer_length": 30, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-1B-PT-stage2-bs1024-jsquad-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-bs1024-jsquad-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "f1", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jsquad.json b/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jsquad.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jsquad.json rename to config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jsquad.json index 41327a8c..504d95bc 100644 --- a/config/constract_llm/eval/pt/Llama-JP-1B-PT-stage2/Llama-JP-1B-PT-stage2_jsquad.json +++ b/config/constract_llm/eval/pt/qa/Llama-JP-1B-PT-stage2/jsquad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-1B-PT-stage2-jsquad-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jsquad-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jsquad-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-1B-PT-stage2-jsquad-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jcsqa-bs2048.json b/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jcsqa-bs2048.json new file mode 100644 index 00000000..677e3c6e --- /dev/null +++ b/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jcsqa-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2-bs2048", + "task_name": "JCSQA", + "trust_remote_code": true, + "dataset_name": "shunk031/JGLUE", + "dataset_config_name": "JCommonsenseQA", + "dataset_revision": "41cc99f1c01d41b1ab13435ae97b7076c2199f4c", + "max_seq_length": 128, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-3B-PT-stage2-bs2048-jcsqa-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jcsqa-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jcsqa.json b/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jcsqa.json similarity index 85% rename from config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jcsqa.json rename to config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jcsqa.json index b520f158..47aff776 100644 --- a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jcsqa.json +++ b/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jcsqa.json @@ -9,8 +9,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-3B-PT-stage2-jcsqa-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jcsqa-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jcsqa-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jcsqa-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jsquad-bs2048.json b/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jsquad-bs2048.json new file mode 100644 index 00000000..0b830e20 --- /dev/null +++ b/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jsquad-bs2048.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2-bs2048", + "task_name": "JSQuAD", + "trust_remote_code": true, + "dataset_name": "shunk031/JGLUE", + "dataset_config_name": "JSQuAD", + "dataset_revision": "41cc99f1c01d41b1ab13435ae97b7076c2199f4c", + "max_seq_length": 384, + "doc_stride": 128, + "n_best_size": 20, + "max_answer_length": 30, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "Llama-JP-3B-PT-stage2-bs2048-jsquad-test", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-bs2048-jsquad-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "f1", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jsquad.json b/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jsquad.json similarity index 86% rename from config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jsquad.json rename to config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jsquad.json index 0182d32e..700ba106 100644 --- a/config/constract_llm/eval/pt/Llama-JP-3B-PT-stage2/Llama-JP-3B-PT-stage2_jsquad.json +++ b/config/constract_llm/eval/pt/qa/Llama-JP-3B-PT-stage2/jsquad.json @@ -12,8 +12,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Llama-JP-3B-PT-stage2-jsquad-test", - "output_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jsquad-test", - "logging_dir": "./data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jsquad-test/logs", + "output_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/Llama-JP-3B-PT-stage2-jsquad-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_csqa.json b/config/constract_llm/eval/pt/qa/Mistral-EN-0.5B-PT-stage2/csqa.json similarity index 85% rename from config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_csqa.json rename to config/constract_llm/eval/pt/qa/Mistral-EN-0.5B-PT-stage2/csqa.json index 05a18fb5..de831f69 100644 --- a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_csqa.json +++ b/config/constract_llm/eval/pt/qa/Mistral-EN-0.5B-PT-stage2/csqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Mistral-EN-0.5B-PT-stage2-csqa-glue", - "output_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_squad.json b/config/constract_llm/eval/pt/qa/Mistral-EN-0.5B-PT-stage2/squad.json similarity index 86% rename from config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_squad.json rename to config/constract_llm/eval/pt/qa/Mistral-EN-0.5B-PT-stage2/squad.json index 67efb6d8..d3b608fa 100644 --- a/config/constract_llm/eval/pt/Mistral-EN-0.5B-PT-stage2/Mistral-EN-0.5B-PT-stage2_squad.json +++ b/config/constract_llm/eval/pt/qa/Mistral-EN-0.5B-PT-stage2/squad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Mistral-EN-0.5B-PT-stage2-squad-glue", - "output_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-squad-glue", - "logging_dir": "./data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-squad-glue", + "logging_dir": "data/outputs/eval/glue/Mistral-EN-0.5B-PT-stage2-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_csqa.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-0.5B-PT-stage2/csqa.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_csqa.json rename to config/constract_llm/eval/pt/qa/ModernBERT-EN-0.5B-PT-stage2/csqa.json index d526e692..440dd4c5 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_csqa.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-0.5B-PT-stage2/csqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-0.5B-PT-stage2-csqa-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-csqa-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_squad.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-0.5B-PT-stage2/squad.json similarity index 86% rename from config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_squad.json rename to config/constract_llm/eval/pt/qa/ModernBERT-EN-0.5B-PT-stage2/squad.json index a9a8ab66..7b0ed5e4 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-0.5B-PT-stage2/ModernBERT-EN-0.5B-PT-stage2_squad.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-0.5B-PT-stage2/squad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-0.5B-PT-stage2-squad-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-squad-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-squad-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-0.5B-PT-stage2-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/csqa-bs1024.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/csqa-bs1024.json new file mode 100644 index 00000000..ceebc2f9 --- /dev/null +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/csqa-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage2-bs1024", + "task_name": "csqa", + "trust_remote_code": true, + "dataset_name": "tau/commonsense_qa", + "dataset_config_name": null, + "dataset_revision": "94630fe30dad47192a8546eb75f094926d47e155", + "max_seq_length": 128, + "force_add_bos_token": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-1B-PT-stage2-bs1024-csqa-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-csqa-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-csqa-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_csqa.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/csqa.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_csqa.json rename to config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/csqa.json index cd298849..898506ef 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_csqa.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/csqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-1B-PT-stage2-csqa-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-csqa-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/squad-bs1024.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/squad-bs1024.json new file mode 100644 index 00000000..fa669b6d --- /dev/null +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/squad-bs1024.json @@ -0,0 +1,55 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage2-bs1024", + "task_name": "squad", + "trust_remote_code": true, + "dataset_name": "rajpurkar/squad", + "dataset_config_name": null, + "dataset_revision": "7b6d24c440a36b6815f21b70d25016731768db1f", + "max_seq_length": 384, + "force_add_bos_token": true, + "doc_stride": 128, + "n_best_size": 20, + "max_answer_length": 30, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-1B-PT-stage2-bs1024-squad-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-squad-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-bs1024-squad-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "f1", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_squad.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/squad.json similarity index 86% rename from config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_squad.json rename to config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/squad.json index b0fcf4c4..3c4cba22 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-1B-PT-stage2/ModernBERT-EN-1B-PT-stage2_squad.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-1B-PT-stage2/squad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-1B-PT-stage2-squad-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-squad-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-squad-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-1B-PT-stage2-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/csqa-bs2048.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/csqa-bs2048.json new file mode 100644 index 00000000..29579d2d --- /dev/null +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/csqa-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-3B-PT-stage2-bs2048", + "task_name": "csqa", + "trust_remote_code": true, + "dataset_name": "tau/commonsense_qa", + "dataset_config_name": null, + "dataset_revision": "94630fe30dad47192a8546eb75f094926d47e155", + "max_seq_length": 128, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-3B-PT-stage2-bs2048-csqa-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-csqa-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-csqa-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_csqa.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/csqa.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_csqa.json rename to config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/csqa.json index 76f45fe8..02e915eb 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_csqa.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/csqa.json @@ -9,8 +9,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-3B-PT-stage2-csqa-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-csqa-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/squad-bs2048.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/squad-bs2048.json new file mode 100644 index 00000000..98cd987a --- /dev/null +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/squad-bs2048.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-3B-PT-stage2-bs2048", + "task_name": "squad", + "trust_remote_code": true, + "dataset_name": "rajpurkar/squad", + "dataset_config_name": null, + "dataset_revision": "7b6d24c440a36b6815f21b70d25016731768db1f", + "max_seq_length": 384, + "doc_stride": 128, + "n_best_size": 20, + "max_answer_length": 30, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-EN-3B-PT-stage2-bs2048-squad-glue", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-squad-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-bs2048-squad-glue/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "f1", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_squad.json b/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/squad.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_squad.json rename to config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/squad.json index d1d80557..4ff52df6 100644 --- a/config/constract_llm/eval/pt/ModernBERT-EN-3B-PT-stage2/ModernBERT-EN-3B-PT-stage2_squad.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-EN-3B-PT-stage2/squad.json @@ -12,8 +12,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-EN-3B-PT-stage2-squad-glue", - "output_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-squad-glue", - "logging_dir": "./data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-squad-glue", + "logging_dir": "data/outputs/eval/glue/ModernBERT-EN-3B-PT-stage2-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jcsqa.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-0.5B-PT-stage2/jcsqa.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jcsqa.json rename to config/constract_llm/eval/pt/qa/ModernBERT-JP-0.5B-PT-stage2/jcsqa.json index 8f65e325..7c8a4651 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jcsqa.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-0.5B-PT-stage2/jcsqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-0.5B-PT-stage2-jcsqa-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jcsqa-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jcsqa-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jcsqa-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jsquad.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-0.5B-PT-stage2/jsquad.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jsquad.json rename to config/constract_llm/eval/pt/qa/ModernBERT-JP-0.5B-PT-stage2/jsquad.json index 1ee80888..39cdbc6d 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-0.5B-PT-stage2/ModernBERT-JP-0.5B-PT-stage2_jsquad.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-0.5B-PT-stage2/jsquad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-0.5B-PT-stage2-jsquad-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jsquad-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jsquad-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-0.5B-PT-stage2-jsquad-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jcsqa-bs1024.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jcsqa-bs1024.json new file mode 100644 index 00000000..20853721 --- /dev/null +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jcsqa-bs1024.json @@ -0,0 +1,52 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-PT-stage2-bs1024", + "task_name": "JCSQA", + "trust_remote_code": true, + "dataset_name": "shunk031/JGLUE", + "dataset_config_name": "JCommonsenseQA", + "dataset_revision": "41cc99f1c01d41b1ab13435ae97b7076c2199f4c", + "max_seq_length": 128, + "force_add_bos_token": true, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-1B-PT-stage2-bs1024-jcsqa-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jcsqa-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jcsqa.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jcsqa.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jcsqa.json rename to config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jcsqa.json index e2d0bf74..bc918e48 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jcsqa.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jcsqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-1B-PT-stage2-jcsqa-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jcsqa-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jcsqa-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jcsqa-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jsquad-bs1024.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jsquad-bs1024.json new file mode 100644 index 00000000..b4be4621 --- /dev/null +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jsquad-bs1024.json @@ -0,0 +1,55 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-PT-stage2-bs1024", + "task_name": "JSQuAD", + "trust_remote_code": true, + "dataset_name": "shunk031/JGLUE", + "dataset_config_name": "JSQuAD", + "dataset_revision": "41cc99f1c01d41b1ab13435ae97b7076c2199f4c", + "max_seq_length": 384, + "force_add_bos_token": true, + "doc_stride": 128, + "n_best_size": 20, + "max_answer_length": 30, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-1B-PT-stage2-bs1024-jsquad-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-bs1024-jsquad-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 1e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "f1", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jsquad.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jsquad.json similarity index 86% rename from config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jsquad.json rename to config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jsquad.json index ebe2a604..6e10821f 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-1B-PT-stage2/ModernBERT-JP-1B-PT-stage2_jsquad.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-1B-PT-stage2/jsquad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-1B-PT-stage2-jsquad-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jsquad-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jsquad-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-1B-PT-stage2-jsquad-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jcsqa-bs2048.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jcsqa-bs2048.json new file mode 100644 index 00000000..0f222faf --- /dev/null +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jcsqa-bs2048.json @@ -0,0 +1,51 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2-bs2048", + "task_name": "JCSQA", + "trust_remote_code": true, + "dataset_name": "shunk031/JGLUE", + "dataset_config_name": "JCommonsenseQA", + "dataset_revision": "41cc99f1c01d41b1ab13435ae97b7076c2199f4c", + "max_seq_length": 128, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-3B-PT-stage2-bs2048-jcsqa-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jcsqa-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "accuracy", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jcsqa.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jcsqa.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jcsqa.json rename to config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jcsqa.json index 6d5ec00b..478775b4 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jcsqa.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jcsqa.json @@ -9,8 +9,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-3B-PT-stage2-jcsqa-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jcsqa-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jcsqa-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jcsqa-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jsquad-bs2048.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jsquad-bs2048.json new file mode 100644 index 00000000..03f4db77 --- /dev/null +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jsquad-bs2048.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2-bs2048", + "task_name": "JSQuAD", + "trust_remote_code": true, + "dataset_name": "shunk031/JGLUE", + "dataset_config_name": "JSQuAD", + "dataset_revision": "41cc99f1c01d41b1ab13435ae97b7076c2199f4c", + "max_seq_length": 384, + "doc_stride": 128, + "n_best_size": 20, + "max_answer_length": 30, + "max_train_samples": null, + "max_eval_samples": null, + "run_name": "ModernBERT-JP-3B-PT-stage2-bs2048-jsquad-test", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-bs2048-jsquad-test/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 16, + "eval_accumulation_steps": 1, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "f1", + "greater_is_better": true, + "load_best_model_at_end": true, + "logging_first_step": true, + "prediction_loss_only": false, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 500, + "eval_steps": 500, + "save_total_limit": 2, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jsquad.json b/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jsquad.json similarity index 85% rename from config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jsquad.json rename to config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jsquad.json index 7ef38ad9..354891fd 100644 --- a/config/constract_llm/eval/pt/ModernBERT-JP-3B-PT-stage2/ModernBERT-JP-3B-PT-stage2_jsquad.json +++ b/config/constract_llm/eval/pt/qa/ModernBERT-JP-3B-PT-stage2/jsquad.json @@ -12,8 +12,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "ModernBERT-JP-3B-PT-stage2-jsquad-test", - "output_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jsquad-test", - "logging_dir": "./data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jsquad-test/logs", + "output_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/ModernBERT-JP-3B-PT-stage2-jsquad-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_csqa.json b/config/constract_llm/eval/pt/qa/Qwen2.5-EN-0.5B/csqa.json similarity index 86% rename from config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_csqa.json rename to config/constract_llm/eval/pt/qa/Qwen2.5-EN-0.5B/csqa.json index fbf70838..938c5b87 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_csqa.json +++ b/config/constract_llm/eval/pt/qa/Qwen2.5-EN-0.5B/csqa.json @@ -9,8 +9,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-EN-0.5B-csqa-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_squad.json b/config/constract_llm/eval/pt/qa/Qwen2.5-EN-0.5B/squad.json similarity index 86% rename from config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_squad.json rename to config/constract_llm/eval/pt/qa/Qwen2.5-EN-0.5B/squad.json index 031f361e..e3758fbb 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-EN-0.5B/Qwen2.5-EN-0.5B_squad.json +++ b/config/constract_llm/eval/pt/qa/Qwen2.5-EN-0.5B/squad.json @@ -12,8 +12,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-EN-0.5B-squad-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-squad-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-EN-0.5B-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-squad-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-EN-0.5B-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_csqa.json b/config/constract_llm/eval/pt/qa/Qwen2.5-Scratch-EN-0.5B-PT-stage2/csqa.json similarity index 85% rename from config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_csqa.json rename to config/constract_llm/eval/pt/qa/Qwen2.5-Scratch-EN-0.5B-PT-stage2/csqa.json index 85360b9f..33cc5a07 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_csqa.json +++ b/config/constract_llm/eval/pt/qa/Qwen2.5-Scratch-EN-0.5B-PT-stage2/csqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-Scratch-EN-0.5B-PT-stage2-csqa-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_squad.json b/config/constract_llm/eval/pt/qa/Qwen2.5-Scratch-EN-0.5B-PT-stage2/squad.json similarity index 85% rename from config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_squad.json rename to config/constract_llm/eval/pt/qa/Qwen2.5-Scratch-EN-0.5B-PT-stage2/squad.json index 8705a194..6b2d0d9d 100644 --- a/config/constract_llm/eval/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2/Qwen2.5-Scratch-EN-0.5B-PT-stage2_squad.json +++ b/config/constract_llm/eval/pt/qa/Qwen2.5-Scratch-EN-0.5B-PT-stage2/squad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Qwen2.5-Scratch-EN-0.5B-PT-stage2-squad-glue", - "output_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-squad-glue", - "logging_dir": "./data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-squad-glue", + "logging_dir": "data/outputs/eval/glue/Qwen2.5-Scratch-EN-0.5B-PT-stage2-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_csqa.json b/config/constract_llm/eval/pt/qa/Sarashina-EN-0.5B/csqa.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_csqa.json rename to config/constract_llm/eval/pt/qa/Sarashina-EN-0.5B/csqa.json index 34d314fb..8a8c694a 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_csqa.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-EN-0.5B/csqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-0.5B-csqa-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_squad.json b/config/constract_llm/eval/pt/qa/Sarashina-EN-0.5B/squad.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_squad.json rename to config/constract_llm/eval/pt/qa/Sarashina-EN-0.5B/squad.json index b6d2e71f..6f0d2e85 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-0.5B/Sarashina-EN-0.5B_squad.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-EN-0.5B/squad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-0.5B-squad-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-squad-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-0.5B-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-squad-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-0.5B-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_csqa.json b/config/constract_llm/eval/pt/qa/Sarashina-EN-1B/csqa.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_csqa.json rename to config/constract_llm/eval/pt/qa/Sarashina-EN-1B/csqa.json index a1015bc5..04627ed0 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_csqa.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-EN-1B/csqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-1B-csqa-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-1B-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-1B-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_squad.json b/config/constract_llm/eval/pt/qa/Sarashina-EN-1B/squad.json similarity index 87% rename from config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_squad.json rename to config/constract_llm/eval/pt/qa/Sarashina-EN-1B/squad.json index a41932d6..68b4c76d 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-1B/Sarashina-EN-1B_squad.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-EN-1B/squad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-1B-squad-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-squad-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-1B-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-1B-squad-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-1B-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_csqa.json b/config/constract_llm/eval/pt/qa/Sarashina-EN-3B/csqa.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_csqa.json rename to config/constract_llm/eval/pt/qa/Sarashina-EN-3B/csqa.json index 7bfbde86..a506d17a 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_csqa.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-EN-3B/csqa.json @@ -9,8 +9,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-3B-csqa-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-csqa-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-csqa-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-3B-csqa-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-3B-csqa-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_squad.json b/config/constract_llm/eval/pt/qa/Sarashina-EN-3B/squad.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_squad.json rename to config/constract_llm/eval/pt/qa/Sarashina-EN-3B/squad.json index 3539e1b2..c0609f65 100644 --- a/config/constract_llm/eval/pt/Sarashina-EN-3B/Sarashina-EN-3B_squad.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-EN-3B/squad.json @@ -12,8 +12,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-EN-3B-squad-glue", - "output_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-squad-glue", - "logging_dir": "./data/outputs/eval/glue/Sarashina-EN-3B-squad-glue/logs", + "output_dir": "data/outputs/eval/glue/Sarashina-EN-3B-squad-glue", + "logging_dir": "data/outputs/eval/glue/Sarashina-EN-3B-squad-glue/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jcsqa.json b/config/constract_llm/eval/pt/qa/Sarashina-JP-0.5B/jcsqa.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jcsqa.json rename to config/constract_llm/eval/pt/qa/Sarashina-JP-0.5B/jcsqa.json index f43d5858..252d6ceb 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jcsqa.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-JP-0.5B/jcsqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-0.5B-jcsqa-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jcsqa-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jcsqa-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jcsqa-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jsquad.json b/config/constract_llm/eval/pt/qa/Sarashina-JP-0.5B/jsquad.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jsquad.json rename to config/constract_llm/eval/pt/qa/Sarashina-JP-0.5B/jsquad.json index fad26e24..9685788d 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-0.5B/Sarashina-JP-0.5B_jsquad.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-JP-0.5B/jsquad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-0.5B-jsquad-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jsquad-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-0.5B-jsquad-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-0.5B-jsquad-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jcsqa.json b/config/constract_llm/eval/pt/qa/Sarashina-JP-1B/jcsqa.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jcsqa.json rename to config/constract_llm/eval/pt/qa/Sarashina-JP-1B/jcsqa.json index 9eefe527..b37cddb1 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jcsqa.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-JP-1B/jcsqa.json @@ -10,8 +10,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-1B-jcsqa-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jcsqa-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jcsqa-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jcsqa-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -48,5 +48,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jsquad.json b/config/constract_llm/eval/pt/qa/Sarashina-JP-1B/jsquad.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jsquad.json rename to config/constract_llm/eval/pt/qa/Sarashina-JP-1B/jsquad.json index bdef5f1c..b1b550fe 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-1B/Sarashina-JP-1B_jsquad.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-JP-1B/jsquad.json @@ -13,8 +13,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-1B-jsquad-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jsquad-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-1B-jsquad-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-1B-jsquad-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jcsqa.json b/config/constract_llm/eval/pt/qa/Sarashina-JP-3B/jcsqa.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jcsqa.json rename to config/constract_llm/eval/pt/qa/Sarashina-JP-3B/jcsqa.json index b2724df9..a9ce51fc 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jcsqa.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-JP-3B/jcsqa.json @@ -9,8 +9,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-3B-jcsqa-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jcsqa-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jcsqa-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jcsqa-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jcsqa-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -47,5 +47,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jsquad.json b/config/constract_llm/eval/pt/qa/Sarashina-JP-3B/jsquad.json similarity index 86% rename from config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jsquad.json rename to config/constract_llm/eval/pt/qa/Sarashina-JP-3B/jsquad.json index 8b527e68..f10de2e8 100644 --- a/config/constract_llm/eval/pt/Sarashina-JP-3B/Sarashina-JP-3B_jsquad.json +++ b/config/constract_llm/eval/pt/qa/Sarashina-JP-3B/jsquad.json @@ -12,8 +12,8 @@ "max_train_samples": null, "max_eval_samples": null, "run_name": "Sarashina-JP-3B-jsquad-test", - "output_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jsquad-test", - "logging_dir": "./data/outputs/eval/jglue/Sarashina-JP-3B-jsquad-test/logs", + "output_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jsquad-test", + "logging_dir": "data/outputs/eval/jglue/Sarashina-JP-3B-jsquad-test/logs", "save_only_model": false, "overwrite_output_dir": true, "do_train": true, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/model/init_model/Llama-0.5B.json b/config/constract_llm/model/init_model/Llama-0.5B.json index 38232bbd..09e034b6 100644 --- a/config/constract_llm/model/init_model/Llama-0.5B.json +++ b/config/constract_llm/model/init_model/Llama-0.5B.json @@ -1,8 +1,8 @@ { - "model_name_or_path": "sbintuitions/sarashina2.2-0.5b", - "output_dir": "./data/model/Llama-JP-0.5B-init", - "model_type": "causal", - "push_to_hub": true, - "private": true, - "seed": 42 + "model_name_or_path": "sbintuitions/sarashina2.2-0.5b", + "output_dir": "data/model/Llama-JP-0.5B-init", + "model_type": "causal", + "push_to_hub": true, + "private": true, + "seed": 42 } diff --git a/config/constract_llm/model/init_model/Llama-1B.json b/config/constract_llm/model/init_model/Llama-1B.json index 9cf8b04b..18c9b0d9 100644 --- a/config/constract_llm/model/init_model/Llama-1B.json +++ b/config/constract_llm/model/init_model/Llama-1B.json @@ -1,6 +1,6 @@ { "model_name_or_path": "sbintuitions/sarashina2.2-1b", - "output_dir": "./data/model/Llama-JP-1B-init", + "output_dir": "data/model/Llama-JP-1B-init", "model_type": "causal", "push_to_hub": true, "private": true, diff --git a/config/constract_llm/model/init_model/Llama-3B.json b/config/constract_llm/model/init_model/Llama-3B.json index d4b131b4..8fe60aad 100644 --- a/config/constract_llm/model/init_model/Llama-3B.json +++ b/config/constract_llm/model/init_model/Llama-3B.json @@ -1,8 +1,8 @@ { - "model_name_or_path": "sbintuitions/sarashina2.2-3b", - "output_dir": "./data/model/Llama-JP-3B-init", - "model_type": "causal", - "push_to_hub": true, - "private": true, - "seed": 42 + "model_name_or_path": "sbintuitions/sarashina2.2-3b", + "output_dir": "data/model/Llama-JP-3B-init", + "model_type": "causal", + "push_to_hub": true, + "private": true, + "seed": 42 } diff --git a/config/constract_llm/model/init_model/Mistral-0.5B.json b/config/constract_llm/model/init_model/Mistral-0.5B.json index c3adb808..ec96ff62 100644 --- a/config/constract_llm/model/init_model/Mistral-0.5B.json +++ b/config/constract_llm/model/init_model/Mistral-0.5B.json @@ -1,7 +1,7 @@ { "model_name_or_path": "instruction-pretrain/InstructLM-500M", "revision": "2158022d3e7be0c4f9668f0a45f8002c434d5432", - "output_dir": "./data/model/Mistral-0.5B-init", + "output_dir": "data/model/Mistral-0.5B-init", "repo_id": "iamtatsuki05/Mistral-0.5B-init", "model_type": "causal", "push_to_hub": true, diff --git a/config/constract_llm/model/init_model/ModernBERT-0.5B.json b/config/constract_llm/model/init_model/ModernBERT-0.5B.json index cf745148..e438c9ee 100644 --- a/config/constract_llm/model/init_model/ModernBERT-0.5B.json +++ b/config/constract_llm/model/init_model/ModernBERT-0.5B.json @@ -1,8 +1,8 @@ { - "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-settings", - "output_dir": "./data/model/ModernBERT-JP-0.5B-init", - "model_type": "masked", - "push_to_hub": true, - "private": true, - "seed": 42 + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-settings", + "output_dir": "data/model/ModernBERT-JP-0.5B-init", + "model_type": "masked", + "push_to_hub": true, + "private": true, + "seed": 42 } diff --git a/config/constract_llm/model/init_model/ModernBERT-1B.json b/config/constract_llm/model/init_model/ModernBERT-1B.json index c320fd1b..edb0a366 100644 --- a/config/constract_llm/model/init_model/ModernBERT-1B.json +++ b/config/constract_llm/model/init_model/ModernBERT-1B.json @@ -1,7 +1,7 @@ { "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-settings", "tokenizer_name_or_path": "sbintuitions/sarashina2.2-1b", - "output_dir": "./data/model/ModernBERT-JP-1B-init", + "output_dir": "data/model/ModernBERT-JP-1B-init", "model_type": "masked", "push_to_hub": true, "private": true, diff --git a/config/constract_llm/model/init_model/ModernBERT-3B.json b/config/constract_llm/model/init_model/ModernBERT-3B.json index 50f8ffdf..01816365 100644 --- a/config/constract_llm/model/init_model/ModernBERT-3B.json +++ b/config/constract_llm/model/init_model/ModernBERT-3B.json @@ -1,8 +1,8 @@ { - "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-settings", - "output_dir": "./data/model/ModernBERT-JP-3B-init", - "model_type": "masked", - "push_to_hub": true, - "private": true, - "seed": 42 + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-settings", + "output_dir": "data/model/ModernBERT-JP-3B-init", + "model_type": "masked", + "push_to_hub": true, + "private": true, + "seed": 42 } diff --git a/config/constract_llm/model/init_model/Qwen2.5-0.5B.json b/config/constract_llm/model/init_model/Qwen2.5-0.5B.json index 193cc5c7..26445dea 100644 --- a/config/constract_llm/model/init_model/Qwen2.5-0.5B.json +++ b/config/constract_llm/model/init_model/Qwen2.5-0.5B.json @@ -1,7 +1,7 @@ { "model_name_or_path": "Qwen/Qwen2.5-0.5B", "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", - "output_dir": "./data/model/Qwen2.5-0.5B-init", + "output_dir": "data/model/Qwen2.5-0.5B-init", "repo_id": "iamtatsuki05/Qwen2.5-0.5B-init", "model_type": "causal", "push_to_hub": true, diff --git a/config/constract_llm/model/init_model/config.json b/config/constract_llm/model/init_model/config.json index d788c610..1e42be27 100644 --- a/config/constract_llm/model/init_model/config.json +++ b/config/constract_llm/model/init_model/config.json @@ -1,6 +1,6 @@ { "model_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "output_dir": "./data/misc/json", + "output_dir": "data/misc/json", "model_type": "causal", "push_to_hub": true, "private": true, diff --git a/config/constract_llm/model/save_custom_model/instructlm_bi_en_05b.json b/config/constract_llm/model/save_custom_model/instructlm_bi_en_05b.json index 1fbda213..7ff740ff 100644 --- a/config/constract_llm/model/save_custom_model/instructlm_bi_en_05b.json +++ b/config/constract_llm/model/save_custom_model/instructlm_bi_en_05b.json @@ -3,7 +3,7 @@ "revision": "2158022d3e7be0c4f9668f0a45f8002c434d5432", "custom_model_type": "mistral", "task_type": "mntp", - "output_dir": "./data/model/InstructLM-Bi-EN-0.5B", + "output_dir": "data/model/InstructLM-Bi-EN-0.5B", "push_to_hub": true, "repo_id": "iamtatsuki05/InstructLM-Bi-EN-0.5B", "private": true diff --git a/config/constract_llm/model/save_custom_model/llama_bi.json b/config/constract_llm/model/save_custom_model/llama_bi.json index 2daf0330..7d00cb7d 100644 --- a/config/constract_llm/model/save_custom_model/llama_bi.json +++ b/config/constract_llm/model/save_custom_model/llama_bi.json @@ -1,9 +1,9 @@ { - "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-PT-stage2", - "custom_model_type": "llama", - "task_type": "lm", - "output_dir": "./data/misc/json", - "push_to_hub": true, - "repo_id": "iamtatsuki05/Llama-Bi-JP-0.5B-PT-stage2", - "private": true + "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-PT-stage2", + "custom_model_type": "llama", + "task_type": "lm", + "output_dir": "data/misc/json", + "push_to_hub": true, + "repo_id": "iamtatsuki05/Llama-Bi-JP-0.5B-PT-stage2", + "private": true } diff --git a/config/constract_llm/model/save_custom_model/llama_bi_1b.json b/config/constract_llm/model/save_custom_model/llama_bi_1b.json index fb718a5f..b88149f3 100644 --- a/config/constract_llm/model/save_custom_model/llama_bi_1b.json +++ b/config/constract_llm/model/save_custom_model/llama_bi_1b.json @@ -2,7 +2,7 @@ "model_name_or_path": "iamtatsuki05/Llama-JP-1B-PT-stage2", "custom_model_type": "llama", "task_type": "lm", - "output_dir": "./data/misc/json", + "output_dir": "data/misc/json", "push_to_hub": true, "repo_id": "iamtatsuki05/Llama-Bi-JP-1B-PT-stage2", "private": true diff --git a/config/constract_llm/model/save_custom_model/llama_bi_1b_bs1024.json b/config/constract_llm/model/save_custom_model/llama_bi_1b_bs1024.json new file mode 100644 index 00000000..e00961ec --- /dev/null +++ b/config/constract_llm/model/save_custom_model/llama_bi_1b_bs1024.json @@ -0,0 +1,9 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-PT-stage2-bs1024", + "custom_model_type": "llama", + "task_type": "lm", + "output_dir": "data/misc/json", + "push_to_hub": true, + "repo_id": "iamtatsuki05/Llama-Bi-JP-1B-PT-stage2-bs1024", + "private": true +} diff --git a/config/constract_llm/model/save_custom_model/llama_bi_3b.json b/config/constract_llm/model/save_custom_model/llama_bi_3b.json index 4675528e..7dc739d7 100644 --- a/config/constract_llm/model/save_custom_model/llama_bi_3b.json +++ b/config/constract_llm/model/save_custom_model/llama_bi_3b.json @@ -1,9 +1,9 @@ { - "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2", - "custom_model_type": "llama", - "task_type": "lm", - "output_dir": "./data/misc/json", - "push_to_hub": true, - "repo_id": "iamtatsuki05/Llama-Bi-JP-3B-PT-stage2", - "private": true + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2", + "custom_model_type": "llama", + "task_type": "lm", + "output_dir": "data/misc/json", + "push_to_hub": true, + "repo_id": "iamtatsuki05/Llama-Bi-JP-3B-PT-stage2", + "private": true } diff --git a/config/constract_llm/model/save_custom_model/llama_bi_3b_bs2048.json b/config/constract_llm/model/save_custom_model/llama_bi_3b_bs2048.json new file mode 100644 index 00000000..b0b2e85f --- /dev/null +++ b/config/constract_llm/model/save_custom_model/llama_bi_3b_bs2048.json @@ -0,0 +1,9 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage2-bs2048", + "custom_model_type": "llama", + "task_type": "lm", + "output_dir": "data/misc/json", + "push_to_hub": true, + "repo_id": "iamtatsuki05/Llama-Bi-JP-3B-PT-stage2-bs2048", + "private": true +} diff --git a/config/constract_llm/model/save_custom_model/llama_bi_en.json b/config/constract_llm/model/save_custom_model/llama_bi_en.json index e62e2ded..87ebf7d3 100644 --- a/config/constract_llm/model/save_custom_model/llama_bi_en.json +++ b/config/constract_llm/model/save_custom_model/llama_bi_en.json @@ -2,7 +2,7 @@ "model_name_or_path": "iamtatsuki05/Llama-EN-0.5B-PT-stage2", "custom_model_type": "llama", "task_type": "lm", - "output_dir": "./data/misc/json", + "output_dir": "data/misc/json", "push_to_hub": true, "repo_id": "iamtatsuki05/Llama-Bi-EN-0.5B-PT-stage2", "private": true diff --git a/config/constract_llm/model/save_custom_model/llama_bi_en_1b.json b/config/constract_llm/model/save_custom_model/llama_bi_en_1b.json index 0a5ec93a..615096e1 100644 --- a/config/constract_llm/model/save_custom_model/llama_bi_en_1b.json +++ b/config/constract_llm/model/save_custom_model/llama_bi_en_1b.json @@ -2,7 +2,7 @@ "model_name_or_path": "iamtatsuki05/Llama-EN-1B-PT-stage2", "custom_model_type": "llama", "task_type": "lm", - "output_dir": "./data/misc/json", + "output_dir": "data/misc/json", "push_to_hub": true, "repo_id": "iamtatsuki05/Llama-Bi-EN-1B-PT-stage2", "private": true diff --git a/config/constract_llm/model/save_custom_model/llama_bi_en_1b_bs1024.json b/config/constract_llm/model/save_custom_model/llama_bi_en_1b_bs1024.json new file mode 100644 index 00000000..5a478ebd --- /dev/null +++ b/config/constract_llm/model/save_custom_model/llama_bi_en_1b_bs1024.json @@ -0,0 +1,9 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-1B-PT-stage2-bs1024", + "custom_model_type": "llama", + "task_type": "lm", + "output_dir": "data/misc/json", + "push_to_hub": true, + "repo_id": "iamtatsuki05/Llama-Bi-EN-1B-PT-stage2-bs1024", + "private": true +} diff --git a/config/constract_llm/model/save_custom_model/llama_bi_en_3b.json b/config/constract_llm/model/save_custom_model/llama_bi_en_3b.json index c1ed70af..98bda91a 100644 --- a/config/constract_llm/model/save_custom_model/llama_bi_en_3b.json +++ b/config/constract_llm/model/save_custom_model/llama_bi_en_3b.json @@ -2,7 +2,7 @@ "model_name_or_path": "iamtatsuki05/Llama-EN-3B-PT-stage2", "custom_model_type": "llama", "task_type": "lm", - "output_dir": "./data/misc/json", + "output_dir": "data/misc/json", "push_to_hub": true, "repo_id": "iamtatsuki05/Llama-Bi-EN-3B-PT-stage2", "private": true diff --git a/config/constract_llm/model/save_custom_model/llama_bi_en_3b_bs2048.json b/config/constract_llm/model/save_custom_model/llama_bi_en_3b_bs2048.json new file mode 100644 index 00000000..ae314473 --- /dev/null +++ b/config/constract_llm/model/save_custom_model/llama_bi_en_3b_bs2048.json @@ -0,0 +1,9 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-EN-3B-PT-stage2-bs2048", + "custom_model_type": "llama", + "task_type": "lm", + "output_dir": "data/misc/json", + "push_to_hub": true, + "repo_id": "iamtatsuki05/Llama-Bi-EN-3B-PT-stage2-bs2048", + "private": true +} diff --git a/config/constract_llm/model/save_custom_model/mistral_bi_en_05b.json b/config/constract_llm/model/save_custom_model/mistral_bi_en_05b.json index 4dc82211..b24480e0 100644 --- a/config/constract_llm/model/save_custom_model/mistral_bi_en_05b.json +++ b/config/constract_llm/model/save_custom_model/mistral_bi_en_05b.json @@ -1,9 +1,9 @@ { - "model_name_or_path": "./data/outputs/Mistral-EN-0.5B-PT-stage2", + "model_name_or_path": "data/outputs/Mistral-EN-0.5B-PT-stage2", "revision": null, "custom_model_type": "mistral", "task_type": "mntp", - "output_dir": "./data/model/Mistral-Bi-EN-0.5B-PT-stage2", + "output_dir": "data/model/Mistral-Bi-EN-0.5B-PT-stage2", "push_to_hub": true, "repo_id": "iamtatsuki05/Mistral-Bi-EN-0.5B-PT-stage2", "private": true diff --git a/config/constract_llm/model/save_custom_model/qwen25_bi_en_05b.json b/config/constract_llm/model/save_custom_model/qwen25_bi_en_05b.json index 0224c46d..f6f3069d 100644 --- a/config/constract_llm/model/save_custom_model/qwen25_bi_en_05b.json +++ b/config/constract_llm/model/save_custom_model/qwen25_bi_en_05b.json @@ -3,7 +3,7 @@ "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "custom_model_type": "qwen2", "task_type": "mntp", - "output_dir": "./data/model/Qwen2.5-Bi-EN-0.5B", + "output_dir": "data/model/Qwen2.5-Bi-EN-0.5B", "push_to_hub": true, "repo_id": "iamtatsuki05/Qwen2.5-Bi-EN-0.5B", "private": true diff --git a/config/constract_llm/model/save_custom_model/qwen25_scratch_bi_en_05b.json b/config/constract_llm/model/save_custom_model/qwen25_scratch_bi_en_05b.json index 8d8ea851..13dc51a4 100644 --- a/config/constract_llm/model/save_custom_model/qwen25_scratch_bi_en_05b.json +++ b/config/constract_llm/model/save_custom_model/qwen25_scratch_bi_en_05b.json @@ -1,9 +1,9 @@ { - "model_name_or_path": "./data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage2", + "model_name_or_path": "data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage2", "revision": null, "custom_model_type": "qwen2", "task_type": "mntp", - "output_dir": "./data/model/Qwen2.5-Scratch-Bi-EN-0.5B-PT-stage2", + "output_dir": "data/model/Qwen2.5-Scratch-Bi-EN-0.5B-PT-stage2", "push_to_hub": true, "repo_id": "iamtatsuki05/Qwen2.5-Scratch-Bi-EN-0.5B-PT-stage2", "private": true diff --git a/config/constract_llm/model/save_custom_model/sarashina_bi.json b/config/constract_llm/model/save_custom_model/sarashina_bi.json index 02bac9ae..5e66a1c8 100644 --- a/config/constract_llm/model/save_custom_model/sarashina_bi.json +++ b/config/constract_llm/model/save_custom_model/sarashina_bi.json @@ -1,9 +1,9 @@ { - "model_name_or_path": "sbintuitions/sarashina2.2-0.5b", - "custom_model_type": "llama", - "task_type": "lm", - "output_dir": "./data/misc/json", - "push_to_hub": true, - "repo_id": "iamtatsuki05/sarashina2.2-Bi-0.5b", - "private": true + "model_name_or_path": "sbintuitions/sarashina2.2-0.5b", + "custom_model_type": "llama", + "task_type": "lm", + "output_dir": "data/misc/json", + "push_to_hub": true, + "repo_id": "iamtatsuki05/sarashina2.2-Bi-0.5b", + "private": true } diff --git a/config/constract_llm/model/save_custom_model/sarashina_bi_1b.json b/config/constract_llm/model/save_custom_model/sarashina_bi_1b.json index 2958bbe8..807f8fd7 100644 --- a/config/constract_llm/model/save_custom_model/sarashina_bi_1b.json +++ b/config/constract_llm/model/save_custom_model/sarashina_bi_1b.json @@ -2,7 +2,7 @@ "model_name_or_path": "sbintuitions/sarashina2.2-1b", "custom_model_type": "llama", "task_type": "lm", - "output_dir": "./data/misc/json", + "output_dir": "data/misc/json", "push_to_hub": true, "repo_id": "iamtatsuki05/sarashina2.2-Bi-1b", "private": true diff --git a/config/constract_llm/model/save_custom_model/sarashina_bi_3b.json b/config/constract_llm/model/save_custom_model/sarashina_bi_3b.json index 388e9250..5a15b23e 100644 --- a/config/constract_llm/model/save_custom_model/sarashina_bi_3b.json +++ b/config/constract_llm/model/save_custom_model/sarashina_bi_3b.json @@ -1,9 +1,9 @@ { - "model_name_or_path": "sbintuitions/sarashina2.2-3b", - "custom_model_type": "llama", - "task_type": "lm", - "output_dir": "./data/misc/json", - "push_to_hub": true, - "repo_id": "iamtatsuki05/sarashina2.2-Bi-3b", - "private": true + "model_name_or_path": "sbintuitions/sarashina2.2-3b", + "custom_model_type": "llama", + "task_type": "lm", + "output_dir": "data/misc/json", + "push_to_hub": true, + "repo_id": "iamtatsuki05/sarashina2.2-Bi-3b", + "private": true } diff --git a/config/constract_llm/tokenizer/add_tokens/config/config.json b/config/constract_llm/tokenizer/add_tokens/config/config.json index 2354dd69..a29caada 100644 --- a/config/constract_llm/tokenizer/add_tokens/config/config.json +++ b/config/constract_llm/tokenizer/add_tokens/config/config.json @@ -1,8 +1,8 @@ { - "tokenizer_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "normal_tokens_config_path": "./config/constract_llm/tokenizer/add_tokens/config/normal_tokens.json", - "special_tokens_config_path": "./config/constract_llm/tokenizer/add_tokens/config/special_tokens.json", - "push_to_hub": true, - "private": true, - "output_dir": "./data/misc/json" + "tokenizer_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "normal_tokens_config_path": "config/constract_llm/tokenizer/add_tokens/config/normal_tokens.json", + "special_tokens_config_path": "config/constract_llm/tokenizer/add_tokens/config/special_tokens.json", + "push_to_hub": true, + "private": true, + "output_dir": "data/misc/json" } diff --git a/config/constract_llm/tokenizer/add_tokens/config/normal_tokens.json b/config/constract_llm/tokenizer/add_tokens/config/normal_tokens.json index 653e0152..f28c8ae4 100644 --- a/config/constract_llm/tokenizer/add_tokens/config/normal_tokens.json +++ b/config/constract_llm/tokenizer/add_tokens/config/normal_tokens.json @@ -1,5 +1,5 @@ [ - "nlp_special_token", - "another_token", - "example_token" + "nlp_special_token", + "another_token", + "example_token" ] diff --git a/config/constract_llm/tokenizer/add_tokens/config/special_tokens.json b/config/constract_llm/tokenizer/add_tokens/config/special_tokens.json index 4b55d04a..ede42d99 100644 --- a/config/constract_llm/tokenizer/add_tokens/config/special_tokens.json +++ b/config/constract_llm/tokenizer/add_tokens/config/special_tokens.json @@ -1,5 +1,5 @@ [ - "", - "", - "" + "", + "", + "" ] diff --git a/config/constract_llm/tokenizer/merge_spm/config.json b/config/constract_llm/tokenizer/merge_spm/config.json index 30601faf..a4e0276c 100644 --- a/config/constract_llm/tokenizer/merge_spm/config.json +++ b/config/constract_llm/tokenizer/merge_spm/config.json @@ -1,7 +1,7 @@ { - "base_tokenizer_name_or_path": "elyza/ELYZA-japanese-Llama-2-7b", - "additional_tokenizer_name_or_path": "./data/misc/json", - "output_dir": "./data/misc/json", - "push_to_hub": true, - "private": true + "base_tokenizer_name_or_path": "elyza/ELYZA-japanese-Llama-2-7b", + "additional_tokenizer_name_or_path": "data/misc/json", + "output_dir": "data/misc/json", + "push_to_hub": true, + "private": true } diff --git a/config/constract_llm/tokenizer/train_tokenizer/config.json b/config/constract_llm/tokenizer/train_tokenizer/config.json index 7aedd5be..b40b3644 100644 --- a/config/constract_llm/tokenizer/train_tokenizer/config.json +++ b/config/constract_llm/tokenizer/train_tokenizer/config.json @@ -1,13 +1,13 @@ { - "dataset_name_or_path": "wikimedia/wikipedia", - "dataset_config": "20231101.ja", - "split": "train", - "text_column": "text", - "vocab_size": 30000, - "min_frequency": 2, - "model_type": "unigram", - "push_to_hub": true, - "private": true, - "output_dir": "./data/misc/json", - "max_train_samples": 10000 + "dataset_name_or_path": "wikimedia/wikipedia", + "dataset_config": "20231101.ja", + "split": "train", + "text_column": "text", + "vocab_size": 30000, + "min_frequency": 2, + "model_type": "unigram", + "push_to_hub": true, + "private": true, + "output_dir": "data/misc/json", + "max_train_samples": 10000 } diff --git a/config/constract_llm/train/ds_config/ds_zero2.json b/config/constract_llm/train/ds_config/ds_zero2.json index 87a1f7eb..54c851c9 100644 --- a/config/constract_llm/train/ds_config/ds_zero2.json +++ b/config/constract_llm/train/ds_config/ds_zero2.json @@ -1,23 +1,23 @@ { - "fp16": { - "enabled": "auto" - }, - "bf16": { - "enabled": "auto" - }, - "zero_optimization": { - "stage": 2, - "allgather_partitions": true, - "allgather_bucket_size": 2e8, - "overlap_comm": true, - "reduce_scatter": true, - "reduce_bucket_size": 2e8, - "contiguous_gradients": true - }, - "gradient_accumulation_steps": "auto", - "gradient_clipping": "auto", - "steps_per_print": 1000, - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": "auto", - "wall_clock_breakdown": false -} \ No newline at end of file + "fp16": { + "enabled": "auto" + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 2, + "allgather_partitions": true, + "allgather_bucket_size": 2e8, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 2e8, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 1000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} diff --git a/config/constract_llm/train/ds_config/ds_zero2_cpu_offload.json b/config/constract_llm/train/ds_config/ds_zero2_cpu_offload.json index 1957457d..622679cf 100644 --- a/config/constract_llm/train/ds_config/ds_zero2_cpu_offload.json +++ b/config/constract_llm/train/ds_config/ds_zero2_cpu_offload.json @@ -1,31 +1,31 @@ { - "fp16": { - "enabled": "auto" + "fp16": { + "enabled": "auto" + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 2, + "offload_optimizer": { + "device": "cpu", + "pin_memory": true }, - "bf16": { - "enabled": "auto" + "offload_param": { + "device": "cpu", + "pin_memory": true }, - "zero_optimization": { - "stage": 2, - "offload_optimizer": { - "device": "cpu", - "pin_memory": true - }, - "offload_param": { - "device": "cpu", - "pin_memory": true - }, - "allgather_partitions": true, - "allgather_bucket_size": 2e8, - "overlap_comm": true, - "reduce_scatter": true, - "reduce_bucket_size": 2e8, - "contiguous_gradients": true - }, - "gradient_accumulation_steps": "auto", - "gradient_clipping": "auto", - "steps_per_print": 1000, - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": "auto", - "wall_clock_breakdown": false -} \ No newline at end of file + "allgather_partitions": true, + "allgather_bucket_size": 2e8, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 2e8, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 1000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} diff --git a/config/constract_llm/train/ds_config/ds_zero2_tp.json b/config/constract_llm/train/ds_config/ds_zero2_tp.json index c2686146..5b890059 100644 --- a/config/constract_llm/train/ds_config/ds_zero2_tp.json +++ b/config/constract_llm/train/ds_config/ds_zero2_tp.json @@ -1,27 +1,27 @@ { - "fp16": { - "enabled": "auto" - }, - "bf16": { - "enabled": "auto" - }, - "zero_optimization": { - "stage": 2, - "allgather_partitions": true, - "allgather_bucket_size": 2e8, - "overlap_comm": true, - "reduce_scatter": true, - "reduce_bucket_size": 2e8, - "contiguous_gradients": true, - "gather_16bit_weights_on_model_save": true - }, - "tensor_parallel":{ - "autotp_size": 4 - }, - "gradient_accumulation_steps": "auto", - "gradient_clipping": "auto", - "steps_per_print": 1000, - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": "auto", - "wall_clock_breakdown": false + "fp16": { + "enabled": "auto" + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 2, + "allgather_partitions": true, + "allgather_bucket_size": 2e8, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 2e8, + "contiguous_gradients": true, + "gather_16bit_weights_on_model_save": true + }, + "tensor_parallel":{ + "autotp_size": 4 + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 1000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false } diff --git a/config/constract_llm/train/ds_config/ds_zero3.json b/config/constract_llm/train/ds_config/ds_zero3.json index c399e8fe..ec6fd95a 100644 --- a/config/constract_llm/train/ds_config/ds_zero3.json +++ b/config/constract_llm/train/ds_config/ds_zero3.json @@ -1,26 +1,26 @@ { - "fp16": { - "enabled": "auto" - }, - "bf16": { - "enabled": "auto" - }, - "zero_optimization": { - "stage": 3, - "overlap_comm": true, - "contiguous_gradients": true, - "sub_group_size": 1e9, - "reduce_bucket_size": "auto", - "stage3_prefetch_bucket_size": "auto", - "stage3_param_persistence_threshold": "auto", - "stage3_max_live_parameters": 1e9, - "stage3_max_reuse_distance": 1e9, - "stage3_gather_16bit_weights_on_model_save": true - }, - "gradient_accumulation_steps": "auto", - "gradient_clipping": "auto", - "steps_per_print": 1000, - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": "auto", - "wall_clock_breakdown": false -} \ No newline at end of file + "fp16": { + "enabled": "auto" + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 3, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 1000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} diff --git a/config/constract_llm/train/ds_config/ds_zero3_cpu_offload.json b/config/constract_llm/train/ds_config/ds_zero3_cpu_offload.json index 6afc68c1..701f0abf 100644 --- a/config/constract_llm/train/ds_config/ds_zero3_cpu_offload.json +++ b/config/constract_llm/train/ds_config/ds_zero3_cpu_offload.json @@ -1,34 +1,34 @@ { - "fp16": { - "enabled": "auto" + "fp16": { + "enabled": "auto" + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 3, + "offload_optimizer": { + "device": "cpu", + "pin_memory": true }, - "bf16": { - "enabled": "auto" + "offload_param": { + "device": "cpu", + "pin_memory": true }, - "zero_optimization": { - "stage": 3, - "offload_optimizer": { - "device": "cpu", - "pin_memory": true - }, - "offload_param": { - "device": "cpu", - "pin_memory": true - }, - "overlap_comm": true, - "contiguous_gradients": true, - "sub_group_size": 1e9, - "reduce_bucket_size": "auto", - "stage3_prefetch_bucket_size": "auto", - "stage3_param_persistence_threshold": "auto", - "stage3_max_live_parameters": 1e9, - "stage3_max_reuse_distance": 1e9, - "stage3_gather_16bit_weights_on_model_save": true - }, - "gradient_accumulation_steps": "auto", - "gradient_clipping": "auto", - "steps_per_print": 1000, - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": "auto", - "wall_clock_breakdown": false -} \ No newline at end of file + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 1000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} diff --git a/config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B.json b/config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B/ft.json similarity index 82% rename from config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B.json rename to config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B/ft.json index fd852ae7..b92f0a6f 100644 --- a/config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B.json +++ b/config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B/ft.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": "negative", "run_name": "Sentence-InstructLM-Bi-EN-0.5B", - "output_dir": "./data/outputs/Sentence-InstructLM-Bi-EN-0.5B", - "logging_dir": "./data/outputs/Sentence-InstructLM-Bi-EN-0.5B/logs", + "output_dir": "data/outputs/Sentence-InstructLM-Bi-EN-0.5B", + "logging_dir": "data/outputs/Sentence-InstructLM-Bi-EN-0.5B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -27,7 +27,7 @@ "loss_scale": 100.0, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 5e-06, + "learning_rate": 5e-6, "per_device_train_batch_size": 32, "loss_cache_mini_batch_size": 32, "per_device_eval_batch_size": 32, @@ -36,10 +36,10 @@ "gather_across_devices": true, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "logging_first_step": true, "prediction_loss_only": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B/pt.json similarity index 82% rename from config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B-PT.json rename to config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B/pt.json index 2d017d7b..7d296617 100644 --- a/config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B-PT.json +++ b/config/constract_llm/train/ft/Sentence-InstructLM-Bi-EN-0.5B/pt.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-InstructLM-Bi-EN-0.5B-PT", - "output_dir": "./data/outputs/Sentence-InstructLM-Bi-EN-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-InstructLM-Bi-EN-0.5B-PT/logs", + "output_dir": "data/outputs/Sentence-InstructLM-Bi-EN-0.5B-PT", + "logging_dir": "data/outputs/Sentence-InstructLM-Bi-EN-0.5B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -27,7 +27,7 @@ "loss_scale": 100.0, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 5e-05, + "learning_rate": 5e-5, "per_device_train_batch_size": 128, "loss_cache_mini_batch_size": 128, "per_device_eval_batch_size": 128, @@ -36,10 +36,10 @@ "gather_across_devices": true, "eval_accumulation_steps": 1, "max_seq_length": 256, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "logging_first_step": true, "prediction_loss_only": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B/ft.json similarity index 86% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B/ft.json index f82a31a3..3558fea0 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B/ft.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": "negative", "run_name": "Sentence-Llama-Bi-EN-0.5B", - "output_dir": "./data/outputs/Sentence-Llama-Bi-EN-0.5B", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-EN-0.5B/logs", + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-0.5B", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-0.5B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B/pt.json similarity index 86% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B-PT.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B/pt.json index b414ae7a..8459c935 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-0.5B/pt.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-Llama-Bi-EN-0.5B-PT", - "output_dir": "./data/outputs/Sentence-Llama-Bi-EN-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-EN-0.5B-PT/logs", + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-0.5B-PT", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-0.5B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/ft-bs1024.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/ft-bs1024.json new file mode 100644 index 00000000..90b68e7f --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/ft-bs1024.json @@ -0,0 +1,66 @@ +{ + "trust_remote_code": true, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": "negative", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-PT-bs8192", + "run_name": "Sentence-Llama-Bi-EN-1B-bs1024", + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-1B-bs1024", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-1B-bs1024/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/ft.json similarity index 85% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/ft.json index a0a941b6..24959a7e 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/ft.json @@ -28,7 +28,7 @@ "per_device_train_batch_size": 64, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 64, - "gradient_accumulation_steps": 2, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, @@ -59,8 +59,8 @@ "batch_sampler": "group_by_label", "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B-PT", "run_name": "Sentence-Llama-Bi-EN-1B", - "output_dir": "./data/outputs/Sentence-Llama-Bi-EN-1B", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-EN-1B/logs", - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-1B", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-1B/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/pt-bs8192.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/pt-bs8192.json new file mode 100644 index 00000000..386fa4ae --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/pt-bs8192.json @@ -0,0 +1,66 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-Bi-EN-1B-PT-stage2-bs1024", + "trust_remote_code": true, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": null, + "run_name": "Sentence-Llama-Bi-EN-1B-PT-bs8192", + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-1B-PT-bs8192", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-1B-PT-bs8192/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 256, + "loss_cache_mini_batch_size": 64, + "per_device_eval_batch_size": 256, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B-PT.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/pt.json similarity index 84% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B-PT.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/pt.json index aebd7265..a28b3842 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-1B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-Llama-Bi-EN-1B-PT", - "output_dir": "./data/outputs/Sentence-Llama-Bi-EN-1B-PT", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-EN-1B-PT/logs", + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-1B-PT", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-1B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 256, "loss_cache_mini_batch_size": 64, "per_device_eval_batch_size": 256, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 2, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, @@ -61,6 +61,6 @@ "deepspeed": null, "evaluator_type": "triplet", "batch_sampler": "group_by_label", - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/ft-bs2048.json similarity index 84% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/ft-bs2048.json index 04a1cf3a..a6020d35 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/ft-bs2048.json @@ -57,10 +57,10 @@ "deepspeed": null, "evaluator_type": "triplet", "batch_sampler": "group_by_label", - "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-3B-PT", - "run_name": "Sentence-Llama-Bi-EN-3B", - "output_dir": "./data/outputs/Sentence-Llama-Bi-EN-3B", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-EN-3B/logs", - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-3B-PT-bs16384", + "run_name": "Sentence-Llama-Bi-EN-3B-bs2048", + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-3B-bs2048", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-3B-bs2048/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/ft.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/ft.json new file mode 100644 index 00000000..63c5ba7e --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/ft.json @@ -0,0 +1,66 @@ +{ + "trust_remote_code": true, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": "negative", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 4, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 1, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-3B-PT", + "run_name": "Sentence-Llama-Bi-EN-3B", + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-3B", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-3B/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/pt-bs16384.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/pt-bs16384.json new file mode 100644 index 00000000..4de20ac3 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/pt-bs16384.json @@ -0,0 +1,66 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-Bi-EN-3B-PT-stage2-bs2048", + "trust_remote_code": true, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": null, + "run_name": "Sentence-Llama-Bi-EN-3B-PT-bs16384", + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-3B-PT-bs16384", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-3B-PT-bs16384/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 512, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 512, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B-PT.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/pt.json similarity index 84% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B-PT.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/pt.json index 8799db21..513f5a69 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-EN-3B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-Llama-Bi-EN-3B-PT", - "output_dir": "./data/outputs/Sentence-Llama-Bi-EN-3B-PT", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-EN-3B-PT/logs", + "output_dir": "data/outputs/Sentence-Llama-Bi-EN-3B-PT", + "logging_dir": "data/outputs/Sentence-Llama-Bi-EN-3B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 512, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 512, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, @@ -61,6 +61,6 @@ "deepspeed": null, "evaluator_type": "triplet", "batch_sampler": "group_by_label", - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B-PT.json deleted file mode 100644 index 5c8f4109..00000000 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B-PT.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-0.5B-PT-stage2", - "trust_remote_code": true, - "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", - "use_all_subset": true, - "use_subsets": null, - "max_subset_samples": 2000000, - "dataset_config_name": null, - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "anchor_column_name": "anc", - "positive_column_name": "pos", - "negative_column_name": null, - "run_name": "Sentence-Llama-Bi-JP-0.5B-PT", - "output_dir": "./data/outputs/Sentence-Llama-Bi-JP-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-JP-0.5B-PT/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 1, - "loss_scale": 100.0, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-5, - "per_device_train_batch_size": 128, - "loss_cache_mini_batch_size": 128, - "per_device_eval_batch_size": 128, - "gradient_accumulation_steps": 4, - "gradient_checkpointing": true, - "gather_across_devices": true, - "eval_accumulation_steps": 1, - "max_seq_length": 256, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 3, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": null, - "evaluator_type": "triplet", - "batch_sampler": "group_by_label" -} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B.json deleted file mode 100644 index e8086fa9..00000000 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-0.5B-PT", - "trust_remote_code": true, - "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", - "use_all_subset": true, - "use_subsets": null, - "max_subset_samples": null, - "dataset_config_name": null, - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "anchor_column_name": "anc", - "positive_column_name": "pos", - "negative_column_name": "neg", - "run_name": "Sentence-Llama-Bi-JP-0.5B", - "output_dir": "./data/outputs/Sentence-Llama-Bi-JP-0.5B", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-JP-0.5B/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 1, - "loss_scale": 100.0, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-6, - "per_device_train_batch_size": 32, - "loss_cache_mini_batch_size": 32, - "per_device_eval_batch_size": 32, - "gradient_accumulation_steps": 2, - "gradient_checkpointing": true, - "gather_across_devices": true, - "eval_accumulation_steps": 1, - "max_seq_length": 512, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 3, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": null, - "evaluator_type": "triplet", - "batch_sampler": "group_by_label" -} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B/ft.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B/ft.json new file mode 100644 index 00000000..2f01804e --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B/ft.json @@ -0,0 +1,63 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-0.5B-PT", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": "neg", + "run_name": "Sentence-Llama-Bi-JP-0.5B", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-0.5B", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-0.5B/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 32, + "loss_cache_mini_batch_size": 32, + "per_device_eval_batch_size": 32, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 3, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B/pt.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B/pt.json new file mode 100644 index 00000000..a123f5ff --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-0.5B/pt.json @@ -0,0 +1,63 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-0.5B-PT-stage2", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": null, + "run_name": "Sentence-Llama-Bi-JP-0.5B-PT", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-0.5B-PT", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-0.5B-PT/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-5, + "per_device_train_batch_size": 128, + "loss_cache_mini_batch_size": 128, + "per_device_eval_batch_size": 128, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 3, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/ft-bs1024.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/ft-bs1024.json new file mode 100644 index 00000000..4c5c7b9f --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/ft-bs1024.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-1B-PT-bs8192", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": "neg", + "run_name": "Sentence-Llama-Bi-JP-1B-bs1024", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-1B-bs1024", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-1B-bs1024/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/ft.json similarity index 91% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/ft.json index cf9333f5..ee79ee1f 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/ft.json @@ -16,8 +16,8 @@ "positive_column_name": "pos", "negative_column_name": "neg", "run_name": "Sentence-Llama-Bi-JP-1B", - "output_dir": "./data/outputs/Sentence-Llama-Bi-JP-1B", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-JP-1B/logs", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-1B", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-1B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 64, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 64, - "gradient_accumulation_steps": 2, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/pt-bs8192.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/pt-bs8192.json new file mode 100644 index 00000000..35ce0725 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/pt-bs8192.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-1B-PT-stage2-bs1024", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": null, + "run_name": "Sentence-Llama-Bi-JP-1B-PT-bs8192", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-1B-PT-bs8192", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-1B-PT-bs8192/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 256, + "loss_cache_mini_batch_size": 64, + "per_device_eval_batch_size": 256, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B-PT.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/pt.json similarity index 91% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B-PT.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/pt.json index b39ce7bd..c2ae5dda 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-1B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "pos", "negative_column_name": null, "run_name": "Sentence-Llama-Bi-JP-1B-PT", - "output_dir": "./data/outputs/Sentence-Llama-Bi-JP-1B-PT", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-JP-1B-PT/logs", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-1B-PT", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-1B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 256, "loss_cache_mini_batch_size": 64, "per_device_eval_batch_size": 256, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 2, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/ft-bs2048.json similarity index 90% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/ft-bs2048.json index 815b4dd9..16bc7a79 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/ft-bs2048.json @@ -1,5 +1,5 @@ { - "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B-PT", + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B-PT-bs16384", "trust_remote_code": true, "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", "use_all_subset": true, @@ -15,9 +15,9 @@ "anchor_column_name": "anc", "positive_column_name": "pos", "negative_column_name": "neg", - "run_name": "Sentence-Llama-Bi-JP-3B", - "output_dir": "./data/outputs/Sentence-Llama-Bi-JP-3B", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-JP-3B/logs", + "run_name": "Sentence-Llama-Bi-JP-3B-bs2048", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-3B-bs2048", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-3B-bs2048/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/ft.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/ft.json new file mode 100644 index 00000000..38a28e7c --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/ft.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B-PT", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": "neg", + "run_name": "Sentence-Llama-Bi-JP-3B", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-3B", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-3B/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 4, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 1, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/pt-bs16384.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/pt-bs16384.json new file mode 100644 index 00000000..97fc7b89 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/pt-bs16384.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-Bi-JP-3B-PT-stage2-bs2048", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": null, + "run_name": "Sentence-Llama-Bi-JP-3B-PT-bs16384", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-3B-PT-bs16384", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-3B-PT-bs16384/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 512, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 512, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B-PT.json b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/pt.json similarity index 91% rename from config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B-PT.json rename to config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/pt.json index 9b437bed..2002105a 100644 --- a/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Llama-Bi-JP-3B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "pos", "negative_column_name": null, "run_name": "Sentence-Llama-Bi-JP-3B-PT", - "output_dir": "./data/outputs/Sentence-Llama-Bi-JP-3B-PT", - "logging_dir": "./data/outputs/Sentence-Llama-Bi-JP-3B-PT/logs", + "output_dir": "data/outputs/Sentence-Llama-Bi-JP-3B-PT", + "logging_dir": "data/outputs/Sentence-Llama-Bi-JP-3B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 512, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 512, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, diff --git a/config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B.json b/config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B/ft.json similarity index 82% rename from config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B.json rename to config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B/ft.json index a39b5027..265030a6 100644 --- a/config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B.json +++ b/config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B/ft.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": "negative", "run_name": "Sentence-Mistral-Bi-EN-0.5B", - "output_dir": "./data/outputs/Sentence-Mistral-Bi-EN-0.5B", - "logging_dir": "./data/outputs/Sentence-Mistral-Bi-EN-0.5B/logs", + "output_dir": "data/outputs/Sentence-Mistral-Bi-EN-0.5B", + "logging_dir": "data/outputs/Sentence-Mistral-Bi-EN-0.5B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -27,7 +27,7 @@ "loss_scale": 100.0, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 5e-06, + "learning_rate": 5e-6, "per_device_train_batch_size": 32, "loss_cache_mini_batch_size": 32, "per_device_eval_batch_size": 32, @@ -36,10 +36,10 @@ "gather_across_devices": true, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "logging_first_step": true, "prediction_loss_only": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B/pt.json similarity index 82% rename from config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B-PT.json rename to config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B/pt.json index 871f51c1..1bb2baad 100644 --- a/config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Mistral-Bi-EN-0.5B/pt.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-Mistral-Bi-EN-0.5B-PT", - "output_dir": "./data/outputs/Sentence-Mistral-Bi-EN-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-Mistral-Bi-EN-0.5B-PT/logs", + "output_dir": "data/outputs/Sentence-Mistral-Bi-EN-0.5B-PT", + "logging_dir": "data/outputs/Sentence-Mistral-Bi-EN-0.5B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -27,7 +27,7 @@ "loss_scale": 100.0, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 5e-05, + "learning_rate": 5e-5, "per_device_train_batch_size": 128, "loss_cache_mini_batch_size": 128, "per_device_eval_batch_size": 128, @@ -36,10 +36,10 @@ "gather_across_devices": true, "eval_accumulation_steps": 1, "max_seq_length": 256, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "logging_first_step": true, "prediction_loss_only": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B/ft.json similarity index 86% rename from config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B/ft.json index 8212a7a5..917f1983 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B/ft.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": "negative", "run_name": "Sentence-ModernBERT-EN-0.5B", - "output_dir": "./data/outputs/Sentence-ModernBERT-EN-0.5B", - "logging_dir": "./data/outputs/Sentence-ModernBERT-EN-0.5B/logs", + "output_dir": "data/outputs/Sentence-ModernBERT-EN-0.5B", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-0.5B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B/pt.json similarity index 86% rename from config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B-PT.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B/pt.json index 029719a4..b48b144e 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B-PT.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-0.5B/pt.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-ModernBERT-EN-0.5B-PT", - "output_dir": "./data/outputs/Sentence-ModernBERT-EN-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-ModernBERT-EN-0.5B-PT/logs", + "output_dir": "data/outputs/Sentence-ModernBERT-EN-0.5B-PT", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-0.5B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/ft-bs1024.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/ft-bs1024.json new file mode 100644 index 00000000..9b5426bc --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/ft-bs1024.json @@ -0,0 +1,66 @@ +{ + "trust_remote_code": false, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": "negative", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-PT-bs8192", + "run_name": "Sentence-ModernBERT-EN-1B-bs1024", + "output_dir": "data/outputs/Sentence-ModernBERT-EN-1B-bs1024", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-1B-bs1024/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/ft.json similarity index 84% rename from config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/ft.json index b1282e79..850619c9 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/ft.json @@ -28,7 +28,7 @@ "per_device_train_batch_size": 64, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 64, - "gradient_accumulation_steps": 2, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, @@ -59,8 +59,8 @@ "batch_sampler": "group_by_label", "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-1B-PT", "run_name": "Sentence-ModernBERT-EN-1B", - "output_dir": "./data/outputs/Sentence-ModernBERT-EN-1B", - "logging_dir": "./data/outputs/Sentence-ModernBERT-EN-1B/logs", - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "output_dir": "data/outputs/Sentence-ModernBERT-EN-1B", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-1B/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/pt-bs8192.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/pt-bs8192.json new file mode 100644 index 00000000..4689324c --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/pt-bs8192.json @@ -0,0 +1,66 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage2-bs1024", + "trust_remote_code": false, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": null, + "run_name": "Sentence-ModernBERT-EN-1B-PT-bs8192", + "output_dir": "data/outputs/Sentence-ModernBERT-EN-1B-PT-bs8192", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-1B-PT-bs8192/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 256, + "loss_cache_mini_batch_size": 64, + "per_device_eval_batch_size": 256, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B-PT.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/pt.json similarity index 84% rename from config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B-PT.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/pt.json index 10f75c98..025b147c 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B-PT.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-1B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-ModernBERT-EN-1B-PT", - "output_dir": "./data/outputs/Sentence-ModernBERT-EN-1B-PT", - "logging_dir": "./data/outputs/Sentence-ModernBERT-EN-1B-PT/logs", + "output_dir": "data/outputs/Sentence-ModernBERT-EN-1B-PT", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-1B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 256, "loss_cache_mini_batch_size": 64, "per_device_eval_batch_size": 256, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 2, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, @@ -61,6 +61,6 @@ "deepspeed": null, "evaluator_type": "triplet", "batch_sampler": "group_by_label", - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/ft-bs2048.json similarity index 83% rename from config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/ft-bs2048.json index 25c8c8f8..67146945 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/ft-bs2048.json @@ -57,10 +57,10 @@ "deepspeed": null, "evaluator_type": "triplet", "batch_sampler": "group_by_label", - "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-3B-PT", - "run_name": "Sentence-ModernBERT-EN-3B", - "output_dir": "./data/outputs/Sentence-ModernBERT-EN-3B", - "logging_dir": "./data/outputs/Sentence-ModernBERT-EN-3B/logs", - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-3B-PT-bs16384", + "run_name": "Sentence-ModernBERT-EN-3B-bs2048", + "output_dir": "data/outputs/Sentence-ModernBERT-EN-3B-bs2048", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-3B-bs2048/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/ft.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/ft.json new file mode 100644 index 00000000..5953d563 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/ft.json @@ -0,0 +1,66 @@ +{ + "trust_remote_code": false, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": "negative", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 4, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 1, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-EN-3B-PT", + "run_name": "Sentence-ModernBERT-EN-3B", + "output_dir": "data/outputs/Sentence-ModernBERT-EN-3B", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-3B/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/pt-bs16384.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/pt-bs16384.json new file mode 100644 index 00000000..ae96a40b --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/pt-bs16384.json @@ -0,0 +1,66 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-3B-PT-stage2-bs2048", + "trust_remote_code": false, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": null, + "run_name": "Sentence-ModernBERT-EN-3B-PT-bs16384", + "output_dir": "data/outputs/Sentence-ModernBERT-EN-3B-PT-bs16384", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-3B-PT-bs16384/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 512, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 512, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B-PT.json b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/pt.json similarity index 84% rename from config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B-PT.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/pt.json index 5185734d..88f7fbb2 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B-PT.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-EN-3B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-ModernBERT-EN-3B-PT", - "output_dir": "./data/outputs/Sentence-ModernBERT-EN-3B-PT", - "logging_dir": "./data/outputs/Sentence-ModernBERT-EN-3B-PT/logs", + "output_dir": "data/outputs/Sentence-ModernBERT-EN-3B-PT", + "logging_dir": "data/outputs/Sentence-ModernBERT-EN-3B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 512, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 512, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, @@ -61,6 +61,6 @@ "deepspeed": null, "evaluator_type": "triplet", "batch_sampler": "group_by_label", - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B-PT.json deleted file mode 100644 index 106942ba..00000000 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B-PT.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-PT-stage2", - "trust_remote_code": false, - "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", - "use_all_subset": true, - "use_subsets": null, - "max_subset_samples": 2000000, - "dataset_config_name": null, - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "anchor_column_name": "anc", - "positive_column_name": "pos", - "negative_column_name": null, - "run_name": "Sentence-ModernBERT-JP-0.5B-PT", - "output_dir": "./data/outputs/Sentence-ModernBERT-JP-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-ModernBERT-JP-0.5B-PT/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 1, - "loss_scale": 100.0, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-5, - "per_device_train_batch_size": 128, - "loss_cache_mini_batch_size": 128, - "per_device_eval_batch_size": 128, - "gradient_accumulation_steps": 4, - "gradient_checkpointing": true, - "gather_across_devices": true, - "eval_accumulation_steps": 1, - "max_seq_length": 256, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 3, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": null, - "evaluator_type": "triplet", - "batch_sampler": "group_by_label" -} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B.json deleted file mode 100644 index 23111f34..00000000 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-0.5B-PT", - "trust_remote_code": false, - "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", - "use_all_subset": true, - "use_subsets": null, - "max_subset_samples": null, - "dataset_config_name": null, - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "anchor_column_name": "anc", - "positive_column_name": "pos", - "negative_column_name": "neg", - "run_name": "Sentence-ModernBERT-JP-0.5B", - "output_dir": "./data/outputs/Sentence-ModernBERT-JP-0.5B", - "logging_dir": "./data/outputs/Sentence-ModernBERT-JP-0.5B/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 1, - "loss_scale": 100.0, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-6, - "per_device_train_batch_size": 32, - "loss_cache_mini_batch_size": 32, - "per_device_eval_batch_size": 32, - "gradient_accumulation_steps": 2, - "gradient_checkpointing": true, - "gather_across_devices": true, - "eval_accumulation_steps": 1, - "max_seq_length": 512, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 3, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": null, - "evaluator_type": "triplet", - "batch_sampler": "group_by_label" -} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/ft.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/ft.json new file mode 100644 index 00000000..ada0197a --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/ft.json @@ -0,0 +1,63 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-0.5B-PT", + "trust_remote_code": false, + "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": "neg", + "run_name": "Sentence-ModernBERT-JP-0.5B", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-0.5B", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-0.5B/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 32, + "loss_cache_mini_batch_size": 32, + "per_device_eval_batch_size": 32, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 3, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/pt.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/pt.json new file mode 100644 index 00000000..e311617f --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/pt.json @@ -0,0 +1,63 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-PT-stage2", + "trust_remote_code": false, + "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": null, + "run_name": "Sentence-ModernBERT-JP-0.5B-PT", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-0.5B-PT", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-0.5B-PT/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-5, + "per_device_train_batch_size": 128, + "loss_cache_mini_batch_size": 128, + "per_device_eval_batch_size": 128, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 3, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/ft-bs1024.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/ft-bs1024.json new file mode 100644 index 00000000..222c6e4e --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/ft-bs1024.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-1B-PT-bs8192", + "trust_remote_code": false, + "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": "neg", + "run_name": "Sentence-ModernBERT-JP-1B-bs1024", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-1B-bs1024", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-1B-bs1024/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/ft.json similarity index 91% rename from config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/ft.json index f2269b12..9ba1b2ed 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/ft.json @@ -16,8 +16,8 @@ "positive_column_name": "pos", "negative_column_name": "neg", "run_name": "Sentence-ModernBERT-JP-1B", - "output_dir": "./data/outputs/Sentence-ModernBERT-JP-1B", - "logging_dir": "./data/outputs/Sentence-ModernBERT-JP-1B/logs", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-1B", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-1B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 64, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 64, - "gradient_accumulation_steps": 2, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/pt-bs8192.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/pt-bs8192.json new file mode 100644 index 00000000..b9f1bd39 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/pt-bs8192.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-PT-stage2-bs1024", + "trust_remote_code": false, + "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": null, + "run_name": "Sentence-ModernBERT-JP-1B-PT-bs8192", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-1B-PT-bs8192", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-1B-PT-bs8192/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 256, + "loss_cache_mini_batch_size": 64, + "per_device_eval_batch_size": 256, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B-PT.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/pt.json similarity index 91% rename from config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B-PT.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/pt.json index d273488c..fdb47b51 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B-PT.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-1B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "pos", "negative_column_name": null, "run_name": "Sentence-ModernBERT-JP-1B-PT", - "output_dir": "./data/outputs/Sentence-ModernBERT-JP-1B-PT", - "logging_dir": "./data/outputs/Sentence-ModernBERT-JP-1B-PT/logs", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-1B-PT", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-1B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 256, "loss_cache_mini_batch_size": 64, "per_device_eval_batch_size": 256, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 2, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/ft-bs2048.json similarity index 89% rename from config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/ft-bs2048.json index 486562bb..56cac417 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/ft-bs2048.json @@ -1,5 +1,5 @@ { - "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-3B-PT", + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-3B-PT-bs16384", "trust_remote_code": false, "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", "use_all_subset": true, @@ -15,9 +15,9 @@ "anchor_column_name": "anc", "positive_column_name": "pos", "negative_column_name": "neg", - "run_name": "Sentence-ModernBERT-JP-3B", - "output_dir": "./data/outputs/Sentence-ModernBERT-JP-3B", - "logging_dir": "./data/outputs/Sentence-ModernBERT-JP-3B/logs", + "run_name": "Sentence-ModernBERT-JP-3B-bs2048", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-3B-bs2048", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-3B-bs2048/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/ft.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/ft.json new file mode 100644 index 00000000..a0b85c27 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/ft.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-3B-PT", + "trust_remote_code": false, + "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": "neg", + "run_name": "Sentence-ModernBERT-JP-3B", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-3B", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-3B/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 4, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 1, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/pt-bs16384.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/pt-bs16384.json new file mode 100644 index 00000000..480021b8 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/pt-bs16384.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage2-bs2048", + "trust_remote_code": false, + "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": null, + "run_name": "Sentence-ModernBERT-JP-3B-PT-bs16384", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-3B-PT-bs16384", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-3B-PT-bs16384/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 512, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 512, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B-PT.json b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/pt.json similarity index 91% rename from config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B-PT.json rename to config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/pt.json index 223fa1ce..249b6ce1 100644 --- a/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B-PT.json +++ b/config/constract_llm/train/ft/Sentence-ModernBERT-JP-3B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "pos", "negative_column_name": null, "run_name": "Sentence-ModernBERT-JP-3B-PT", - "output_dir": "./data/outputs/Sentence-ModernBERT-JP-3B-PT", - "logging_dir": "./data/outputs/Sentence-ModernBERT-JP-3B-PT/logs", + "output_dir": "data/outputs/Sentence-ModernBERT-JP-3B-PT", + "logging_dir": "data/outputs/Sentence-ModernBERT-JP-3B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 512, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 512, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, diff --git a/config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B.json b/config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B/ft.json similarity index 82% rename from config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B.json rename to config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B/ft.json index 38bd644a..8c8690a3 100644 --- a/config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B.json +++ b/config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B/ft.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": "negative", "run_name": "Sentence-Qwen2.5-Bi-EN-0.5B", - "output_dir": "./data/outputs/Sentence-Qwen2.5-Bi-EN-0.5B", - "logging_dir": "./data/outputs/Sentence-Qwen2.5-Bi-EN-0.5B/logs", + "output_dir": "data/outputs/Sentence-Qwen2.5-Bi-EN-0.5B", + "logging_dir": "data/outputs/Sentence-Qwen2.5-Bi-EN-0.5B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -27,7 +27,7 @@ "loss_scale": 100.0, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 5e-06, + "learning_rate": 5e-6, "per_device_train_batch_size": 32, "loss_cache_mini_batch_size": 32, "per_device_eval_batch_size": 32, @@ -36,10 +36,10 @@ "gather_across_devices": true, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "logging_first_step": true, "prediction_loss_only": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B/pt.json similarity index 82% rename from config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B-PT.json rename to config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B/pt.json index e36e553a..aea1b23c 100644 --- a/config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Qwen2.5-Bi-EN-0.5B/pt.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-Qwen2.5-Bi-EN-0.5B-PT", - "output_dir": "./data/outputs/Sentence-Qwen2.5-Bi-EN-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-Qwen2.5-Bi-EN-0.5B-PT/logs", + "output_dir": "data/outputs/Sentence-Qwen2.5-Bi-EN-0.5B-PT", + "logging_dir": "data/outputs/Sentence-Qwen2.5-Bi-EN-0.5B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -27,7 +27,7 @@ "loss_scale": 100.0, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 5e-05, + "learning_rate": 5e-5, "per_device_train_batch_size": 128, "loss_cache_mini_batch_size": 128, "per_device_eval_batch_size": 128, @@ -36,10 +36,10 @@ "gather_across_devices": true, "eval_accumulation_steps": 1, "max_seq_length": 256, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "logging_first_step": true, "prediction_loss_only": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B.json b/config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/ft.json similarity index 82% rename from config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B.json rename to config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/ft.json index c507fd52..9e42fd9f 100644 --- a/config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B.json +++ b/config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/ft.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": "negative", "run_name": "Sentence-Qwen2.5-Scratch-Bi-EN-0.5B", - "output_dir": "./data/outputs/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B", - "logging_dir": "./data/outputs/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/logs", + "output_dir": "data/outputs/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B", + "logging_dir": "data/outputs/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -27,7 +27,7 @@ "loss_scale": 100.0, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 5e-06, + "learning_rate": 5e-6, "per_device_train_batch_size": 32, "loss_cache_mini_batch_size": 32, "per_device_eval_batch_size": 32, @@ -36,10 +36,10 @@ "gather_across_devices": true, "eval_accumulation_steps": 1, "max_seq_length": 512, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "logging_first_step": true, "prediction_loss_only": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/pt.json similarity index 82% rename from config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B-PT.json rename to config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/pt.json index 4ec2e867..c4836873 100644 --- a/config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B/pt.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-Qwen2.5-Scratch-Bi-EN-0.5B-PT", - "output_dir": "./data/outputs/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B-PT/logs", + "output_dir": "data/outputs/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B-PT", + "logging_dir": "data/outputs/Sentence-Qwen2.5-Scratch-Bi-EN-0.5B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -27,7 +27,7 @@ "loss_scale": 100.0, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 5e-05, + "learning_rate": 5e-5, "per_device_train_batch_size": 128, "loss_cache_mini_batch_size": 128, "per_device_eval_batch_size": 128, @@ -36,10 +36,10 @@ "gather_across_devices": true, "eval_accumulation_steps": 1, "max_seq_length": 256, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "logging_first_step": true, "prediction_loss_only": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B-PT.json deleted file mode 100644 index dc14f7bf..00000000 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B-PT.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-0.5b", - "trust_remote_code": true, - "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", - "use_all_subset": true, - "use_subsets": null, - "max_subset_samples": 2000000, - "dataset_config_name": null, - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "anchor_column_name": "anc", - "positive_column_name": "pos", - "negative_column_name": null, - "run_name": "Sentence-Sarashina-Bi-0.5B-PT", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-0.5B-PT/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 1, - "loss_scale": 100.0, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-5, - "per_device_train_batch_size": 128, - "loss_cache_mini_batch_size": 128, - "per_device_eval_batch_size": 128, - "gradient_accumulation_steps": 4, - "gradient_checkpointing": true, - "gather_across_devices": true, - "eval_accumulation_steps": 1, - "max_seq_length": 256, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 3, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": null, - "evaluator_type": "triplet", - "batch_sampler": "group_by_label" -} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B.json deleted file mode 100644 index b7dc640b..00000000 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-0.5B-PT", - "trust_remote_code": true, - "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", - "use_all_subset": true, - "use_subsets": null, - "max_subset_samples": null, - "dataset_config_name": null, - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "anchor_column_name": "anc", - "positive_column_name": "pos", - "negative_column_name": "neg", - "run_name": "Sentence-Sarashina-Bi-0.5B", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-0.5B", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-0.5B/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 1, - "loss_scale": 100.0, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-6, - "per_device_train_batch_size": 32, - "loss_cache_mini_batch_size": 32, - "per_device_eval_batch_size": 32, - "gradient_accumulation_steps": 2, - "gradient_checkpointing": true, - "gather_across_devices": true, - "eval_accumulation_steps": 1, - "max_seq_length": 512, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 3, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": null, - "evaluator_type": "triplet", - "batch_sampler": "group_by_label" -} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B/ft.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B/ft.json new file mode 100644 index 00000000..dcb6ef44 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B/ft.json @@ -0,0 +1,63 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-0.5B-PT", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": "neg", + "run_name": "Sentence-Sarashina-Bi-0.5B", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-0.5B", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-0.5B/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-6, + "per_device_train_batch_size": 32, + "loss_cache_mini_batch_size": 32, + "per_device_eval_batch_size": 32, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 3, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B/pt.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B/pt.json new file mode 100644 index 00000000..90416022 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-0.5B/pt.json @@ -0,0 +1,63 @@ +{ + "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-0.5b", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": null, + "run_name": "Sentence-Sarashina-Bi-0.5B-PT", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-0.5B-PT", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-0.5B-PT/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-5, + "per_device_train_batch_size": 128, + "loss_cache_mini_batch_size": 128, + "per_device_eval_batch_size": 128, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 3, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B/ft.json similarity index 86% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B/ft.json index 98ceb093..876fa989 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B/ft.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": "negative", "run_name": "Sentence-Sarashina-Bi-EN-0.5B", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-0.5B", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-0.5B/logs", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-0.5B", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-0.5B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B-PT.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B/pt.json similarity index 86% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B-PT.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B/pt.json index e29c3223..25c9cb12 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-0.5B/pt.json @@ -15,8 +15,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-Sarashina-Bi-EN-0.5B-PT", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-0.5B-PT", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-0.5B-PT/logs", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-0.5B-PT", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-0.5B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -61,6 +61,6 @@ "evaluator_type": "triplet", "batch_sampler": "group_by_label", "streaming": false, - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/ft-bs1024.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/ft-bs1024.json new file mode 100644 index 00000000..2230002e --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/ft-bs1024.json @@ -0,0 +1,66 @@ +{ + "trust_remote_code": true, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": "negative", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-PT-bs8192", + "run_name": "Sentence-Sarashina-Bi-EN-1B-bs1024", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-1B-bs1024", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-1B-bs1024/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/ft.json similarity index 84% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/ft.json index 32391386..80642811 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/ft.json @@ -28,7 +28,7 @@ "per_device_train_batch_size": 64, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 64, - "gradient_accumulation_steps": 2, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, @@ -59,8 +59,8 @@ "batch_sampler": "group_by_label", "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-1B-PT", "run_name": "Sentence-Sarashina-Bi-EN-1B", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-1B", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-1B/logs", - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-1B", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-1B/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/pt-bs8192.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/pt-bs8192.json new file mode 100644 index 00000000..a4430fff --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/pt-bs8192.json @@ -0,0 +1,66 @@ +{ + "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-1b", + "trust_remote_code": true, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": null, + "run_name": "Sentence-Sarashina-Bi-EN-1B-PT-bs8192", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-1B-PT-bs8192", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-1B-PT-bs8192/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 256, + "loss_cache_mini_batch_size": 64, + "per_device_eval_batch_size": 256, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B-PT.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/pt.json similarity index 84% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B-PT.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/pt.json index 8f6d9fc1..14630eaa 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-1B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-Sarashina-Bi-EN-1B-PT", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-1B-PT", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-1B-PT/logs", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-1B-PT", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-1B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 256, "loss_cache_mini_batch_size": 64, "per_device_eval_batch_size": 256, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 2, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, @@ -61,6 +61,6 @@ "deepspeed": null, "evaluator_type": "triplet", "batch_sampler": "group_by_label", - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/ft-bs2048.json similarity index 83% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/ft-bs2048.json index b9cf3608..6f4b0e99 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/ft-bs2048.json @@ -57,10 +57,10 @@ "deepspeed": null, "evaluator_type": "triplet", "batch_sampler": "group_by_label", - "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-3B-PT", - "run_name": "Sentence-Sarashina-Bi-EN-3B", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-3B", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-3B/logs", - "train_file": "./data/processed/english_embedding/ft/train.jsonl", - "validation_file": "./data/processed/english_embedding/ft/validation.jsonl" + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-3B-PT-bs16384", + "run_name": "Sentence-Sarashina-Bi-EN-3B-bs2048", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-3B-bs2048", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-3B-bs2048/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/ft.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/ft.json new file mode 100644 index 00000000..b09b8657 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/ft.json @@ -0,0 +1,66 @@ +{ + "trust_remote_code": true, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": "negative", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 4, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 1, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-EN-3B-PT", + "run_name": "Sentence-Sarashina-Bi-EN-3B", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-3B", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-3B/logs", + "train_file": "data/processed/english_embedding/ft/train.jsonl", + "validation_file": "data/processed/english_embedding/ft/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/pt-bs16384.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/pt-bs16384.json new file mode 100644 index 00000000..08957abd --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/pt-bs16384.json @@ -0,0 +1,66 @@ +{ + "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-3b", + "trust_remote_code": true, + "dataset_name": null, + "use_all_subset": false, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anchor", + "positive_column_name": "positive", + "negative_column_name": null, + "run_name": "Sentence-Sarashina-Bi-EN-3B-PT-bs16384", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-3B-PT-bs16384", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-3B-PT-bs16384/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 512, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 512, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label", + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B-PT.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/pt.json similarity index 84% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B-PT.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/pt.json index 1e3eeb70..f5119629 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-EN-3B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "positive", "negative_column_name": null, "run_name": "Sentence-Sarashina-Bi-EN-3B-PT", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-3B-PT", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-EN-3B-PT/logs", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-EN-3B-PT", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-EN-3B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 512, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 512, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, @@ -61,6 +61,6 @@ "deepspeed": null, "evaluator_type": "triplet", "batch_sampler": "group_by_label", - "train_file": "./data/processed/english_embedding/wsl/train.jsonl", - "validation_file": "./data/processed/english_embedding/wsl/validation.jsonl" + "train_file": "data/processed/english_embedding/wsl/train.jsonl", + "validation_file": "data/processed/english_embedding/wsl/validation.jsonl" } diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/ft-bs1024.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/ft-bs1024.json new file mode 100644 index 00000000..c3689353 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/ft-bs1024.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-1B-PT-bs8192", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": "neg", + "run_name": "Sentence-Sarashina-Bi-JP-1B-bs1024", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-JP-1B-bs1024", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-JP-1B-bs1024/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/ft.json similarity index 91% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/ft.json index a6392470..bc42ac00 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/ft.json @@ -16,8 +16,8 @@ "positive_column_name": "pos", "negative_column_name": "neg", "run_name": "Sentence-Sarashina-Bi-JP-1B", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-JP-1B", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-JP-1B/logs", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-JP-1B", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-JP-1B/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 64, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 64, - "gradient_accumulation_steps": 2, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/pt-bs8192.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/pt-bs8192.json new file mode 100644 index 00000000..e2dd11b8 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/pt-bs8192.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-1b", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": null, + "run_name": "Sentence-Sarashina-Bi-JP-1B-PT-bs8192", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-JP-1B-PT-bs8192", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-JP-1B-PT-bs8192/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 256, + "loss_cache_mini_batch_size": 64, + "per_device_eval_batch_size": 256, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B-PT.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/pt.json similarity index 91% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B-PT.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/pt.json index c795cfa1..bde2373b 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-1B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "pos", "negative_column_name": null, "run_name": "Sentence-Sarashina-Bi-JP-1B-PT", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-JP-1B-PT", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-JP-1B-PT/logs", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-JP-1B-PT", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-JP-1B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 256, "loss_cache_mini_batch_size": 64, "per_device_eval_batch_size": 256, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 2, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/ft-bs2048.json similarity index 89% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/ft-bs2048.json index a760646f..795828ec 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/ft-bs2048.json @@ -1,5 +1,5 @@ { - "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-3B-PT", + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-3B-PT-bs16384", "trust_remote_code": true, "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", "use_all_subset": true, @@ -15,9 +15,9 @@ "anchor_column_name": "anc", "positive_column_name": "pos", "negative_column_name": "neg", - "run_name": "Sentence-Sarashina-Bi-JP-3B", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-JP-3B", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-JP-3B/logs", + "run_name": "Sentence-Sarashina-Bi-JP-3B-bs2048", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-JP-3B-bs2048", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-JP-3B-bs2048/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/ft.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/ft.json new file mode 100644 index 00000000..ae08d790 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/ft.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/Sentence-Sarashina-Bi-JP-3B-PT", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-v3-dataset-ft", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": null, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": "neg", + "run_name": "Sentence-Sarashina-Bi-JP-3B", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-JP-3B", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-JP-3B/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-6, + "per_device_train_batch_size": 64, + "loss_cache_mini_batch_size": 4, + "per_device_eval_batch_size": 64, + "gradient_accumulation_steps": 1, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 512, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/pt-bs16384.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/pt-bs16384.json new file mode 100644 index 00000000..8dbb38d7 --- /dev/null +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/pt-bs16384.json @@ -0,0 +1,64 @@ +{ + "model_name_or_path": "iamtatsuki05/sarashina2.2-Bi-3b", + "trust_remote_code": true, + "dataset_name": "cl-nagoya/ruri-dataset-v2-pt", + "use_all_subset": true, + "use_subsets": null, + "max_subset_samples": 2000000, + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "anchor_column_name": "anc", + "positive_column_name": "pos", + "negative_column_name": null, + "run_name": "Sentence-Sarashina-Bi-JP-3B-PT-bs16384", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-JP-3B-PT-bs16384", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-JP-3B-PT-bs16384/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "loss_scale": 100.0, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 512, + "loss_cache_mini_batch_size": 16, + "per_device_eval_batch_size": 512, + "gradient_accumulation_steps": 4, + "gradient_checkpointing": true, + "gather_across_devices": true, + "eval_accumulation_steps": 1, + "max_seq_length": 256, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 20, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": null, + "evaluator_type": "triplet", + "batch_sampler": "group_by_label" +} diff --git a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B-PT.json b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/pt.json similarity index 91% rename from config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B-PT.json rename to config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/pt.json index 79aaec12..027b9b3d 100644 --- a/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B-PT.json +++ b/config/constract_llm/train/ft/Sentence-Sarashina-Bi-JP-3B/pt.json @@ -16,8 +16,8 @@ "positive_column_name": "pos", "negative_column_name": null, "run_name": "Sentence-Sarashina-Bi-JP-3B-PT", - "output_dir": "./data/outputs/Sentence-Sarashina-Bi-JP-3B-PT", - "logging_dir": "./data/outputs/Sentence-Sarashina-Bi-JP-3B-PT/logs", + "output_dir": "data/outputs/Sentence-Sarashina-Bi-JP-3B-PT", + "logging_dir": "data/outputs/Sentence-Sarashina-Bi-JP-3B-PT/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -32,7 +32,7 @@ "per_device_train_batch_size": 512, "loss_cache_mini_batch_size": 16, "per_device_eval_batch_size": 512, - "gradient_accumulation_steps": 4, + "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "gather_across_devices": true, "eval_accumulation_steps": 1, diff --git a/config/constract_llm/train/pt/Llama-EN-0.5B-PT-stage1.json b/config/constract_llm/train/pt/Llama-EN-0.5B/stage1.json similarity index 88% rename from config/constract_llm/train/pt/Llama-EN-0.5B-PT-stage1.json rename to config/constract_llm/train/pt/Llama-EN-0.5B/stage1.json index 2fa1a577..44ea3216 100644 --- a/config/constract_llm/train/pt/Llama-EN-0.5B-PT-stage1.json +++ b/config/constract_llm/train/pt/Llama-EN-0.5B/stage1.json @@ -10,8 +10,8 @@ "validation_split_percentage": 1, "text_column_name": "text", "run_name": "Llama-EN-0.5B-PT-stage1", - "output_dir": "./data/outputs/Llama-EN-0.5B-PT-stage1", - "logging_dir": "./data/outputs/Llama-EN-0.5B-PT-stage1/logs", + "output_dir": "data/outputs/Llama-EN-0.5B-PT-stage1", + "logging_dir": "data/outputs/Llama-EN-0.5B-PT-stage1/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -49,6 +49,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "streaming": false } diff --git a/config/constract_llm/train/pt/Llama-EN-0.5B-PT-stage2.json b/config/constract_llm/train/pt/Llama-EN-0.5B/stage2.json similarity index 88% rename from config/constract_llm/train/pt/Llama-EN-0.5B-PT-stage2.json rename to config/constract_llm/train/pt/Llama-EN-0.5B/stage2.json index 27f36007..8a35e405 100644 --- a/config/constract_llm/train/pt/Llama-EN-0.5B-PT-stage2.json +++ b/config/constract_llm/train/pt/Llama-EN-0.5B/stage2.json @@ -10,8 +10,8 @@ "dataset_config_name": "en", "text_column_name": "text", "run_name": "Llama-EN-0.5B-PT-stage2", - "output_dir": "./data/outputs/Llama-EN-0.5B-PT-stage2", - "logging_dir": "./data/outputs/Llama-EN-0.5B-PT-stage2/logs", + "output_dir": "data/outputs/Llama-EN-0.5B-PT-stage2", + "logging_dir": "data/outputs/Llama-EN-0.5B-PT-stage2/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -49,6 +49,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "streaming": false } diff --git a/config/constract_llm/train/pt/Llama-EN-1B/stage1-bs8192.json b/config/constract_llm/train/pt/Llama-EN-1B/stage1-bs8192.json new file mode 100644 index 00000000..75e50075 --- /dev/null +++ b/config/constract_llm/train/pt/Llama-EN-1B/stage1-bs8192.json @@ -0,0 +1,54 @@ +{ + "dataset_name": "HuggingFaceFW/fineweb-edu", + "dataset_config_name": "sample-100BT", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 10, + "text_column_name": "text", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 0.2, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-4, + "per_device_train_batch_size": 8, + "per_device_eval_batch_size": 4, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", + "trust_remote_code": true, + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-init", + "run_name": "Llama-EN-1B-PT-stage1-bs8192", + "output_dir": "data/outputs/Llama-EN-1B-PT-stage1-bs8192", + "logging_dir": "data/outputs/Llama-EN-1B-PT-stage1-bs8192/logs" +} diff --git a/config/constract_llm/train/pt/Llama-EN-1B-PT-stage1.json b/config/constract_llm/train/pt/Llama-EN-1B/stage1.json similarity index 84% rename from config/constract_llm/train/pt/Llama-EN-1B-PT-stage1.json rename to config/constract_llm/train/pt/Llama-EN-1B/stage1.json index 78e1a7af..3403c4d3 100644 --- a/config/constract_llm/train/pt/Llama-EN-1B-PT-stage1.json +++ b/config/constract_llm/train/pt/Llama-EN-1B/stage1.json @@ -17,10 +17,10 @@ "num_train_epochs": 0.2, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 0.0003, + "learning_rate": 3e-4, "per_device_train_batch_size": 8, "per_device_eval_batch_size": 4, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 64, "eval_accumulation_steps": 1, "max_seq_length": 1024, "weight_decay": 1e-5, @@ -45,10 +45,10 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "trust_remote_code": true, "model_name_or_path": "iamtatsuki05/Llama-JP-1B-init", "run_name": "Llama-EN-1B-PT-stage1", - "output_dir": "./data/outputs/Llama-EN-1B-PT-stage1", - "logging_dir": "./data/outputs/Llama-EN-1B-PT-stage1/logs" + "output_dir": "data/outputs/Llama-EN-1B-PT-stage1", + "logging_dir": "data/outputs/Llama-EN-1B-PT-stage1/logs" } diff --git a/config/constract_llm/train/pt/Llama-EN-1B/stage2-bs1024.json b/config/constract_llm/train/pt/Llama-EN-1B/stage2-bs1024.json new file mode 100644 index 00000000..e00f29fd --- /dev/null +++ b/config/constract_llm/train/pt/Llama-EN-1B/stage2-bs1024.json @@ -0,0 +1,54 @@ +{ + "dataset_name": "google/wiki40b", + "dataset_config_name": "en", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", + "trust_remote_code": true, + "model_name_or_path": "iamtatsuki05/Llama-EN-1B-PT-stage1-bs8192", + "run_name": "Llama-EN-1B-PT-stage2-bs1024", + "output_dir": "data/outputs/Llama-EN-1B-PT-stage2-bs1024", + "logging_dir": "data/outputs/Llama-EN-1B-PT-stage2-bs1024/logs" +} diff --git a/config/constract_llm/train/pt/Llama-EN-1B-PT-stage2.json b/config/constract_llm/train/pt/Llama-EN-1B/stage2.json similarity index 86% rename from config/constract_llm/train/pt/Llama-EN-1B-PT-stage2.json rename to config/constract_llm/train/pt/Llama-EN-1B/stage2.json index 4718443c..256c7f0e 100644 --- a/config/constract_llm/train/pt/Llama-EN-1B-PT-stage2.json +++ b/config/constract_llm/train/pt/Llama-EN-1B/stage2.json @@ -20,7 +20,7 @@ "learning_rate": 3e-5, "per_device_train_batch_size": 1, "per_device_eval_batch_size": 1, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 64, "eval_accumulation_steps": 1, "max_seq_length": 8192, "weight_decay": 1e-5, @@ -45,10 +45,10 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "trust_remote_code": true, "model_name_or_path": "iamtatsuki05/Llama-EN-1B-PT-stage1", "run_name": "Llama-EN-1B-PT-stage2", - "output_dir": "./data/outputs/Llama-EN-1B-PT-stage2", - "logging_dir": "./data/outputs/Llama-EN-1B-PT-stage2/logs" + "output_dir": "data/outputs/Llama-EN-1B-PT-stage2", + "logging_dir": "data/outputs/Llama-EN-1B-PT-stage2/logs" } diff --git a/config/constract_llm/train/pt/Llama-EN-3B/stage1-bs16384.json b/config/constract_llm/train/pt/Llama-EN-3B/stage1-bs16384.json new file mode 100644 index 00000000..48451053 --- /dev/null +++ b/config/constract_llm/train/pt/Llama-EN-3B/stage1-bs16384.json @@ -0,0 +1,54 @@ +{ + "dataset_name": "HuggingFaceFW/fineweb-edu", + "dataset_config_name": "sample-100BT", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 10, + "text_column_name": "text", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 0.45, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-4, + "per_device_train_batch_size": 4, + "per_device_eval_batch_size": 4, + "gradient_accumulation_steps": 512, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", + "trust_remote_code": true, + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-init", + "run_name": "Llama-EN-3B-PT-stage1-bs16384", + "output_dir": "data/outputs/Llama-EN-3B-PT-stage1-bs16384", + "logging_dir": "data/outputs/Llama-EN-3B-PT-stage1-bs16384/logs" +} diff --git a/config/constract_llm/train/pt/Llama-EN-3B-PT-stage1.json b/config/constract_llm/train/pt/Llama-EN-3B/stage1.json similarity index 84% rename from config/constract_llm/train/pt/Llama-EN-3B-PT-stage1.json rename to config/constract_llm/train/pt/Llama-EN-3B/stage1.json index ab50ffd6..59dc244b 100644 --- a/config/constract_llm/train/pt/Llama-EN-3B-PT-stage1.json +++ b/config/constract_llm/train/pt/Llama-EN-3B/stage1.json @@ -17,10 +17,10 @@ "num_train_epochs": 0.45, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 0.0002, + "learning_rate": 2e-4, "per_device_train_batch_size": 4, "per_device_eval_batch_size": 4, - "gradient_accumulation_steps": 512, + "gradient_accumulation_steps": 128, "eval_accumulation_steps": 1, "max_seq_length": 1024, "weight_decay": 1e-5, @@ -45,10 +45,10 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "trust_remote_code": true, "model_name_or_path": "iamtatsuki05/Llama-JP-3B-init", "run_name": "Llama-EN-3B-PT-stage1", - "output_dir": "./data/outputs/Llama-EN-3B-PT-stage1", - "logging_dir": "./data/outputs/Llama-EN-3B-PT-stage1/logs" + "output_dir": "data/outputs/Llama-EN-3B-PT-stage1", + "logging_dir": "data/outputs/Llama-EN-3B-PT-stage1/logs" } diff --git a/config/constract_llm/train/pt/Llama-EN-3B/stage2-bs2048.json b/config/constract_llm/train/pt/Llama-EN-3B/stage2-bs2048.json new file mode 100644 index 00000000..01999bd8 --- /dev/null +++ b/config/constract_llm/train/pt/Llama-EN-3B/stage2-bs2048.json @@ -0,0 +1,54 @@ +{ + "dataset_name": "google/wiki40b", + "dataset_config_name": "en", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", + "trust_remote_code": true, + "model_name_or_path": "iamtatsuki05/Llama-EN-3B-PT-stage1-bs16384", + "run_name": "Llama-EN-3B-PT-stage2-bs2048", + "output_dir": "data/outputs/Llama-EN-3B-PT-stage2-bs2048", + "logging_dir": "data/outputs/Llama-EN-3B-PT-stage2-bs2048/logs" +} diff --git a/config/constract_llm/train/pt/Llama-EN-3B-PT-stage2.json b/config/constract_llm/train/pt/Llama-EN-3B/stage2.json similarity index 86% rename from config/constract_llm/train/pt/Llama-EN-3B-PT-stage2.json rename to config/constract_llm/train/pt/Llama-EN-3B/stage2.json index d45aee95..3d01b0bb 100644 --- a/config/constract_llm/train/pt/Llama-EN-3B-PT-stage2.json +++ b/config/constract_llm/train/pt/Llama-EN-3B/stage2.json @@ -20,7 +20,7 @@ "learning_rate": 2e-5, "per_device_train_batch_size": 2, "per_device_eval_batch_size": 2, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 32, "eval_accumulation_steps": 1, "max_seq_length": 8192, "weight_decay": 1e-5, @@ -45,10 +45,10 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "trust_remote_code": true, "model_name_or_path": "iamtatsuki05/Llama-EN-3B-PT-stage1", "run_name": "Llama-EN-3B-PT-stage2", - "output_dir": "./data/outputs/Llama-EN-3B-PT-stage2", - "logging_dir": "./data/outputs/Llama-EN-3B-PT-stage2/logs" + "output_dir": "data/outputs/Llama-EN-3B-PT-stage2", + "logging_dir": "data/outputs/Llama-EN-3B-PT-stage2/logs" } diff --git a/config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage1.json b/config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage1.json deleted file mode 100644 index 0fc7c324..00000000 --- a/config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage1.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-init", - "trust_remote_code": true, - "dataset_name": "hotchpotch/fineweb-2-edu-japanese", - "dataset_config_name": "sample_10BT", - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "text_column_name": "text", - "run_name": "Llama-JP-0.5B-PT-stage1", - "output_dir": "./data/outputs/Llama-JP-0.5B-PT-stage1", - "logging_dir": "./data/outputs/Llama-JP-0.5B-PT-stage1/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 1, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-4, - "per_device_train_batch_size": 16, - "per_device_eval_batch_size": 16, - "gradient_accumulation_steps": 32, - "eval_accumulation_steps": 1, - "max_seq_length": 1024, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "metric_for_best_model": "eval_loss", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 50, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" -} diff --git a/config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage2.json b/config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage2.json deleted file mode 100644 index 23233e1e..00000000 --- a/config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage2.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-PT-stage1", - "trust_remote_code": true, - "dataset_name": "fujiki/wiki40b_ja", - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "dataset_config_name": null, - "text_column_name": "text", - "run_name": "Llama-JP-0.5B-PT-stage2", - "output_dir": "./data/outputs/Llama-JP-0.5B-PT-stage2", - "logging_dir": "./data/outputs/Llama-JP-0.5B-PT-stage2/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 2, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-5, - "per_device_train_batch_size": 2, - "per_device_eval_batch_size": 2, - "gradient_accumulation_steps": 32, - "eval_accumulation_steps": 1, - "max_seq_length": 8192, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "eval_strategy": "steps", - "metric_for_best_model": "eval_loss", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 50, - "eval_steps": 50, - "save_total_limit": 50, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" -} diff --git a/config/constract_llm/train/pt/Llama-JP-0.5B/stage1.json b/config/constract_llm/train/pt/Llama-JP-0.5B/stage1.json new file mode 100644 index 00000000..80731160 --- /dev/null +++ b/config/constract_llm/train/pt/Llama-JP-0.5B/stage1.json @@ -0,0 +1,53 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-init", + "trust_remote_code": true, + "dataset_name": "hotchpotch/fineweb-2-edu-japanese", + "dataset_config_name": "sample_10BT", + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "run_name": "Llama-JP-0.5B-PT-stage1", + "output_dir": "data/outputs/Llama-JP-0.5B-PT-stage1", + "logging_dir": "data/outputs/Llama-JP-0.5B-PT-stage1/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-4, + "per_device_train_batch_size": 16, + "per_device_eval_batch_size": 16, + "gradient_accumulation_steps": 32, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/Llama-JP-0.5B/stage2.json b/config/constract_llm/train/pt/Llama-JP-0.5B/stage2.json new file mode 100644 index 00000000..c1dfa80f --- /dev/null +++ b/config/constract_llm/train/pt/Llama-JP-0.5B/stage2.json @@ -0,0 +1,53 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-0.5B-PT-stage1", + "trust_remote_code": true, + "dataset_name": "fujiki/wiki40b_ja", + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "dataset_config_name": null, + "text_column_name": "text", + "run_name": "Llama-JP-0.5B-PT-stage2", + "output_dir": "data/outputs/Llama-JP-0.5B-PT-stage2", + "logging_dir": "data/outputs/Llama-JP-0.5B-PT-stage2/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 2, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 32, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/Llama-JP-1B/stage1-bs8192.json b/config/constract_llm/train/pt/Llama-JP-1B/stage1-bs8192.json new file mode 100644 index 00000000..4d22ce3c --- /dev/null +++ b/config/constract_llm/train/pt/Llama-JP-1B/stage1-bs8192.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-init", + "trust_remote_code": true, + "dataset_name": "hotchpotch/fineweb-2-edu-japanese", + "dataset_config_name": "default", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 10, + "text_column_name": "text", + "run_name": "Llama-JP-1B-PT-stage1-bs8192", + "output_dir": "data/outputs/Llama-JP-1B-PT-stage1-bs8192", + "logging_dir": "data/outputs/Llama-JP-1B-PT-stage1-bs8192/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 0.25, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-4, + "per_device_train_batch_size": 8, + "per_device_eval_batch_size": 4, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/Llama-JP-1B-PT-stage1.json b/config/constract_llm/train/pt/Llama-JP-1B/stage1.json similarity index 84% rename from config/constract_llm/train/pt/Llama-JP-1B-PT-stage1.json rename to config/constract_llm/train/pt/Llama-JP-1B/stage1.json index 2d5aaaba..60afd3e9 100644 --- a/config/constract_llm/train/pt/Llama-JP-1B-PT-stage1.json +++ b/config/constract_llm/train/pt/Llama-JP-1B/stage1.json @@ -11,8 +11,8 @@ "validation_split_percentage": 10, "text_column_name": "text", "run_name": "Llama-JP-1B-PT-stage1", - "output_dir": "./data/outputs/Llama-JP-1B-PT-stage1", - "logging_dir": "./data/outputs/Llama-JP-1B-PT-stage1/logs", + "output_dir": "data/outputs/Llama-JP-1B-PT-stage1", + "logging_dir": "data/outputs/Llama-JP-1B-PT-stage1/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -22,10 +22,10 @@ "num_train_epochs": 0.25, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 0.0003, + "learning_rate": 3e-4, "per_device_train_batch_size": 8, "per_device_eval_batch_size": 4, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 64, "eval_accumulation_steps": 1, "max_seq_length": 1024, "weight_decay": 1e-5, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/train/pt/Llama-JP-1B/stage2-bs1024.json b/config/constract_llm/train/pt/Llama-JP-1B/stage2-bs1024.json new file mode 100644 index 00000000..5889f875 --- /dev/null +++ b/config/constract_llm/train/pt/Llama-JP-1B/stage2-bs1024.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-1B-PT-stage1-bs8192", + "trust_remote_code": true, + "dataset_name": "fujiki/wiki40b_ja", + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "run_name": "Llama-JP-1B-PT-stage2-bs1024", + "output_dir": "data/outputs/Llama-JP-1B-PT-stage2-bs1024", + "logging_dir": "data/outputs/Llama-JP-1B-PT-stage2-bs1024/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 6, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/Llama-JP-1B-PT-stage2.json b/config/constract_llm/train/pt/Llama-JP-1B/stage2.json similarity index 86% rename from config/constract_llm/train/pt/Llama-JP-1B-PT-stage2.json rename to config/constract_llm/train/pt/Llama-JP-1B/stage2.json index 4283d1e3..0c6fbb36 100644 --- a/config/constract_llm/train/pt/Llama-JP-1B-PT-stage2.json +++ b/config/constract_llm/train/pt/Llama-JP-1B/stage2.json @@ -11,8 +11,8 @@ "validation_split_percentage": 1, "text_column_name": "text", "run_name": "Llama-JP-1B-PT-stage2", - "output_dir": "./data/outputs/Llama-JP-1B-PT-stage2", - "logging_dir": "./data/outputs/Llama-JP-1B-PT-stage2/logs", + "output_dir": "data/outputs/Llama-JP-1B-PT-stage2", + "logging_dir": "data/outputs/Llama-JP-1B-PT-stage2/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -25,7 +25,7 @@ "learning_rate": 3e-5, "per_device_train_batch_size": 1, "per_device_eval_batch_size": 1, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 64, "eval_accumulation_steps": 1, "max_seq_length": 8192, "weight_decay": 1e-5, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/train/pt/Llama-JP-3B/stage1-bs16384.json b/config/constract_llm/train/pt/Llama-JP-3B/stage1-bs16384.json new file mode 100644 index 00000000..ab2ca0b2 --- /dev/null +++ b/config/constract_llm/train/pt/Llama-JP-3B/stage1-bs16384.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-init", + "trust_remote_code": true, + "dataset_name": "hotchpotch/fineweb-2-edu-japanese", + "dataset_config_name": "default", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 10, + "text_column_name": "text", + "run_name": "Llama-JP-3B-PT-stage1-bs16384", + "output_dir": "data/outputs/Llama-JP-3B-PT-stage1-bs16384", + "logging_dir": "data/outputs/Llama-JP-3B-PT-stage1-bs16384/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 0.8, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-4, + "per_device_train_batch_size": 4, + "per_device_eval_batch_size": 4, + "gradient_accumulation_steps": 512, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/Llama-JP-3B-PT-stage1.json b/config/constract_llm/train/pt/Llama-JP-3B/stage1.json similarity index 86% rename from config/constract_llm/train/pt/Llama-JP-3B-PT-stage1.json rename to config/constract_llm/train/pt/Llama-JP-3B/stage1.json index 17cda790..a2862d80 100644 --- a/config/constract_llm/train/pt/Llama-JP-3B-PT-stage1.json +++ b/config/constract_llm/train/pt/Llama-JP-3B/stage1.json @@ -11,8 +11,8 @@ "validation_split_percentage": 10, "text_column_name": "text", "run_name": "Llama-JP-3B-PT-stage1", - "output_dir": "./data/outputs/Llama-JP-3B-PT-stage1", - "logging_dir": "./data/outputs/Llama-JP-3B-PT-stage1/logs", + "output_dir": "data/outputs/Llama-JP-3B-PT-stage1", + "logging_dir": "data/outputs/Llama-JP-3B-PT-stage1/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -25,7 +25,7 @@ "learning_rate": 2e-4, "per_device_train_batch_size": 4, "per_device_eval_batch_size": 4, - "gradient_accumulation_steps": 512, + "gradient_accumulation_steps": 128, "eval_accumulation_steps": 1, "max_seq_length": 1024, "weight_decay": 1e-5, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/train/pt/Llama-JP-3B/stage2-bs2048.json b/config/constract_llm/train/pt/Llama-JP-3B/stage2-bs2048.json new file mode 100644 index 00000000..006d9d0a --- /dev/null +++ b/config/constract_llm/train/pt/Llama-JP-3B/stage2-bs2048.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/Llama-JP-3B-PT-stage1-bs16384", + "trust_remote_code": true, + "dataset_name": "fujiki/wiki40b_ja", + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "run_name": "Llama-JP-3B-PT-stage2-bs2048", + "output_dir": "data/outputs/Llama-JP-3B-PT-stage2-bs2048", + "logging_dir": "data/outputs/Llama-JP-3B-PT-stage2-bs2048/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 6, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/Llama-JP-3B-PT-stage2.json b/config/constract_llm/train/pt/Llama-JP-3B/stage2.json similarity index 86% rename from config/constract_llm/train/pt/Llama-JP-3B-PT-stage2.json rename to config/constract_llm/train/pt/Llama-JP-3B/stage2.json index dca8ecc4..f4d34339 100644 --- a/config/constract_llm/train/pt/Llama-JP-3B-PT-stage2.json +++ b/config/constract_llm/train/pt/Llama-JP-3B/stage2.json @@ -11,8 +11,8 @@ "validation_split_percentage": 1, "text_column_name": "text", "run_name": "Llama-JP-3B-PT-stage2", - "output_dir": "./data/outputs/Llama-JP-3B-PT-stage2", - "logging_dir": "./data/outputs/Llama-JP-3B-PT-stage2/logs", + "output_dir": "data/outputs/Llama-JP-3B-PT-stage2", + "logging_dir": "data/outputs/Llama-JP-3B-PT-stage2/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -25,7 +25,7 @@ "learning_rate": 2e-5, "per_device_train_batch_size": 2, "per_device_eval_batch_size": 2, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 32, "eval_accumulation_steps": 1, "max_seq_length": 8192, "weight_decay": 1e-5, @@ -50,5 +50,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/train/pt/Mistral-EN-0.5B-PT-stage1.json b/config/constract_llm/train/pt/Mistral-EN-0.5B/stage1.json similarity index 85% rename from config/constract_llm/train/pt/Mistral-EN-0.5B-PT-stage1.json rename to config/constract_llm/train/pt/Mistral-EN-0.5B/stage1.json index e1ce8fa6..034b6455 100644 --- a/config/constract_llm/train/pt/Mistral-EN-0.5B-PT-stage1.json +++ b/config/constract_llm/train/pt/Mistral-EN-0.5B/stage1.json @@ -12,8 +12,8 @@ "validation_split_percentage": 1, "text_column_name": "text", "run_name": "Mistral-EN-0.5B-PT-stage1", - "output_dir": "./data/outputs/Mistral-EN-0.5B-PT-stage1", - "logging_dir": "./data/outputs/Mistral-EN-0.5B-PT-stage1/logs", + "output_dir": "data/outputs/Mistral-EN-0.5B-PT-stage1", + "logging_dir": "data/outputs/Mistral-EN-0.5B-PT-stage1/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -23,16 +23,16 @@ "num_train_epochs": 1, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 0.0005, + "learning_rate": 5e-4, "per_device_train_batch_size": 16, "per_device_eval_batch_size": 16, "gradient_accumulation_steps": 32, "eval_accumulation_steps": 1, "max_seq_length": 1024, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "eval_loss", "logging_first_step": true, @@ -51,6 +51,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "streaming": false } diff --git a/config/constract_llm/train/pt/Mistral-EN-0.5B-PT-stage2.json b/config/constract_llm/train/pt/Mistral-EN-0.5B/stage2.json similarity index 88% rename from config/constract_llm/train/pt/Mistral-EN-0.5B-PT-stage2.json rename to config/constract_llm/train/pt/Mistral-EN-0.5B/stage2.json index 793e7160..bdddbffa 100644 --- a/config/constract_llm/train/pt/Mistral-EN-0.5B-PT-stage2.json +++ b/config/constract_llm/train/pt/Mistral-EN-0.5B/stage2.json @@ -10,8 +10,8 @@ "dataset_config_name": "en", "text_column_name": "text", "run_name": "Mistral-EN-0.5B-PT-stage2", - "output_dir": "./data/outputs/Mistral-EN-0.5B-PT-stage2", - "logging_dir": "./data/outputs/Mistral-EN-0.5B-PT-stage2/logs", + "output_dir": "data/outputs/Mistral-EN-0.5B-PT-stage2", + "logging_dir": "data/outputs/Mistral-EN-0.5B-PT-stage2/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -49,6 +49,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "streaming": false } diff --git a/config/constract_llm/train/pt/ModernBERT-EN-0.5B-PT-stage1.json b/config/constract_llm/train/pt/ModernBERT-EN-0.5B/stage1.json similarity index 88% rename from config/constract_llm/train/pt/ModernBERT-EN-0.5B-PT-stage1.json rename to config/constract_llm/train/pt/ModernBERT-EN-0.5B/stage1.json index 095a2c07..913bd6f3 100644 --- a/config/constract_llm/train/pt/ModernBERT-EN-0.5B-PT-stage1.json +++ b/config/constract_llm/train/pt/ModernBERT-EN-0.5B/stage1.json @@ -9,8 +9,8 @@ "validation_split_percentage": 1, "text_column_name": "text", "run_name": "ModernBERT-EN-0.5B-PT-stage1", - "output_dir": "./data/outputs/ModernBERT-EN-0.5B-PT-stage1", - "logging_dir": "./data/outputs/ModernBERT-EN-0.5B-PT-stage1/logs", + "output_dir": "data/outputs/ModernBERT-EN-0.5B-PT-stage1", + "logging_dir": "data/outputs/ModernBERT-EN-0.5B-PT-stage1/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -50,6 +50,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "streaming": false } diff --git a/config/constract_llm/train/pt/ModernBERT-EN-0.5B-PT-stage2.json b/config/constract_llm/train/pt/ModernBERT-EN-0.5B/stage2.json similarity index 87% rename from config/constract_llm/train/pt/ModernBERT-EN-0.5B-PT-stage2.json rename to config/constract_llm/train/pt/ModernBERT-EN-0.5B/stage2.json index ed209503..025f0eaf 100644 --- a/config/constract_llm/train/pt/ModernBERT-EN-0.5B-PT-stage2.json +++ b/config/constract_llm/train/pt/ModernBERT-EN-0.5B/stage2.json @@ -9,8 +9,8 @@ "dataset_config_name": "en", "text_column_name": "text", "run_name": "ModernBERT-EN-0.5B-PT-stage2", - "output_dir": "./data/outputs/ModernBERT-EN-0.5B-PT-stage2", - "logging_dir": "./data/outputs/ModernBERT-EN-0.5B-PT-stage2/logs", + "output_dir": "data/outputs/ModernBERT-EN-0.5B-PT-stage2", + "logging_dir": "data/outputs/ModernBERT-EN-0.5B-PT-stage2/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -50,6 +50,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "streaming": false } diff --git a/config/constract_llm/train/pt/ModernBERT-EN-1B/stage1-bs8192.json b/config/constract_llm/train/pt/ModernBERT-EN-1B/stage1-bs8192.json new file mode 100644 index 00000000..d330e62d --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-EN-1B/stage1-bs8192.json @@ -0,0 +1,55 @@ +{ + "dataset_name": "HuggingFaceFW/fineweb-edu", + "dataset_config_name": "sample-100BT", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 10, + "text_column_name": "text", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 0.2, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-4, + "per_device_train_batch_size": 8, + "per_device_eval_batch_size": 4, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", + "mlm_probability": 0.3, + "line_by_line": true, + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-init", + "run_name": "ModernBERT-EN-1B-PT-stage1-bs8192", + "output_dir": "data/outputs/ModernBERT-EN-1B-PT-stage1-bs8192", + "logging_dir": "data/outputs/ModernBERT-EN-1B-PT-stage1-bs8192/logs" +} diff --git a/config/constract_llm/train/pt/ModernBERT-EN-1B-PT-stage1.json b/config/constract_llm/train/pt/ModernBERT-EN-1B/stage1.json similarity index 84% rename from config/constract_llm/train/pt/ModernBERT-EN-1B-PT-stage1.json rename to config/constract_llm/train/pt/ModernBERT-EN-1B/stage1.json index d07338ca..c7ea38c6 100644 --- a/config/constract_llm/train/pt/ModernBERT-EN-1B-PT-stage1.json +++ b/config/constract_llm/train/pt/ModernBERT-EN-1B/stage1.json @@ -17,10 +17,10 @@ "num_train_epochs": 0.2, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 0.0003, + "learning_rate": 3e-4, "per_device_train_batch_size": 8, "per_device_eval_batch_size": 4, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 64, "eval_accumulation_steps": 1, "max_seq_length": 1024, "weight_decay": 1e-5, @@ -45,11 +45,11 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "mlm_probability": 0.3, "line_by_line": true, "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-init", "run_name": "ModernBERT-EN-1B-PT-stage1", - "output_dir": "./data/outputs/ModernBERT-EN-1B-PT-stage1", - "logging_dir": "./data/outputs/ModernBERT-EN-1B-PT-stage1/logs" + "output_dir": "data/outputs/ModernBERT-EN-1B-PT-stage1", + "logging_dir": "data/outputs/ModernBERT-EN-1B-PT-stage1/logs" } diff --git a/config/constract_llm/train/pt/ModernBERT-EN-1B/stage2-bs1024.json b/config/constract_llm/train/pt/ModernBERT-EN-1B/stage2-bs1024.json new file mode 100644 index 00000000..dde2faf5 --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-EN-1B/stage2-bs1024.json @@ -0,0 +1,55 @@ +{ + "dataset_name": "google/wiki40b", + "dataset_config_name": "en", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", + "mlm_probability": 0.3, + "line_by_line": true, + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage1-bs8192", + "run_name": "ModernBERT-EN-1B-PT-stage2-bs1024", + "output_dir": "data/outputs/ModernBERT-EN-1B-PT-stage2-bs1024", + "logging_dir": "data/outputs/ModernBERT-EN-1B-PT-stage2-bs1024/logs" +} diff --git a/config/constract_llm/train/pt/ModernBERT-EN-1B-PT-stage2.json b/config/constract_llm/train/pt/ModernBERT-EN-1B/stage2.json similarity index 85% rename from config/constract_llm/train/pt/ModernBERT-EN-1B-PT-stage2.json rename to config/constract_llm/train/pt/ModernBERT-EN-1B/stage2.json index 4b027498..6967de60 100644 --- a/config/constract_llm/train/pt/ModernBERT-EN-1B-PT-stage2.json +++ b/config/constract_llm/train/pt/ModernBERT-EN-1B/stage2.json @@ -20,7 +20,7 @@ "learning_rate": 3e-5, "per_device_train_batch_size": 1, "per_device_eval_batch_size": 1, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 64, "eval_accumulation_steps": 1, "max_seq_length": 8192, "weight_decay": 1e-5, @@ -45,11 +45,11 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "mlm_probability": 0.3, "line_by_line": true, "model_name_or_path": "iamtatsuki05/ModernBERT-EN-1B-PT-stage1", "run_name": "ModernBERT-EN-1B-PT-stage2", - "output_dir": "./data/outputs/ModernBERT-EN-1B-PT-stage2", - "logging_dir": "./data/outputs/ModernBERT-EN-1B-PT-stage2/logs" + "output_dir": "data/outputs/ModernBERT-EN-1B-PT-stage2", + "logging_dir": "data/outputs/ModernBERT-EN-1B-PT-stage2/logs" } diff --git a/config/constract_llm/train/pt/ModernBERT-EN-3B/stage1-bs16384.json b/config/constract_llm/train/pt/ModernBERT-EN-3B/stage1-bs16384.json new file mode 100644 index 00000000..7f8e4511 --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-EN-3B/stage1-bs16384.json @@ -0,0 +1,55 @@ +{ + "dataset_name": "HuggingFaceFW/fineweb-edu", + "dataset_config_name": "sample-100BT", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 10, + "text_column_name": "text", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 0.45, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-4, + "per_device_train_batch_size": 4, + "per_device_eval_batch_size": 4, + "gradient_accumulation_steps": 512, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", + "mlm_probability": 0.3, + "line_by_line": true, + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-init", + "run_name": "ModernBERT-EN-3B-PT-stage1-bs16384", + "output_dir": "data/outputs/ModernBERT-EN-3B-PT-stage1-bs16384", + "logging_dir": "data/outputs/ModernBERT-EN-3B-PT-stage1-bs16384/logs" +} diff --git a/config/constract_llm/train/pt/ModernBERT-EN-3B-PT-stage1.json b/config/constract_llm/train/pt/ModernBERT-EN-3B/stage1.json similarity index 84% rename from config/constract_llm/train/pt/ModernBERT-EN-3B-PT-stage1.json rename to config/constract_llm/train/pt/ModernBERT-EN-3B/stage1.json index 6afaa712..86a00920 100644 --- a/config/constract_llm/train/pt/ModernBERT-EN-3B-PT-stage1.json +++ b/config/constract_llm/train/pt/ModernBERT-EN-3B/stage1.json @@ -17,10 +17,10 @@ "num_train_epochs": 0.45, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 0.0002, + "learning_rate": 2e-4, "per_device_train_batch_size": 4, "per_device_eval_batch_size": 4, - "gradient_accumulation_steps": 512, + "gradient_accumulation_steps": 128, "eval_accumulation_steps": 1, "max_seq_length": 1024, "weight_decay": 1e-5, @@ -45,11 +45,11 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "mlm_probability": 0.3, "line_by_line": true, "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-init", "run_name": "ModernBERT-EN-3B-PT-stage1", - "output_dir": "./data/outputs/ModernBERT-EN-3B-PT-stage1", - "logging_dir": "./data/outputs/ModernBERT-EN-3B-PT-stage1/logs" + "output_dir": "data/outputs/ModernBERT-EN-3B-PT-stage1", + "logging_dir": "data/outputs/ModernBERT-EN-3B-PT-stage1/logs" } diff --git a/config/constract_llm/train/pt/ModernBERT-EN-3B/stage2-bs2048.json b/config/constract_llm/train/pt/ModernBERT-EN-3B/stage2-bs2048.json new file mode 100644 index 00000000..4b2ce822 --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-EN-3B/stage2-bs2048.json @@ -0,0 +1,55 @@ +{ + "dataset_name": "google/wiki40b", + "dataset_config_name": "en", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 3, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", + "mlm_probability": 0.3, + "line_by_line": true, + "model_name_or_path": "iamtatsuki05/ModernBERT-EN-3B-PT-stage1-bs16384", + "run_name": "ModernBERT-EN-3B-PT-stage2-bs2048", + "output_dir": "data/outputs/ModernBERT-EN-3B-PT-stage2-bs2048", + "logging_dir": "data/outputs/ModernBERT-EN-3B-PT-stage2-bs2048/logs" +} diff --git a/config/constract_llm/train/pt/ModernBERT-EN-3B-PT-stage2.json b/config/constract_llm/train/pt/ModernBERT-EN-3B/stage2.json similarity index 85% rename from config/constract_llm/train/pt/ModernBERT-EN-3B-PT-stage2.json rename to config/constract_llm/train/pt/ModernBERT-EN-3B/stage2.json index 6e905d26..8c95a42d 100644 --- a/config/constract_llm/train/pt/ModernBERT-EN-3B-PT-stage2.json +++ b/config/constract_llm/train/pt/ModernBERT-EN-3B/stage2.json @@ -20,7 +20,7 @@ "learning_rate": 2e-5, "per_device_train_batch_size": 2, "per_device_eval_batch_size": 2, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 32, "eval_accumulation_steps": 1, "max_seq_length": 8192, "weight_decay": 1e-5, @@ -45,11 +45,11 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "mlm_probability": 0.3, "line_by_line": true, "model_name_or_path": "iamtatsuki05/ModernBERT-EN-3B-PT-stage1", "run_name": "ModernBERT-EN-3B-PT-stage2", - "output_dir": "./data/outputs/ModernBERT-EN-3B-PT-stage2", - "logging_dir": "./data/outputs/ModernBERT-EN-3B-PT-stage2/logs" + "output_dir": "data/outputs/ModernBERT-EN-3B-PT-stage2", + "logging_dir": "data/outputs/ModernBERT-EN-3B-PT-stage2/logs" } diff --git a/config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage1.json b/config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage1.json deleted file mode 100644 index 7d0ce1eb..00000000 --- a/config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage1.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-init", - "dataset_name": "hotchpotch/fineweb-2-edu-japanese", - "dataset_config_name": "sample_10BT", - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "text_column_name": "text", - "run_name": "ModernBERT-JP-0.5B-PT-stage1", - "output_dir": "./data/outputs/ModernBERT-JP-0.5B-PT-stage1", - "logging_dir": "./data/outputs/ModernBERT-JP-0.5B-PT-stage1/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 1, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-4, - "per_device_train_batch_size": 16, - "per_device_eval_batch_size": 16, - "gradient_accumulation_steps": 32, - "eval_accumulation_steps": 1, - "max_seq_length": 1024, - "mlm_probability": 0.3, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "line_by_line": true, - "eval_strategy": "steps", - "metric_for_best_model": "eval_loss", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 100, - "eval_steps": 100, - "save_total_limit": 50, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" -} diff --git a/config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage2.json b/config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage2.json deleted file mode 100644 index d55ee855..00000000 --- a/config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage2.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-PT-stage1", - "dataset_name": "fujiki/wiki40b_ja", - "preprocessing_num_workers": 128, - "dataloader_num_workers": 32, - "max_train_samples": null, - "max_eval_samples": 1000, - "validation_split_percentage": 1, - "dataset_config_name": null, - "text_column_name": "text", - "run_name": "ModernBERT-JP-0.5B-PT-stage2", - "output_dir": "./data/outputs/ModernBERT-JP-0.5B-PT-stage2", - "logging_dir": "./data/outputs/ModernBERT-JP-0.5B-PT-stage2/logs", - "save_only_model": false, - "overwrite_output_dir": true, - "do_train": true, - "do_eval": true, - "do_predict": false, - "ddp_timeout": 86400, - "num_train_epochs": 2, - "optim": "schedule_free_radam", - "lr_scheduler_type": "constant", - "learning_rate": 5e-5, - "per_device_train_batch_size": 2, - "per_device_eval_batch_size": 2, - "gradient_accumulation_steps": 32, - "eval_accumulation_steps": 1, - "max_seq_length": 8192, - "mlm_probability": 0.3, - "weight_decay": 1e-5, - "adam_beta1": 0.9, - "adam_beta2": 0.98, - "adam_epsilon": 1e-6, - "line_by_line": true, - "eval_strategy": "steps", - "metric_for_best_model": "eval_loss", - "logging_first_step": true, - "prediction_loss_only": true, - "eval_on_start": true, - "logging_steps": 1, - "save_steps": 50, - "eval_steps": 50, - "save_total_limit": 50, - "load_best_model_at_end": false, - "report_to": "all", - "include_num_input_tokens_seen": true, - "include_tokens_per_second": true, - "fp16": false, - "fp16_full_eval": false, - "bf16": true, - "bf16_full_eval": true, - "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" -} diff --git a/config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage1.json b/config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage1.json new file mode 100644 index 00000000..fd872a91 --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage1.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-init", + "dataset_name": "hotchpotch/fineweb-2-edu-japanese", + "dataset_config_name": "sample_10BT", + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "run_name": "ModernBERT-JP-0.5B-PT-stage1", + "output_dir": "data/outputs/ModernBERT-JP-0.5B-PT-stage1", + "logging_dir": "data/outputs/ModernBERT-JP-0.5B-PT-stage1/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 1, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-4, + "per_device_train_batch_size": 16, + "per_device_eval_batch_size": 16, + "gradient_accumulation_steps": 32, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "mlm_probability": 0.3, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "line_by_line": true, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage2.json b/config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage2.json new file mode 100644 index 00000000..4e4706a5 --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage2.json @@ -0,0 +1,54 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-0.5B-PT-stage1", + "dataset_name": "fujiki/wiki40b_ja", + "preprocessing_num_workers": 128, + "dataloader_num_workers": 32, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "dataset_config_name": null, + "text_column_name": "text", + "run_name": "ModernBERT-JP-0.5B-PT-stage2", + "output_dir": "data/outputs/ModernBERT-JP-0.5B-PT-stage2", + "logging_dir": "data/outputs/ModernBERT-JP-0.5B-PT-stage2/logs", + "save_only_model": false, + "overwrite_output_dir": true, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 2, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 5e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 32, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "mlm_probability": 0.3, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "line_by_line": true, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/ModernBERT-JP-1B/stage1-bs8192.json b/config/constract_llm/train/pt/ModernBERT-JP-1B/stage1-bs8192.json new file mode 100644 index 00000000..123b42e0 --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-JP-1B/stage1-bs8192.json @@ -0,0 +1,55 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-init", + "dataset_name": "hotchpotch/fineweb-2-edu-japanese", + "dataset_config_name": "default", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 10, + "text_column_name": "text", + "run_name": "ModernBERT-JP-1B-PT-stage1-bs8192", + "output_dir": "data/outputs/ModernBERT-JP-1B-PT-stage1-bs8192", + "logging_dir": "data/outputs/ModernBERT-JP-1B-PT-stage1-bs8192/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 0.25, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-4, + "per_device_train_batch_size": 8, + "per_device_eval_batch_size": 4, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "mlm_probability": 0.3, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "line_by_line": true, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/ModernBERT-JP-1B-PT-stage1.json b/config/constract_llm/train/pt/ModernBERT-JP-1B/stage1.json similarity index 84% rename from config/constract_llm/train/pt/ModernBERT-JP-1B-PT-stage1.json rename to config/constract_llm/train/pt/ModernBERT-JP-1B/stage1.json index 0272060b..6274d90f 100644 --- a/config/constract_llm/train/pt/ModernBERT-JP-1B-PT-stage1.json +++ b/config/constract_llm/train/pt/ModernBERT-JP-1B/stage1.json @@ -10,8 +10,8 @@ "validation_split_percentage": 10, "text_column_name": "text", "run_name": "ModernBERT-JP-1B-PT-stage1", - "output_dir": "./data/outputs/ModernBERT-JP-1B-PT-stage1", - "logging_dir": "./data/outputs/ModernBERT-JP-1B-PT-stage1/logs", + "output_dir": "data/outputs/ModernBERT-JP-1B-PT-stage1", + "logging_dir": "data/outputs/ModernBERT-JP-1B-PT-stage1/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -21,10 +21,10 @@ "num_train_epochs": 0.25, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 0.0003, + "learning_rate": 3e-4, "per_device_train_batch_size": 8, "per_device_eval_batch_size": 4, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 64, "eval_accumulation_steps": 1, "max_seq_length": 1024, "mlm_probability": 0.3, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/train/pt/ModernBERT-JP-1B/stage2-bs1024.json b/config/constract_llm/train/pt/ModernBERT-JP-1B/stage2-bs1024.json new file mode 100644 index 00000000..0a18eb6b --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-JP-1B/stage2-bs1024.json @@ -0,0 +1,55 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-1B-PT-stage1-bs8192", + "dataset_name": "fujiki/wiki40b_ja", + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "run_name": "ModernBERT-JP-1B-PT-stage2-bs1024", + "output_dir": "data/outputs/ModernBERT-JP-1B-PT-stage2-bs1024", + "logging_dir": "data/outputs/ModernBERT-JP-1B-PT-stage2-bs1024/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 6, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 3e-5, + "per_device_train_batch_size": 1, + "per_device_eval_batch_size": 1, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "mlm_probability": 0.3, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "line_by_line": true, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/ModernBERT-JP-1B-PT-stage2.json b/config/constract_llm/train/pt/ModernBERT-JP-1B/stage2.json similarity index 85% rename from config/constract_llm/train/pt/ModernBERT-JP-1B-PT-stage2.json rename to config/constract_llm/train/pt/ModernBERT-JP-1B/stage2.json index d0e0a0d2..38c02622 100644 --- a/config/constract_llm/train/pt/ModernBERT-JP-1B-PT-stage2.json +++ b/config/constract_llm/train/pt/ModernBERT-JP-1B/stage2.json @@ -10,8 +10,8 @@ "validation_split_percentage": 1, "text_column_name": "text", "run_name": "ModernBERT-JP-1B-PT-stage2", - "output_dir": "./data/outputs/ModernBERT-JP-1B-PT-stage2", - "logging_dir": "./data/outputs/ModernBERT-JP-1B-PT-stage2/logs", + "output_dir": "data/outputs/ModernBERT-JP-1B-PT-stage2", + "logging_dir": "data/outputs/ModernBERT-JP-1B-PT-stage2/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -24,7 +24,7 @@ "learning_rate": 3e-5, "per_device_train_batch_size": 1, "per_device_eval_batch_size": 1, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 64, "eval_accumulation_steps": 1, "max_seq_length": 8192, "mlm_probability": 0.3, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/train/pt/ModernBERT-JP-3B/stage1-bs16384.json b/config/constract_llm/train/pt/ModernBERT-JP-3B/stage1-bs16384.json new file mode 100644 index 00000000..dc9ecb87 --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-JP-3B/stage1-bs16384.json @@ -0,0 +1,55 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-init", + "dataset_name": "hotchpotch/fineweb-2-edu-japanese", + "dataset_config_name": "default", + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 10, + "text_column_name": "text", + "run_name": "ModernBERT-JP-3B-PT-stage1-bs16384", + "output_dir": "data/outputs/ModernBERT-JP-3B-PT-stage1-bs16384", + "logging_dir": "data/outputs/ModernBERT-JP-3B-PT-stage1-bs16384/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 0.8, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-4, + "per_device_train_batch_size": 4, + "per_device_eval_batch_size": 4, + "gradient_accumulation_steps": 512, + "eval_accumulation_steps": 1, + "max_seq_length": 1024, + "mlm_probability": 0.3, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "line_by_line": true, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 100, + "eval_steps": 100, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/ModernBERT-JP-3B-PT-stage1.json b/config/constract_llm/train/pt/ModernBERT-JP-3B/stage1.json similarity index 86% rename from config/constract_llm/train/pt/ModernBERT-JP-3B-PT-stage1.json rename to config/constract_llm/train/pt/ModernBERT-JP-3B/stage1.json index 0aca69a9..2214b374 100644 --- a/config/constract_llm/train/pt/ModernBERT-JP-3B-PT-stage1.json +++ b/config/constract_llm/train/pt/ModernBERT-JP-3B/stage1.json @@ -10,8 +10,8 @@ "validation_split_percentage": 10, "text_column_name": "text", "run_name": "ModernBERT-JP-3B-PT-stage1", - "output_dir": "./data/outputs/ModernBERT-JP-3B-PT-stage1", - "logging_dir": "./data/outputs/ModernBERT-JP-3B-PT-stage1/logs", + "output_dir": "data/outputs/ModernBERT-JP-3B-PT-stage1", + "logging_dir": "data/outputs/ModernBERT-JP-3B-PT-stage1/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -24,7 +24,7 @@ "learning_rate": 2e-4, "per_device_train_batch_size": 4, "per_device_eval_batch_size": 4, - "gradient_accumulation_steps": 512, + "gradient_accumulation_steps": 128, "eval_accumulation_steps": 1, "max_seq_length": 1024, "mlm_probability": 0.3, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/train/pt/ModernBERT-JP-3B/stage2-bs2048.json b/config/constract_llm/train/pt/ModernBERT-JP-3B/stage2-bs2048.json new file mode 100644 index 00000000..04b56047 --- /dev/null +++ b/config/constract_llm/train/pt/ModernBERT-JP-3B/stage2-bs2048.json @@ -0,0 +1,55 @@ +{ + "model_name_or_path": "iamtatsuki05/ModernBERT-JP-3B-PT-stage1-bs16384", + "dataset_name": "fujiki/wiki40b_ja", + "dataset_config_name": null, + "streaming": false, + "preprocessing_num_workers": 32, + "dataloader_num_workers": 16, + "max_train_samples": null, + "max_eval_samples": 1000, + "validation_split_percentage": 1, + "text_column_name": "text", + "run_name": "ModernBERT-JP-3B-PT-stage2-bs2048", + "output_dir": "data/outputs/ModernBERT-JP-3B-PT-stage2-bs2048", + "logging_dir": "data/outputs/ModernBERT-JP-3B-PT-stage2-bs2048/logs", + "save_only_model": false, + "overwrite_output_dir": false, + "do_train": true, + "do_eval": true, + "do_predict": false, + "ddp_timeout": 86400, + "num_train_epochs": 6, + "optim": "schedule_free_radam", + "lr_scheduler_type": "constant", + "learning_rate": 2e-5, + "per_device_train_batch_size": 2, + "per_device_eval_batch_size": 2, + "gradient_accumulation_steps": 128, + "eval_accumulation_steps": 1, + "max_seq_length": 8192, + "mlm_probability": 0.3, + "weight_decay": 1e-5, + "adam_beta1": 0.9, + "adam_beta2": 0.98, + "adam_epsilon": 1e-6, + "line_by_line": true, + "eval_strategy": "steps", + "metric_for_best_model": "eval_loss", + "logging_first_step": true, + "prediction_loss_only": true, + "eval_on_start": true, + "logging_steps": 1, + "save_steps": 50, + "eval_steps": 50, + "save_total_limit": 50, + "load_best_model_at_end": false, + "report_to": "all", + "include_num_input_tokens_seen": true, + "include_tokens_per_second": true, + "fp16": false, + "fp16_full_eval": false, + "bf16": true, + "bf16_full_eval": true, + "attn_implementation": "flash_attention_2", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" +} diff --git a/config/constract_llm/train/pt/ModernBERT-JP-3B-PT-stage2.json b/config/constract_llm/train/pt/ModernBERT-JP-3B/stage2.json similarity index 85% rename from config/constract_llm/train/pt/ModernBERT-JP-3B-PT-stage2.json rename to config/constract_llm/train/pt/ModernBERT-JP-3B/stage2.json index fa21d718..befe2fbd 100644 --- a/config/constract_llm/train/pt/ModernBERT-JP-3B-PT-stage2.json +++ b/config/constract_llm/train/pt/ModernBERT-JP-3B/stage2.json @@ -10,8 +10,8 @@ "validation_split_percentage": 1, "text_column_name": "text", "run_name": "ModernBERT-JP-3B-PT-stage2", - "output_dir": "./data/outputs/ModernBERT-JP-3B-PT-stage2", - "logging_dir": "./data/outputs/ModernBERT-JP-3B-PT-stage2/logs", + "output_dir": "data/outputs/ModernBERT-JP-3B-PT-stage2", + "logging_dir": "data/outputs/ModernBERT-JP-3B-PT-stage2/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -24,7 +24,7 @@ "learning_rate": 2e-5, "per_device_train_batch_size": 2, "per_device_eval_batch_size": 2, - "gradient_accumulation_steps": 128, + "gradient_accumulation_steps": 32, "eval_accumulation_steps": 1, "max_seq_length": 8192, "mlm_probability": 0.3, @@ -51,5 +51,5 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json" + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json" } diff --git a/config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage1.json b/config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B/stage1.json similarity index 84% rename from config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage1.json rename to config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B/stage1.json index 990b3466..f319a9b8 100644 --- a/config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage1.json +++ b/config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B/stage1.json @@ -12,8 +12,8 @@ "validation_split_percentage": 1, "text_column_name": "text", "run_name": "Qwen2.5-Scratch-EN-0.5B-PT-stage1", - "output_dir": "./data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage1", - "logging_dir": "./data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage1/logs", + "output_dir": "data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage1", + "logging_dir": "data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage1/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -23,16 +23,16 @@ "num_train_epochs": 1, "optim": "schedule_free_radam", "lr_scheduler_type": "constant", - "learning_rate": 0.0005, + "learning_rate": 5e-4, "per_device_train_batch_size": 16, "per_device_eval_batch_size": 16, "gradient_accumulation_steps": 32, "eval_accumulation_steps": 1, "max_seq_length": 1024, - "weight_decay": 1e-05, + "weight_decay": 1e-5, "adam_beta1": 0.9, "adam_beta2": 0.98, - "adam_epsilon": 1e-06, + "adam_epsilon": 1e-6, "eval_strategy": "steps", "metric_for_best_model": "eval_loss", "logging_first_step": true, @@ -51,6 +51,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "streaming": false } diff --git a/config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2.json b/config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B/stage2.json similarity index 87% rename from config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2.json rename to config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B/stage2.json index 5206eac0..8680965a 100644 --- a/config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B-PT-stage2.json +++ b/config/constract_llm/train/pt/Qwen2.5-Scratch-EN-0.5B/stage2.json @@ -10,8 +10,8 @@ "dataset_config_name": "en", "text_column_name": "text", "run_name": "Qwen2.5-Scratch-EN-0.5B-PT-stage2", - "output_dir": "./data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage2", - "logging_dir": "./data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage2/logs", + "output_dir": "data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage2", + "logging_dir": "data/outputs/Qwen2.5-Scratch-EN-0.5B-PT-stage2/logs", "save_only_model": false, "overwrite_output_dir": false, "do_train": true, @@ -49,6 +49,6 @@ "bf16": true, "bf16_full_eval": true, "attn_implementation": "flash_attention_2", - "deepspeed": "./config/constract_llm/train/ds_config/ds_zero2.json", + "deepspeed": "config/constract_llm/train/ds_config/ds_zero2.json", "streaming": false } diff --git a/scripts/constract_llm/eval/analysis/README_JA.md b/scripts/constract_llm/eval/analysis/README_JA.md index fc69bf23..9d5d1c46 100644 --- a/scripts/constract_llm/eval/analysis/README_JA.md +++ b/scripts/constract_llm/eval/analysis/README_JA.md @@ -106,7 +106,7 @@ CKA は比較前に、両点の `analysis.json` の設定(language / num_example - 実行中に対処した問題: - `Sentence-Sarashina-Bi-{EN,JP}-1B-PT` の run root `model.safetensors` が破損 (保存時 truncation)→ HF の同一モデルへ差し替えて解析(checkpoint と HF は無事)。 - - `Sentence-Llama-Bi-JP-1B`(FT 段)の checkpoint が auto_map ローカル参照なのに + - `Sentence-Llama-Bi-JP-1B-bs1024`(FT 段)の checkpoint が auto_map ローカル参照なのに `modeling_bidirectional_llama.py` を同梱していない → source からコピーして解決。 ## 検証状況 diff --git a/scripts/constract_llm/eval/analysis/compute_pairwise_cka.py b/scripts/constract_llm/eval/analysis/compute_pairwise_cka.py index 09341a2e..efd51cd5 100644 --- a/scripts/constract_llm/eval/analysis/compute_pairwise_cka.py +++ b/scripts/constract_llm/eval/analysis/compute_pairwise_cka.py @@ -19,18 +19,18 @@ import numpy as np PAIRS = [ - ('Sentence-ModernBERT-JP-1B@final', 'Sentence-Sarashina-Bi-JP-1B@final'), - ('Sentence-Llama-Bi-JP-1B@final', 'Sentence-Sarashina-Bi-JP-1B@final'), - ('Sentence-Sarashina-Bi-JP-1B-PT@final', 'Sentence-Sarashina-Bi-JP-1B@final'), - ('sarashina2.2-Bi-1b@base-ja', 'Sentence-Sarashina-Bi-JP-1B-PT@final'), - ('ModernBERT-JP-1B-PT-stage2@base-ja', 'Sentence-ModernBERT-JP-1B@final'), - ('Llama-Bi-JP-1B-PT-stage2@base-ja', 'Sentence-Llama-Bi-JP-1B@final'), - ('Sentence-ModernBERT-EN-1B@final', 'Sentence-Sarashina-Bi-EN-1B@final'), - ('Sentence-Llama-Bi-EN-1B@final', 'Sentence-Sarashina-Bi-EN-1B@final'), - ('Sentence-Sarashina-Bi-EN-1B-PT@final', 'Sentence-Sarashina-Bi-EN-1B@final'), - ('sarashina2.2-Bi-1b@base-en', 'Sentence-Sarashina-Bi-EN-1B-PT@final'), - ('ModernBERT-EN-1B-PT-stage2@base-en', 'Sentence-ModernBERT-EN-1B@final'), - ('Llama-Bi-EN-1B-PT-stage2@base-en', 'Sentence-Llama-Bi-EN-1B@final'), + ('Sentence-ModernBERT-JP-1B-bs1024@final', 'Sentence-Sarashina-Bi-JP-1B-bs1024@final'), + ('Sentence-Llama-Bi-JP-1B-bs1024@final', 'Sentence-Sarashina-Bi-JP-1B-bs1024@final'), + ('Sentence-Sarashina-Bi-JP-1B-PT-bs8192@final', 'Sentence-Sarashina-Bi-JP-1B-bs1024@final'), + ('sarashina2.2-Bi-1b@base-ja', 'Sentence-Sarashina-Bi-JP-1B-PT-bs8192@final'), + ('ModernBERT-JP-1B-PT-stage2-bs1024@base-ja', 'Sentence-ModernBERT-JP-1B-bs1024@final'), + ('Llama-Bi-JP-1B-PT-stage2-bs1024@base-ja', 'Sentence-Llama-Bi-JP-1B-bs1024@final'), + ('Sentence-ModernBERT-EN-1B-bs1024@final', 'Sentence-Sarashina-Bi-EN-1B-bs1024@final'), + ('Sentence-Llama-Bi-EN-1B-bs1024@final', 'Sentence-Sarashina-Bi-EN-1B-bs1024@final'), + ('Sentence-Sarashina-Bi-EN-1B-PT-bs8192@final', 'Sentence-Sarashina-Bi-EN-1B-bs1024@final'), + ('sarashina2.2-Bi-1b@base-en', 'Sentence-Sarashina-Bi-EN-1B-PT-bs8192@final'), + ('ModernBERT-EN-1B-PT-stage2-bs1024@base-en', 'Sentence-ModernBERT-EN-1B-bs1024@final'), + ('Llama-Bi-EN-1B-PT-stage2-bs1024@base-en', 'Sentence-Llama-Bi-EN-1B-bs1024@final'), ] diff --git a/scripts/constract_llm/eval/isotropic/README.md b/scripts/constract_llm/eval/isotropic/README.md index b0ec2bd3..28f1cb98 100644 --- a/scripts/constract_llm/eval/isotropic/README.md +++ b/scripts/constract_llm/eval/isotropic/README.md @@ -14,13 +14,13 @@ The script supports configuration via JSON/YAML/TOML files or command-line argum ```bash # Compute both alignment and uniformity -python scripts/constract_llm/eval/isotropic/eval.py main --config_file_path=config/constract_llm/eval/isotropic/example.json +python scripts/constract_llm/eval/isotropic/eval.py main --config_file_path=config/constract_llm/eval/ft/isotropic/example.json # Compute only alignment -python scripts/constract_llm/eval/isotropic/eval.py alignment --config_file_path=config/constract_llm/eval/isotropic/example.json +python scripts/constract_llm/eval/isotropic/eval.py alignment --config_file_path=config/constract_llm/eval/ft/isotropic/example.json # Compute only uniformity -python scripts/constract_llm/eval/isotropic/eval.py uniformity --config_file_path=config/constract_llm/eval/isotropic/example.json +python scripts/constract_llm/eval/isotropic/eval.py uniformity --config_file_path=config/constract_llm/eval/ft/isotropic/example.json ``` You can also override config parameters via CLI: diff --git a/scripts/constract_llm/eval/isotropic/README_JA.md b/scripts/constract_llm/eval/isotropic/README_JA.md index e5973b96..ec9d8e70 100644 --- a/scripts/constract_llm/eval/isotropic/README_JA.md +++ b/scripts/constract_llm/eval/isotropic/README_JA.md @@ -14,13 +14,13 @@ ```bash # アライメントとユニフォーミティを両方計算 -python scripts/constract_llm/eval/isotropic/eval.py main --config_file_path=config/constract_llm/eval/isotropic/example.json +python scripts/constract_llm/eval/isotropic/eval.py main --config_file_path=config/constract_llm/eval/ft/isotropic/example.json # アライメントのみ計算 -python scripts/constract_llm/eval/isotropic/eval.py alignment --config_file_path=config/constract_llm/eval/isotropic/example.json +python scripts/constract_llm/eval/isotropic/eval.py alignment --config_file_path=config/constract_llm/eval/ft/isotropic/example.json # ユニフォーミティのみ計算 -python scripts/constract_llm/eval/isotropic/eval.py uniformity --config_file_path=config/constract_llm/eval/isotropic/example.json +python scripts/constract_llm/eval/isotropic/eval.py uniformity --config_file_path=config/constract_llm/eval/ft/isotropic/example.json ``` コマンドライン引数で設定値を上書きすることも可能です。 diff --git a/scripts/constract_llm/train/README.md b/scripts/constract_llm/train/README.md index f44f082d..fbe65d50 100644 --- a/scripts/constract_llm/train/README.md +++ b/scripts/constract_llm/train/README.md @@ -42,7 +42,7 @@ This directory provides launch scripts for language-model pre-training and Sente All launchers expose [Google Fire](https://github.com/google/python-fire) CLIs that accept JSON/YAML/TOML configs. Basic usage: ```bash -python scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage1.json +python scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage1.json ``` For multi-GPU runs, prepend `uv run torchrun --standalone --nnodes 1 --nproc-per-node $NUM_GPU` to the command above. @@ -52,27 +52,27 @@ For multi-GPU runs, prepend `uv run torchrun --standalone --nnodes 1 --nproc-per Configuration templates live under `config/constract_llm/train/`: - Pre-training (`pt/`): - - `Llama-JP-0.5B-PT-stage1.json` - - `Llama-JP-0.5B-PT-stage2.json` - - `Llama-JP-3B-PT-stage1.json` - - `Llama-JP-3B-PT-stage2.json` - - `ModernBERT-JP-0.5B-PT-stage1.json` - - `ModernBERT-JP-0.5B-PT-stage2.json` - - `ModernBERT-JP-3B-PT-stage1.json` - - `ModernBERT-JP-3B-PT-stage2.json` + - `Llama-JP-0.5B/stage1.json` + - `Llama-JP-0.5B/stage2.json` + - `Llama-JP-3B/stage1.json` (total batch matched to 0.5B) / `Llama-JP-3B/stage1-bs16384.json` + - `Llama-JP-3B/stage2.json` (total batch matched to 0.5B) / `Llama-JP-3B/stage2-bs2048.json` + - `ModernBERT-JP-0.5B/stage1.json` + - `ModernBERT-JP-0.5B/stage2.json` + - `ModernBERT-JP-3B/stage1.json` (total batch matched to 0.5B) / `ModernBERT-JP-3B/stage1-bs16384.json` + - `ModernBERT-JP-3B/stage2.json` (total batch matched to 0.5B) / `ModernBERT-JP-3B/stage2-bs2048.json` - Fine-tuning (`ft/`): - - `Sentence-Llama-Bi-JP-0.5B-PT.json` - - `Sentence-Llama-Bi-JP-0.5B.json` - - `Sentence-Llama-Bi-JP-3B-PT.json` - - `Sentence-Llama-Bi-JP-3B.json` - - `Sentence-ModernBERT-JP-0.5B-PT.json` - - `Sentence-ModernBERT-JP-0.5B.json` - - `Sentence-ModernBERT-JP-3B-PT.json` - - `Sentence-ModernBERT-JP-3B.json` - - `Sentence-Sarashina-Bi-0.5B-PT.json` - - `Sentence-Sarashina-Bi-0.5B.json` - - `Sentence-Sarashina-Bi-JP-3B-PT.json` - - `Sentence-Sarashina-Bi-JP-3B.json` + - `Sentence-Llama-Bi-JP-0.5B/pt.json` + - `Sentence-Llama-Bi-JP-0.5B/ft.json` + - `Sentence-Llama-Bi-JP-3B/pt.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-JP-3B/pt-bs16384.json` + - `Sentence-Llama-Bi-JP-3B/ft.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-JP-3B/ft-bs2048.json` + - `Sentence-ModernBERT-JP-0.5B/pt.json` + - `Sentence-ModernBERT-JP-0.5B/ft.json` + - `Sentence-ModernBERT-JP-3B/pt.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-JP-3B/pt-bs16384.json` + - `Sentence-ModernBERT-JP-3B/ft.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-JP-3B/ft-bs2048.json` + - `Sentence-Sarashina-Bi-0.5B/pt.json` + - `Sentence-Sarashina-Bi-0.5B/ft.json` + - `Sentence-Sarashina-Bi-JP-3B/pt.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-JP-3B/pt-bs16384.json` + - `Sentence-Sarashina-Bi-JP-3B/ft.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-JP-3B/ft-bs2048.json` - DeepSpeed configs: `config/constract_llm/train/ds_config/` Copy a template, adjust hyperparameters, and pass the path to the relevant launch script. diff --git a/scripts/constract_llm/train/README_JA.md b/scripts/constract_llm/train/README_JA.md index 3a4f50ef..a8ef673a 100644 --- a/scripts/constract_llm/train/README_JA.md +++ b/scripts/constract_llm/train/README_JA.md @@ -42,7 +42,7 @@ すべてのランチャーは [Google Fire](https://github.com/google/python-fire) を利用した CLI を公開しており、JSON/YAML/TOML の設定ファイルを受け取ります。 ```bash -python scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage1.json +python scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage1.json ``` マルチ GPU の場合は `uv run torchrun --standalone --nnodes 1 --nproc-per-node $NUM_GPU` をコマンド前に付けて実行してください。 @@ -52,27 +52,27 @@ python scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/M 設定テンプレートは `config/constract_llm/train/` 配下に用意されています。 - 事前学習（`pt/`）： - - `Llama-JP-0.5B-PT-stage1.json` - - `Llama-JP-0.5B-PT-stage2.json` - - `Llama-JP-3B-PT-stage1.json` - - `Llama-JP-3B-PT-stage2.json` - - `ModernBERT-JP-0.5B-PT-stage1.json` - - `ModernBERT-JP-0.5B-PT-stage2.json` - - `ModernBERT-JP-3B-PT-stage1.json` - - `ModernBERT-JP-3B-PT-stage2.json` + - `Llama-JP-0.5B/stage1.json` + - `Llama-JP-0.5B/stage2.json` + - `Llama-JP-3B/stage1.json`(total batch を 0.5B と統一) / `Llama-JP-3B/stage1-bs16384.json` + - `Llama-JP-3B/stage2.json`(total batch を 0.5B と統一) / `Llama-JP-3B/stage2-bs2048.json` + - `ModernBERT-JP-0.5B/stage1.json` + - `ModernBERT-JP-0.5B/stage2.json` + - `ModernBERT-JP-3B/stage1.json`(total batch を 0.5B と統一) / `ModernBERT-JP-3B/stage1-bs16384.json` + - `ModernBERT-JP-3B/stage2.json`(total batch を 0.5B と統一) / `ModernBERT-JP-3B/stage2-bs2048.json` - ファインチューニング（`ft/`）： - - `Sentence-Llama-Bi-JP-0.5B-PT.json` - - `Sentence-Llama-Bi-JP-0.5B.json` - - `Sentence-Llama-Bi-JP-3B-PT.json` - - `Sentence-Llama-Bi-JP-3B.json` - - `Sentence-ModernBERT-JP-0.5B-PT.json` - - `Sentence-ModernBERT-JP-0.5B.json` - - `Sentence-ModernBERT-JP-3B-PT.json` - - `Sentence-ModernBERT-JP-3B.json` - - `Sentence-Sarashina-Bi-0.5B-PT.json` - - `Sentence-Sarashina-Bi-0.5B.json` - - `Sentence-Sarashina-Bi-JP-3B-PT.json` - - `Sentence-Sarashina-Bi-JP-3B.json` + - `Sentence-Llama-Bi-JP-0.5B/pt.json` + - `Sentence-Llama-Bi-JP-0.5B/ft.json` + - `Sentence-Llama-Bi-JP-3B/pt.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-JP-3B/pt-bs16384.json` + - `Sentence-Llama-Bi-JP-3B/ft.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-JP-3B/ft-bs2048.json` + - `Sentence-ModernBERT-JP-0.5B/pt.json` + - `Sentence-ModernBERT-JP-0.5B/ft.json` + - `Sentence-ModernBERT-JP-3B/pt.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-JP-3B/pt-bs16384.json` + - `Sentence-ModernBERT-JP-3B/ft.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-JP-3B/ft-bs2048.json` + - `Sentence-Sarashina-Bi-0.5B/pt.json` + - `Sentence-Sarashina-Bi-0.5B/ft.json` + - `Sentence-Sarashina-Bi-JP-3B/pt.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-JP-3B/pt-bs16384.json` + - `Sentence-Sarashina-Bi-JP-3B/ft.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-JP-3B/ft-bs2048.json` - DeepSpeed 設定: `config/constract_llm/train/ds_config/` テンプレートをコピーしてハイパーパラメータを調整し、対応するランチャーに渡してください。 diff --git a/scripts/constract_llm/train/ft/README.md b/scripts/constract_llm/train/ft/README.md index 9653cc2d..0de01b9c 100644 --- a/scripts/constract_llm/train/ft/README.md +++ b/scripts/constract_llm/train/ft/README.md @@ -22,7 +22,7 @@ The `run_st.py` script is used for fine-tuning Sentence Transformer models, whic ### Usage ```bash -python scripts/constract_llm/train/ft/run_st.py config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B.json +python scripts/constract_llm/train/ft/run_st.py config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/ft.json ``` For multi-GPU training: @@ -32,7 +32,7 @@ uv run torchrun \ --standalone \ --nnodes 1 \ --nproc-per-node $NUM_GPU \ - scripts/constract_llm/train/ft/run_st.py config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B.json + scripts/constract_llm/train/ft/run_st.py config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/ft.json ``` ### Custom Parameters @@ -63,23 +63,23 @@ For detailed parameter information, refer to the data classes in `src/mirei/cons Configuration files for fine-tuning are stored in `config/constract_llm/train/ft/`: -- `Sentence-Llama-Bi-JP-0.5B-PT.json` - Configuration for fine-tuning pre-trained Llama-Bi-JP-0.5B as a Sentence Transformer -- `Sentence-Llama-Bi-JP-0.5B.json` - Configuration for fine-tuning Llama-Bi-JP-0.5B as a Sentence Transformer -- `Sentence-Llama-Bi-EN-0.5B-PT.json` / `Sentence-Llama-Bi-EN-0.5B.json` - English WSL and FT configurations for Llama-Bi-0.5B -- `Sentence-Llama-Bi-JP-1B-PT.json` / `Sentence-Llama-Bi-JP-1B.json` - Japanese WSL and FT configurations for Llama-Bi-1B -- `Sentence-Llama-Bi-EN-1B-PT.json` / `Sentence-Llama-Bi-EN-1B.json` - English WSL and FT configurations for Llama-Bi-1B -- `Sentence-Llama-Bi-JP-3B-PT.json` / `Sentence-Llama-Bi-JP-3B.json` - Japanese WSL and FT configurations for Llama-Bi-3B -- `Sentence-Llama-Bi-EN-3B-PT.json` / `Sentence-Llama-Bi-EN-3B.json` - English WSL and FT configurations for Llama-Bi-3B -- `Sentence-ModernBERT-JP-0.5B-PT.json` - Configuration for fine-tuning pre-trained ModernBERT-JP-0.5B as a Sentence Transformer -- `Sentence-ModernBERT-JP-0.5B.json` - Configuration for fine-tuning ModernBERT-JP-0.5B as a Sentence Transformer -- `Sentence-ModernBERT-EN-0.5B-PT.json` / `Sentence-ModernBERT-EN-0.5B.json` - English WSL and FT configurations for ModernBERT-0.5B -- `Sentence-ModernBERT-JP-1B-PT.json` / `Sentence-ModernBERT-JP-1B.json` - Japanese WSL and FT configurations for ModernBERT-1B -- `Sentence-ModernBERT-EN-1B-PT.json` / `Sentence-ModernBERT-EN-1B.json` - English WSL and FT configurations for ModernBERT-1B -- `Sentence-ModernBERT-JP-3B-PT.json` / `Sentence-ModernBERT-JP-3B.json` - Japanese WSL and FT configurations for ModernBERT-3B -- `Sentence-ModernBERT-EN-3B-PT.json` / `Sentence-ModernBERT-EN-3B.json` - English WSL and FT configurations for ModernBERT-3B -- `Sentence-Sarashina-Bi-0.5B-PT.json` / `Sentence-Sarashina-Bi-0.5B.json` - Japanese WSL and FT configurations for Sarashina-Bi-0.5B -- `Sentence-Sarashina-Bi-EN-0.5B-PT.json` / `Sentence-Sarashina-Bi-EN-0.5B.json` - English WSL and FT configurations for Sarashina-Bi-0.5B -- `Sentence-Sarashina-Bi-JP-1B-PT.json` / `Sentence-Sarashina-Bi-JP-1B.json` - Japanese WSL and FT configurations for Sarashina-Bi-1B -- `Sentence-Sarashina-Bi-EN-1B-PT.json` / `Sentence-Sarashina-Bi-EN-1B.json` - English WSL and FT configurations for Sarashina-Bi-1B -- `Sentence-Sarashina-Bi-JP-3B-PT.json` / `Sentence-Sarashina-Bi-JP-3B.json` - Japanese WSL and FT configurations for Sarashina-Bi-3B -- `Sentence-Sarashina-Bi-EN-3B-PT.json` / `Sentence-Sarashina-Bi-EN-3B.json` - English WSL and FT configurations for Sarashina-Bi-3B +- `Sentence-Llama-Bi-JP-0.5B/pt.json` - Configuration for fine-tuning pre-trained Llama-Bi-JP-0.5B as a Sentence Transformer +- `Sentence-Llama-Bi-JP-0.5B/ft.json` - Configuration for fine-tuning Llama-Bi-JP-0.5B as a Sentence Transformer +- `Sentence-Llama-Bi-EN-0.5B/pt.json` / `Sentence-Llama-Bi-EN-0.5B/ft.json` - English WSL and FT configurations for Llama-Bi-0.5B +- `Sentence-Llama-Bi-JP-1B/pt.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-JP-1B/pt-bs8192.json` / `Sentence-Llama-Bi-JP-1B/ft.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-JP-1B/ft-bs1024.json` - Japanese WSL and FT configurations for Llama-Bi-1B +- `Sentence-Llama-Bi-EN-1B/pt.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-EN-1B/pt-bs8192.json` / `Sentence-Llama-Bi-EN-1B/ft.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-EN-1B/ft-bs1024.json` - English WSL and FT configurations for Llama-Bi-1B +- `Sentence-Llama-Bi-JP-3B/pt.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-JP-3B/pt-bs16384.json` / `Sentence-Llama-Bi-JP-3B/ft.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-JP-3B/ft-bs2048.json` - Japanese WSL and FT configurations for Llama-Bi-3B +- `Sentence-Llama-Bi-EN-3B/pt.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-EN-3B/pt-bs16384.json` / `Sentence-Llama-Bi-EN-3B/ft.json` (total batch matched to 0.5B) / `Sentence-Llama-Bi-EN-3B/ft-bs2048.json` - English WSL and FT configurations for Llama-Bi-3B +- `Sentence-ModernBERT-JP-0.5B/pt.json` - Configuration for fine-tuning pre-trained ModernBERT-JP-0.5B as a Sentence Transformer +- `Sentence-ModernBERT-JP-0.5B/ft.json` - Configuration for fine-tuning ModernBERT-JP-0.5B as a Sentence Transformer +- `Sentence-ModernBERT-EN-0.5B/pt.json` / `Sentence-ModernBERT-EN-0.5B/ft.json` - English WSL and FT configurations for ModernBERT-0.5B +- `Sentence-ModernBERT-JP-1B/pt.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-JP-1B/pt-bs8192.json` / `Sentence-ModernBERT-JP-1B/ft.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-JP-1B/ft-bs1024.json` - Japanese WSL and FT configurations for ModernBERT-1B +- `Sentence-ModernBERT-EN-1B/pt.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-EN-1B/pt-bs8192.json` / `Sentence-ModernBERT-EN-1B/ft.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-EN-1B/ft-bs1024.json` - English WSL and FT configurations for ModernBERT-1B +- `Sentence-ModernBERT-JP-3B/pt.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-JP-3B/pt-bs16384.json` / `Sentence-ModernBERT-JP-3B/ft.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-JP-3B/ft-bs2048.json` - Japanese WSL and FT configurations for ModernBERT-3B +- `Sentence-ModernBERT-EN-3B/pt.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-EN-3B/pt-bs16384.json` / `Sentence-ModernBERT-EN-3B/ft.json` (total batch matched to 0.5B) / `Sentence-ModernBERT-EN-3B/ft-bs2048.json` - English WSL and FT configurations for ModernBERT-3B +- `Sentence-Sarashina-Bi-0.5B/pt.json` / `Sentence-Sarashina-Bi-0.5B/ft.json` - Japanese WSL and FT configurations for Sarashina-Bi-0.5B +- `Sentence-Sarashina-Bi-EN-0.5B/pt.json` / `Sentence-Sarashina-Bi-EN-0.5B/ft.json` - English WSL and FT configurations for Sarashina-Bi-0.5B +- `Sentence-Sarashina-Bi-JP-1B/pt.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-JP-1B/pt-bs8192.json` / `Sentence-Sarashina-Bi-JP-1B/ft.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-JP-1B/ft-bs1024.json` - Japanese WSL and FT configurations for Sarashina-Bi-1B +- `Sentence-Sarashina-Bi-EN-1B/pt.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-EN-1B/pt-bs8192.json` / `Sentence-Sarashina-Bi-EN-1B/ft.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-EN-1B/ft-bs1024.json` - English WSL and FT configurations for Sarashina-Bi-1B +- `Sentence-Sarashina-Bi-JP-3B/pt.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-JP-3B/pt-bs16384.json` / `Sentence-Sarashina-Bi-JP-3B/ft.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-JP-3B/ft-bs2048.json` - Japanese WSL and FT configurations for Sarashina-Bi-3B +- `Sentence-Sarashina-Bi-EN-3B/pt.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-EN-3B/pt-bs16384.json` / `Sentence-Sarashina-Bi-EN-3B/ft.json` (total batch matched to 0.5B) / `Sentence-Sarashina-Bi-EN-3B/ft-bs2048.json` - English WSL and FT configurations for Sarashina-Bi-3B diff --git a/scripts/constract_llm/train/ft/README_JA.md b/scripts/constract_llm/train/ft/README_JA.md index 83a73edb..707d178d 100644 --- a/scripts/constract_llm/train/ft/README_JA.md +++ b/scripts/constract_llm/train/ft/README_JA.md @@ -21,7 +21,7 @@ ### 使用方法 ```bash -python scripts/constract_llm/train/ft/run_st.py config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B.json +python scripts/constract_llm/train/ft/run_st.py config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/ft.json ``` マルチGPU学習の場合： @@ -31,7 +31,7 @@ uv run torchrun \ --standalone \ --nnodes 1 \ --nproc-per-node $NUM_GPU \ - scripts/constract_llm/train/ft/run_st.py config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B.json + scripts/constract_llm/train/ft/run_st.py config/constract_llm/train/ft/Sentence-ModernBERT-JP-0.5B/ft.json ``` ### カスタムパラメータ @@ -62,23 +62,23 @@ uv run torchrun \ FTの設定ファイルは`config/constract_llm/train/ft/`に格納されています： -- `Sentence-Llama-Bi-JP-0.5B-PT.json` - 事前学習済みLlama-Bi-JP-0.5BをSentence TransformerとしてWSLするための設定 -- `Sentence-Llama-Bi-JP-0.5B.json` - Llama-Bi-JP-0.5BをSentence TransformerとしてFTするための設定 -- `Sentence-Llama-Bi-EN-0.5B-PT.json` / `Sentence-Llama-Bi-EN-0.5B.json` - Llama-Bi-0.5Bの英語WSL/FT設定 -- `Sentence-Llama-Bi-JP-1B-PT.json` / `Sentence-Llama-Bi-JP-1B.json` - Llama-Bi-1Bの日本語WSL/FT設定 -- `Sentence-Llama-Bi-EN-1B-PT.json` / `Sentence-Llama-Bi-EN-1B.json` - Llama-Bi-1Bの英語WSL/FT設定 -- `Sentence-Llama-Bi-JP-3B-PT.json` / `Sentence-Llama-Bi-JP-3B.json` - Llama-Bi-3Bの日本語WSL/FT設定 -- `Sentence-Llama-Bi-EN-3B-PT.json` / `Sentence-Llama-Bi-EN-3B.json` - Llama-Bi-3Bの英語WSL/FT設定 -- `Sentence-ModernBERT-JP-0.5B-PT.json` - 事前学習済みModernBERT-JP-0.5BをSentence TransformerとしてWSLするための設定 -- `Sentence-ModernBERT-JP-0.5B.json` - ModernBERT-JP-0.5BをSentence TransformerとしてFTするための設定 -- `Sentence-ModernBERT-EN-0.5B-PT.json` / `Sentence-ModernBERT-EN-0.5B.json` - ModernBERT-0.5Bの英語WSL/FT設定 -- `Sentence-ModernBERT-JP-1B-PT.json` / `Sentence-ModernBERT-JP-1B.json` - ModernBERT-1Bの日本語WSL/FT設定 -- `Sentence-ModernBERT-EN-1B-PT.json` / `Sentence-ModernBERT-EN-1B.json` - ModernBERT-1Bの英語WSL/FT設定 -- `Sentence-ModernBERT-JP-3B-PT.json` / `Sentence-ModernBERT-JP-3B.json` - ModernBERT-3Bの日本語WSL/FT設定 -- `Sentence-ModernBERT-EN-3B-PT.json` / `Sentence-ModernBERT-EN-3B.json` - ModernBERT-3Bの英語WSL/FT設定 -- `Sentence-Sarashina-Bi-0.5B-PT.json` / `Sentence-Sarashina-Bi-0.5B.json` - Sarashina-Bi-0.5Bの日本語WSL/FT設定 -- `Sentence-Sarashina-Bi-EN-0.5B-PT.json` / `Sentence-Sarashina-Bi-EN-0.5B.json` - Sarashina-Bi-0.5Bの英語WSL/FT設定 -- `Sentence-Sarashina-Bi-JP-1B-PT.json` / `Sentence-Sarashina-Bi-JP-1B.json` - Sarashina-Bi-1Bの日本語WSL/FT設定 -- `Sentence-Sarashina-Bi-EN-1B-PT.json` / `Sentence-Sarashina-Bi-EN-1B.json` - Sarashina-Bi-1Bの英語WSL/FT設定 -- `Sentence-Sarashina-Bi-JP-3B-PT.json` / `Sentence-Sarashina-Bi-JP-3B.json` - Sarashina-Bi-3Bの日本語WSL/FT設定 -- `Sentence-Sarashina-Bi-EN-3B-PT.json` / `Sentence-Sarashina-Bi-EN-3B.json` - Sarashina-Bi-3Bの英語WSL/FT設定 +- `Sentence-Llama-Bi-JP-0.5B/pt.json` - 事前学習済みLlama-Bi-JP-0.5BをSentence TransformerとしてWSLするための設定 +- `Sentence-Llama-Bi-JP-0.5B/ft.json` - Llama-Bi-JP-0.5BをSentence TransformerとしてFTするための設定 +- `Sentence-Llama-Bi-EN-0.5B/pt.json` / `Sentence-Llama-Bi-EN-0.5B/ft.json` - Llama-Bi-0.5Bの英語WSL/FT設定 +- `Sentence-Llama-Bi-JP-1B/pt.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-JP-1B/pt-bs8192.json` / `Sentence-Llama-Bi-JP-1B/ft.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-JP-1B/ft-bs1024.json` - Llama-Bi-1Bの日本語WSL/FT設定 +- `Sentence-Llama-Bi-EN-1B/pt.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-EN-1B/pt-bs8192.json` / `Sentence-Llama-Bi-EN-1B/ft.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-EN-1B/ft-bs1024.json` - Llama-Bi-1Bの英語WSL/FT設定 +- `Sentence-Llama-Bi-JP-3B/pt.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-JP-3B/pt-bs16384.json` / `Sentence-Llama-Bi-JP-3B/ft.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-JP-3B/ft-bs2048.json` - Llama-Bi-3Bの日本語WSL/FT設定 +- `Sentence-Llama-Bi-EN-3B/pt.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-EN-3B/pt-bs16384.json` / `Sentence-Llama-Bi-EN-3B/ft.json`(total batch を 0.5B と統一) / `Sentence-Llama-Bi-EN-3B/ft-bs2048.json` - Llama-Bi-3Bの英語WSL/FT設定 +- `Sentence-ModernBERT-JP-0.5B/pt.json` - 事前学習済みModernBERT-JP-0.5BをSentence TransformerとしてWSLするための設定 +- `Sentence-ModernBERT-JP-0.5B/ft.json` - ModernBERT-JP-0.5BをSentence TransformerとしてFTするための設定 +- `Sentence-ModernBERT-EN-0.5B/pt.json` / `Sentence-ModernBERT-EN-0.5B/ft.json` - ModernBERT-0.5Bの英語WSL/FT設定 +- `Sentence-ModernBERT-JP-1B/pt.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-JP-1B/pt-bs8192.json` / `Sentence-ModernBERT-JP-1B/ft.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-JP-1B/ft-bs1024.json` - ModernBERT-1Bの日本語WSL/FT設定 +- `Sentence-ModernBERT-EN-1B/pt.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-EN-1B/pt-bs8192.json` / `Sentence-ModernBERT-EN-1B/ft.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-EN-1B/ft-bs1024.json` - ModernBERT-1Bの英語WSL/FT設定 +- `Sentence-ModernBERT-JP-3B/pt.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-JP-3B/pt-bs16384.json` / `Sentence-ModernBERT-JP-3B/ft.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-JP-3B/ft-bs2048.json` - ModernBERT-3Bの日本語WSL/FT設定 +- `Sentence-ModernBERT-EN-3B/pt.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-EN-3B/pt-bs16384.json` / `Sentence-ModernBERT-EN-3B/ft.json`(total batch を 0.5B と統一) / `Sentence-ModernBERT-EN-3B/ft-bs2048.json` - ModernBERT-3Bの英語WSL/FT設定 +- `Sentence-Sarashina-Bi-0.5B/pt.json` / `Sentence-Sarashina-Bi-0.5B/ft.json` - Sarashina-Bi-0.5Bの日本語WSL/FT設定 +- `Sentence-Sarashina-Bi-EN-0.5B/pt.json` / `Sentence-Sarashina-Bi-EN-0.5B/ft.json` - Sarashina-Bi-0.5Bの英語WSL/FT設定 +- `Sentence-Sarashina-Bi-JP-1B/pt.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-JP-1B/pt-bs8192.json` / `Sentence-Sarashina-Bi-JP-1B/ft.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-JP-1B/ft-bs1024.json` - Sarashina-Bi-1Bの日本語WSL/FT設定 +- `Sentence-Sarashina-Bi-EN-1B/pt.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-EN-1B/pt-bs8192.json` / `Sentence-Sarashina-Bi-EN-1B/ft.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-EN-1B/ft-bs1024.json` - Sarashina-Bi-1Bの英語WSL/FT設定 +- `Sentence-Sarashina-Bi-JP-3B/pt.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-JP-3B/pt-bs16384.json` / `Sentence-Sarashina-Bi-JP-3B/ft.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-JP-3B/ft-bs2048.json` - Sarashina-Bi-3Bの日本語WSL/FT設定 +- `Sentence-Sarashina-Bi-EN-3B/pt.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-EN-3B/pt-bs16384.json` / `Sentence-Sarashina-Bi-EN-3B/ft.json`(total batch を 0.5B と統一) / `Sentence-Sarashina-Bi-EN-3B/ft-bs2048.json` - Sarashina-Bi-3Bの英語WSL/FT設定 diff --git a/scripts/constract_llm/train/pt/README.md b/scripts/constract_llm/train/pt/README.md index 90aa7dfd..47b0535e 100644 --- a/scripts/constract_llm/train/pt/README.md +++ b/scripts/constract_llm/train/pt/README.md @@ -24,7 +24,7 @@ The `run_mlm.py` script is used for pre-training models using the Masked Languag ### Usage ```bash -python scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage1.json +python scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage1.json ``` For multi-GPU training: @@ -34,7 +34,7 @@ uv run torchrun \ --standalone \ --nnodes 1 \ --nproc-per-node $NUM_GPU \ - scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage1.json + scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage1.json ``` ### Custom Parameters @@ -53,7 +53,7 @@ The `run_mntp.py` script is used for pre-training models using the Masked Next T ### Usage ```bash -python scripts/constract_llm/train/pt/run_mntp.py config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage1.json +python scripts/constract_llm/train/pt/run_mntp.py config/constract_llm/train/pt/Llama-JP-0.5B/stage1.json ``` For multi-GPU training: @@ -63,7 +63,7 @@ uv run torchrun \ --standalone \ --nnodes 1 \ --nproc-per-node $NUM_GPU \ - scripts/constract_llm/train/pt/run_mntp.py config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage1.json + scripts/constract_llm/train/pt/run_mntp.py config/constract_llm/train/pt/Llama-JP-0.5B/stage1.json ``` ### Custom Parameters @@ -81,22 +81,22 @@ For detailed parameter information, refer to the data classes in `src/mirei/cons Configuration files for pre-training are stored in `config/constract_llm/train/pt/`: -- `Llama-JP-0.5B-PT-stage1.json` - Configuration for stage 1 pre-training of Llama-JP-0.5B -- `Llama-JP-0.5B-PT-stage2.json` - Configuration for stage 2 pre-training of Llama-JP-0.5B -- `Llama-EN-0.5B-PT-stage1.json` - Configuration for stage 1 English pre-training of Llama-0.5B -- `Llama-EN-0.5B-PT-stage2.json` - Configuration for stage 2 English pre-training of Llama-0.5B -- `Llama-JP-1B-PT-stage1.json` / `Llama-JP-1B-PT-stage2.json` - Configuration for intermediate-scale Japanese pre-training of Llama-1B -- `Llama-EN-1B-PT-stage1.json` / `Llama-EN-1B-PT-stage2.json` - Configuration for intermediate-scale English pre-training of Llama-1B -- `Llama-JP-3B-PT-stage1.json` / `Llama-JP-3B-PT-stage2.json` - Configuration for scaled Japanese pre-training of Llama-3B -- `Llama-EN-3B-PT-stage1.json` / `Llama-EN-3B-PT-stage2.json` - Configuration for scaled English pre-training of Llama-3B -- `ModernBERT-JP-0.5B-PT-stage1.json` - Configuration for stage 1 pre-training of ModernBERT-JP-0.5B -- `ModernBERT-JP-0.5B-PT-stage2.json` - Configuration for stage 2 pre-training of ModernBERT-JP-0.5B -- `ModernBERT-EN-0.5B-PT-stage1.json` - Configuration for stage 1 English pre-training of ModernBERT-0.5B -- `ModernBERT-EN-0.5B-PT-stage2.json` - Configuration for stage 2 English pre-training of ModernBERT-0.5B -- `ModernBERT-JP-1B-PT-stage1.json` / `ModernBERT-JP-1B-PT-stage2.json` - Configuration for intermediate-scale Japanese pre-training of ModernBERT-1B -- `ModernBERT-EN-1B-PT-stage1.json` / `ModernBERT-EN-1B-PT-stage2.json` - Configuration for intermediate-scale English pre-training of ModernBERT-1B -- `ModernBERT-JP-3B-PT-stage1.json` / `ModernBERT-JP-3B-PT-stage2.json` - Configuration for scaled Japanese pre-training of ModernBERT-3B -- `ModernBERT-EN-3B-PT-stage1.json` / `ModernBERT-EN-3B-PT-stage2.json` - Configuration for scaled English pre-training of ModernBERT-3B +- `Llama-JP-0.5B/stage1.json` - Configuration for stage 1 pre-training of Llama-JP-0.5B +- `Llama-JP-0.5B/stage2.json` - Configuration for stage 2 pre-training of Llama-JP-0.5B +- `Llama-EN-0.5B/stage1.json` - Configuration for stage 1 English pre-training of Llama-0.5B +- `Llama-EN-0.5B/stage2.json` - Configuration for stage 2 English pre-training of Llama-0.5B +- `Llama-JP-1B/stage1.json` (total batch matched to 0.5B) / `Llama-JP-1B/stage1-bs8192.json` / `Llama-JP-1B/stage2.json` (total batch matched to 0.5B) / `Llama-JP-1B/stage2-bs1024.json` - Configuration for intermediate-scale Japanese pre-training of Llama-1B +- `Llama-EN-1B/stage1.json` (total batch matched to 0.5B) / `Llama-EN-1B/stage1-bs8192.json` / `Llama-EN-1B/stage2.json` (total batch matched to 0.5B) / `Llama-EN-1B/stage2-bs1024.json` - Configuration for intermediate-scale English pre-training of Llama-1B +- `Llama-JP-3B/stage1.json` (total batch matched to 0.5B) / `Llama-JP-3B/stage1-bs16384.json` / `Llama-JP-3B/stage2.json` (total batch matched to 0.5B) / `Llama-JP-3B/stage2-bs2048.json` - Configuration for scaled Japanese pre-training of Llama-3B +- `Llama-EN-3B/stage1.json` (total batch matched to 0.5B) / `Llama-EN-3B/stage1-bs16384.json` / `Llama-EN-3B/stage2.json` (total batch matched to 0.5B) / `Llama-EN-3B/stage2-bs2048.json` - Configuration for scaled English pre-training of Llama-3B +- `ModernBERT-JP-0.5B/stage1.json` - Configuration for stage 1 pre-training of ModernBERT-JP-0.5B +- `ModernBERT-JP-0.5B/stage2.json` - Configuration for stage 2 pre-training of ModernBERT-JP-0.5B +- `ModernBERT-EN-0.5B/stage1.json` - Configuration for stage 1 English pre-training of ModernBERT-0.5B +- `ModernBERT-EN-0.5B/stage2.json` - Configuration for stage 2 English pre-training of ModernBERT-0.5B +- `ModernBERT-JP-1B/stage1.json` (total batch matched to 0.5B) / `ModernBERT-JP-1B/stage1-bs8192.json` / `ModernBERT-JP-1B/stage2.json` (total batch matched to 0.5B) / `ModernBERT-JP-1B/stage2-bs1024.json` - Configuration for intermediate-scale Japanese pre-training of ModernBERT-1B +- `ModernBERT-EN-1B/stage1.json` (total batch matched to 0.5B) / `ModernBERT-EN-1B/stage1-bs8192.json` / `ModernBERT-EN-1B/stage2.json` (total batch matched to 0.5B) / `ModernBERT-EN-1B/stage2-bs1024.json` - Configuration for intermediate-scale English pre-training of ModernBERT-1B +- `ModernBERT-JP-3B/stage1.json` (total batch matched to 0.5B) / `ModernBERT-JP-3B/stage1-bs16384.json` / `ModernBERT-JP-3B/stage2.json` (total batch matched to 0.5B) / `ModernBERT-JP-3B/stage2-bs2048.json` - Configuration for scaled Japanese pre-training of ModernBERT-3B +- `ModernBERT-EN-3B/stage1.json` (total batch matched to 0.5B) / `ModernBERT-EN-3B/stage1-bs16384.json` / `ModernBERT-EN-3B/stage2.json` (total batch matched to 0.5B) / `ModernBERT-EN-3B/stage2-bs2048.json` - Configuration for scaled English pre-training of ModernBERT-3B ## Causal Language Modeling (CLM) @@ -115,7 +115,7 @@ The `run_clm.py` script is used for pre-training and fine-tuning models using th ### Usage ```bash -python scripts/constract_llm/train/pt/run_clm.py config/constract_llm/train/pt/YourCLMConfig.json +python scripts/constract_llm/train/pt/run_clm.py config/constract_llm/train/pt/YourCLMConfig/stage2.json ``` For multi-GPU training: @@ -125,7 +125,7 @@ uv run torchrun \ --standalone \ --nnodes 1 \ --nproc-per-node $NUM_GPU \ - scripts/constract_llm/train/pt/run_clm.py config/constract_llm/train/pt/YourCLMConfig.json + scripts/constract_llm/train/pt/run_clm.py config/constract_llm/train/pt/YourCLMConfig/stage2.json ``` ### Custom Parameters diff --git a/scripts/constract_llm/train/pt/README_JA.md b/scripts/constract_llm/train/pt/README_JA.md index dd43666e..2243a020 100644 --- a/scripts/constract_llm/train/pt/README_JA.md +++ b/scripts/constract_llm/train/pt/README_JA.md @@ -24,7 +24,7 @@ ### 使用方法 ```bash -python scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage1.json +python scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage1.json ``` マルチGPU学習の場合： @@ -34,7 +34,7 @@ uv run torchrun \ --standalone \ --nnodes 1 \ --nproc-per-node $NUM_GPU \ - scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B-PT-stage1.json + scripts/constract_llm/train/pt/run_mlm.py config/constract_llm/train/pt/ModernBERT-JP-0.5B/stage1.json ``` ### カスタムパラメータ @@ -53,7 +53,7 @@ uv run torchrun \ ### 使用方法 ```bash -python scripts/constract_llm/train/pt/run_mntp.py config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage1.json +python scripts/constract_llm/train/pt/run_mntp.py config/constract_llm/train/pt/Llama-JP-0.5B/stage1.json ``` マルチGPU学習の場合： @@ -63,7 +63,7 @@ uv run torchrun \ --standalone \ --nnodes 1 \ --nproc-per-node $NUM_GPU \ - scripts/constract_llm/train/pt/run_mntp.py config/constract_llm/train/pt/Llama-JP-0.5B-PT-stage1.json + scripts/constract_llm/train/pt/run_mntp.py config/constract_llm/train/pt/Llama-JP-0.5B/stage1.json ``` ### カスタムパラメータ @@ -81,22 +81,22 @@ uv run torchrun \ 事前学習の設定ファイルは`config/constract_llm/train/pt/`に格納されています： -- `Llama-JP-0.5B-PT-stage1.json` - Llama-JP-0.5Bのステージ1事前学習の設定 -- `Llama-JP-0.5B-PT-stage2.json` - Llama-JP-0.5Bのステージ2事前学習の設定 -- `Llama-EN-0.5B-PT-stage1.json` - Llama-0.5Bの英語ステージ1事前学習の設定 -- `Llama-EN-0.5B-PT-stage2.json` - Llama-0.5Bの英語ステージ2事前学習の設定 -- `Llama-JP-1B-PT-stage1.json` / `Llama-JP-1B-PT-stage2.json` - Llama-1Bの日本語中間スケール実験設定 -- `Llama-EN-1B-PT-stage1.json` / `Llama-EN-1B-PT-stage2.json` - Llama-1Bの英語中間スケール実験設定 -- `Llama-JP-3B-PT-stage1.json` / `Llama-JP-3B-PT-stage2.json` - Llama-3Bの日本語スケール実験設定 -- `Llama-EN-3B-PT-stage1.json` / `Llama-EN-3B-PT-stage2.json` - Llama-3Bの英語スケール実験設定 -- `ModernBERT-JP-0.5B-PT-stage1.json` - ModernBERT-JP-0.5Bのステージ1事前学習の設定 -- `ModernBERT-JP-0.5B-PT-stage2.json` - ModernBERT-JP-0.5Bのステージ2事前学習の設定 -- `ModernBERT-EN-0.5B-PT-stage1.json` - ModernBERT-0.5Bの英語ステージ1事前学習の設定 -- `ModernBERT-EN-0.5B-PT-stage2.json` - ModernBERT-0.5Bの英語ステージ2事前学習の設定 -- `ModernBERT-JP-1B-PT-stage1.json` / `ModernBERT-JP-1B-PT-stage2.json` - ModernBERT-1Bの日本語中間スケール実験設定 -- `ModernBERT-EN-1B-PT-stage1.json` / `ModernBERT-EN-1B-PT-stage2.json` - ModernBERT-1Bの英語中間スケール実験設定 -- `ModernBERT-JP-3B-PT-stage1.json` / `ModernBERT-JP-3B-PT-stage2.json` - ModernBERT-3Bの日本語スケール実験設定 -- `ModernBERT-EN-3B-PT-stage1.json` / `ModernBERT-EN-3B-PT-stage2.json` - ModernBERT-3Bの英語スケール実験設定 +- `Llama-JP-0.5B/stage1.json` - Llama-JP-0.5Bのステージ1事前学習の設定 +- `Llama-JP-0.5B/stage2.json` - Llama-JP-0.5Bのステージ2事前学習の設定 +- `Llama-EN-0.5B/stage1.json` - Llama-0.5Bの英語ステージ1事前学習の設定 +- `Llama-EN-0.5B/stage2.json` - Llama-0.5Bの英語ステージ2事前学習の設定 +- `Llama-JP-1B/stage1.json`(total batch を 0.5B と統一) / `Llama-JP-1B/stage1-bs8192.json` / `Llama-JP-1B/stage2.json`(total batch を 0.5B と統一) / `Llama-JP-1B/stage2-bs1024.json` - Llama-1Bの日本語中間スケール実験設定 +- `Llama-EN-1B/stage1.json`(total batch を 0.5B と統一) / `Llama-EN-1B/stage1-bs8192.json` / `Llama-EN-1B/stage2.json`(total batch を 0.5B と統一) / `Llama-EN-1B/stage2-bs1024.json` - Llama-1Bの英語中間スケール実験設定 +- `Llama-JP-3B/stage1.json`(total batch を 0.5B と統一) / `Llama-JP-3B/stage1-bs16384.json` / `Llama-JP-3B/stage2.json`(total batch を 0.5B と統一) / `Llama-JP-3B/stage2-bs2048.json` - Llama-3Bの日本語スケール実験設定 +- `Llama-EN-3B/stage1.json`(total batch を 0.5B と統一) / `Llama-EN-3B/stage1-bs16384.json` / `Llama-EN-3B/stage2.json`(total batch を 0.5B と統一) / `Llama-EN-3B/stage2-bs2048.json` - Llama-3Bの英語スケール実験設定 +- `ModernBERT-JP-0.5B/stage1.json` - ModernBERT-JP-0.5Bのステージ1事前学習の設定 +- `ModernBERT-JP-0.5B/stage2.json` - ModernBERT-JP-0.5Bのステージ2事前学習の設定 +- `ModernBERT-EN-0.5B/stage1.json` - ModernBERT-0.5Bの英語ステージ1事前学習の設定 +- `ModernBERT-EN-0.5B/stage2.json` - ModernBERT-0.5Bの英語ステージ2事前学習の設定 +- `ModernBERT-JP-1B/stage1.json`(total batch を 0.5B と統一) / `ModernBERT-JP-1B/stage1-bs8192.json` / `ModernBERT-JP-1B/stage2.json`(total batch を 0.5B と統一) / `ModernBERT-JP-1B/stage2-bs1024.json` - ModernBERT-1Bの日本語中間スケール実験設定 +- `ModernBERT-EN-1B/stage1.json`(total batch を 0.5B と統一) / `ModernBERT-EN-1B/stage1-bs8192.json` / `ModernBERT-EN-1B/stage2.json`(total batch を 0.5B と統一) / `ModernBERT-EN-1B/stage2-bs1024.json` - ModernBERT-1Bの英語中間スケール実験設定 +- `ModernBERT-JP-3B/stage1.json`(total batch を 0.5B と統一) / `ModernBERT-JP-3B/stage1-bs16384.json` / `ModernBERT-JP-3B/stage2.json`(total batch を 0.5B と統一) / `ModernBERT-JP-3B/stage2-bs2048.json` - ModernBERT-3Bの日本語スケール実験設定 +- `ModernBERT-EN-3B/stage1.json`(total batch を 0.5B と統一) / `ModernBERT-EN-3B/stage1-bs16384.json` / `ModernBERT-EN-3B/stage2.json`(total batch を 0.5B と統一) / `ModernBERT-EN-3B/stage2-bs2048.json` - ModernBERT-3Bの英語スケール実験設定 ## Causal Language Modeling (CLM) @@ -116,7 +116,7 @@ uv run torchrun \ ### 使用方法 ```bash -python scripts/constract_llm/train/pt/run_clm.py config/constract_llm/train/pt/YourCLMConfig.json +python scripts/constract_llm/train/pt/run_clm.py config/constract_llm/train/pt/YourCLMConfig/stage2.json ``` マルチGPU学習の場合： @@ -126,7 +126,7 @@ uv run torchrun \ --standalone \ --nnodes 1 \ --nproc-per-node $NUM_GPU \ - scripts/constract_llm/train/pt/run_clm.py config/constract_llm/train/pt/YourCLMConfig.json + scripts/constract_llm/train/pt/run_clm.py config/constract_llm/train/pt/YourCLMConfig/stage2.json ``` ### カスタムパラメータ