GradCraft/tokenize.cpp at main · horecdev/GradCraft · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#include "gradc/gradc.hpp"

int main() {
    try {
        std::string vocab_dir = "C:/Local Projects/autograd_cpp/data/vocab";
        std::string dataset_dir = "C:/Local Projects/autograd_cpp/data/datasets";
        std::string vocab_path = vocab_dir + "/vocab.bin";
        std::string output_path = dataset_dir + "/cosmo_cpp_edit.bin";

        std::filesystem::create_directories(vocab_dir);
        std::filesystem::create_directories(dataset_dir);

        std::vector<std::string> all_files;
        for (int i = 0; i < 614; ++i) {
            all_files.push_back(std::format("C:/Local Projects/autograd_cpp/data/raw_data/cosmopedia/cosmo_chunk_{:03d}.txt", i));
            all_files.push_back(std::format("C:/Local Projects/autograd_cpp/data/raw_data/cpp/cpp_chunk_{:03d}.txt", i));
        }

        if (!std::filesystem::exists(vocab_path)) {
            gradc::TokenManager::create_vocab_out_of_files(vocab_path, all_files);
        }
        else {
            std::cout << "Vocab already exists. Skipping creating." << std::endl;
        }

        std::cout << "Encoding dataset." << std::endl;
        gradc::TokenManager::encode_dataset(output_path, vocab_path, all_files);

        std::cout << "Tokenization complete.\n";
        return 0;
    }
    catch (const std::exception& e) {
        std::cerr << "Fatal Error: " << e.what() << std::endl;
        return 1;
    }
}