CC-bench-tools/fvecs_to_bin.cpp at main · DataSysResearch/CC-bench-tools · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>

void block_convert_float(std::ifstream &reader, std::ofstream &writer,
                         float *read_buf, float *write_buf, size_t npts,
                         size_t ndims) {
    reader.read((char *)read_buf,
                npts * (ndims * sizeof(float) + sizeof(uint32_t)));
    for (size_t i = 0; i < npts; i++) {
        memcpy(write_buf + i * ndims, (read_buf + i * (ndims + 1)) + 1,
               ndims * sizeof(float));
    }
    writer.write((char *)write_buf, npts * ndims * sizeof(float));
}

void block_convert_byte(std::ifstream &reader, std::ofstream &writer,
                        uint8_t *read_buf, uint8_t *write_buf, size_t npts,
                        size_t ndims) {
    reader.read((char *)read_buf,
                npts * (ndims * sizeof(uint8_t) + sizeof(uint32_t)));
    for (size_t i = 0; i < npts; i++) {
        memcpy(write_buf + i * ndims,
               (read_buf + i * (ndims + sizeof(uint32_t))) + sizeof(uint32_t),
               ndims * sizeof(uint8_t));
    }
    writer.write((char *)write_buf, npts * ndims * sizeof(uint8_t));
}

int main(int argc, char **argv) {
    if (argc != 4) {
        std::cout << argv[0] << " <float/int8/uint8> input_vecs output_bin"
                  << std::endl;
        exit(-1);
    }

    int datasize = sizeof(float);

    if (strcmp(argv[1], "uint8") == 0 || strcmp(argv[1], "int8") == 0) {
        datasize = sizeof(uint8_t);
    } else if (strcmp(argv[1], "float") != 0) {
        std::cout << "Error: type not supported. Use float/int8/uint8"
                  << std::endl;
        exit(-1);
    }

    std::ifstream reader(argv[2], std::ios::binary | std::ios::ate);
    size_t fsize = reader.tellg();
    reader.seekg(0, std::ios::beg);

    uint32_t ndims_u32;
    reader.read((char *)&ndims_u32, sizeof(uint32_t));
    reader.seekg(0, std::ios::beg);
    size_t ndims = (size_t)ndims_u32;
    size_t npts = fsize / ((ndims * datasize) + sizeof(uint32_t));
    std::cout << "Dataset: #pts = " << npts << ", # dims = " << ndims
              << std::endl;

    size_t blk_size = 131072;
    size_t nblks = (npts + blk_size - 1) / blk_size;
    std::cout << "# blks: " << nblks << std::endl;
    std::ofstream writer(argv[3], std::ios::binary);
    int32_t npts_s32 = (int32_t)npts;
    int32_t ndims_s32 = (int32_t)ndims;
    writer.write((char *)&npts_s32, sizeof(int32_t));
    writer.write((char *)&ndims_s32, sizeof(int32_t));

    size_t chunknpts = std::min(npts, blk_size);
    uint8_t *read_buf =
        new uint8_t[chunknpts * ((ndims * datasize) + sizeof(uint32_t))];
    uint8_t *write_buf = new uint8_t[chunknpts * ndims * datasize];

    for (size_t i = 0; i < nblks; i++) {
        size_t cblk_size = std::min(npts - i * blk_size, blk_size);
        if (datasize == sizeof(float)) {
            block_convert_float(reader, writer, (float *)read_buf,
                                (float *)write_buf, cblk_size, ndims);
        } else {
            block_convert_byte(reader, writer, read_buf, write_buf, cblk_size,
                               ndims);
        }
        std::cout << "Block #" << i << " written" << std::endl;
    }

    delete[] read_buf;
    delete[] write_buf;

    reader.close();
    writer.close();
}