descriptors/tokenizer.cpp at master · ishaanam/descriptors · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
#include "tokenizer.h"

/*
    Converts a string stream to a vector of tokens, each with a
    given TokenType.
*/

std::vector<Token> Tokenizer(std::stringstream& ss)
{
    char c;
    Token current_token;

    std::vector<Token> tokens;

    while (ss>>c)
    {
        switch (c)
        {
            case '#':{
                current_token.type = TokenType::CHECKSUM;
                break;
            }
            case '(':{
                if (current_token.type == TokenType::KEY) current_token.type = TokenType::FUNCTION;
                tokens.push_back(current_token);
                current_token.reset();
                break;
            }
            case ')':{
                if (!current_token.is_empty()) tokens.push_back(current_token);
                current_token.raw_token = ")";
                current_token.type = TokenType::END_FUNC;
                tokens.push_back(current_token);
                current_token.reset();
                break;
            }
            case ',':{
                if (!current_token.is_empty())
                {
                    tokens.push_back(current_token);
                    current_token.reset();
                }
                break;
            }
            case '_':
            case '[':
            case ']':
            case '/': {
                current_token.raw_token += c;
                if (current_token.type == TokenType::UNKNOWN) current_token.type = TokenType::KEY;
                break;
            }
            default: {
                if (std::isalnum(c))
                {
                    current_token.raw_token += c;
                    if (current_token.type == TokenType::UNKNOWN) current_token.type = TokenType::KEY;
                }
            }
        }
    }
    // add last token to vector
    if (!current_token.is_empty()) tokens.push_back(current_token);

    return tokens;
}