bompiler/tokenizer.js at main · ryanntannn/bompiler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
/**
 * Converts a C code string into an array of tokens
 * @param {string} input
 */
export function tokenizer(input) {
  let current = 0;
  let tokens = [];

  while (current < input.length) {
    let char = input[current];

    let LETTERS = /[a-zA-Z]/;

    if (LETTERS.test(char)) {
      let value = "";

      while (LETTERS.test(char)) {
        value += char;
        char = input[++current];
      }

      // check for keywords
      switch (value) {
        case "int":
          tokens.push({ type: "keyword", value });
          break;
        case "return":
          tokens.push({ type: "keyword", value });
          break;
        case "if":
          tokens.push({ type: "keyword", value });
          break;
        case "else":
          tokens.push({ type: "keyword", value });
          break;
        case "while":
          tokens.push({ type: "keyword", value });
          break;
        default:
          tokens.push({ type: "name", value });
      }
      continue;
    }

    let NUMBERS = /[0-9]/;
    if (NUMBERS.test(char)) {
      let value = "";

      while (NUMBERS.test(char)) {
        value += char;
        char = input[++current];
      }

      tokens.push({ type: "number", value });
      continue;
    }

    let WHITESPACE = /\s/;
    if (WHITESPACE.test(char)) {
      current++;
      continue;
    }

    if (char === "(") {
      tokens.push({ type: "paren", value: "(" });
      current++;
      continue;
    }

    if (char === ")") {
      tokens.push({ type: "paren", value: ")" });
      current++;
      continue;
    }

    if (char === "{") {
      tokens.push({ type: "brace", value: "{" });
      current++;
      continue;
    }

    if (char === "}") {
      tokens.push({ type: "brace", value: "}" });
      current++;
      continue;
    }

    if (char === ",") {
      tokens.push({ type: "comma", value: "," });
      current++;
      continue;
    }

    if (char === "=") {
      tokens.push({ type: "assign", value: "=" });
      current++;
      continue;
    }

    if (char === ";") {
      tokens.push({ type: "semicolon", value: ";" });
      current++;
      continue;
    }

    throw new TypeError("I dont know what this character is: " + char);
  }

  return tokens;
}