ToyParser/test_lexer.py at master · joseflores314/ToyParser · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
from lexer import tokenize, Token, TokenType, InvalidTokenException, TokenGenerator
from typing import List
import pytest


def assert_tokenize(token_list: List[Token], input_string: str) -> None:
    assert token_list == list(tokenize(input_string))


def test_sum():
    input_string: str = "(+ 1 2)"
    token_list: List[Token] = [
        Token(
            TokenType.LEFT_PARENTHESIS,
            '('),
        Token(
            TokenType.OPERATOR,
            '+'),
        Token(
            TokenType.NUMBER,
            '1'),
        Token(
            TokenType.NUMBER,
            '2'),
        Token(
            TokenType.RIGHT_PARENTHESIS,
            ')'),
    ]
    assert_tokenize(token_list, input_string)


def test_long_token():
    input_string: str = "(- 1544 8693)"
    token_list: List[Token] = [
        Token(
            TokenType.LEFT_PARENTHESIS,
            '('),
        Token(
            TokenType.OPERATOR,
            '-'),
        Token(
            TokenType.NUMBER,
            '1544'),
        Token(
            TokenType.NUMBER,
            '8693'),
        Token(
            TokenType.RIGHT_PARENTHESIS,
            ')'),
    ]
    assert_tokenize(token_list, input_string)


def test_invalid_variable():
    input_string: str = "$1234"
    with pytest.raises(InvalidTokenException):
        list(tokenize(input_string))


def test_invalid_number():
    input_string: str = "12a5"
    with pytest.raises(InvalidTokenException):
        list(tokenize(input_string))


def test_nested_expression():
    input_string: str = "(* (/ 4 2) 1)"
    token_list: List[Token] = [
        Token(
            TokenType.LEFT_PARENTHESIS,
            '('),
        Token(
            TokenType.OPERATOR,
            '*'),
        Token(
            TokenType.LEFT_PARENTHESIS,
            '('),
        Token(
            TokenType.OPERATOR,
            '/'),
        Token(
            TokenType.NUMBER,
            '4'),
        Token(
            TokenType.NUMBER,
            '2'),
        Token(
            TokenType.RIGHT_PARENTHESIS,
            ')'),
        Token(
            TokenType.NUMBER,
            '1'),
        Token(
            TokenType.RIGHT_PARENTHESIS,
            ')'),
    ]
    assert_tokenize(token_list, input_string)


def test_incomplete_variable():
    input_string: str = "a4"
    with pytest.raises(InvalidTokenException):
        list(tokenize(input_string))


def test_expression_with_variables():
    input_string: str = "(- $var1 $var2)"
    token_list: List[Token] = [
        Token(
            TokenType.LEFT_PARENTHESIS,
            '('),
        Token(
            TokenType.OPERATOR,
            '-'),
        Token(
            TokenType.VARIABLE,
            '$var1'),
        Token(
            TokenType.VARIABLE,
            '$var2'),
        Token(
            TokenType.RIGHT_PARENTHESIS,
            ')'),
    ]
    assert_tokenize(token_list, input_string)


def test_white_spaces():
    input_string: str = "    $var1       "
    token_list: List[Token] = [
        Token(
            TokenType.VARIABLE,
            '$var1'),
    ]
    assert_tokenize(token_list, input_string)


def test_no_floating_point_numbers():
    input_string: str = "12.34"
    with pytest.raises(InvalidTokenException):
        list(tokenize(input_string))


def test_no_white_spaces_expression():
    input_string: str = "(+$var1$var2)"
    token_list: List[Token] = [
        Token(
            TokenType.LEFT_PARENTHESIS,
            '('),
        Token(
            TokenType.OPERATOR,
            '+'),
        Token(
            TokenType.VARIABLE,
            '$var1'),
        Token(
            TokenType.VARIABLE,
            '$var2'),
        Token(
            TokenType.RIGHT_PARENTHESIS,
            ')'),
    ]
    assert_tokenize(token_list, input_string)