Skip to content
This repository was archived by the owner on Sep 6, 2024. It is now read-only.

Commit d0c4212

Browse files
jonathantsangjonathantsang
committed
form nfas from file text
1 parent 04bb578 commit d0c4212

3 files changed

Lines changed: 104 additions & 50 deletions

File tree

‎src/watcompiler/Tokens.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
BRACKET { } ( ) [ ]
2+
BOOLEAN true false
3+
KEYWORD abstract default if private this boolean do implements protected break double import public throws throw byte else instanceof return transient case extends int short try catch interface static void char finally final long strictfp volatile class float native super while const for new switch continue goto package synchronized
4+
UNARYOPERATOR + ++ - -- ! ~
5+
BINARYOPERATOR == * / % < << > >> >>> & ^ | != >= <=
6+
ASSIGNMENTOPERATOR = *= /= %= += -= <<= >>= >>>= &= ^= |=
7+
TERMINAL ; " ,
8+

‎src/watcompiler/re.clj‎

Lines changed: 70 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
(ns watcompiler.re
22
(:require [clojure.set :refer :all]
33
[watcompiler.nfa :refer :all]
4-
[watcompiler.lang :refer :all])
4+
[watcompiler.lang :refer :all]
5+
[clojure.string :as str])
56
(:import [watcompiler.nfa NFA]))
67

78
;; Merging multiple nfas
@@ -62,7 +63,7 @@
6263
[& arguments]
6364
(let
6465
[stateS (gensym :s)
65-
class (first arguments)
66+
class (keyword (first arguments)) ;; Conver to a keyword since it reads strings from a file
6667
args (rest arguments)
6768
;; Key: string for keyword, Value: NFA for that keyword
6869
strings-nfas (into (sorted-map) (for [nfa-name args]
@@ -85,6 +86,24 @@
8586
all-transitions
8687
all-accept-priorities)))
8788

89+
;; Reading the file
90+
(def readFile
91+
(with-open [rdr (clojure.java.io/reader "src/watcompiler/Tokens.txt")]
92+
(reduce conj [] (line-seq rdr))))
93+
94+
;; Splitting the lines by space
95+
(def splitLines
96+
(into []
97+
(for [x readFile]
98+
(str/split x #" "))))
99+
100+
(def fileFormed-nfa
101+
(let [nfas
102+
(into []
103+
(for [x splitLines]
104+
(apply form-multiple-nfas x)))]
105+
(apply merge-nfas nfas)))
106+
88107
;; NFAs for types
89108

90109
;; Integer literal
@@ -100,12 +119,48 @@
100119
(make-transition-NFA [[stateS state1 e]
101120
[state1 state2 DIGITS-NONZERO]
102121
[state2 state2 DIGITS]]))))
122+
123+
;; String literal
124+
;; \".*\" (\ shown for escaping ")
125+
(def string-literal-nfa
126+
(let [stateS (gensym :S)
127+
state1 (gensym :1)
128+
state2 (gensym :2)]
129+
(make-NFA (into #{} )
130+
#{stateS state1 state2}
131+
stateS
132+
{state2 (list :STRING-LITERAL 0)}
133+
(make-transition-NFA [[stateS state1 "\""]
134+
[state1 state1 UPPER-ALPHABET]
135+
[state1 state1 LOWER-ALPHABET]
136+
[state1 state2 "\""]]))))
137+
138+
;; Identifiers
139+
;; [a-zA-Z][a-zA-Z0-9]*
140+
(def identifier-nfa
141+
(let [stateS (gensym :S)
142+
state1 (gensym :s1)
143+
state2 (gensym :s2)]
144+
(make-NFA (into #{} )
145+
#{stateS state1 state2}
146+
stateS
147+
{state1 (list :IDENTIFIER 1)
148+
state2 (list :IDENTIFIER 1)}
149+
(make-transition-NFA [[stateS state1 UPPER-ALPHABET]
150+
[stateS state1 LOWER-ALPHABET]
151+
[state1 state2 UPPER-ALPHABET]
152+
[state1 state2 LOWER-ALPHABET]
153+
[state2 state2 UPPER-ALPHABET]
154+
[state2 state2 LOWER-ALPHABET]
155+
[state2 state2 DIGITS]]))))
103156
;; Operators
104157
(def operators-nfa
105158
(form-multiple-nfas :OPERATOR ">" "<" "<<" ">>" ">>>" "<<<" ">>>=" ">>="
106159
">=" "<=" "&" "&=" "=" "==" "!" "!=" "^=" "^" "+" "+="
107160
"++" "-" "-=" "--" "*" "*=" "/" "/=" "%" "%="))
108161

162+
;; white space?
163+
109164
;; Keywords nfa
110165
(def keywords-nfa
111166
(form-multiple-nfas :KEYWORD
@@ -166,37 +221,20 @@
166221
(def bracket-nfa
167222
(form-multiple-nfas :BRACKET "{" "}" "(" ")" "[" "]"))
168223

169-
170-
;; Identifiers
171-
;; [a-zA-Z][a-zA-Z0-9]*
172-
(def identifier-nfa
173-
(let [stateS (gensym :S)
174-
state1 (gensym :s1)
175-
state2 (gensym :s2)]
176-
(make-NFA (into #{} )
177-
#{stateS state1 state2}
178-
stateS
179-
{state1 (list :IDENTIFIER 1)
180-
state2 (list :IDENTIFIER 1)}
181-
(make-transition-NFA [[stateS state1 UPPER-ALPHABET]
182-
[stateS state1 LOWER-ALPHABET]
183-
[state1 state2 UPPER-ALPHABET]
184-
[state1 state2 LOWER-ALPHABET]
185-
[state2 state2 UPPER-ALPHABET]
186-
[state2 state2 LOWER-ALPHABET]
187-
[state2 state2 DIGITS]]))))
188-
189224
;; complete nfa from all of the individual RE nfas
190225
;; int-literal
191-
;; operators
192-
;; boolean
193-
;; keywords
194-
;; brackets
195-
;; identifier
226+
;; string-literal
227+
;; identifiers
228+
;; file specified nfas:
229+
;; BRACKET
230+
;; BOOLEAN
231+
;; KEYWORD
232+
;; UNARYOPERATOR
233+
;; BINARYOPERATOR
234+
;; ASSIGNMENTOPERATOR
235+
;; TERMINAL
196236
(def complete-nfa
197237
(merge-nfas integer-literal-nfa
198-
operators-nfa
199-
boolean-nfa
200-
keywords-nfa
201-
bracket-nfa
202-
identifier-nfa))
238+
string-literal-nfa
239+
identifier-nfa
240+
fileFormed-nfa))

‎test/watcompiler/re_test.clj‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,15 @@
44
[watcompiler.re :refer :all])
55
(:import [watcompiler.nfa NFA]))
66

7+
;; Form the NFAs from a file
8+
(deftest read-file
9+
(def lines readFile)
10+
(def formed fileFormed-nfa)
11+
12+
(is (= :BRACKET (run-NFA formed "]")))
13+
(is (= :BOOLEAN (run-NFA formed "true")))
14+
(is (= :BRACKET (run-NFA formed "{"))))
15+
716
;; Test forming multiple nfas from multiple strings
817
(deftest multiple-nfas-function-test
918
(def full-nfa (form-multiple-nfas :KEYWORD "int" "if"))
@@ -12,7 +21,6 @@
1221
(is (= false (run-NFA full-nfa "in")))
1322
(is (= false (run-NFA full-nfa "nt"))))
1423

15-
1624
;; Test function forming individual nfa
1725
(deftest function-test
1826
(def int-nfa-test (string-to-nfa "int" :INT))
@@ -75,7 +83,7 @@
7583
(is (= :KEYWORD (run-NFA complete-nfa "int")))
7684
(is (= :KEYWORD (run-NFA complete-nfa "synchronized")))
7785
(is (= :INTEGER (run-NFA complete-nfa "109")))
78-
(is (= :OPERATOR (run-NFA complete-nfa "++")))
86+
(is (= :UNARYOPERATOR (run-NFA complete-nfa "++")))
7987
(is (= :BOOLEAN (run-NFA complete-nfa "true")))
8088
(is (= :BOOLEAN (run-NFA complete-nfa "false"))))
8189

@@ -134,19 +142,19 @@
134142
;; Integer
135143
(is (= :INTEGER (run-NFA complete-nfa "109")))
136144
;; Operators
137-
(is (= :OPERATOR (run-NFA complete-nfa "+")))
138-
(is (= :OPERATOR (run-NFA complete-nfa "++")))
139-
(is (= :OPERATOR (run-NFA complete-nfa ">")))
140-
(is (= :OPERATOR (run-NFA complete-nfa ">=")))
141-
(is (= :OPERATOR (run-NFA complete-nfa ">>")))
142-
(is (= :OPERATOR (run-NFA complete-nfa ">>=")))
143-
(is (= :OPERATOR (run-NFA complete-nfa ">>>")))
144-
(is (= :OPERATOR (run-NFA complete-nfa ">>>=")))
145-
(is (= :OPERATOR (run-NFA complete-nfa "&")))
146-
(is (= :OPERATOR (run-NFA complete-nfa "^=")))
147-
(is (= :OPERATOR (run-NFA complete-nfa "^")))
148-
(is (= :OPERATOR (run-NFA complete-nfa "<<")))
149-
(is (= :OPERATOR (run-NFA complete-nfa "=")))
150-
(is (= :OPERATOR (run-NFA complete-nfa "==")))
151-
(is (= :OPERATOR (run-NFA complete-nfa "!")))
152-
(is (= :OPERATOR (run-NFA complete-nfa "!="))))
145+
(is (= :UNARYOPERATOR (run-NFA complete-nfa "+")))
146+
(is (= :UNARYOPERATOR (run-NFA complete-nfa "++")))
147+
(is (= :BINARYOPERATOR (run-NFA complete-nfa ">")))
148+
(is (= :BINARYOPERATOR (run-NFA complete-nfa ">=")))
149+
(is (= :BINARYOPERATOR (run-NFA complete-nfa ">>")))
150+
(is (= :ASSIGNMENTOPERATOR (run-NFA complete-nfa ">>=")))
151+
(is (= :BINARYOPERATOR (run-NFA complete-nfa ">>>")))
152+
(is (= :ASSIGNMENTOPERATOR (run-NFA complete-nfa ">>>=")))
153+
(is (= :BINARYOPERATOR (run-NFA complete-nfa "&")))
154+
(is (= :ASSIGNMENTOPERATOR (run-NFA complete-nfa "^=")))
155+
(is (= :BINARYOPERATOR (run-NFA complete-nfa "^")))
156+
(is (= :BINARYOPERATOR (run-NFA complete-nfa "<<")))
157+
(is (= :ASSIGNMENTOPERATOR (run-NFA complete-nfa "=")))
158+
(is (= :BINARYOPERATOR (run-NFA complete-nfa "==")))
159+
(is (= :UNARYOPERATOR (run-NFA complete-nfa "!")))
160+
(is (= :BINARYOPERATOR (run-NFA complete-nfa "!="))))

0 commit comments

Comments
 (0)