escapes, escapes

Author: jonathantsang

src/watcompiler/Tokens.txt

@@ -1,8 +1,13 @@
 BRACKET { } ( ) [ ]
-BOOLEAN true false
 KEYWORD abstract default if private this boolean do implements protected break double import public throws throw byte else instanceof return transient case extends int short try catch interface static void char finally final long strictfp volatile class float native super while const for new switch continue goto package synchronized
-UNARYOPERATOR + ++ - -- ! ~
-BINARYOPERATOR ==  * / % < << > >> >>> & ^ | != >= <=
+UNARYOPERATOR ++ - -- ! ~
+BINARYOPERATOR ==  * / % < << > >> >>> & ^ | != >= <= +
 ASSIGNMENTOPERATOR = *= /= %= += -= <<= >>= >>>= &= ^= |=
-TERMINAL ; " ,
-
+TERMINAL ; ,
+BOOLEAN-LITERAL true false
+NULL-LITERAL null
+INT-LITERAL <numbers>
+STRING-LITERAL <string of anything>
+CHARACTER-LITERAL <character or escape>
+IDENTIFIER <alphanumeric>
+WHITESPACE <spaces,tabs,newlines>

src/watcompiler/lang.clj

@@ -21,6 +21,12 @@
 (def DIGITS-NONZERO
   (char-range \1 \9))
 
+(def ALL-ASCII
+  (char-range 32 126))
+
+(def ESCAPABLE
+  [\b \t \n \f \r \" \' \\])
+
 (def S-PLUS \+)
 (def S-MINUS \-)
 (def S-STAR \*)

src/watcompiler/nfa.clj

@@ -79,7 +79,7 @@
       transition-map
       (let [[s-from s-to alphabets] (first remaining)]
         (recur (rest remaining)
-               (if (seq? alphabets)
+               (if (or (seq? alphabets) (vector? alphabets))
                  (reduce #(add-to-map %1 (list s-from %2) s-to)
                          transition-map
                          alphabets)

src/watcompiler/re.clj

@@ -30,10 +30,9 @@
 
 ;; Parses a string to form the nfa
 (defn string-to-nfa
-  [word wordtype]
+  [wordtype word]
   (let
     [stateS (gensym :s)
-
       ;; List of substrings of word, stored as strings
       states-map (set (rest (reductions str (str) word)))
 
@@ -51,7 +50,7 @@
       transitions-map (into #{ [stateS (get gensym-map (str (first (seq word))) \a) (first (seq word))] }
                       (for [v (partition 2 1 (vals gensym-map))]
                            [(first v) (second v) (get states-char-map (second v))]))]
-    (make-NFA (into #{} )
+    (make-NFA (into #{} (concat (seq word)))
               states-map
               stateS
               accept-states-map
@@ -67,7 +66,7 @@
      args (rest arguments)
      ;; Key: string for keyword, Value: NFA for that keyword
      strings-nfas (into (sorted-map) (for [nfa-name args]
-                                          [nfa-name (string-to-nfa nfa-name class)]))
+                                          [nfa-name (string-to-nfa class nfa-name)]))
      all-states (apply union (map :states (vals strings-nfas)))
      all-accept-states (apply union (map :accept-states (vals strings-nfas)))
      merged-transitions (apply merge (map :transitions (vals strings-nfas)))
@@ -88,21 +87,24 @@
 
 ;; Reading the file
 (def readFile
-   (with-open [rdr (clojure.java.io/reader "src/watcompiler/Tokens.txt")]
-     (reduce conj [] (line-seq rdr))))
-
-;; Splitting the lines by space
-(def splitLines
   (into []
-  (for [x readFile]
-       (str/split x #" "))))
+    (for [line (str/split-lines (slurp "src/watcompiler/Tokens.txt"))]
+      (str/split line #" "))))
 
 (def fileFormed-nfa
-  (let [nfas
-    (into []
-    (for [x splitLines]
-         (apply form-multiple-nfas x)))]
-    (apply merge-nfas nfas)))
+  (let [nfas (into []
+               (for [x readFile]
+                 ;; Check for regex based NFAs
+                 (if
+                   (or
+                     (if (= (first x) "IDENTIFIER") true false)
+                     (if (= (first x) "INT-LITERAL") true false)
+                     (if (= (first x) "STRING-LITERAL") true false)
+                     (if (= (first x) "CHARACTER-LITERAL") true false)
+                     (if (= (first x) "WHITESPACE") true false))
+                 nil
+                 (apply form-multiple-nfas x))))]
+  (apply merge-nfas (remove nil? nfas))))
 
 ;; NFAs for types
 
@@ -112,36 +114,59 @@
   (let [stateS (gensym :S)
         state1 (gensym :1)
         state2 (gensym :2)]
-  (make-NFA (into #{} )
+  (make-NFA (into #{} (concat [\0] DIGITS DIGITS-NONZERO))
             #{stateS state1 state2}
             stateS
-            {state2 (list :INTEGER 0)}
-            (make-transition-NFA [[stateS state1 e]
-                                  [state1 state2 DIGITS-NONZERO]
+            {state1 (list :INTEGER-LITERAL 0)
+             state2 (list :INTEGER-LITERAL 0)}
+            (make-transition-NFA [[stateS state1 \0]
+                                  [stateS state2 DIGITS-NONZERO]
                                   [state2 state2 DIGITS]]))))
 
 ;; String literal
-;; \".*\" (\ shown for escaping ")
+;; \"(\\[btnfr\"\'\\] | ALL-ASCII)*\" (\ shown for escaping ")
+;; aka \"(.*)\" with escapes inside
 (def string-literal-nfa
   (let [stateS (gensym :S)
         state1 (gensym :1)
-        state2 (gensym :2)]
-  (make-NFA (into #{} )
-            #{stateS state1 state2}
+        state2 (gensym :2)
+        state3 (gensym :3)]
+  (make-NFA (into #{} (concat [\'] ALL-ASCII [\\] ESCAPABLE))
+            #{stateS state1 state2 state3}
+            stateS
+            {state3 (list :STRING-LITERAL 0)}
+            (make-transition-NFA [[stateS state1 \"]
+                                  [state1 state1 ALL-ASCII]
+                                  [state1 state2 \\]
+                                  [state2 state1 ESCAPABLE]
+                                  [state1 state3 \"]]))))
+
+;; Character literal
+;; \'(\\ESCAPABLE | ALL-ASCII)*\' (\ shown for escaping ")
+;; aka \'(.*)\' with escapes inside
+(def character-literal-nfa
+  (let [stateS (gensym :S)
+        state1 (gensym :1)
+        state2 (gensym :2)
+        state3 (gensym :3)
+        state4 (gensym :4)]
+  (make-NFA (into #{} (concat [\'] ALL-ASCII [\\] ESCAPABLE))
+            #{stateS state1 state2 state3 state4}
             stateS
-            {state2 (list :STRING-LITERAL 0)}
-            (make-transition-NFA [[stateS state1 "\""]
-                                  [state1 state1 UPPER-ALPHABET]
-                                  [state1 state1 LOWER-ALPHABET]
-                                  [state1 state2 "\""]]))))
+            {state4 (list :CHARACTER-LITERAL 0)}
+            (make-transition-NFA [[stateS state1 \']
+                                  [state1 state3 ALL-ASCII]
+                                  [state1 state2 \\]
+                                  [state2 state3 ESCAPABLE]
+                                  [state3 state4 \']]))))
 
 ;; Identifiers
 ;; [a-zA-Z][a-zA-Z0-9]*
 (def identifier-nfa
   (let [stateS        (gensym :S)
         state1        (gensym :s1)
         state2        (gensym :s2)]
-  (make-NFA (into #{} )
+  (make-NFA (into #{} (concat UPPER-ALPHABET LOWER-ALPHABET DIGITS))
             #{stateS state1 state2}
             stateS
             {state1  (list :IDENTIFIER 1)
@@ -153,88 +178,24 @@
                                   [state2 state2 UPPER-ALPHABET]
                                   [state2 state2 LOWER-ALPHABET]
                                   [state2 state2 DIGITS]]))))
-;; Operators
-(def operators-nfa
-  (form-multiple-nfas :OPERATOR ">" "<" "<<" ">>" ">>>" "<<<" ">>>=" ">>="
-                      ">=" "<=" "&" "&=" "=" "==" "!" "!=" "^=" "^" "+" "+="
-                      "++" "-" "-=" "--" "*" "*=" "/" "/=" "%" "%="))
-
-;; white space?
-
-;; Keywords nfa
-(def keywords-nfa
-  (form-multiple-nfas :KEYWORD
-                      "abstract"
-                      "default"
-                      "if"
-                      "private"
-                      "this"
-                      "boolean"
-                      "do"
-                      "implements"
-                      "protected"
-                      "break"
-                      "double"
-                      "import"
-                      "public"
-                      "throws"
-                      "throw"
-                      "byte"
-                      "else"
-                      "instanceof"
-                      "return"
-                      "transient"
-                      "case"
-                      "extends"
-                      "int"
-                      "short"
-                      "try"
-                      "catch"
-                      "interface"
-                      "static"
-                      "void"
-                      "char"
-                      "finally"
-                      "final"
-                      "long"
-                      "strictfp"
-                      "volatile"
-                      "class"
-                      "float"
-                      "native"
-                      "super"
-                      "while"
-                      "const"
-                      "for"
-                      "new"
-                      "switch"
-                      "continue"
-                      "goto"
-                      "package"
-                      "synchronized"))
-
-;; Booleans
-(def boolean-nfa
-  (form-multiple-nfas :BOOLEAN "true" "false"))
-
-;; Brackets
-(def bracket-nfa
-  (form-multiple-nfas :BRACKET "{" "}" "(" ")" "[" "]"))
+
+;; Whitespace
+;; [space tab newline]+
+(def whitespace-nfa
+  (let [stateS        (gensym :S)
+        state1        (gensym :s1)]
+  (make-NFA (into #{} WHITESPACE)
+            #{stateS state1}
+            stateS
+            {state1  (list :WHITESPACE 0)}
+            (make-transition-NFA [[stateS state1 WHITESPACE]
+                                  [state1 state1 WHITESPACE]]))))
 
 ;; complete nfa from all of the individual RE nfas
-;; int-literal
-;; string-literal
-;; identifiers
-;; file specified nfas:
-;; BRACKET
-;; BOOLEAN
-;; KEYWORD
-;; UNARYOPERATOR
-;; BINARYOPERATOR
-;; ASSIGNMENTOPERATOR
-;; TERMINAL
 (def complete-nfa
   (merge-nfas integer-literal-nfa
               string-literal-nfa
+              character-literal-nfa
               identifier-nfa
+              whitespace-nfa
               fileFormed-nfa))