Never render raw control bytes from Form.str

CHANGELOG.md

@@ -3,6 +3,16 @@
 Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/);
 the project follows [Semantic Versioning](https://semver.org/).
 
+## [Unreleased]
+
+- `Form.str` no longer emits raw control bytes. String literals render
+  `\a`, `\b`, `\v` and `\f` as escapes instead of dropping them to the
+  raw byte, and any other control byte renders as `\uXXXX`; character
+  literals do the same for control codes without a name. Writing
+  `Form.str` output back over a source file no longer corrupts it. A
+  literal newline inside a string is still emitted raw, unchanged, and a
+  `\n` escape is still rewritten to a literal newline.
+
 ## [0.3.8]
 
 - String escapes now pass through the reference compiler's semantics

carp-reader.carp

@@ -61,15 +61,35 @@ nested. Synthesized nodes (reader-macro expansions) carry
           (StringBuf.append-str &sb (Array.unsafe-nth segs i))))
       (StringBuf.to-string &sb)))
 
+  (private control-code?)
+  (hidden control-code?)
+  (defn control-code? [i] (or (Int.< i 32) (Int.= i 127)))
+
   (private escape-string-byte)
   (hidden escape-string-byte)
   (defn escape-string-byte [b]
-    (cond
-      (Char.= b \\) @"\\\\"
-      (Char.= b \") @"\\\""
-      (Char.= b \tab) @"\\t"
-      (Char.= b \return) @"\\r"
-      (String.from-bytes &[(Byte.from-int (Int.bit-and 255 (Char.to-int b)))])))
+    (let [i (Int.bit-and 255 (Char.to-int b))]
+      (cond
+        (Char.= b \\)
+        @"\\\\"
+        (Char.= b \")
+        @"\\\""
+        (Int.= i 7)
+        @"\\a"
+        (Int.= i 8)
+        @"\\b"
+        (Int.= i 9)
+        @"\\t"
+        (Int.= i 11)
+        @"\\v"
+        (Int.= i 12)
+        @"\\f"
+        (Int.= i 13)
+        @"\\r"
+        ; \xHH is greedy in the reference reader, so \u is the only safe escape.
+        (and (control-code? i) (Int./= i 10))
+        (fmt "\\u%04x" i)
+        (String.from-bytes &[(Byte.from-int i)]))))
 
   (private escape-string)
   (hidden escape-string)
@@ -112,7 +132,7 @@ nested. Synthesized nodes (reader-macro expansions) carry
       ; codepoints above the BMP need \U + 8 digits; \u only re-parses 4.
       (Int.> (Char.to-int c) 65535)
       (fmt "\\U%08x" (Char.to-int c))
-      (Int.> (Char.to-int c) 127)
+      (or (Int.> (Char.to-int c) 127) (control-code? (Char.to-int c)))
       (fmt "\\u%04x" (Char.to-int c))
       (fmt "\\%c" c)))
 
@@ -169,7 +189,9 @@ nested. Synthesized nodes (reader-macro expansions) carry
 
   (doc str "renders a `Form` back to source-like text. Round-trips
   for syntactic content; whitespace around comments is left to the
-  caller's formatting pass.")
+  caller's formatting pass. Non-printable bytes are always escaped, so
+  the output is safe to write back into a source file; the one
+  exception is a literal newline inside a string, which stays raw.")
   (defn str [a]
     (match-ref a
       (Itg n) (Int.str @n)

docs/Form.html

@@ -438,7 +438,9 @@ <h3 id="str">
                     <p class="doc">
                         <p>renders a <code>Form</code> back to source-like text. Round-trips
 for syntactic content; whitespace around comments is left to the
-caller's formatting pass.</p>
+caller's formatting pass. Non-printable bytes are always escaped, so
+the output is safe to write back into a source file; the one
+exception is a literal newline inside a string, which stays raw.</p>
 
                     </p>
                 </div>

test/carp-reader.carp

@@ -24,6 +24,45 @@
 (defn is-float? [s]
   (let [loc (first-form s)] (match-ref (Located.form &loc) (Flt _) true _ false)))
 
+; a literal newline is the one control byte Form.str deliberately emits raw.
+(defn printable-source? [s]
+  (let-do [bs (String.to-bytes s)
+           ok true]
+    (for [i 0 (Array.length &bs)]
+      (let [v (Byte.to-int @(Array.unsafe-nth &bs i))]
+        (when (and (or (Int.< v 32) (Int.= v 127)) (Int./= v 10))
+          (set! ok false))))
+    ok))
+
+(defn string-byte-round-trips? [b]
+  (let [want (String.from-bytes &[(Byte.from-int b)])
+        rendered (Form.str &(Form.Str (String.from-bytes &[(Byte.from-int b)])))]
+    (and (printable-source? &rendered)
+         (match (Reader.parse-form &rendered)
+           (Result.Success l)
+             (match-ref (Located.form &l) (Str s) (= s &want) _ false)
+           (Result.Error _) false))))
+
+(defn char-code-round-trips? [i]
+  (let [rendered (Form.str &(Form.Chr (Char.from-int i)))]
+    (and (printable-source? &rendered)
+         (match (Reader.parse-form &rendered)
+           (Result.Success l)
+             (match-ref (Located.form &l)
+               (Chr c) (Int.= (Char.to-int @c) i)
+               _ false)
+           (Result.Error _) false))))
+
+(defn bad-string-bytes []
+  (let-do [n 0]
+    (for [b 1 256] (unless (string-byte-round-trips? b) (set! n (Int.inc n))))
+    n))
+
+(defn bad-char-codes []
+  (let-do [n 0]
+    (for [i 1 256] (unless (char-code-round-trips? i) (set! n (Int.inc n))))
+    n))
+
 (deftest t
   (assert-equal t 1 (Array.length &(unsafe-parse "42")) "parse: single integer")
   (assert-equal t "42" &(Located.str &(first-form "42")) "round-trip: integer")
@@ -234,5 +273,30 @@ b\"")) "string with literal newline stays as literal newline")
     (parse-error? "\"\\U0000GHIJ\"")
     "\\U with non-hex chars errors")
 
+  (assert-equal t
+    0
+    (bad-string-bytes)
+    "every string byte 1-255 renders printably and re-parses identically")
+  (assert-equal t
+    0
+    (bad-char-codes)
+    "every char code 1-255 renders printably and re-parses identically")
+  (assert-equal t
+    "\"\\a\\b\\f\\v\""
+    &(Located.str &(first-form "\"\\a\\b\\f\\v\""))
+    "round-trip: \\a \\b \\f \\v keep their escapes")
+  (assert-equal t
+    "\"\\u001b\""
+    &(Located.str &(first-form "\"\\u001b\""))
+    "unnamed control byte in a string renders as \\u")
+  (assert-equal t
+    "\\u0007"
+    &(Located.str &(first-form "\\u0007"))
+    "unnamed control byte as a char literal renders as \\u")
+  (assert-equal t
+    "\\backspace"
+    &(Located.str &(first-form "\\backspace"))
+    "named control char literal keeps its name")
+
   (assert-true t (parse-error? "(") "unbalanced open paren errors")
   (assert-true t (parse-error? "\"unterminated") "unterminated string errors"))