diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2b8368b..2785ef9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,16 @@
Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/);
the project follows [Semantic Versioning](https://semver.org/).
+## [Unreleased]
+
+- `Form.str` no longer emits raw control bytes. String literals render
+ `\a`, `\b`, `\v` and `\f` as escapes instead of dropping them to the
+ raw byte, and any other control byte renders as `\uXXXX`; character
+ literals do the same for control codes without a name. Writing
+ `Form.str` output back over a source file no longer corrupts it. A
+ literal newline inside a string is still emitted raw, unchanged, and a
+ `\n` escape is still rewritten to a literal newline.
+
## [0.3.8]
- String escapes now pass through the reference compiler's semantics
diff --git a/carp-reader.carp b/carp-reader.carp
index 4f2633c..1c441df 100644
--- a/carp-reader.carp
+++ b/carp-reader.carp
@@ -61,15 +61,35 @@ nested. Synthesized nodes (reader-macro expansions) carry
(StringBuf.append-str &sb (Array.unsafe-nth segs i))))
(StringBuf.to-string &sb)))
+ (private control-code?)
+ (hidden control-code?)
+ (defn control-code? [i] (or (Int.< i 32) (Int.= i 127)))
+
(private escape-string-byte)
(hidden escape-string-byte)
(defn escape-string-byte [b]
- (cond
- (Char.= b \\) @"\\\\"
- (Char.= b \") @"\\\""
- (Char.= b \tab) @"\\t"
- (Char.= b \return) @"\\r"
- (String.from-bytes &[(Byte.from-int (Int.bit-and 255 (Char.to-int b)))])))
+ (let [i (Int.bit-and 255 (Char.to-int b))]
+ (cond
+ (Char.= b \\)
+ @"\\\\"
+ (Char.= b \")
+ @"\\\""
+ (Int.= i 7)
+ @"\\a"
+ (Int.= i 8)
+ @"\\b"
+ (Int.= i 9)
+ @"\\t"
+ (Int.= i 11)
+ @"\\v"
+ (Int.= i 12)
+ @"\\f"
+ (Int.= i 13)
+ @"\\r"
+ ; \xHH is greedy in the reference reader, so \u is the only safe escape.
+ (and (control-code? i) (Int./= i 10))
+ (fmt "\\u%04x" i)
+ (String.from-bytes &[(Byte.from-int i)]))))
(private escape-string)
(hidden escape-string)
@@ -112,7 +132,7 @@ nested. Synthesized nodes (reader-macro expansions) carry
; codepoints above the BMP need \U + 8 digits; \u only re-parses 4.
(Int.> (Char.to-int c) 65535)
(fmt "\\U%08x" (Char.to-int c))
- (Int.> (Char.to-int c) 127)
+ (or (Int.> (Char.to-int c) 127) (control-code? (Char.to-int c)))
(fmt "\\u%04x" (Char.to-int c))
(fmt "\\%c" c)))
@@ -169,7 +189,9 @@ nested. Synthesized nodes (reader-macro expansions) carry
(doc str "renders a `Form` back to source-like text. Round-trips
for syntactic content; whitespace around comments is left to the
- caller's formatting pass.")
+ caller's formatting pass. Non-printable bytes are always escaped, so
+ the output is safe to write back into a source file; the one
+ exception is a literal newline inside a string, which stays raw.")
(defn str [a]
(match-ref a
(Itg n) (Int.str @n)
diff --git a/docs/Form.html b/docs/Form.html
index 5d6c50a..affdda8 100644
--- a/docs/Form.html
+++ b/docs/Form.html
@@ -438,7 +438,9 @@
renders a Form back to source-like text. Round-trips
for syntactic content; whitespace around comments is left to the
-caller's formatting pass.
+caller's formatting pass. Non-printable bytes are always escaped, so
+the output is safe to write back into a source file; the one
+exception is a literal newline inside a string, which stays raw.
diff --git a/test/carp-reader.carp b/test/carp-reader.carp
index ea06635..65e6a12 100644
--- a/test/carp-reader.carp
+++ b/test/carp-reader.carp
@@ -24,6 +24,45 @@
(defn is-float? [s]
(let [loc (first-form s)] (match-ref (Located.form &loc) (Flt _) true _ false)))
+; a literal newline is the one control byte Form.str deliberately emits raw.
+(defn printable-source? [s]
+ (let-do [bs (String.to-bytes s)
+ ok true]
+ (for [i 0 (Array.length &bs)]
+ (let [v (Byte.to-int @(Array.unsafe-nth &bs i))]
+ (when (and (or (Int.< v 32) (Int.= v 127)) (Int./= v 10))
+ (set! ok false))))
+ ok))
+
+(defn string-byte-round-trips? [b]
+ (let [want (String.from-bytes &[(Byte.from-int b)])
+ rendered (Form.str &(Form.Str (String.from-bytes &[(Byte.from-int b)])))]
+ (and (printable-source? &rendered)
+ (match (Reader.parse-form &rendered)
+ (Result.Success l)
+ (match-ref (Located.form &l) (Str s) (= s &want) _ false)
+ (Result.Error _) false))))
+
+(defn char-code-round-trips? [i]
+ (let [rendered (Form.str &(Form.Chr (Char.from-int i)))]
+ (and (printable-source? &rendered)
+ (match (Reader.parse-form &rendered)
+ (Result.Success l)
+ (match-ref (Located.form &l)
+ (Chr c) (Int.= (Char.to-int @c) i)
+ _ false)
+ (Result.Error _) false))))
+
+(defn bad-string-bytes []
+ (let-do [n 0]
+ (for [b 1 256] (unless (string-byte-round-trips? b) (set! n (Int.inc n))))
+ n))
+
+(defn bad-char-codes []
+ (let-do [n 0]
+ (for [i 1 256] (unless (char-code-round-trips? i) (set! n (Int.inc n))))
+ n))
+
(deftest t
(assert-equal t 1 (Array.length &(unsafe-parse "42")) "parse: single integer")
(assert-equal t "42" &(Located.str &(first-form "42")) "round-trip: integer")
@@ -234,5 +273,30 @@ b\"")) "string with literal newline stays as literal newline")
(parse-error? "\"\\U0000GHIJ\"")
"\\U with non-hex chars errors")
+ (assert-equal t
+ 0
+ (bad-string-bytes)
+ "every string byte 1-255 renders printably and re-parses identically")
+ (assert-equal t
+ 0
+ (bad-char-codes)
+ "every char code 1-255 renders printably and re-parses identically")
+ (assert-equal t
+ "\"\\a\\b\\f\\v\""
+ &(Located.str &(first-form "\"\\a\\b\\f\\v\""))
+ "round-trip: \\a \\b \\f \\v keep their escapes")
+ (assert-equal t
+ "\"\\u001b\""
+ &(Located.str &(first-form "\"\\u001b\""))
+ "unnamed control byte in a string renders as \\u")
+ (assert-equal t
+ "\\u0007"
+ &(Located.str &(first-form "\\u0007"))
+ "unnamed control byte as a char literal renders as \\u")
+ (assert-equal t
+ "\\backspace"
+ &(Located.str &(first-form "\\backspace"))
+ "named control char literal keeps its name")
+
(assert-true t (parse-error? "(") "unbalanced open paren errors")
(assert-true t (parse-error? "\"unterminated") "unterminated string errors"))