feat: use tokens for more accurate sourcemaps (alternative approach)

.changeset/token-delimiters.md

@@ -0,0 +1,5 @@
+---
+'esrap': minor
+---
+
+feat: add a `tokens` option to `print`. Pass the parser's tokens (e.g. Acorn's `onToken` array, or `ast.tokens` from typescript-eslint) and punctuation and keywords written by visitors — brackets, parentheses, braces, `else`, `extends`, `as` and so on — are mapped to their location in the source

README.md

@@ -109,7 +109,7 @@ code; // `[[foo][bar]]`
 
 The `context` API has several methods:
 
-- `context.write(data: string, node?: BaseNode)` — add a string. If `node` is provided and has a standard `loc` property (with `start` and `end` properties each with a `line` and `column`), a sourcemap mapping will be created
+- `context.write(data: string, node?: BaseNode)` — add a string. If `node` is provided and has a standard `loc` property (with `start` and `end` properties each with a `line` and `column`), a sourcemap mapping will be created. Otherwise, if `tokens` were passed to `print` and `data` (ignoring surrounding whitespace) is the next token in the source, it will be mapped to that token — so write tokens individually (e.g. `')'` then `' {'` rather than `') {'`)
 - `context.indent()` — increase the indentation level, typically before adding a newline
 - `context.newline()` — self-explanatory
 - `context.space()` — adds a space character, if it doesn't immediately follow a newline
@@ -143,6 +143,13 @@ const { code, map } = print(ast, ts(), {
   // as a VLQ string, rather than an unencoded array. Defaults to `true`
   sourceMapEncodeMappings: false,
 
+  // The parser's tokens, if `ast` came from parsing source code. Used to
+  // create accurate mappings for punctuation and keywords. Tokens must have
+  // `loc` properties — e.g. Acorn's `onToken` array (with `locations: true`),
+  // or `ast.tokens` from espree/@typescript-eslint/parser (with
+  // `tokens: true, loc: true`)
+  tokens,
+
   // String to use for indentation — defaults to '\t'
   indent: '  '
 });

src/context.js

@@ -1,4 +1,5 @@
 /** @import { BaseNode, Command, Visitors } from './types.js' */
+/** @import { Tokens } from './tokens.js' */
 
 export const margin = 0;
 export const newline = 1;
@@ -9,6 +10,7 @@ export const space = 4;
 export class Context {
 	#visitors;
 	#commands;
+	#tokens;
 	#has_newline = false;
 
 	multiline = false;
@@ -17,10 +19,12 @@ export class Context {
 	 *
 	 * @param {Visitors} visitors
 	 * @param {Command[]} commands
+	 * @param {Tokens | null} tokens
 	 */
-	constructor(visitors, commands = []) {
+	constructor(visitors, commands = [], tokens = null) {
 		this.#visitors = visitors;
 		this.#commands = commands;
+		this.#tokens = tokens;
 	}
 
 	indent() {
@@ -56,15 +60,31 @@ export class Context {
 	}
 
 	/**
-	 *
+	 * Write `content`. If `node` is provided, `content` is mapped to its location. Otherwise, if `tokens`
+	 * were passed to `print` and `content` (ignoring surrounding whitespace) is the next token
+	 * in the source, it is mapped to that token's location.
 	 * @param {string} content
 	 * @param {BaseNode} [node]
 	 */
 	write(content, node) {
+		const value = this.#tokens && content.trim();
+		const token = value
+			? this.#tokens?.consume(value, node?.loc?.start, node?.loc?.end)
+			: undefined;
+
 		if (node?.loc) {
 			this.location(node.loc.start.line, node.loc.start.column);
 			this.#commands.push(content);
 			this.location(node.loc.end.line, node.loc.end.column);
+		} else if (token && value) {
+			const start = content.indexOf(value);
+			const end = start + value.length;
+
+			if (start > 0) this.#commands.push(content.slice(0, start));
+			this.location(token.start.line, token.start.column);
+			this.#commands.push(value);
+			this.location(token.end.line, token.end.column);
+			if (end < content.length) this.#commands.push(content.slice(end));
 		} else {
 			this.#commands.push(content);
 		}
@@ -103,13 +123,18 @@ export class Context {
 			throw new Error(message);
 		}
 
+		const tokens = this.#tokens;
+		const previous = tokens?.enter(node);
+
 		if (this.#visitors._) {
 			// @ts-ignore
 			this.#visitors._(node, this, (node) => visitor(node, this));
 		} else {
 			// @ts-ignore
 			visitor(node, this);
 		}
+
+		if (previous) tokens?.exit(node, previous);
 	}
 
 	empty() {
@@ -121,7 +146,7 @@ export class Context {
 	}
 
 	new() {
-		return new Context(this.#visitors);
+		return new Context(this.#visitors, [], this.#tokens);
 	}
 }
 

src/index.js

@@ -1,6 +1,7 @@
 /** @import { BaseNode, Command, Location, Visitors, PrintOptions } from './types.js' */
 import { encode } from '@jridgewell/sourcemap-codec';
 import { Context, dedent, indent, margin, newline, space } from './context.js';
+import { Tokens } from './tokens.js';
 
 /** @type {(str: string) => string} str */
 let btoa = () => {
@@ -55,8 +56,10 @@ export function print(node, visitors, opts = {}) {
 	/** @type {Command[]} */
 	const commands = [];
 
+	const tokens = opts.tokens ? new Tokens(opts.tokens) : null;
+
 	// @ts-ignore some nonsense I don't understand
-	const context = new Context(visitors, commands);
+	const context = new Context(visitors, commands, tokens);
 
 	context.visit(node);