diff --git a/.changeset/token-delimiters.md b/.changeset/token-delimiters.md new file mode 100644 index 00000000..921fec25 --- /dev/null +++ b/.changeset/token-delimiters.md @@ -0,0 +1,5 @@ +--- +'esrap': minor +--- + +feat: add a `tokens` option to `print`. Pass the parser's tokens (e.g. Acorn's `onToken` array, or `ast.tokens` from typescript-eslint) and punctuation and keywords written by visitors — brackets, parentheses, braces, `else`, `extends`, `as` and so on — are mapped to their location in the source diff --git a/README.md b/README.md index 0fa7c295..dbdb3cdc 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ code; // `[[foo][bar]]` The `context` API has several methods: -- `context.write(data: string, node?: BaseNode)` — add a string. If `node` is provided and has a standard `loc` property (with `start` and `end` properties each with a `line` and `column`), a sourcemap mapping will be created +- `context.write(data: string, node?: BaseNode)` — add a string. If `node` is provided and has a standard `loc` property (with `start` and `end` properties each with a `line` and `column`), a sourcemap mapping will be created. Otherwise, if `tokens` were passed to `print` and `data` (ignoring surrounding whitespace) is the next token in the source, it will be mapped to that token — so write tokens individually (e.g. `')'` then `' {'` rather than `') {'`) - `context.indent()` — increase the indentation level, typically before adding a newline - `context.newline()` — self-explanatory - `context.space()` — adds a space character, if it doesn't immediately follow a newline @@ -143,6 +143,13 @@ const { code, map } = print(ast, ts(), { // as a VLQ string, rather than an unencoded array. Defaults to `true` sourceMapEncodeMappings: false, + // The parser's tokens, if `ast` came from parsing source code. Used to + // create accurate mappings for punctuation and keywords. Tokens must have + // `loc` properties — e.g. Acorn's `onToken` array (with `locations: true`), + // or `ast.tokens` from espree/@typescript-eslint/parser (with + // `tokens: true, loc: true`) + tokens, + // String to use for indentation — defaults to '\t' indent: ' ' }); diff --git a/src/context.js b/src/context.js index 7808a5cd..7cda63ec 100644 --- a/src/context.js +++ b/src/context.js @@ -1,4 +1,5 @@ /** @import { BaseNode, Command, Visitors } from './types.js' */ +/** @import { Tokens } from './tokens.js' */ export const margin = 0; export const newline = 1; @@ -9,6 +10,7 @@ export const space = 4; export class Context { #visitors; #commands; + #tokens; #has_newline = false; multiline = false; @@ -17,10 +19,12 @@ export class Context { * * @param {Visitors} visitors * @param {Command[]} commands + * @param {Tokens | null} tokens */ - constructor(visitors, commands = []) { + constructor(visitors, commands = [], tokens = null) { this.#visitors = visitors; this.#commands = commands; + this.#tokens = tokens; } indent() { @@ -56,15 +60,31 @@ export class Context { } /** - * + * Write `content`. If `node` is provided, `content` is mapped to its location. Otherwise, if `tokens` + * were passed to `print` and `content` (ignoring surrounding whitespace) is the next token + * in the source, it is mapped to that token's location. * @param {string} content * @param {BaseNode} [node] */ write(content, node) { + const value = this.#tokens && content.trim(); + const token = value + ? this.#tokens?.consume(value, node?.loc?.start, node?.loc?.end) + : undefined; + if (node?.loc) { this.location(node.loc.start.line, node.loc.start.column); this.#commands.push(content); this.location(node.loc.end.line, node.loc.end.column); + } else if (token && value) { + const start = content.indexOf(value); + const end = start + value.length; + + if (start > 0) this.#commands.push(content.slice(0, start)); + this.location(token.start.line, token.start.column); + this.#commands.push(value); + this.location(token.end.line, token.end.column); + if (end < content.length) this.#commands.push(content.slice(end)); } else { this.#commands.push(content); } @@ -103,6 +123,9 @@ export class Context { throw new Error(message); } + const tokens = this.#tokens; + const previous = tokens?.enter(node); + if (this.#visitors._) { // @ts-ignore this.#visitors._(node, this, (node) => visitor(node, this)); @@ -110,6 +133,8 @@ export class Context { // @ts-ignore visitor(node, this); } + + if (previous) tokens?.exit(node, previous); } empty() { @@ -121,7 +146,7 @@ export class Context { } new() { - return new Context(this.#visitors); + return new Context(this.#visitors, [], this.#tokens); } } diff --git a/src/index.js b/src/index.js index ff108abb..a830ca4f 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,7 @@ /** @import { BaseNode, Command, Location, Visitors, PrintOptions } from './types.js' */ import { encode } from '@jridgewell/sourcemap-codec'; import { Context, dedent, indent, margin, newline, space } from './context.js'; +import { Tokens } from './tokens.js'; /** @type {(str: string) => string} str */ let btoa = () => { @@ -55,8 +56,10 @@ export function print(node, visitors, opts = {}) { /** @type {Command[]} */ const commands = []; + const tokens = opts.tokens ? new Tokens(opts.tokens) : null; + // @ts-ignore some nonsense I don't understand - const context = new Context(visitors, commands); + const context = new Context(visitors, commands, tokens); context.visit(node); diff --git a/src/languages/ts/index.js b/src/languages/ts/index.js index efb0d1ff..559ff058 100644 --- a/src/languages/ts/index.js +++ b/src/languages/ts/index.js @@ -133,15 +133,16 @@ function write_comment(comment, context) { if (comment.type === 'Line') { context.write(`//${comment.value}`); } else { - context.write('/*'); const lines = comment.value.split('\n'); + // write the delimiters with the first and last lines, so that the content + // of a one-line comment can't be mistaken for a source token (e.g. `/* ( */`) for (let i = 0; i < lines.length; i += 1) { if (i > 0) context.newline(); - context.write(lines[i]); + const line = (i === 0 ? '/*' : '') + lines[i] + (i === lines.length - 1 ? '*/' : ''); + context.write(line); } - context.write('*/'); if (lines.length > 1) context.newline(); } } @@ -335,7 +336,8 @@ export default (options = {}) => { const attributes = node.attributes ?? node.assertions; if (!attributes || attributes.length === 0) return; - context.write(node.attributes ? ' with { ' : ' assert { '); + context.write(node.attributes ? ' with ' : ' assert '); + context.write('{ '); for (let i = 0; i < attributes.length; i += 1) { const { key, value } = attributes[i]; @@ -701,7 +703,8 @@ export default (options = {}) => { */ 'FunctionDeclaration|FunctionExpression': (node, context) => { if (node.async) context.write('async '); - context.write(node.generator ? 'function* ' : 'function '); + context.write('function'); + context.write(node.generator ? '* ' : ' '); if (node.id) track_binding(node.id); @@ -956,7 +959,11 @@ export default (options = {}) => { node.type in EXPRESSIONS_PRECEDENCE && !BINDINGS.has(node) ); - if (!has_preceding_decorator(node)) { + // the node starts here unless its decorators are printed later (they + // precede `loc.start`) or were printed already and `loc.start` is + // one of them (Acorn ranges a class from decorators written before + // its `export`; typescript-estree from `class`, which then starts here) + if (has_preceding_decorator(node) === printed_decorators.has(node)) { context.location(node.loc.start.line, node.loc.start.column); } } @@ -1130,9 +1137,11 @@ export default (options = {}) => { token(context, 'do', node); context.write(' '); context.visit(node.body); - context.write(' while ('); + context.write(' while '); + context.write('('); context.visit(node.test); - context.write(');'); + context.write(')'); + context.write(';'); }, EmptyStatement(node, context) { @@ -1142,7 +1151,8 @@ export default (options = {}) => { ExportAllDeclaration(node, context) { token(context, 'export', node); - context.write(node.exportKind === 'type' ? ' type * ' : ' * '); + if (node.exportKind === 'type') context.write(' type'); + context.write(' * '); if (node.exported) { context.write('as '); @@ -1167,6 +1177,8 @@ export default (options = {}) => { }); } + // the declaration's decorators came first, so the node's start is written here + if (node.loc) context.location(node.loc.start.line, node.loc.start.column); token(context, 'export', node); context.write(' default '); @@ -1194,6 +1206,8 @@ export default (options = {}) => { }); } + // the declaration's decorators came first, so the node's start is written here + if (node.loc) context.location(node.loc.start.line, node.loc.start.column); token(context, 'export', node); context.write(' '); @@ -1591,7 +1605,8 @@ export default (options = {}) => { SpreadElement: shared['RestElement|SpreadElement'], StaticBlock(node, context) { - context.write('static {'); + context.write('static '); + context.write('{'); context.indent(); context.newline(); @@ -1632,7 +1647,8 @@ export default (options = {}) => { context.write(' ('); context.visit(node.discriminant); - context.write(') {'); + context.write(') '); + context.write('{'); context.indent(); let first = true; @@ -1672,7 +1688,8 @@ export default (options = {}) => { for (let i = 0; i < expressions.length; i++) { const raw = quasis[i].value.raw; - context.write(raw + '${'); + context.write(raw); + context.write('${'); context.visit(expressions[i]); context.write('}'); @@ -1681,7 +1698,8 @@ export default (options = {}) => { const raw = quasis[quasis.length - 1].value.raw; - context.write(raw + '`'); + context.write(raw); + context.write('`'); if (/\n/.test(raw)) context.multiline = true; }, @@ -1892,7 +1910,8 @@ export default (options = {}) => { TSArrayType(node, context) { context.visit(node.elementType); - context.write('[]'); + context.write('['); + context.write(']'); }, TSTypeAnnotation(node, context) { @@ -1936,14 +1955,16 @@ export default (options = {}) => { for (let i = 0; i < types.length; i++) { const raw = quasis[i].value.raw; - context.write(raw + '${'); + context.write(raw); + context.write('${'); context.visit(types[i]); context.write('}'); if (/\n/.test(raw)) context.multiline = true; } const raw = quasis[quasis.length - 1].value.raw; - context.write(raw + '`'); + context.write(raw); + context.write('`'); if (/\n/.test(raw)) context.multiline = true; }, @@ -1988,14 +2009,16 @@ export default (options = {}) => { }, TSExportAssignment(node, context) { - context.write('export = '); + context.write('export '); + context.write('= '); context.visit(node.expression); context.write(';'); }, TSNamespaceExportDeclaration(node, context) { token(context, 'export', node); - context.write(' as namespace '); + context.write(' as '); + context.write('namespace '); context.visit(node.id); context.write(';'); }, @@ -2118,9 +2141,8 @@ export default (options = {}) => { // `readonly` / `+readonly` / `-readonly` modifier if (node.readonly) { - context.write( - node.readonly === '-' ? '-readonly ' : node.readonly === '+' ? '+readonly ' : 'readonly ' - ); + if (node.readonly === '-' || node.readonly === '+') context.write(node.readonly); + context.write('readonly '); } context.write('['); @@ -2248,9 +2270,11 @@ export default (options = {}) => { TSConstructorType: shared['TSFunctionType|TSConstructorType'], TSExternalModuleReference(node, context) { - context.write('require('); + context.write('require'); + context.write('('); context.visit(node.expression); - context.write(');'); + context.write(')'); + context.write(';'); }, TSIndexedAccessType(node, context) { @@ -2569,7 +2593,7 @@ function is_decorator_expression(node) { * @param {TSESTree.Node & { decorators: TSESTree.Decorator[] | undefined }} node */ function block_decorators(context, node) { - if (!node.decorators) return; + if (!node.decorators || printed_decorators.has(node)) return; for (const decorator of node.decorators) { context.visit(decorator); @@ -2598,6 +2622,12 @@ function inline_decorators(context, node) { } } +/** + * Declarations whose decorators an export visitor has already printed + * @type {WeakSet} + */ +const printed_decorators = new WeakSet(); + /** * Visit an exported declaration minus its decorators, which have already been printed * @param {Context} context @@ -2605,18 +2635,9 @@ function inline_decorators(context, node) { */ function visit_without_decorators(context, node) { if ('decorators' in node && node.decorators && node.decorators.length > 0) { - const { decorators, loc } = node; - - // Temporarily remove decorators so ClassDeclaration doesn't print them again - node.decorators = []; - // @ts-expect-error - node.loc = null; - + printed_decorators.add(node); context.visit(node); - node.decorators = decorators; - node.loc = loc; - - if (loc) context.location(loc.end.line, loc.end.column); + printed_decorators.delete(node); } else { context.visit(node); } @@ -2835,10 +2856,14 @@ function handle_var_declarator(node, context, no_in) { * @param {TSESTree.Node} node */ function has_preceding_decorator(node) { - let n = - ((node.type === 'ExportNamedDeclaration' || node.type === 'ExportDefaultDeclaration') && - node.declaration) || - node; + if (node.type === 'ExportNamedDeclaration' || node.type === 'ExportDefaultDeclaration') { + // the declaration's decorators are printed before `export`, whichever + // side of it they were written on + const d = node.declaration; + return !!(d && 'decorators' in d && d.decorators && d.decorators.length > 0); + } + + let n = node; if ('parameter' in n && 'decorators' in n.parameter) { n = n.parameter; @@ -2878,7 +2903,13 @@ function handle_var_declaration(node, context, no_in = false) { context.append(child_context); - token(child_context, node.declare ? `declare ${node.kind}` : node.kind, node); + if (node.declare) { + token(child_context, 'declare', node); + child_context.write(' '); + child_context.write(node.kind); + } else { + token(child_context, node.kind, node); + } child_context.write(' '); child_context.append(open); diff --git a/src/public.ts b/src/public.ts index 5daff171..ffb01687 100644 --- a/src/public.ts +++ b/src/public.ts @@ -1,2 +1,2 @@ -export type { PrintOptions, Visitors } from './types.js'; +export type { PrintOptions, Token, Visitors } from './types.js'; export * from './index.js'; diff --git a/src/tokens.js b/src/tokens.js new file mode 100644 index 00000000..abc95657 --- /dev/null +++ b/src/tokens.js @@ -0,0 +1,235 @@ +/** @import { BaseNode, Token } from './types.js' */ + +/** + * @typedef {{ line: number, column: number }} Position + * @typedef {{ value: string, start: Position, end: Position, pair: number }} Entry + * @typedef {{ lo: number, hi: number, openers: Opener[] }} Frame + * @typedef {{ value: string, pair: number }} Opener + */ + +/** Acorn/Babel token types that don't correspond to anything written verbatim */ +const IGNORED = new Set([ + 'string', + 'num', + 'bigint', + 'decimal', + 'regexp', + 'template', + 'invalidTemplate', + 'privateId', + 'jsxText', + 'eof' +]); + +/** ESLint-style token types (espree, typescript-eslint) whose `value` is written verbatim */ +const VERBATIM = new Set([ + 'Punctuator', + 'Keyword', + 'Identifier', + 'Boolean', + 'Null', + 'JSXIdentifier' +]); + +/** @type {Record} */ +const OPENERS = { ')': '(', ']': '[', '}': '{' }; + +/** + * The text a parser token stands for, or `null` if it's not something a printer writes as-is. + * Acorn and Babel describe tokens with a `type` object whose `label` is the punctuator (or + * `name` for identifiers and contextual keywords), with operators and words in `value`. + * ESLint-style tokens have a string `type` and the source text in `value`. + * @param {Token} token + * @returns {string | null} + */ +function text(token) { + const { type, value } = token; + + if (typeof type === 'object' && type !== null) { + const label = type.label; + if (label === undefined || IGNORED.has(label)) return null; + if (label === 'jsxTagStart') return '<'; + if (label === 'jsxTagEnd') return '>'; + return typeof value === 'string' ? value : label; + } + + if (typeof type === 'string' && VERBATIM.has(type) && typeof value === 'string') { + return value; + } + + return null; +} + +/** + * @param {Position} a + * @param {Position} b + */ +function compare(a, b) { + return a.line - b.line || a.column - b.column; +} + +/** + * Tracks which of the parser's tokens have been written, so that `context.write` + * can map a token to its source location. Each `context.visit` narrows the search to + * the tokens inside the visited node; each written token advances the cursor past it. + */ +export class Tokens { + /** @type {Entry[]} */ + #entries = []; + + /** @type {Frame} */ + #frame = { lo: 0, hi: 0, openers: [] }; + + /** @param {readonly Token[]} tokens */ + constructor(tokens) { + /** @type {Entry[]} */ + const entries = []; + + for (const token of tokens) { + if (!token.loc) continue; + const value = text(token); + if (value === null) continue; + entries.push({ value, start: token.loc.start, end: token.loc.end, pair: -1 }); + } + + // parsers that backtrack (e.g. acorn-typescript) can emit tokens more than once, or out + // of order. Keep the last token emitted at each position, and drop any that overlap + entries.sort((a, b) => compare(a.start, b.start)); + + for (const entry of entries) { + const previous = this.#entries[this.#entries.length - 1]; + + if (previous && compare(previous.start, entry.start) === 0) { + this.#entries[this.#entries.length - 1] = entry; + } else if (!previous || compare(previous.end, entry.start) <= 0) { + this.#entries.push(entry); + } + } + + // pair up brackets, so that a closing bracket can be found from its opening bracket + // regardless of any redundant parentheses or trailing commas in between + /** @type {number[]} */ + const stack = []; + + for (let i = 0; i < this.#entries.length; i += 1) { + const { value } = this.#entries[i]; + + if (value === '(' || value === '[' || value === '{' || value === '${') { + stack.push(i); + } else if (value in OPENERS) { + const opener = stack.pop(); + if (opener === undefined) continue; + + const expected = this.#entries[opener].value; + if (expected === OPENERS[value] || (value === '}' && expected === '${')) { + this.#entries[opener].pair = i; + this.#entries[i].pair = opener; + } + } + } + } + + /** + * Bound the search to the tokens within `node`. If `node` has no location, disable + * searching until a descendant with a location is visited + * @param {BaseNode} node + * @returns {Frame} the previous state, to be passed to `exit` + */ + enter(node) { + const previous = this.#frame; + + if (node.loc) { + let lo = this.#search(node.loc.start); + const hi = this.#search(node.loc.end); + + // if the parent has already written tokens inside this node (e.g. the decorators + // of an exported class, which are printed before `export`), skip them + if (previous.lo > lo && previous.lo < hi) lo = previous.lo; + + this.#frame = { lo, hi, openers: [] }; + } else { + this.#frame = { lo: 0, hi: 0, openers: [] }; + } + + return previous; + } + + /** + * Restore the state from before `enter`, advancing the cursor past `node` + * @param {BaseNode} node + * @param {Frame} previous + */ + exit(node, previous) { + this.#frame = previous; + if (node.loc) this.skip(node.loc.end); + } + + /** + * Advance the cursor past `position` + * @param {Position} position + */ + skip(position) { + this.#frame.lo = Math.max(this.#frame.lo, this.#search(position)); + } + + /** + * If `value` is the next unwritten token, consume it and return it. A closing + * bracket matches the partner of the opening bracket that was last written + * @param {string} value + * @param {Position} [from] only match tokens starting at or after this position + * @param {Position} [to] only match tokens ending at or before this position + * @returns {Entry | undefined} + */ + consume(value, from, to) { + const frame = this.#frame; + const is_opener = value === '(' || value === '[' || value === '{' || value === '${'; + + if (value in OPENERS) { + const opener = frame.openers[frame.openers.length - 1]; + + if (opener && (opener.value === OPENERS[value] || (value === '}' && opener.value === '${'))) { + frame.openers.pop(); + + const i = opener.pair; + if (i === -1 || i >= frame.hi) return; + + frame.lo = Math.max(frame.lo, i + 1); + return this.#entries[i]; + } + } + + const entry = frame.lo < frame.hi ? this.#entries[frame.lo] : undefined; + + const match = + entry !== undefined && + entry.value === value && + (!from || compare(entry.start, from) >= 0) && + (!to || compare(entry.end, to) <= 0); + + if (is_opener) { + frame.openers.push({ value, pair: match ? entry.pair : -1 }); + } + + if (match) { + frame.lo += 1; + return entry; + } + } + + /** + * The index of the first token starting at or after `position` + * @param {Position} position + */ + #search(position) { + let lo = 0; + let hi = this.#entries.length; + + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (compare(this.#entries[mid].start, position) < 0) lo = mid + 1; + else hi = mid; + } + + return lo; + } +} diff --git a/src/types.ts b/src/types.ts index e8628032..553cba40 100644 --- a/src/types.ts +++ b/src/types.ts @@ -45,4 +45,23 @@ export interface PrintOptions { sourceMapContent?: string; sourceMapEncodeMappings?: boolean; // default true indent?: string; // default tab + /** + * The parser's tokens — e.g. Acorn's `onToken` array, or `ast.tokens` from + * typescript-eslint or espree. When provided, punctuation and keywords + * written by visitors are mapped to their location in the source + */ + tokens?: readonly Token[]; +} + +/** + * A token produced by a parser. Acorn and Babel tokens have a `type` object + * with a `label`; ESLint-style tokens have a string `type` and a `value` + */ +export interface Token { + type?: string | { label?: string }; + value?: unknown; + loc?: null | { + start: { line: number; column: number }; + end: { line: number; column: number }; + }; } diff --git a/test/common.js b/test/common.js index 55483eac..33bb1425 100644 --- a/test/common.js +++ b/test/common.js @@ -10,7 +10,7 @@ export const acornTs = acorn.Parser.extend(tsPlugin()); export const acornTsx = acorn.Parser.extend(tsPlugin({ jsx: true })); /** @param {string} input - * @param {{ jsxMode?: boolean, sourceType?: 'module' | 'script', preserveParens?: boolean, fileExtension?: string }} opts + * @param {{ jsxMode?: boolean, sourceType?: 'module' | 'script', preserveParens?: boolean, fileExtension?: string, tokens?: any[] }} opts */ export function acornParse(input, opts = {}) { const jsx = opts.jsxMode ?? false; @@ -24,6 +24,7 @@ export function acornParse(input, opts = {}) { sourceType, locations: true, preserveParens: opts.preserveParens ?? false, + onToken: opts.tokens, onComment: (block, value, start, end, startLoc, endLoc) => { if (block && /\n/.test(value)) { let a = start; diff --git a/test/samples/ts-declare/expected.ts.map b/test/samples/ts-declare/expected.ts.map index 2104c512..b82ae0ce 100644 --- a/test/samples/ts-declare/expected.ts.map +++ b/test/samples/ts-declare/expected.ts.map @@ -7,5 +7,5 @@ "sourcesContent": [ "declare function a(): void;\ndeclare const b: number;\ndeclare class c {}\n\nexport {};\n" ], - "mappings": "AAAA,iBAAiB,CAAC,EAAE,EAAE,IAAI,CAAC;;AAC3B,aAAa,CAAC,CAAC,EAAE,MAAM,CAAC;;AACxB,cAAc,CAAC,CAAC,CAAC,CAAC;;AAElB,MAAM,IAAI,AACV" + "mappings": "AAAA,iBAAiB,CAAC,EAAE,EAAE,IAAI,CAAC;;AAC3B,OAAO,OAAO,CAAC,EAAE,MAAM,CAAC;;AACxB,cAAc,CAAC,CAAC,CAAC,CAAC;;AAElB,MAAM,IAAI,AACV" } \ No newline at end of file diff --git a/test/samples/ts-decorator-export-comment/expected.ts.map b/test/samples/ts-decorator-export-comment/expected.ts.map index 91ed8ecb..e0e5d3cf 100644 --- a/test/samples/ts-decorator-export-comment/expected.ts.map +++ b/test/samples/ts-decorator-export-comment/expected.ts.map @@ -7,5 +7,5 @@ "sourcesContent": [ "@dec\nexport /* c */ declare class X {}\n" ], - "mappings": "AAAA,CAAC,GAAG;MACE,uBAAuB,CAAC,CAAC,CAAC,CAAC,AACjC" + "mappings": "AAAA,CAAC,GAAG;AACJ,MAAM,SAAS,cAAc,CAAC,CAAC,CAAC,CAAC,AACjC" } \ No newline at end of file diff --git a/test/samples/ts-decorators-class/expected.ts.map b/test/samples/ts-decorators-class/expected.ts.map index 49a621ae..f1fb9bf7 100644 --- a/test/samples/ts-decorators-class/expected.ts.map +++ b/test/samples/ts-decorators-class/expected.ts.map @@ -7,5 +7,5 @@ "sourcesContent": [ "export class Entity {\n\tconstructor(\n\t\tprivate readonly name: string,\n\t\tprivate readonly info: boolean\n\t) {}\n}\n\n@Entity('users', { info: true })\nexport class User {}\n\n@Entity('categories', { info: false })\nclass Category {}\n\n@Entity('tasks', { info: true })\nexport class Task {}\n\n@Entity('default', { info: true })\nexport default class Default {}\n\nconst u = new User();\nconst c = new Category();\nconst t = new Task();\n" ], - "mappings": "AAAA,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC;CACpB,WAAW;EACV,iBAAiB,IAAI,EAAE,MAAM;EAC7B,iBAAiB,IAAI,EAAE,OAAO;GAC7B,CAAC,CAAC;AACL,CAAC;;AAED,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;MAC1B,OAAO,IAAI,CAAC,CAAC,CAAC;;AAEpB,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;MAChC,QAAQ,CAAC,CAAC,CAAC;;AAEjB,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;MAC1B,OAAO,IAAI,CAAC,CAAC,CAAC;;AAEpB,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;MAC5B,eAAe,OAAO,CAAC,CAAC,CAAC,CAAA;;AAE/B,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;AACrB,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;AACzB,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,AACrB" + "mappings": "AAAA,MAAM,CAAC,MAAM,MAAM,CAAC,CAAC;CACpB,WAAW;EACV,iBAAiB,IAAI,EAAE,MAAM;EAC7B,iBAAiB,IAAI,EAAE,OAAO;GAC7B,CAAC,CAAC;AACL,CAAC;;AAED,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAChC,MAAM,OAAO,IAAI,CAAC,CAAC,CAAC;;AAEpB,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;MAChC,QAAQ,CAAC,CAAC,CAAC;;AAEjB,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAChC,MAAM,OAAO,IAAI,CAAC,CAAC,CAAC;;AAEpB,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAClC,MAAM,eAAe,OAAO,CAAC,CAAC,CAAC,CAAA;;AAE/B,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;AACrB,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;AACzB,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,AACrB" } \ No newline at end of file diff --git a/test/sourcemap-keywords.test.js b/test/sourcemap-keywords.test.js index 8fe28f87..c525f0db 100644 --- a/test/sourcemap-keywords.test.js +++ b/test/sourcemap-keywords.test.js @@ -103,6 +103,17 @@ test.each([ expect(segment.slice(2)).toEqual([gen_line, gen_col]); }); +/** + * @param {string} code + * @param {number} index + * @param {[number, number, number, number][][]} mappings + */ +function mappingsAtIndex(code, index, mappings) { + const { gen_line, gen_col } = generatedLineColumn(code, index); + const line = mappings[gen_line] ?? []; + return line.filter((segment) => segment[0] === gen_col); +} + /** * @param {string} code * @param {string} needle @@ -128,3 +139,395 @@ test.each([ expect(sourcePositionsAt(code, needle, mappings)).toContainEqual([gen_line, gen_col]); } ); + +/** + * @param {string} code + * @param {string} needle + * @param {[number, number, number, number][][]} mappings + */ +function mappingsAtSubstring(code, needle, mappings) { + const index = code.indexOf(needle); + expect(index >= 0, `needle not in output: ${JSON.stringify(needle)}`).toBe(true); + return mappingsAtIndex(code, index, mappings); +} + +/** + * Prints `source` with the parser's tokens passed to the printer (or `opts.tokens`) + * @param {string} source + * @param {{ tokens?: boolean | any[], sourceType?: 'module' | 'script' }} [opts] + */ +function mappedWithTokens(source, opts = {}) { + /** @type {any[]} */ + const tokens = []; + const { ast, comments } = acornParse(source, { + sourceType: opts.sourceType ?? 'module', + jsxMode: false, + fileExtension: 'ts', + tokens + }); + const { code, map } = print(ast, ts({ comments }), { + sourceMapSource: 'input.ts', + sourceMapContent: source, + sourceMapEncodeMappings: false, + tokens: opts.tokens === true ? tokens : opts.tokens || undefined + }); + const mappings = /** @type {[number, number, number, number][][]} */ ( + /** @type {unknown} */ (map.mappings) + ); + return { code, mappings, tokens }; +} + +/** + * Every `delimiters` character in the output maps to the matching token in the + * source, and the output is the same as without tokens. + * @param {string} source + * @param {string[]} delimiters + * @param {{ sourceType?: 'module' | 'script' }} [opts] + */ +function expectExactDelimiterMappings(source, delimiters, opts = {}) { + const without = mappedWithTokens(source, { sourceType: opts.sourceType }); + const { code, mappings, tokens } = mappedWithTokens(source, { + tokens: true, + sourceType: opts.sourceType + }); + expect(code).toBe(without.code); + + // a backtracking parser can emit a token twice + const seen = new Set(); + /** @type {{ value: string, start: { line: number, column: number } }[]} */ + const source_delimiters = []; + for (const token of tokens) { + const value = token.type.label; + if (!token.loc || !delimiters.includes(value === '${' ? '{' : value)) continue; + const key = `${value}:${token.loc.start.line}:${token.loc.start.column}`; + if (seen.has(key)) continue; + seen.add(key); + source_delimiters.push({ value, start: token.loc.start }); + } + + /** @type {{ value: string, index: number }[]} */ + const generated_delimiters = []; + for (let i = 0; i < code.length; i += 1) { + if (code[i] === '$' && code[i + 1] === '{' && delimiters.includes('{')) { + generated_delimiters.push({ value: '${', index: i }); + i += 1; + } else if (delimiters.includes(code[i])) { + generated_delimiters.push({ value: code[i], index: i }); + } + } + + expect(generated_delimiters.map((d) => d.value)).toEqual(source_delimiters.map((d) => d.value)); + for (let i = 0; i < generated_delimiters.length; i += 1) { + const { start } = source_delimiters[i]; + expect( + mappingsAtIndex(code, generated_delimiters[i].index, mappings).map((s) => s.slice(2)), + `${generated_delimiters[i].value} at ${start.line}:${start.column}` + ).toContainEqual([start.line - 1, start.column]); + } +} + +test.each([ + ['object property', `const object = { [\nkey\n]: value };`], + ['object method', `const object = { async *[\nmethod\n]() {} };`], + ['object getter', `const object = { get [\nkey\n]() { return 1; } };`], + ['object setter', `const object = { set [\nkey\n](value) {} };`], + ['class method', `class Example { [\nmethod\n]() {} }`], + ['class field', `class Example { [\nfield\n] = 1; }`], + ['type property signature', `type Example = { [\nkey\n]: string }`], + ['type method signature', `type Example = { [\nmethod\n](): void }`], + ['comments inside the brackets', `const object = { [ /* before */\nkey\n/* after */ ]: value };`], + ['array expression', `const values = [ /* before */\nvalue\n/* after */ ];`], + ['array pattern', `const [ /* before */\nvalue\n/* after */ ] = values;`], + ['computed member access', `const value = object[ /* before */\nkey\n/* after */ ];`], + ['array type', `type Values = Item /* before */ [ /* after */ ];`], + ['index signature', `type Dictionary = { [ /* before */ key: string /* after */ ]: number };`], + ['readonly index signature', `type Dictionary = { readonly [ key: string ]: number };`], + [ + 'mapped type', + `type Selected = { [ /* before */ K in Keys as Rename /* after */ ]: string };` + ], + ['tuple type', `type Pair = [ /* before */ First, Second /* after */ ];`], + ['indexed access type', `type Value = Object[ /* before */ Key /* after */ ];`] +])('tokens map square brackets for %s', (_name, source) => { + expectExactDelimiterMappings(source, ['[', ']']); +}); + +test.each([ + ['call', `const result = fn /* before */ ( /* inside */ value /* after */ );`, {}], + [ + 'new expression', + `const result = new Thing /* before */ ( /* inside */ value /* after */ );`, + {} + ], + [ + 'function declaration', + `function example /* before */ ( /* inside */ value /* after */ ) {}`, + {} + ], + ['async arrow function', `const fn = async /* before */ ( value ) => value;`, {}], + ['arrow function', `const callback = ( /* before */ value /* after */ ) => value;`, {}], + ['empty arrow function', `const callback = /* before */ () => value;`, {}], + ['class method', `class Example { method /* before */ ( value /* after */ ) {} }`, {}], + ['object method', `const object = { method /* before */ ( value /* after */ ) {} };`, {}], + ['nested method delimiters', `class Example { [getKey()] /* before */ (value = make()) {} }`, {}], + ['nested parameter delimiters', `function example(value = make(1)) {}`, {}], + [ + 'trailing comma on its own line', + `function example(\n\tfirst: number,\n\tsecond: string,\n) {}`, + {} + ], + ['call signature', `type Example = { ( /* before */ value: string /* after */ ): void };`, {}], + [ + 'construct signature', + `type Example = { new ( /* before */ value: string /* after */ ): object };`, + {} + ], + ['function type', `type Example = ( /* before */ value: string /* after */ ) => void;`, {}], + [ + 'constructor type', + `type Example = new ( /* before */ value: string /* after */ ) => object;`, + {} + ], + [ + 'method signature', + `type Example = { method( /* before */ value: string /* after */ ): void };`, + {} + ], + [ + 'declare function', + `declare function example( /* before */ value: string /* after */ ): void;`, + {} + ], + ['for statement', `for /* before */ (let i = 0; i < 1; i += 1) {}`, {}], + ['for-of statement', `for /* before */ (const value of values) {}`, {}], + ['if statement', `if /* before */ (condition /* after */) {}`, {}], + ['while statement', `while /* before */ (condition /* after */) {}`, {}], + ['do-while statement', `do {} while /* before */ (condition /* after */);`, {}], + ['with statement', `with /* before */ (object /* after */) {}`, { sourceType: 'script' }], + ['switch statement', `switch /* before */ (value /* after */) { default: break; }`, {}], + ['catch clause', `try {} catch /* before */ (error /* after */) {}`, {}], + ['import expression', `const module = import( /* before */ 'module' /* after */ );`, {}], + ['external module reference', `import Alias = require( /* before */ 'module' /* after */ );`, {}], + ['import type', `type Value = import( /* before */ 'module' /* after */ ).Value;`, {}] +])('tokens map parentheses for %s', (_name, source, opts) => { + expectExactDelimiterMappings(source, ['(', ')'], { + sourceType: 'sourceType' in opts && opts.sourceType === 'script' ? 'script' : undefined + }); +}); + +test.each([ + ['named import', `import { /* before */ value /* after */ } from 'module';`], + ['named export', `const value = 1;\nexport { /* before */ value /* after */ };`], + ['empty named export', `export { /* inside */ };`], + ['import attributes', `import value from 'module' with { type: 'json' };`], + ['static block', `class Example { static /* before */ { value; } }`], + ['switch body', `switch (value) /* before */ { default: break; }`], + ['enum body', `enum Example /* before */ { Value }`], + ['interface body', `interface Example /* before */ { value: string }`], + ['namespace body', `namespace Example /* before */ { export const value = 1; }`], + ['template interpolation', 'const result = `before ${ /* before */ value /* after */ } after`;'], + ['template-literal type interpolation', 'type Result = `before ${Value} after`;'] +])('tokens map curly braces for %s', (_name, source) => { + expectExactDelimiterMappings(source, ['{', '}']); +}); + +test('control-flow parentheses are the outer authored ones', () => { + const source = `if /* before */ ((condition)) {}`; + const { code, mappings, tokens } = mappedWithTokens(source, { tokens: true }); + const opens = tokens.filter((token) => token.type.label === '('); + const closes = tokens.filter((token) => token.type.label === ')'); + + expect(mappingsAtSubstring(code, '(', mappings).map((s) => s.slice(2))).toContainEqual([ + opens[0].loc.start.line - 1, + opens[0].loc.start.column + ]); + expect(mappingsAtSubstring(code, ')', mappings).map((s) => s.slice(2))).toContainEqual([ + closes[closes.length - 1].loc.start.line - 1, + closes[closes.length - 1].loc.start.column + ]); +}); + +test('delimiters inside a node are left unmapped without tokens', () => { + const source = `const object = { [ key ]: value };`; + const { code, mappings } = mappedWithTokens(source); + expect(mappingsAtSubstring(code, ']', mappings).map((s) => s.slice(2))).not.toContainEqual([ + 0, + source.indexOf(']') + ]); +}); + +test('closing delimiters at a node end are mapped without tokens', () => { + const source = `const value = object[key];`; + const { code, mappings } = mappedWithTokens(source); + expect(mappingsAtSubstring(code, ']', mappings).map((s) => s.slice(2))).toContainEqual([ + 0, + source.indexOf(']') + ]); +}); + +test('tokens without locations leave mappings unchanged', () => { + const source = `const object = { [key]: fn(value) };`; + const { mappings, tokens } = mappedWithTokens(source); + const bare = tokens.map(({ type, value }) => ({ type, value })); + expect(mappedWithTokens(source, { tokens: bare }).mappings).toEqual(mappings); + expect(mappedWithTokens(source, { tokens: [] }).mappings).toEqual(mappings); +}); + +test('tokens out of source order still locate delimiters', () => { + // acorn-typescript backtracks and re-emits tokens, so its stream is not monotone + const source = `const object = { [ key ]: fn( value ) };`; + const { mappings, tokens } = mappedWithTokens(source, { tokens: true }); + const reversed = [...tokens].reverse(); + expect(mappedWithTokens(source, { tokens: reversed }).mappings).toEqual(mappings); +}); + +test('ESLint-style tokens locate delimiters and keywords', () => { + const source = `const object = { [ key ]: fn( value ) };\nasync /* c */ function f() {}`; + const { mappings, tokens } = mappedWithTokens(source, { tokens: true }); + const eslint_style = tokens.map((token) => ({ + type: token.type.keyword + ? 'Keyword' + : token.type.label === 'name' + ? 'Identifier' + : 'Punctuator', + value: token.type.label === 'name' || token.type.keyword ? token.value : token.type.label, + loc: token.loc + })); + expect(mappedWithTokens(source, { tokens: eslint_style }).mappings).toEqual(mappings); +}); + +/** + * Every `words` keyword in the output maps to the matching token in the source, + * and the output is the same as without tokens. + * @param {string} source + * @param {string[]} words + */ +function expectExactKeywordMappings(source, words) { + const without = mappedWithTokens(source); + const { code, mappings, tokens } = mappedWithTokens(source, { tokens: true }); + expect(code).toBe(without.code); + + const seen = new Set(); + /** @type {{ value: string, start: { line: number, column: number } }[]} */ + const source_words = []; + for (const token of tokens) { + const value = token.type.label === 'name' ? token.value : token.type.label; + if (!token.loc || !words.includes(value)) continue; + const key = `${token.loc.start.line}:${token.loc.start.column}`; + if (seen.has(key)) continue; + seen.add(key); + source_words.push({ value, start: token.loc.start }); + } + + const pattern = new RegExp(`\\b(${words.join('|')})\\b`, 'g'); + const generated_words = [...code.matchAll(pattern)].map((m) => ({ + value: m[0], + index: /** @type {number} */ (m.index) + })); + + expect(generated_words.map((w) => w.value)).toEqual(source_words.map((w) => w.value)); + for (let i = 0; i < generated_words.length; i += 1) { + const { start } = source_words[i]; + expect( + mappingsAtIndex(code, generated_words[i].index, mappings).map((s) => s.slice(2)), + `${generated_words[i].value} at ${start.line}:${start.column}` + ).toContainEqual([start.line - 1, start.column]); + } +} + +test.each([ + ['async function', `async /* c */ function f() {}`, ['async', 'function']], + ['async generator', `async /* c */ function* f() {}`, ['async', 'function']], + ['function expression', `const f = async /* c */ function () {};`, ['async', 'function']], + ['async arrow', `const f = async /* c */ ( x ) => x;`, ['async']], + ['declare function', `declare /* c */ function f(): void;`, ['declare', 'function']], + [ + 'class modifiers', + `declare /* c */ abstract /* c */ class K {}`, + ['declare', 'abstract', 'class'] + ], + ['decorated class', `@dec /* c */ class K {}`, ['class']], + ['export default class', `export /* c */ default /* c */ class {}`, ['default', 'class']], + [ + 'class heritage', + `class K extends /* c */ B implements /* c */ I {}`, + ['extends', 'implements'] + ], + [ + 'member modifiers', + `class K extends B {\n\tstatic /* c */ async /* c */ *m() {}\n\tprivate /* c */ readonly p = 1;\n\tget /* c */ g() { return 1; }\n\t@dec /* c */ override /* c */ n() {}\n}`, + ['static', 'async', 'private', 'readonly', 'get', 'override'] + ], + [ + 'object accessors', + `const o = { get /* c */ p() { return 1; }, async /* c */ m() {} };`, + ['get', 'async'] + ], + ['else', `if (a) {} /* c */ else /* c */ {}`, ['else']], + ['finally', `try {} finally /* c */ {}`, ['finally']], + ['finally after catch', `try {} catch {} /* c */ finally {}`, ['finally']], + ['do-while', `do {} /* c */ while /* c */ (a);`, ['while']], + [ + 'for await of', + `async function f() { for /* c */ await /* c */ (const x of /* c */ y) {} }`, + ['async', 'function', 'await', 'of'] + ], + ['for in', `for (const k /* c */ in o) {}`, ['in']], + ['import type', `import /* c */ type { T } from 'm';`, ['type']], + ['import type specifier', `import { type /* c */ T, a as /* c */ b } from 'm';`, ['type', 'as']], + ['export type', `type T = 1;\nexport /* c */ type { T };`, ['type']], + ['export star as', `export * as /* c */ ns from 'm';`, ['as']], + ['export specifier as', `const b = 1;\nexport { b as /* c */ c };`, ['as']], + ['enum', `declare /* c */ const /* c */ enum E {}`, ['declare', 'const', 'enum']], + ['namespace', `declare /* c */ namespace N {}`, ['declare', 'namespace']], + ['module', `declare /* c */ module 'm' {}`, ['declare', 'module']], + [ + 'interface', + `declare /* c */ interface I extends /* c */ J {}`, + ['declare', 'interface', 'extends'] + ], + ['type alias', `declare /* c */ type T = string;`, ['declare', 'type']], + [ + 'mapped type', + `type M = { -readonly /* c */ [K in /* c */ keyof T as /* c */ R]: T[K] };`, + ['readonly', 'in', 'as'] + ], + ['as expression', `const v = x /* c */ as /* c */ T;`, ['as']], + ['satisfies expression', `const v = x /* c */ satisfies /* c */ T;`, ['satisfies']], + [ + 'parameter properties', + `class P { constructor(private /* c */ readonly x: number) {} }`, + ['private', 'readonly'] + ], + ['constructor type', `type F = abstract /* c */ new () => object;`, ['abstract', 'new']] +])('tokens map keywords after a node start for %s', (_name, source, words) => { + expectExactKeywordMappings(source, words); +}); + +test.each([ + ['decorator before export', `@dec export class K {}`], + ['decorator after export', `export @dec class K {}`], + ['decorator before export default', `@dec export default class {}`], + ['decorator after export default', `export default @dec class {}`] +])('`export` of a decorated class maps to its source position: %s', (_name, statement) => { + // a leading statement keeps the Program's own start mapping off the tested line + const source = `let a;\n${statement}`; + const { code, mappings } = mapped(source); + const segment = mappingAtSubstring(code, 'export', mappings); + expect(segment.slice(2)).toEqual([1, statement.indexOf('export')]); + // the printed decorator maps to itself, not to `export` + expect(sourcePositionsAt(code, '@dec', mappings)).toEqual([[1, statement.indexOf('@dec')]]); +}); + +test.each([ + ['decorator before export', `@dec /* c */ export /* c */ class K {}`, ['class']], + ['decorator after export', `export /* c */ @dec /* c */ class K {}`, ['class']], + [ + 'decorator before export default', + `@dec export /* c */ default /* c */ class {}`, + ['default', 'class'] + ], + ['decorated member', `class K { @dec /* c */ static /* c */ m() {} }`, ['static']] +])('tokens map the keywords of a decorated declaration: %s', (_name, source, words) => { + expectExactKeywordMappings(source, words); +});