diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 883aa4ebe..f46c623c7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -87,8 +87,8 @@ importers: specifier: 1.0.14 version: 1.0.14 web-tree-sitter: - specifier: 0.24.5 - version: 0.24.5 + specifier: 0.27.0 + version: 0.27.0 zod: specifier: 3.25.76 version: 3.25.76 @@ -2125,8 +2125,8 @@ packages: resolution: {integrity: sha512-eUt8f1z2N2IEUDBsKaNapkz7jl5QpskN2Y0G01T/ItMxBxw1fJwvtySGB9QMecatne8jFIWJGWI61dWjyTLQsw==} hasBin: true - web-tree-sitter@0.24.5: - resolution: {integrity: sha512-+J/2VSHN8J47gQUAvF8KDadrfz6uFYVjxoxbKWDoXVsH2u7yLdarCnIURnrMA6uSRkgX3SdmqM5BOoQjPdSh5w==} + web-tree-sitter@0.27.0: + resolution: {integrity: sha512-XK08gj6RwTMQatAG7uVRP8MunqotL/XC19vHgkSPKmELgbGPBj4ECvB8haHOUnyj6ls2B8t42UTro14zxGgAHg==} webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -4424,7 +4424,7 @@ snapshots: dependencies: vscode-languageserver-protocol: 3.17.3 - web-tree-sitter@0.24.5: {} + web-tree-sitter@0.27.0: {} webidl-conversions@3.0.1: {} diff --git a/scripts/upgrade-tree-sitter.sh b/scripts/upgrade-tree-sitter.sh index 588afd49d..14fb9fe16 100755 --- a/scripts/upgrade-tree-sitter.sh +++ b/scripts/upgrade-tree-sitter.sh @@ -1,14 +1,46 @@ #!/usr/bin/env bash -set -euox pipefail +set -euo pipefail -cd server -pnpm add web-tree-sitter -pnpm add --save-dev tree-sitter-cli https://github.com/tree-sitter/tree-sitter-bash -npx tree-sitter build --wasm node_modules/tree-sitter-bash +repo_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +build_dir=$(mktemp -d) +trap 'rm -rf "$build_dir"' EXIT -curl 'https://api.github.com/repos/tree-sitter/tree-sitter-bash/commits/master' | jq .commit.url > parser.info -echo "tree-sitter-cli $(cat package.json | jq '.devDependencies["tree-sitter-cli"]')" >> parser.info +cli_version=0.27.0 +# web-tree-sitter 0.27.x supports parser ABIs 13 through 15. +parser_abi=15 -pnpm remove tree-sitter-cli tree-sitter-bash +git clone --depth 1 https://github.com/tree-sitter/tree-sitter-bash "$build_dir/grammar" +cd "$build_dir/grammar" +parser_commit=$(git rev-parse HEAD) +pnpm --package="tree-sitter-cli@$cli_version" dlx tree-sitter generate --abi "$parser_abi" +pnpm --package="tree-sitter-cli@$cli_version" dlx tree-sitter build --wasm --output "$build_dir/tree-sitter-bash.wasm" + +# Check the artifact with the server's installed runtime before replacing it. +node - "$repo_dir" "$build_dir/tree-sitter-bash.wasm" <<'JS' +const [repoDir, wasmPath] = process.argv.slice(2) +const { Language, Parser } = require(require.resolve('web-tree-sitter', { paths: [`${repoDir}/server`] })) + +async function verify() { + await Parser.init() + const parser = new Parser() + parser.setLanguage(await Language.load(wasmPath)) + const tree = parser.parse('echo "$HOME"\n') + if (!tree || tree.rootNode.hasError) { + throw new Error('The generated Bash parser failed its smoke test') + } + tree.delete() + parser.delete() +} + +verify().catch((error) => { + console.error(error) + process.exitCode = 1 +}) +JS + +printf '"https://api.github.com/repos/tree-sitter/tree-sitter-bash/git/commits/%s"\ntree-sitter-cli "%s"\nparser ABI %s\n' \ + "$parser_commit" "$cli_version" "$parser_abi" > "$build_dir/parser.info" +cp "$build_dir/tree-sitter-bash.wasm" "$repo_dir/server/tree-sitter-bash.wasm" +cp "$build_dir/parser.info" "$repo_dir/server/parser.info" diff --git a/server/package.json b/server/package.json index 5ddb4b241..30da0b4ef 100644 --- a/server/package.json +++ b/server/package.json @@ -24,7 +24,7 @@ "turndown": "7.2.4", "vscode-languageserver": "8.1.0", "vscode-languageserver-textdocument": "1.0.14", - "web-tree-sitter": "0.24.5", + "web-tree-sitter": "0.27.0", "zod": "3.25.76" }, "scripts": { diff --git a/server/parser.info b/server/parser.info index d562de535..aec920537 100644 --- a/server/parser.info +++ b/server/parser.info @@ -1,2 +1,3 @@ -"https://api.github.com/repos/tree-sitter/tree-sitter-bash/git/commits/c8713e50f0bd77d080832fc61ad128bc8f2934e9" -tree-sitter-cli "0.23.0" +"https://api.github.com/repos/tree-sitter/tree-sitter-bash/git/commits/a06c2e4415e9bc0346c6b86d401879ffb44058f7" +tree-sitter-cli "0.27.0" +parser ABI 15 diff --git a/server/src/__tests__/analyzer.test.ts b/server/src/__tests__/analyzer.test.ts index 23dc676ef..899ceeb93 100644 --- a/server/src/__tests__/analyzer.test.ts +++ b/server/src/__tests__/analyzer.test.ts @@ -1,5 +1,7 @@ import { pathToFileURL } from 'node:url' +import { Parser } from 'web-tree-sitter' + import { FIXTURE_DOCUMENT, FIXTURE_FOLDER, @@ -55,6 +57,21 @@ async function getAnalyzer({ } describe('analyze', () => { + it('reports a failed parse before analyzing a missing tree', async () => { + const analyzer = await getAnalyzer({}) + const parse = jest.spyOn(Parser.prototype, 'parse').mockReturnValueOnce(null) + try { + expect(() => + analyzer.analyze({ + uri: CURRENT_URI, + document: FIXTURE_DOCUMENT.INSTALL, + }), + ).toThrow(`Failed to parse ${CURRENT_URI}: no syntax tree returned`) + } finally { + parse.mockRestore() + } + }) + it('returns an empty list of diagnostics for a file with no parsing errors', async () => { const analyzer = await getAnalyzer({}) const diagnostics = analyzer.analyze({ diff --git a/server/src/__tests__/parser.test.ts b/server/src/__tests__/parser.test.ts new file mode 100644 index 000000000..21b5aabe3 --- /dev/null +++ b/server/src/__tests__/parser.test.ts @@ -0,0 +1,36 @@ +import { initializeParser } from '../parser' + +describe('initializeParser', () => { + it('loads the bundled grammar without changing global fetch', async () => { + const originalFetch = global.fetch + expect(originalFetch).toEqual(expect.any(Function)) + + const parser = await initializeParser() + try { + expect(global.fetch).toBe(originalFetch) + expect(parser.language?.name).toBe('bash') + } finally { + parser.delete() + } + }) + + it('parses arithmetic commands and preserves UTF-16 positions after emoji', async () => { + const parser = await initializeParser() + const tree = parser.parse('echo "🦀" "$HOME"\n((count = 1, count += 2))\n') + try { + expect(tree).not.toBeNull() + expect(tree!.rootNode.hasError).toBe(false) + expect(tree!.rootNode.descendantsOfType('compound_statement')).toHaveLength(1) + + const home = tree!.rootNode.descendantsOfType('variable_name')[0] + expect(home.text).toBe('HOME') + expect(home.startIndex).toBe(12) + expect(home.endIndex).toBe(16) + expect(home.startPosition).toEqual({ row: 0, column: 12 }) + expect(home.endPosition).toEqual({ row: 0, column: 16 }) + } finally { + tree?.delete() + parser.delete() + } + }) +}) diff --git a/server/src/analyser.ts b/server/src/analyser.ts index 3fc7cde16..5d2fd5cf4 100644 --- a/server/src/analyser.ts +++ b/server/src/analyser.ts @@ -5,7 +5,7 @@ import * as url from 'url' import { isDeepStrictEqual } from 'util' import * as LSP from 'vscode-languageserver/node' import { TextDocument } from 'vscode-languageserver-textdocument' -import * as Parser from 'web-tree-sitter' +import { Node as SyntaxNode, Parser, Point, Tree } from 'web-tree-sitter' import { flattenArray } from './util/array' import { @@ -29,7 +29,7 @@ type AnalyzedDocument = { globalDeclarations: GlobalDeclarations sourcedUris: Set sourceCommands: sourcing.SourceCommand[] - tree: Parser.Tree + tree: Tree } /** @@ -75,6 +75,9 @@ export default class Analyzer { const fileContent = document.getText() const tree = this.parser.parse(fileContent) + if (!tree) { + throw new Error(`Failed to parse ${uri}: no syntax tree returned`) + } const globalDeclarations = getGlobalDeclarations({ tree, uri }) @@ -304,7 +307,7 @@ export default class Analyzer { boundary: params.position.line, } let parent = this.parentScope(node) - let declaration: Parser.SyntaxNode | null | undefined + let declaration: SyntaxNode | null | undefined let continueSearching = false // Search for local declaration within parents @@ -409,7 +412,7 @@ export default class Analyzer { const locations: LSP.Location[] = [] TreeSitterUtil.forEach(tree.rootNode, (n) => { - let namedNode: Parser.SyntaxNode | null = null + let namedNode: SyntaxNode | null = null if (TreeSitterUtil.isReference(n)) { // NOTE: a reference can be a command, variable, function, etc. @@ -476,7 +479,7 @@ export default class Analyzer { : baseNode.startPosition const ignoredRanges: LSP.Range[] = [] - const filterVariables = (n: Parser.SyntaxNode) => { + const filterVariables = (n: SyntaxNode) => { if ( n.text !== word || (n.type === 'word' && !TreeSitterUtil.isVariableInReadCommand(n)) @@ -534,7 +537,7 @@ export default class Analyzer { return includeDeclaration } - const filterFunctions = (n: Parser.SyntaxNode) => { + const filterFunctions = (n: SyntaxNode) => { const text = n.type === 'function_definition' ? n.firstNamedChild?.text : n.text if (text !== word) { return false @@ -1030,7 +1033,7 @@ export default class Analyzer { * `function_definition`'s body, this only returns a `function_definition` if * its body is a `compound_statement`. */ - private parentScope(node: Parser.SyntaxNode): Parser.SyntaxNode | null { + private parentScope(node: SyntaxNode): SyntaxNode | null { return TreeSitterUtil.findParent( node, (n) => @@ -1042,11 +1045,7 @@ export default class Analyzer { /** * Find the node at the given point. */ - private nodeAtPoint( - uri: string, - line: number, - column: number, - ): Parser.SyntaxNode | null { + private nodeAtPoint(uri: string, line: number, column: number): SyntaxNode | null { const tree = this.uriToAnalyzedDocument[uri]?.tree if (!tree?.rootNode) { @@ -1057,11 +1056,7 @@ export default class Analyzer { return tree.rootNode.descendantForPosition({ row: line, column }) } - private nodeAtPoints( - uri: string, - start: Parser.Point, - end: Parser.Point, - ): Parser.SyntaxNode | null { + private nodeAtPoints(uri: string, start: Point, end: Point): SyntaxNode | null { const rootNode = this.uriToAnalyzedDocument[uri]?.tree.rootNode if (!rootNode) { diff --git a/server/src/parser.ts b/server/src/parser.ts index f4627c9f8..dc766cdcc 100644 --- a/server/src/parser.ts +++ b/server/src/parser.ts @@ -1,15 +1,7 @@ -import * as Parser from 'web-tree-sitter' - -const _global: any = global +import { readFile } from 'fs/promises' +import { Language, Parser } from 'web-tree-sitter' export async function initializeParser(): Promise { - if (_global.fetch) { - // NOTE: temporary workaround for emscripten node 18 support. - // emscripten is used for compiling tree-sitter to wasm. - // https://github.com/emscripten-core/emscripten/issues/16915 - delete _global.fetch - } - await Parser.init() const parser = new Parser() @@ -17,9 +9,10 @@ export async function initializeParser(): Promise { * See https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web#generate-wasm-language-files * * To compile and use a new tree-sitter-bash version: - * sh scripts/upgrade-tree-sitter.sh + * bash scripts/upgrade-tree-sitter.sh */ - const lang = await Parser.Language.load(`${__dirname}/../tree-sitter-bash.wasm`) + const wasm = await readFile(`${__dirname}/../tree-sitter-bash.wasm`) + const lang = await Language.load(wasm) parser.setLanguage(lang) return parser diff --git a/server/src/util/__tests__/sourcing.test.ts b/server/src/util/__tests__/sourcing.test.ts index cb9359b5e..948ac5154 100644 --- a/server/src/util/__tests__/sourcing.test.ts +++ b/server/src/util/__tests__/sourcing.test.ts @@ -1,6 +1,6 @@ import * as fs from 'fs' import * as os from 'os' -import * as Parser from 'web-tree-sitter' +import { Parser } from 'web-tree-sitter' import { FIXTURE_FOLDER, REPO_ROOT_FOLDER } from '../../../../testing/fixtures' import { initializeParser } from '../../parser' @@ -23,7 +23,7 @@ describe('getSourcedUris', () => { const sourceCommands = getSourceCommands({ fileUri, rootPath: null, - tree: parser.parse(fileContent), + tree: parser.parse(fileContent)!, }) expect(sourceCommands).toEqual([]) }) @@ -130,7 +130,7 @@ describe('getSourcedUris', () => { const sourceCommands = getSourceCommands({ fileUri, rootPath: null, - tree: parser.parse(fileContent), + tree: parser.parse(fileContent)!, }) const sourcedUris = new Set( @@ -185,7 +185,7 @@ describe('getSourcedUris', () => { const sourceCommands = getSourceCommands({ fileUri, rootPath: REPO_ROOT_FOLDER, - tree: parser.parse(fileContent), + tree: parser.parse(fileContent)!, }) const sourcedUris = new Set( @@ -240,7 +240,7 @@ describe('getSourcedUris', () => { const sourceCommands = getSourceCommands({ fileUri: `${FIXTURE_FOLDER}bats/sourcing.bats`, rootPath: REPO_ROOT_FOLDER, - tree: parser.parse(fileContent), + tree: parser.parse(fileContent)!, }) const sourcedUris = new Set( @@ -285,7 +285,7 @@ describe('getSourcedUris', () => { const sourceCommands = getSourceCommands({ fileUri: `${FIXTURE_FOLDER}bats/not-a-bats-file.sh`, rootPath: REPO_ROOT_FOLDER, - tree: parser.parse(fileContent), + tree: parser.parse(fileContent)!, }) expect(sourceCommands).toEqual([]) diff --git a/server/src/util/declarations.ts b/server/src/util/declarations.ts index 53b9a61d3..696bff676 100644 --- a/server/src/util/declarations.ts +++ b/server/src/util/declarations.ts @@ -1,5 +1,5 @@ import * as LSP from 'vscode-languageserver/node' -import * as Parser from 'web-tree-sitter' +import { Node as SyntaxNode, Tree } from 'web-tree-sitter' import * as TreeSitterUtil from './tree-sitter' @@ -35,7 +35,7 @@ export function getGlobalDeclarations({ tree, uri, }: { - tree: Parser.Tree + tree: Tree uri: string }): GlobalDeclarations { const globalDeclarations: GlobalDeclarations = {} @@ -63,7 +63,7 @@ export function getAllDeclarationsInTree({ tree, uri, }: { - tree: Parser.Tree + tree: Tree uri: string }): LSP.SymbolInformation[] { const symbols: LSP.SymbolInformation[] = [] @@ -90,14 +90,14 @@ export function getLocalDeclarations({ rootNode, uri, }: { - node: Parser.SyntaxNode | null - rootNode: Parser.SyntaxNode + node: SyntaxNode | null + rootNode: SyntaxNode uri: string }): Declarations { const declarations: Declarations = {} // Bottom up traversal to capture all local and scoped declarations - const walk = (node: Parser.SyntaxNode | null) => { + const walk = (node: SyntaxNode | null) => { // NOTE: there is also node.walk if (node) { for (const childNode of node.children) { @@ -163,7 +163,7 @@ function getAllGlobalVariableDeclarations({ rootNode, }: { uri: string - rootNode: Parser.SyntaxNode + rootNode: SyntaxNode }) { const declarations: Declarations = {} @@ -192,7 +192,7 @@ function nodeToSymbolInformation({ node, uri, }: { - node: Parser.SyntaxNode + node: SyntaxNode uri: string }): LSP.SymbolInformation | null { const named = node.firstNamedChild @@ -220,7 +220,7 @@ function getDeclarationSymbolFromNode({ node, uri, }: { - node: Parser.SyntaxNode + node: SyntaxNode uri: string }): LSP.SymbolInformation | null { if (TreeSitterUtil.isDefinition(node)) { @@ -253,7 +253,7 @@ export type FindDeclarationParams = { /** * The node where the search will start. */ - baseNode: Parser.SyntaxNode + baseNode: SyntaxNode symbolInfo: { position: LSP.Position uri: string @@ -284,7 +284,7 @@ export function findDeclarationUsingGlobalSemantics({ symbolInfo: { position, uri, word, kind }, otherInfo: { currentUri, boundary }, }: FindDeclarationParams) { - let declaration: Parser.SyntaxNode | null | undefined + let declaration: SyntaxNode | null | undefined let continueSearching = false TreeSitterUtil.forEach(baseNode, (n) => { @@ -398,7 +398,7 @@ export function findDeclarationUsingLocalSemantics({ symbolInfo: { position, word }, otherInfo: { boundary }, }: FindDeclarationParams) { - let declaration: Parser.SyntaxNode | null | undefined + let declaration: SyntaxNode | null | undefined let continueSearching = false TreeSitterUtil.forEach(baseNode, (n) => { @@ -448,8 +448,8 @@ export function findDeclarationUsingLocalSemantics({ * skipped and a higher scope should be checked for the original declaration. */ function isDefinedVariableInExpression( - definition: Parser.SyntaxNode, - variable: Parser.SyntaxNode, + definition: SyntaxNode, + variable: SyntaxNode, position: LSP.Position, ): boolean { return ( diff --git a/server/src/util/sourcing.ts b/server/src/util/sourcing.ts index c0639491b..bf9295ac5 100644 --- a/server/src/util/sourcing.ts +++ b/server/src/util/sourcing.ts @@ -1,7 +1,7 @@ import * as fs from 'fs' import * as path from 'path' import * as LSP from 'vscode-languageserver' -import * as Parser from 'web-tree-sitter' +import { Node as SyntaxNode, Tree } from 'web-tree-sitter' import { parseShellCheckDirective } from '../shellcheck/directive' import { discriminate } from './discriminate' @@ -34,7 +34,7 @@ export function getSourceCommands({ }: { fileUri: string rootPath: string | null - tree: Parser.Tree + tree: Tree }): SourceCommand[] { const sourceCommands: SourceCommand[] = [] @@ -67,7 +67,7 @@ function getSourcedPathInfoFromNode({ node, isBatsFile, }: { - node: Parser.SyntaxNode + node: SyntaxNode isBatsFile: boolean }): null | { sourcedPath?: string; parseError?: string } { const sourcingCommands = isBatsFile @@ -218,7 +218,7 @@ function resolveSourcedUri({ * Returns null if the source path can't be statically determined after stripping a segment. * Note: If a non-concatenation node is passed, null will be returned. This is likely a programmer error. */ -function resolveSourceFromConcatenation(node: Parser.SyntaxNode): string | null { +function resolveSourceFromConcatenation(node: SyntaxNode): string | null { if (node.type !== 'concatenation') return null const stringValue = TreeSitterUtil.resolveStaticString(node) if (stringValue !== null) return stringValue // This string is fully static. diff --git a/server/src/util/tree-sitter.ts b/server/src/util/tree-sitter.ts index 613a15576..27b70fbc4 100644 --- a/server/src/util/tree-sitter.ts +++ b/server/src/util/tree-sitter.ts @@ -1,5 +1,5 @@ import * as LSP from 'vscode-languageserver/node' -import { SyntaxNode } from 'web-tree-sitter' +import { Node as SyntaxNode } from 'web-tree-sitter' /** * Recursively iterate over all nodes in a tree. diff --git a/server/tree-sitter-bash.wasm b/server/tree-sitter-bash.wasm index 28e2869fa..1e298412c 100755 Binary files a/server/tree-sitter-bash.wasm and b/server/tree-sitter-bash.wasm differ