Upgrade Bash grammar and web-tree-sitter runtime

scripts/upgrade-tree-sitter.sh

@@ -1,14 +1,46 @@
 #!/usr/bin/env bash
 
-set -euox pipefail
+set -euo pipefail
 
-cd server
-pnpm add web-tree-sitter
-pnpm add --save-dev tree-sitter-cli https://github.com/tree-sitter/tree-sitter-bash
-npx tree-sitter build --wasm node_modules/tree-sitter-bash
+repo_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+build_dir=$(mktemp -d)
+trap 'rm -rf "$build_dir"' EXIT
 
-curl 'https://api.github.com/repos/tree-sitter/tree-sitter-bash/commits/master' | jq .commit.url > parser.info
-echo "tree-sitter-cli $(cat package.json | jq '.devDependencies["tree-sitter-cli"]')" >> parser.info
+cli_version=0.27.0
+# web-tree-sitter 0.27.x supports parser ABIs 13 through 15.
+parser_abi=15
 
-pnpm remove tree-sitter-cli tree-sitter-bash
+git clone --depth 1 https://github.com/tree-sitter/tree-sitter-bash "$build_dir/grammar"
+cd "$build_dir/grammar"
+parser_commit=$(git rev-parse HEAD)
 
+pnpm --package="tree-sitter-cli@$cli_version" dlx tree-sitter generate --abi "$parser_abi"
+pnpm --package="tree-sitter-cli@$cli_version" dlx tree-sitter build --wasm --output "$build_dir/tree-sitter-bash.wasm"
+
+# Check the artifact with the server's installed runtime before replacing it.
+node - "$repo_dir" "$build_dir/tree-sitter-bash.wasm" <<'JS'
+const [repoDir, wasmPath] = process.argv.slice(2)
+const { Language, Parser } = require(require.resolve('web-tree-sitter', { paths: [`${repoDir}/server`] }))
+
+async function verify() {
+  await Parser.init()
+  const parser = new Parser()
+  parser.setLanguage(await Language.load(wasmPath))
+  const tree = parser.parse('echo "$HOME"\n')
+  if (!tree || tree.rootNode.hasError) {
+    throw new Error('The generated Bash parser failed its smoke test')
+  }
+  tree.delete()
+  parser.delete()
+}
+
+verify().catch((error) => {
+  console.error(error)
+  process.exitCode = 1
+})
+JS
+
+printf '"https://api.github.com/repos/tree-sitter/tree-sitter-bash/git/commits/%s"\ntree-sitter-cli "%s"\nparser ABI %s\n' \
+  "$parser_commit" "$cli_version" "$parser_abi" > "$build_dir/parser.info"
+cp "$build_dir/tree-sitter-bash.wasm" "$repo_dir/server/tree-sitter-bash.wasm"
+cp "$build_dir/parser.info" "$repo_dir/server/parser.info"

server/package.json

@@ -24,7 +24,7 @@
     "turndown": "7.2.4",
     "vscode-languageserver": "8.1.0",
     "vscode-languageserver-textdocument": "1.0.14",
-    "web-tree-sitter": "0.24.5",
+    "web-tree-sitter": "0.27.0",
     "zod": "3.25.76"
   },
   "scripts": {

server/parser.info

@@ -1,2 +1,3 @@
-"https://api.github.com/repos/tree-sitter/tree-sitter-bash/git/commits/c8713e50f0bd77d080832fc61ad128bc8f2934e9"
-tree-sitter-cli "0.23.0"
+"https://api.github.com/repos/tree-sitter/tree-sitter-bash/git/commits/a06c2e4415e9bc0346c6b86d401879ffb44058f7"
+tree-sitter-cli "0.27.0"
+parser ABI 15

server/src/__tests__/analyzer.test.ts

@@ -1,5 +1,7 @@
 import { pathToFileURL } from 'node:url'
 
+import { Parser } from 'web-tree-sitter'
+
 import {
   FIXTURE_DOCUMENT,
   FIXTURE_FOLDER,
@@ -55,6 +57,21 @@ async function getAnalyzer({
 }
 
 describe('analyze', () => {
+  it('reports a failed parse before analyzing a missing tree', async () => {
+    const analyzer = await getAnalyzer({})
+    const parse = jest.spyOn(Parser.prototype, 'parse').mockReturnValueOnce(null)
+    try {
+      expect(() =>
+        analyzer.analyze({
+          uri: CURRENT_URI,
+          document: FIXTURE_DOCUMENT.INSTALL,
+        }),
+      ).toThrow(`Failed to parse ${CURRENT_URI}: no syntax tree returned`)
+    } finally {
+      parse.mockRestore()
+    }
+  })
+
   it('returns an empty list of diagnostics for a file with no parsing errors', async () => {
     const analyzer = await getAnalyzer({})
     const diagnostics = analyzer.analyze({

server/src/__tests__/parser.test.ts

@@ -0,0 +1,36 @@
+import { initializeParser } from '../parser'
+
+describe('initializeParser', () => {
+  it('loads the bundled grammar without changing global fetch', async () => {
+    const originalFetch = global.fetch
+    expect(originalFetch).toEqual(expect.any(Function))
+
+    const parser = await initializeParser()
+    try {
+      expect(global.fetch).toBe(originalFetch)
+      expect(parser.language?.name).toBe('bash')
+    } finally {
+      parser.delete()
+    }
+  })
+
+  it('parses arithmetic commands and preserves UTF-16 positions after emoji', async () => {
+    const parser = await initializeParser()
+    const tree = parser.parse('echo "🦀" "$HOME"\n((count = 1, count += 2))\n')
+    try {
+      expect(tree).not.toBeNull()
+      expect(tree!.rootNode.hasError).toBe(false)
+      expect(tree!.rootNode.descendantsOfType('compound_statement')).toHaveLength(1)
+
+      const home = tree!.rootNode.descendantsOfType('variable_name')[0]
+      expect(home.text).toBe('HOME')
+      expect(home.startIndex).toBe(12)
+      expect(home.endIndex).toBe(16)
+      expect(home.startPosition).toEqual({ row: 0, column: 12 })
+      expect(home.endPosition).toEqual({ row: 0, column: 16 })
+    } finally {
+      tree?.delete()
+      parser.delete()
+    }
+  })
+})

server/src/analyser.ts

@@ -5,7 +5,7 @@ import * as url from 'url'
 import { isDeepStrictEqual } from 'util'
 import * as LSP from 'vscode-languageserver/node'
 import { TextDocument } from 'vscode-languageserver-textdocument'
-import * as Parser from 'web-tree-sitter'
+import { Node as SyntaxNode, Parser, Point, Tree } from 'web-tree-sitter'
 
 import { flattenArray } from './util/array'
 import {
@@ -29,7 +29,7 @@ type AnalyzedDocument = {
   globalDeclarations: GlobalDeclarations
   sourcedUris: Set<string>
   sourceCommands: sourcing.SourceCommand[]
-  tree: Parser.Tree
+  tree: Tree
 }
 
 /**
@@ -75,6 +75,9 @@ export default class Analyzer {
     const fileContent = document.getText()
 
     const tree = this.parser.parse(fileContent)
+    if (!tree) {
+      throw new Error(`Failed to parse ${uri}: no syntax tree returned`)
+    }
 
     const globalDeclarations = getGlobalDeclarations({ tree, uri })
 
@@ -304,7 +307,7 @@ export default class Analyzer {
       boundary: params.position.line,
     }
     let parent = this.parentScope(node)
-    let declaration: Parser.SyntaxNode | null | undefined
+    let declaration: SyntaxNode | null | undefined
     let continueSearching = false
 
     // Search for local declaration within parents
@@ -409,7 +412,7 @@ export default class Analyzer {
     const locations: LSP.Location[] = []
 
     TreeSitterUtil.forEach(tree.rootNode, (n) => {
-      let namedNode: Parser.SyntaxNode | null = null
+      let namedNode: SyntaxNode | null = null
 
       if (TreeSitterUtil.isReference(n)) {
         // NOTE: a reference can be a command, variable, function, etc.
@@ -476,7 +479,7 @@ export default class Analyzer {
       : baseNode.startPosition
 
     const ignoredRanges: LSP.Range[] = []
-    const filterVariables = (n: Parser.SyntaxNode) => {
+    const filterVariables = (n: SyntaxNode) => {
       if (
         n.text !== word ||
         (n.type === 'word' && !TreeSitterUtil.isVariableInReadCommand(n))
@@ -534,7 +537,7 @@ export default class Analyzer {
 
       return includeDeclaration
     }
-    const filterFunctions = (n: Parser.SyntaxNode) => {
+    const filterFunctions = (n: SyntaxNode) => {
       const text = n.type === 'function_definition' ? n.firstNamedChild?.text : n.text
       if (text !== word) {
         return false
@@ -1030,7 +1033,7 @@ export default class Analyzer {
    * `function_definition`'s body, this only returns a `function_definition` if
    * its body is a `compound_statement`.
    */
-  private parentScope(node: Parser.SyntaxNode): Parser.SyntaxNode | null {
+  private parentScope(node: SyntaxNode): SyntaxNode | null {
     return TreeSitterUtil.findParent(
       node,
       (n) =>
@@ -1042,11 +1045,7 @@ export default class Analyzer {
   /**
    * Find the node at the given point.
    */
-  private nodeAtPoint(
-    uri: string,
-    line: number,
-    column: number,
-  ): Parser.SyntaxNode | null {
+  private nodeAtPoint(uri: string, line: number, column: number): SyntaxNode | null {
     const tree = this.uriToAnalyzedDocument[uri]?.tree
 
     if (!tree?.rootNode) {
@@ -1057,11 +1056,7 @@ export default class Analyzer {
     return tree.rootNode.descendantForPosition({ row: line, column })
   }
 
-  private nodeAtPoints(
-    uri: string,
-    start: Parser.Point,
-    end: Parser.Point,
-  ): Parser.SyntaxNode | null {
+  private nodeAtPoints(uri: string, start: Point, end: Point): SyntaxNode | null {
     const rootNode = this.uriToAnalyzedDocument[uri]?.tree.rootNode
 
     if (!rootNode) {

server/src/parser.ts

@@ -1,25 +1,18 @@
-import * as Parser from 'web-tree-sitter'
-
-const _global: any = global
+import { readFile } from 'fs/promises'
+import { Language, Parser } from 'web-tree-sitter'
 
 export async function initializeParser(): Promise<Parser> {
-  if (_global.fetch) {
-    // NOTE: temporary workaround for emscripten node 18 support.
-    // emscripten is used for compiling tree-sitter to wasm.
-    // https://github.com/emscripten-core/emscripten/issues/16915
-    delete _global.fetch
-  }
-
   await Parser.init()
   const parser = new Parser()
 
   /**
    * See https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web#generate-wasm-language-files
    *
    * To compile and use a new tree-sitter-bash version:
-   *    sh scripts/upgrade-tree-sitter.sh
+   *    bash scripts/upgrade-tree-sitter.sh
    */
-  const lang = await Parser.Language.load(`${__dirname}/../tree-sitter-bash.wasm`)
+  const wasm = await readFile(`${__dirname}/../tree-sitter-bash.wasm`)
+  const lang = await Language.load(wasm)
 
   parser.setLanguage(lang)
   return parser

server/src/util/__tests__/sourcing.test.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs'
 import * as os from 'os'
-import * as Parser from 'web-tree-sitter'
+import { Parser } from 'web-tree-sitter'
 
 import { FIXTURE_FOLDER, REPO_ROOT_FOLDER } from '../../../../testing/fixtures'
 import { initializeParser } from '../../parser'
@@ -23,7 +23,7 @@ describe('getSourcedUris', () => {
     const sourceCommands = getSourceCommands({
       fileUri,
       rootPath: null,
-      tree: parser.parse(fileContent),
+      tree: parser.parse(fileContent)!,
     })
     expect(sourceCommands).toEqual([])
   })
@@ -130,7 +130,7 @@ describe('getSourcedUris', () => {
     const sourceCommands = getSourceCommands({
       fileUri,
       rootPath: null,
-      tree: parser.parse(fileContent),
+      tree: parser.parse(fileContent)!,
     })
 
     const sourcedUris = new Set(
@@ -185,7 +185,7 @@ describe('getSourcedUris', () => {
     const sourceCommands = getSourceCommands({
       fileUri,
       rootPath: REPO_ROOT_FOLDER,
-      tree: parser.parse(fileContent),
+      tree: parser.parse(fileContent)!,
     })
 
     const sourcedUris = new Set(
@@ -240,7 +240,7 @@ describe('getSourcedUris', () => {
     const sourceCommands = getSourceCommands({
       fileUri: `${FIXTURE_FOLDER}bats/sourcing.bats`,
       rootPath: REPO_ROOT_FOLDER,
-      tree: parser.parse(fileContent),
+      tree: parser.parse(fileContent)!,
     })
 
     const sourcedUris = new Set(
@@ -285,7 +285,7 @@ describe('getSourcedUris', () => {
     const sourceCommands = getSourceCommands({
       fileUri: `${FIXTURE_FOLDER}bats/not-a-bats-file.sh`,
       rootPath: REPO_ROOT_FOLDER,
-      tree: parser.parse(fileContent),
+      tree: parser.parse(fileContent)!,
     })
 
     expect(sourceCommands).toEqual([])