From f6b5457b6164e9dbf7fb90c4d33e93028a2c05fa Mon Sep 17 00:00:00 2001 From: Ryan Luker Date: Sun, 8 Mar 2026 18:09:24 -0700 Subject: [PATCH 1/9] Add retry option to launchArgs in runTest Attempt to add automatic retries to the ci builds to prevent failure email spam. --- test/runTest.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/runTest.ts b/test/runTest.ts index 29911ff..ca331d9 100644 --- a/test/runTest.ts +++ b/test/runTest.ts @@ -10,7 +10,10 @@ async function main() { version: "insiders", extensionDevelopmentPath, extensionTestsPath, - launchArgs: ["example/example.code-workspace"], + launchArgs: [ + "example/example.code-workspace", + "--retries 3", + ], }) console.info("Success!"); From 030db61493448a176c159a47e241aca33bd944e1 Mon Sep 17 00:00:00 2001 From: Ryan Luker Date: Sun, 8 Mar 2026 18:14:39 -0700 Subject: [PATCH 2/9] Try to add retry another way --- test/extension.test.ts | 3 ++- test/runTest.ts | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/extension.test.ts b/test/extension.test.ts index 1c74c30..f6de10c 100644 --- a/test/extension.test.ts +++ b/test/extension.test.ts @@ -9,7 +9,8 @@ import { Gutters, PREVIEW_COMMAND } from "../src/extension/gutters"; suite("Extension Tests", function () { const disposables: vscode.Disposable[] = []; - + this.retries(3); + afterEach(() => { // Clear mocks after each test to avoid cascading failures due to one test failing sinon.restore(); diff --git a/test/runTest.ts b/test/runTest.ts index ca331d9..29911ff 100644 --- a/test/runTest.ts +++ b/test/runTest.ts @@ -10,10 +10,7 @@ async function main() { version: "insiders", extensionDevelopmentPath, extensionTestsPath, - launchArgs: [ - "example/example.code-workspace", - "--retries 3", - ], + launchArgs: ["example/example.code-workspace"], }) console.info("Success!"); From 2cedfdced072b631fcfe208de4f5b7ab76016995 Mon Sep 17 00:00:00 2001 From: Ryan Luker Date: Sun, 8 Mar 2026 18:18:18 -0700 Subject: [PATCH 3/9] Add doc around retry addition --- test/extension.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/extension.test.ts b/test/extension.test.ts index f6de10c..aa9e639 100644 --- a/test/extension.test.ts +++ b/test/extension.test.ts @@ -9,6 +9,9 @@ import { Gutters, PREVIEW_COMMAND } from "../src/extension/gutters"; suite("Extension Tests", function () { const disposables: vscode.Disposable[] = []; + + // Adds a couple extra retries as the macos nightly builds can sometimes + // fail on the E2E tests. this.retries(3); afterEach(() => { From 96b654095577b2f306a766a00c2d873d11e6b5a9 Mon Sep 17 00:00:00 2001 From: Ryan Luker Date: Sun, 8 Mar 2026 18:22:06 -0700 Subject: [PATCH 4/9] Add failure temp to test retry --- test/extension.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extension.test.ts b/test/extension.test.ts index aa9e639..efcb431 100644 --- a/test/extension.test.ts +++ b/test/extension.test.ts @@ -72,7 +72,7 @@ suite("Extension Tests", function () { test("Run display coverage on a test file that has coverages generated remotely @integration", async () => { const decorationSpy = sinon.spy(Renderer.prototype, "setDecorationsForEditor"); - + expect(true).to.be.false; const testCoverage = await vscode.workspace.findFiles("**/remote-test-coverage.js", "**/node_modules/**"); const testDocument = await vscode.workspace.openTextDocument(testCoverage[0]); await vscode.window.showTextDocument(testDocument); From d986cf729340fcf6e0370afb29170258d3007e21 Mon Sep 17 00:00:00 2001 From: Ryan Luker Date: Sun, 8 Mar 2026 18:32:53 -0700 Subject: [PATCH 5/9] Add retries somewhere else --- test/extension.test.ts | 5 ----- test/index.ts | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/extension.test.ts b/test/extension.test.ts index efcb431..9bb4836 100644 --- a/test/extension.test.ts +++ b/test/extension.test.ts @@ -9,11 +9,6 @@ import { Gutters, PREVIEW_COMMAND } from "../src/extension/gutters"; suite("Extension Tests", function () { const disposables: vscode.Disposable[] = []; - - // Adds a couple extra retries as the macos nightly builds can sometimes - // fail on the E2E tests. - this.retries(3); - afterEach(() => { // Clear mocks after each test to avoid cascading failures due to one test failing sinon.restore(); diff --git a/test/index.ts b/test/index.ts index b73242a..51f3b91 100644 --- a/test/index.ts +++ b/test/index.ts @@ -4,9 +4,12 @@ import { resolve } from "path"; export function run(): Promise { // Create the mocha test + // NOTE: Adds a couple extra retries as the macos nightly builds + // can sometimes fail on the E2E tests. const mocha = new Mocha({ color: true, ui: "tdd", + retries: 3, }); // Apply regex to run subset of tests (integration vs unit) From 62179c651c7fd1b445c8e2477aca98422d5b770a Mon Sep 17 00:00:00 2001 From: Ryan Luker Date: Sun, 8 Mar 2026 18:40:31 -0700 Subject: [PATCH 6/9] Move around the retry again --- test/extension.test.ts | 6 +++++- test/index.ts | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/extension.test.ts b/test/extension.test.ts index 9bb4836..2a189f3 100644 --- a/test/extension.test.ts +++ b/test/extension.test.ts @@ -8,7 +8,7 @@ import { StatusBarToggler } from "../src/extension/statusbartoggler"; import { Gutters, PREVIEW_COMMAND } from "../src/extension/gutters"; suite("Extension Tests", function () { - const disposables: vscode.Disposable[] = []; + const disposables: v afterEach(() => { // Clear mocks after each test to avoid cascading failures due to one test failing sinon.restore(); @@ -66,6 +66,10 @@ suite("Extension Tests", function () { }); test("Run display coverage on a test file that has coverages generated remotely @integration", async () => { + // NOTE: Adds a couple extra retries as the macos nightly builds + // can sometimes fail on this E2E test. + this.retries(3); + const decorationSpy = sinon.spy(Renderer.prototype, "setDecorationsForEditor"); expect(true).to.be.false; const testCoverage = await vscode.workspace.findFiles("**/remote-test-coverage.js", "**/node_modules/**"); diff --git a/test/index.ts b/test/index.ts index 51f3b91..b73242a 100644 --- a/test/index.ts +++ b/test/index.ts @@ -4,12 +4,9 @@ import { resolve } from "path"; export function run(): Promise { // Create the mocha test - // NOTE: Adds a couple extra retries as the macos nightly builds - // can sometimes fail on the E2E tests. const mocha = new Mocha({ color: true, ui: "tdd", - retries: 3, }); // Apply regex to run subset of tests (integration vs unit) From a1e77e16742a6220214a05c6800d85637b80b662 Mon Sep 17 00:00:00 2001 From: Ryan Luker Date: Sun, 8 Mar 2026 18:43:10 -0700 Subject: [PATCH 7/9] temp --- test/extension.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/extension.test.ts b/test/extension.test.ts index 2a189f3..f49442f 100644 --- a/test/extension.test.ts +++ b/test/extension.test.ts @@ -8,7 +8,8 @@ import { StatusBarToggler } from "../src/extension/statusbartoggler"; import { Gutters, PREVIEW_COMMAND } from "../src/extension/gutters"; suite("Extension Tests", function () { - const disposables: v + const disposables: vscode.Disposable[] = []; + afterEach(() => { // Clear mocks after each test to avoid cascading failures due to one test failing sinon.restore(); @@ -70,6 +71,8 @@ suite("Extension Tests", function () { // can sometimes fail on this E2E test. this.retries(3); + const decorationSpy = sinon.spy(Renderer.prototype, "setDecorationsForEditor"); + expect(true).to.be.false const decorationSpy = sinon.spy(Renderer.prototype, "setDecorationsForEditor"); expect(true).to.be.false; const testCoverage = await vscode.workspace.findFiles("**/remote-test-coverage.js", "**/node_modules/**"); From e4e6f8c0b5d9a6b3311a2a7872d66599eafcb460 Mon Sep 17 00:00:00 2001 From: Ryan Luker Date: Sun, 8 Mar 2026 18:46:10 -0700 Subject: [PATCH 8/9] Remove extra --- test/extension.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/extension.test.ts b/test/extension.test.ts index f49442f..095213a 100644 --- a/test/extension.test.ts +++ b/test/extension.test.ts @@ -73,8 +73,7 @@ suite("Extension Tests", function () { const decorationSpy = sinon.spy(Renderer.prototype, "setDecorationsForEditor"); expect(true).to.be.false - const decorationSpy = sinon.spy(Renderer.prototype, "setDecorationsForEditor"); - expect(true).to.be.false; + const testCoverage = await vscode.workspace.findFiles("**/remote-test-coverage.js", "**/node_modules/**"); const testDocument = await vscode.workspace.openTextDocument(testCoverage[0]); await vscode.window.showTextDocument(testDocument); From e2f429965ccd0ec42d4edbc096f672ae5f02777b Mon Sep 17 00:00:00 2001 From: Ryan Luker Date: Sun, 8 Mar 2026 18:50:38 -0700 Subject: [PATCH 9/9] New area --- test/extension.test.ts | 7 +------ test/index.ts | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/test/extension.test.ts b/test/extension.test.ts index 095213a..f28c61a 100644 --- a/test/extension.test.ts +++ b/test/extension.test.ts @@ -67,13 +67,8 @@ suite("Extension Tests", function () { }); test("Run display coverage on a test file that has coverages generated remotely @integration", async () => { - // NOTE: Adds a couple extra retries as the macos nightly builds - // can sometimes fail on this E2E test. - this.retries(3); - - const decorationSpy = sinon.spy(Renderer.prototype, "setDecorationsForEditor"); expect(true).to.be.false - + const decorationSpy = sinon.spy(Renderer.prototype, "setDecorationsForEditor"); const testCoverage = await vscode.workspace.findFiles("**/remote-test-coverage.js", "**/node_modules/**"); const testDocument = await vscode.workspace.openTextDocument(testCoverage[0]); await vscode.window.showTextDocument(testDocument); diff --git a/test/index.ts b/test/index.ts index b73242a..cda437b 100644 --- a/test/index.ts +++ b/test/index.ts @@ -25,6 +25,7 @@ export function run(): Promise { files.forEach((f) => mocha.addFile(resolve(testsRoot, f))); try { // Run the mocha test + mocha.retries(3); mocha.run((failures) => { if (failures > 0) { e(new Error(`${failures} tests failed.`));