-
-
Notifications
You must be signed in to change notification settings - Fork 431
Report Grok failures instead of Claude's missing response #1043
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,13 +37,14 @@ struct InstalledAITests { | |
| defer { fixture.tearDown() } | ||
| openCodeCatalogCarriesModelVariants() | ||
| cursorCatalogParsesListModels() | ||
| grokCatalogParsesListedModels() | ||
| statusJSONRecognizesLogin() | ||
| versionKeepsPrereleaseAndBuild() | ||
| await openCodeRunsWithoutToolsAndDeletesItsSession(fixture) | ||
| claudeDiscoveryReadsTheCLIsOwnModelList() | ||
| await claudeRunsWithoutToolsOrHistory(fixture) | ||
| await grokRunsWithoutToolsAndDeletesItsSession(fixture) | ||
| grokCatalogParsesListedModels() | ||
| await grokDiscoveryRequiresLoginAndFiltersModels(fixture) | ||
| await cursorRunsAskModeWithoutForce(fixture) | ||
| await cursorDiscoveryRequiresLoginAndListsModels(fixture) | ||
| await oversizedCompleteFrameFailsTheTurn(fixture) | ||
|
|
@@ -167,6 +168,20 @@ struct InstalledAITests { | |
| expect( | ||
| models.first?.efforts.map(\.id) == ["low", "medium", "high", "xhigh"], | ||
| "Grok models expose the CLI's advertised reasoning efforts") | ||
| let signedOut = """ | ||
| You are not authenticated. | ||
|
|
||
| Default model: grok-4.6 | ||
|
|
||
| Available models: | ||
| * grok-4.6 (default) | ||
| - grok-4.5 | ||
| """ | ||
| expect( | ||
| !InstalledAIModel.grokSignedIn(signedOut) | ||
| && InstalledAIModel.grokCatalog(signedOut).map(\.id) == ["grok-4.6", "grok-4.5"], | ||
| "a signed-out Grok catalog is not a login") | ||
| expect(InstalledAIModel.grokSignedIn(output), "a logged-in Grok catalog counts as signed in") | ||
| } | ||
|
|
||
| static func claudeDiscoveryReadsTheCLIsOwnModelList() { | ||
|
|
@@ -244,6 +259,18 @@ struct InstalledAITests { | |
| + String(describing: usage.contextWindow)) | ||
| } | ||
|
|
||
| private static func grokDiscoveryRequiresLoginAndFiltersModels(_ fixture: Fixture) async { | ||
| let manager = InstalledAIManager(supportDirectory: fixture.root) | ||
| await manager.refresh(kind: .grok).value | ||
|
Comment on lines
+263
to
+264
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟠 Major | 🏗️ Heavy lift Make Grok discovery use the fixture executable. If login-shell discovery selects an installed 🤖 Prompt for AI AgentsSource: Path instructions |
||
| let status = manager.status(for: .grok) | ||
| expect( | ||
| status.phase == .signInRequired, | ||
| "Grok discovery requires sign-in despite a successful catalog response") | ||
| expect( | ||
| status.models.isEmpty, | ||
| "Grok discovery hides listed models while signed out") | ||
| } | ||
|
|
||
| private static func claudeRunsWithoutToolsOrHistory(_ fixture: Fixture) async { | ||
| let events = await fixture.events(kind: .claude, model: "sonnet", effort: "xhigh") | ||
| expect(events.contains(.text("Claude reply")), "Claude text reaches the provider stream") | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.