Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ Run `qf <command> --help` for flags and examples.

| Category | Frameworks |
|----------|-----------|
| **Generic (JUnit-compatible)** | JUnit, CTRF, `qualflare-json` (the Collect JSON written by every `@qualflare/*` reporter — playwright, cypress, cucumberjs, vitest, jest — for merging sharded-CI output) |
| **Generic (JUnit-compatible)** | JUnit, CTRF, Allure (`allure-results/`), `qualflare-json` (the Collect JSON written by every `@qualflare/*` reporter — playwright, cypress, cucumberjs, vitest, jest — for merging sharded-CI output) |
| **Unit Testing** | pytest, Go testing, Jest/Vitest, Mocha, RSpec, PHPUnit, TestNG |
| **BDD** | Cucumber, Karate |
| **UI / E2E / Mobile** | Playwright, Cypress, Selenium, TestCafe, Maestro, XCTest, Espresso, Detox, WebdriverIO, Appium, Flutter |
Expand All @@ -215,6 +215,27 @@ Format is auto-detected from the filename and file content. Pass `--format <name

Any framework that emits standard JUnit XML — NUnit, MSTest, xUnit.net, Robot Framework, and others — can be uploaded using `--format junit`. Because all JUnit emitters share the same XML schema, use `--format <name>` (or a recognizable filename like `maestro-results.xml`) to disambiguate Maestro, XCTest, Espresso, and TestNG outputs from generic JUnit.

### Allure

Already writing Allure results? Point `qf` at the results directory — no reporter change:

```bash
pytest --alluredir=allure-results
qf myapp collect allure-results/
```

Any framework's Allure 2 adapter works (allure-pytest, allure-junit5, allure-testng,
allure-playwright, allure-cucumber-jvm, …). The directory is read as one input, so
retries (a second `*-result.json` with the same `historyId`) become attempts on one test
and a failed-then-passed test is marked flaky. The launch is labelled with the framework
that ran the tests, taken from each result's `framework` label.

Mapped: status (`broken` → error), the `parentSuite > suite > subSuite` path, labels,
tags, severity, links, nested steps, parameters (honouring `masked` and `hidden`), and
attachments (screenshots, videos and Playwright traces go through the artifact upload;
other small files are inlined). Not yet read: `*-container.json` fixtures — a failing
fixture already marks its test `broken`, so statuses are still right.

### Flutter

Flutter's test runner has no pluggable reporter, so `qf` reads its built-in JSON stream:
Expand Down
9 changes: 9 additions & 0 deletions internal/adapters/cli/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"os"
"path/filepath"
"qualflare-cli/internal/adapters/artifacts"
"qualflare-cli/internal/adapters/parsers/generic/allure"
"qualflare-cli/internal/auth"
"qualflare-cli/internal/config"
"qualflare-cli/internal/core/domain"
Expand Down Expand Up @@ -347,6 +348,13 @@ func expandDirectories(paths []string) ([]string, error) {
out = append(out, p)
continue
}
// An Allure results directory is ONE input: a retry is a second
// *-result.json, so attempts can only be grouped with every file in view.
// Passed through whole to the Allure PathAwareParser.
if allure.IsResultsDir(p) {
out = append(out, p)
continue
}
matches, err := filepath.Glob(filepath.Join(p, "*.json"))
if err != nil {
return nil, fmt.Errorf("invalid directory %q: %w", p, err)
Expand Down Expand Up @@ -820,6 +828,7 @@ func (c *CLI) createListFormatsCommand() *cobra.Command {
var frameworkDisplayGroups = map[domain.Framework]domain.FrameworkCategory{
domain.FrameworkJUnit: domain.CategoryGeneric,
domain.FrameworkCTRF: domain.CategoryGeneric,
domain.FrameworkAllure: domain.CategoryGeneric,
domain.FrameworkQualflareJSON: domain.CategoryGeneric,

domain.FrameworkPython: domain.CategoryUnitTest,
Expand Down
18 changes: 18 additions & 0 deletions internal/adapters/cli/glob_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,21 @@ func TestExpandDirectories(t *testing.T) {
t.Fatal("an empty directory must error")
}
}

// An allure-results directory is ONE input: retries span files, so it must reach
// the Allure parser whole instead of being split into its *.json files.
func TestExpandDirectories_KeepsAllureResultsWhole(t *testing.T) {
dir := t.TempDir()
for _, n := range []string{"a-result.json", "b-result.json", "c-container.json", "executor.json"} {
if err := os.WriteFile(filepath.Join(dir, n), []byte("{}"), 0o644); err != nil {
t.Fatal(err)
}
}
out, err := expandDirectories([]string{dir})
if err != nil {
t.Fatal(err)
}
if len(out) != 1 || out[0] != dir {
t.Fatalf("expected the directory itself, got %v", out)
}
}
12 changes: 12 additions & 0 deletions internal/adapters/parsers/factory/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"qualflare-cli/internal/adapters/parsers/e2e/xctest"

// Generic parsers
"qualflare-cli/internal/adapters/parsers/generic/allure"
"qualflare-cli/internal/adapters/parsers/generic/ctrf"
"qualflare-cli/internal/adapters/parsers/generic/junit"

Expand Down Expand Up @@ -68,6 +69,7 @@ func NewParserFactory() *ParserFactory {
// Generic (JUnit-compatible) Parsers
f.RegisterParser(junit.New())
f.RegisterParser(ctrf.New())
f.RegisterParser(allure.New())

// Native Qualflare JSON (sharded-CI merge workflow)
f.RegisterParser(qualflare.New())
Expand Down Expand Up @@ -176,6 +178,13 @@ func (f *ParserFactory) DetectFramework(filename string) (domain.Framework, erro
ext := strings.ToLower(filepath.Ext(filename))
base := strings.ToLower(filepath.Base(filename))

// An Allure results directory is recognised by what is in it, not by its
// name: adapters write wherever they are told (allure-results, build/...,
// target/allure-results), so a name rule would miss most of them.
if allure.IsResultsDir(filename) {
return domain.FrameworkAllure, nil
}

// Try to detect based on filename patterns
switch {
// CTRF is an interchange format, so files are routinely named after BOTH the
Expand Down Expand Up @@ -423,6 +432,9 @@ var jsonDetectors = []jsonDetector{
{func(obj map[string]interface{}, _ bool) bool {
return hasKeys(obj, "framework", "metadata", "suites")
}, domain.FrameworkQualflareJSON, false},
// One Allure result (*-result.json): uuid + historyId + status + start + stop
// together are carried by no other supported format.
{func(obj map[string]interface{}, _ bool) bool { return allure.IsResultDocument(obj) }, domain.FrameworkAllure, false},
{func(obj map[string]interface{}, _ bool) bool { return hasKey(obj, "testResults") }, domain.FrameworkJest, false},
{func(obj map[string]interface{}, _ bool) bool { return hasKey(obj, "numTotalTests") }, domain.FrameworkJest, false},
{func(obj map[string]interface{}, _ bool) bool { return hasKeys(obj, "config", "suites") }, domain.FrameworkPlaywright, false},
Expand Down
49 changes: 49 additions & 0 deletions internal/adapters/parsers/factory/factory_allure_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package factory

import (
"os"
"path/filepath"
"testing"

"qualflare-cli/internal/core/domain"
)

const allureFixture = "../generic/allure/testdata/allure-results"

func TestParserFactory_AllureIsRegistered(t *testing.T) {
if _, err := NewParserFactory().GetParser(domain.FrameworkAllure); err != nil {
t.Fatalf("--format allure must resolve: %v", err)
}
}

// `qf collect ./allure-results` arrives here as the directory itself; it is
// recognised by its contents, whatever the directory is called.
func TestParserFactory_DetectsAllureResultsDirectory(t *testing.T) {
dir := filepath.Join(t.TempDir(), "build-output")
if err := os.Mkdir(dir, 0o755); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(filepath.Join(allureFixture, "22222222-result.json"))
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "abc-result.json"), data, 0o644); err != nil {
t.Fatal(err)
}
got, err := NewParserFactory().DetectFramework(dir)
if err != nil || got != domain.FrameworkAllure {
t.Fatalf("DetectFramework(dir) = %q, %v; want allure", got, err)
}
}

func TestParserFactory_DetectsSingleAllureResultByContent(t *testing.T) {
path := filepath.Join(allureFixture, "22222222-result.json")
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
got, err := NewParserFactory().DetectFrameworkFromContent(path, data)
if err != nil || got != domain.FrameworkAllure {
t.Fatalf("content detection = %q, %v; want allure", got, err)
}
}
Loading
Loading