diff --git a/README.md b/README.md index 638041a..ef54b29 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ Run `qf --help` for flags and examples. | Category | Frameworks | |----------|-----------| -| **Generic (JUnit-compatible)** | JUnit, CTRF, `qualflare-json` (the Collect JSON written by every `@qualflare/*` reporter — playwright, cypress, cucumberjs, vitest, jest — for merging sharded-CI output) | +| **Generic (JUnit-compatible)** | JUnit, CTRF, Allure (`allure-results/`), `qualflare-json` (the Collect JSON written by every `@qualflare/*` reporter — playwright, cypress, cucumberjs, vitest, jest — for merging sharded-CI output) | | **Unit Testing** | pytest, Go testing, Jest/Vitest, Mocha, RSpec, PHPUnit, TestNG | | **BDD** | Cucumber, Karate | | **UI / E2E / Mobile** | Playwright, Cypress, Selenium, TestCafe, Maestro, XCTest, Espresso, Detox, WebdriverIO, Appium, Flutter | @@ -215,6 +215,27 @@ Format is auto-detected from the filename and file content. Pass `--format ` (or a recognizable filename like `maestro-results.xml`) to disambiguate Maestro, XCTest, Espresso, and TestNG outputs from generic JUnit. +### Allure + +Already writing Allure results? Point `qf` at the results directory — no reporter change: + +```bash +pytest --alluredir=allure-results +qf myapp collect allure-results/ +``` + +Any framework's Allure 2 adapter works (allure-pytest, allure-junit5, allure-testng, +allure-playwright, allure-cucumber-jvm, …). The directory is read as one input, so +retries (a second `*-result.json` with the same `historyId`) become attempts on one test +and a failed-then-passed test is marked flaky. The launch is labelled with the framework +that ran the tests, taken from each result's `framework` label. + +Mapped: status (`broken` → error), the `parentSuite > suite > subSuite` path, labels, +tags, severity, links, nested steps, parameters (honouring `masked` and `hidden`), and +attachments (screenshots, videos and Playwright traces go through the artifact upload; +other small files are inlined). Not yet read: `*-container.json` fixtures — a failing +fixture already marks its test `broken`, so statuses are still right. + ### Flutter Flutter's test runner has no pluggable reporter, so `qf` reads its built-in JSON stream: diff --git a/internal/adapters/cli/command.go b/internal/adapters/cli/command.go index a578539..3c9b4a1 100644 --- a/internal/adapters/cli/command.go +++ b/internal/adapters/cli/command.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "qualflare-cli/internal/adapters/artifacts" + "qualflare-cli/internal/adapters/parsers/generic/allure" "qualflare-cli/internal/auth" "qualflare-cli/internal/config" "qualflare-cli/internal/core/domain" @@ -347,6 +348,13 @@ func expandDirectories(paths []string) ([]string, error) { out = append(out, p) continue } + // An Allure results directory is ONE input: a retry is a second + // *-result.json, so attempts can only be grouped with every file in view. + // Passed through whole to the Allure PathAwareParser. + if allure.IsResultsDir(p) { + out = append(out, p) + continue + } matches, err := filepath.Glob(filepath.Join(p, "*.json")) if err != nil { return nil, fmt.Errorf("invalid directory %q: %w", p, err) @@ -820,6 +828,7 @@ func (c *CLI) createListFormatsCommand() *cobra.Command { var frameworkDisplayGroups = map[domain.Framework]domain.FrameworkCategory{ domain.FrameworkJUnit: domain.CategoryGeneric, domain.FrameworkCTRF: domain.CategoryGeneric, + domain.FrameworkAllure: domain.CategoryGeneric, domain.FrameworkQualflareJSON: domain.CategoryGeneric, domain.FrameworkPython: domain.CategoryUnitTest, diff --git a/internal/adapters/cli/glob_test.go b/internal/adapters/cli/glob_test.go index 773dc6f..7c500de 100644 --- a/internal/adapters/cli/glob_test.go +++ b/internal/adapters/cli/glob_test.go @@ -71,3 +71,21 @@ func TestExpandDirectories(t *testing.T) { t.Fatal("an empty directory must error") } } + +// An allure-results directory is ONE input: retries span files, so it must reach +// the Allure parser whole instead of being split into its *.json files. +func TestExpandDirectories_KeepsAllureResultsWhole(t *testing.T) { + dir := t.TempDir() + for _, n := range []string{"a-result.json", "b-result.json", "c-container.json", "executor.json"} { + if err := os.WriteFile(filepath.Join(dir, n), []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + } + out, err := expandDirectories([]string{dir}) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || out[0] != dir { + t.Fatalf("expected the directory itself, got %v", out) + } +} diff --git a/internal/adapters/parsers/factory/factory.go b/internal/adapters/parsers/factory/factory.go index fd5b9a1..83a0fb9 100644 --- a/internal/adapters/parsers/factory/factory.go +++ b/internal/adapters/parsers/factory/factory.go @@ -31,6 +31,7 @@ import ( "qualflare-cli/internal/adapters/parsers/e2e/xctest" // Generic parsers + "qualflare-cli/internal/adapters/parsers/generic/allure" "qualflare-cli/internal/adapters/parsers/generic/ctrf" "qualflare-cli/internal/adapters/parsers/generic/junit" @@ -68,6 +69,7 @@ func NewParserFactory() *ParserFactory { // Generic (JUnit-compatible) Parsers f.RegisterParser(junit.New()) f.RegisterParser(ctrf.New()) + f.RegisterParser(allure.New()) // Native Qualflare JSON (sharded-CI merge workflow) f.RegisterParser(qualflare.New()) @@ -176,6 +178,13 @@ func (f *ParserFactory) DetectFramework(filename string) (domain.Framework, erro ext := strings.ToLower(filepath.Ext(filename)) base := strings.ToLower(filepath.Base(filename)) + // An Allure results directory is recognised by what is in it, not by its + // name: adapters write wherever they are told (allure-results, build/..., + // target/allure-results), so a name rule would miss most of them. + if allure.IsResultsDir(filename) { + return domain.FrameworkAllure, nil + } + // Try to detect based on filename patterns switch { // CTRF is an interchange format, so files are routinely named after BOTH the @@ -423,6 +432,9 @@ var jsonDetectors = []jsonDetector{ {func(obj map[string]interface{}, _ bool) bool { return hasKeys(obj, "framework", "metadata", "suites") }, domain.FrameworkQualflareJSON, false}, + // One Allure result (*-result.json): uuid + historyId + status + start + stop + // together are carried by no other supported format. + {func(obj map[string]interface{}, _ bool) bool { return allure.IsResultDocument(obj) }, domain.FrameworkAllure, false}, {func(obj map[string]interface{}, _ bool) bool { return hasKey(obj, "testResults") }, domain.FrameworkJest, false}, {func(obj map[string]interface{}, _ bool) bool { return hasKey(obj, "numTotalTests") }, domain.FrameworkJest, false}, {func(obj map[string]interface{}, _ bool) bool { return hasKeys(obj, "config", "suites") }, domain.FrameworkPlaywright, false}, diff --git a/internal/adapters/parsers/factory/factory_allure_test.go b/internal/adapters/parsers/factory/factory_allure_test.go new file mode 100644 index 0000000..4fdaf14 --- /dev/null +++ b/internal/adapters/parsers/factory/factory_allure_test.go @@ -0,0 +1,49 @@ +package factory + +import ( + "os" + "path/filepath" + "testing" + + "qualflare-cli/internal/core/domain" +) + +const allureFixture = "../generic/allure/testdata/allure-results" + +func TestParserFactory_AllureIsRegistered(t *testing.T) { + if _, err := NewParserFactory().GetParser(domain.FrameworkAllure); err != nil { + t.Fatalf("--format allure must resolve: %v", err) + } +} + +// `qf collect ./allure-results` arrives here as the directory itself; it is +// recognised by its contents, whatever the directory is called. +func TestParserFactory_DetectsAllureResultsDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "build-output") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(allureFixture, "22222222-result.json")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "abc-result.json"), data, 0o644); err != nil { + t.Fatal(err) + } + got, err := NewParserFactory().DetectFramework(dir) + if err != nil || got != domain.FrameworkAllure { + t.Fatalf("DetectFramework(dir) = %q, %v; want allure", got, err) + } +} + +func TestParserFactory_DetectsSingleAllureResultByContent(t *testing.T) { + path := filepath.Join(allureFixture, "22222222-result.json") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + got, err := NewParserFactory().DetectFrameworkFromContent(path, data) + if err != nil || got != domain.FrameworkAllure { + t.Fatalf("content detection = %q, %v; want allure", got, err) + } +} diff --git a/internal/adapters/parsers/generic/allure/allure.go b/internal/adapters/parsers/generic/allure/allure.go new file mode 100644 index 0000000..6ddef25 --- /dev/null +++ b/internal/adapters/parsers/generic/allure/allure.go @@ -0,0 +1,570 @@ +// Package allure parses Allure 2 results (an allure-results/ directory). +// +// Allure is a FORMAT, not a test framework: every framework's Allure adapter +// (allure-pytest, allure-junit5, allure-playwright, allure-cucumber-jvm, …) +// writes the same files into one directory — +// +// -result.json one per test EXECUTION (a retry is a second file) +// -container.json fixtures (befores/afters) and grouping +// -attachment. files referenced by results and steps +// executor.json, categories.json, environment.properties +// +// so this parser takes the whole directory (ports.PathAwareParser) rather than +// one file at a time: retries can only be recognised across files. +// +// Out of scope for now: containers. A failing fixture already makes Allure +// mark the affected test "broken", which maps to StatusError, so statuses stay +// right; only the fixture's own steps are not shown. +package allure + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "qualflare-cli/internal/core/domain" +) + +const ( + resultSuffix = "-result.json" + suiteSep = " > " + defaultSuite = "Allure results" + maskedValue = "[masked]" + maxInlineSize = 1 << 20 // non-media attachments up to 1 MiB travel inline + + propParamPrefix = "allureParam." + propFullName = "allureFullName" +) + +// result is the subset of an Allure 2 *-result.json this parser reads. +type result struct { + UUID string `json:"uuid"` + HistoryID string `json:"historyId"` + TestCaseID string `json:"testCaseId"` + FullName string `json:"fullName"` + Name string `json:"name"` + Status string `json:"status"` + StatusDetails statusDetails `json:"statusDetails"` + Start int64 `json:"start"` + Stop int64 `json:"stop"` + Labels []label `json:"labels"` + Links []link `json:"links"` + Parameters []parameter `json:"parameters"` + Steps []step `json:"steps"` + Attachments []attachment `json:"attachments"` +} + +type statusDetails struct { + Message string `json:"message"` + Trace string `json:"trace"` + Flaky bool `json:"flaky"` +} + +type label struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type link struct { + Type string `json:"type"` + Name string `json:"name"` + URL string `json:"url"` +} + +type parameter struct { + Name string `json:"name"` + Value string `json:"value"` + Excluded bool `json:"excluded"` + Mode string `json:"mode"` // "", "default", "masked", "hidden" +} + +type step struct { + Name string `json:"name"` + Status string `json:"status"` + StatusDetails statusDetails `json:"statusDetails"` + Start int64 `json:"start"` + Stop int64 `json:"stop"` + Parameters []parameter `json:"parameters"` + Steps []step `json:"steps"` + Attachments []attachment `json:"attachments"` +} + +type attachment struct { + Name string `json:"name"` + Source string `json:"source"` + Type string `json:"type"` +} + +// Parser implements ports.PathAwareParser for Allure results. +type Parser struct{} + +// New creates an Allure results parser. +func New() *Parser { return &Parser{} } + +// GetFramework returns the framework identifier. +func (p *Parser) GetFramework() domain.Framework { return domain.FrameworkAllure } + +// SupportedFileExtensions returns the file extensions this parser handles. +func (p *Parser) SupportedFileExtensions() []string { return []string{".json"} } + +// IsResultsDir reports whether path is a directory holding Allure results. +func IsResultsDir(path string) bool { + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + return false + } + matches, _ := filepath.Glob(filepath.Join(path, "*"+resultSuffix)) + return len(matches) > 0 +} + +// IsResultDocument reports whether a decoded JSON object looks like one Allure +// result: the fields no other supported format carries together. +func IsResultDocument(obj map[string]interface{}) bool { + _, uuid := obj["uuid"] + _, history := obj["historyId"] + _, status := obj["status"] + _, start := obj["start"] + _, stop := obj["stop"] + return uuid && history && status && start && stop +} + +// Parse reads a single *-result.json. Attachments are not resolved: a bare +// reader has no directory to resolve them against. Pass the directory instead. +func (p *Parser) Parse(reader io.Reader) (*domain.Suite, error) { + var r result + if err := json.NewDecoder(reader).Decode(&r); err != nil { + return nil, fmt.Errorf("allure: invalid result JSON: %w", err) + } + if r.Status == "" { + return nil, errors.New("allure: document has no status; not an Allure result") + } + return buildSuite([]result{r}, ""), nil +} + +// ParsePath parses an allure-results directory, or one result file. +func (p *Parser) ParsePath(path string) (*domain.Suite, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if !info.IsDir() { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + return p.Parse(f) + } + + files, err := filepath.Glob(filepath.Join(path, "*"+resultSuffix)) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, fmt.Errorf("allure: %s contains no *%s files", path, resultSuffix) + } + sort.Strings(files) + + results := make([]result, 0, len(files)) + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("allure: %w", err) + } + var r result + if err := json.Unmarshal(data, &r); err != nil { + return nil, fmt.Errorf("allure: %s: %w", filepath.Base(f), err) + } + results = append(results, r) + } + + dir, err := filepath.Abs(path) + if err != nil { + dir = path + } + return buildSuite(results, dir), nil +} + +// buildSuite turns results into one Suite. dir resolves attachments; "" skips them. +func buildSuite(results []result, dir string) *domain.Suite { + groups, order := groupByTest(results) + + suite := &domain.Suite{Properties: map[string]string{}} + var first, last int64 + producers := map[domain.Framework]struct{}{} + topSuites := map[string]struct{}{} + + for _, key := range order { + attempts := groups[key] + final := attempts[len(attempts)-1] + c := buildCase(final, dir) + + if len(attempts) > 1 { + retries := len(attempts) - 1 + c.RetryCount = &retries + suite.Retries += retries + for i, a := range attempts { + started := time.UnixMilli(a.Start) + c.Attempts = append(c.Attempts, domain.Attempt{ + Number: i + 1, + Status: mapStatus(a.Status), + Duration: duration(a.Start, a.Stop), + StartedAt: &started, + UID: a.UUID, + Message: a.StatusDetails.Message, + Trace: a.StatusDetails.Trace, + }) + } + } + if isFlaky(attempts) { + flaky := true + c.IsFlaky = &flaky + suite.Flaky++ + } + suite.Cases = append(suite.Cases, c) + + for _, a := range attempts { + if first == 0 || (a.Start > 0 && a.Start < first) { + first = a.Start + } + if a.Stop > last { + last = a.Stop + } + } + if f, ok := frameworkForLabel(labelValue(final.Labels, "framework")); ok { + producers[f] = struct{}{} + } + if top := firstNonEmpty(labelValue(final.Labels, "parentSuite"), labelValue(final.Labels, "suite")); top != "" { + topSuites[top] = struct{}{} + } + } + + suite.Name = defaultSuite + if len(topSuites) == 1 { + for name := range topSuites { + suite.Name = name + } + } + suite.Category = domain.CategoryGeneric + if len(producers) == 1 { + for f := range producers { + suite.Properties[domain.PropSourceFramework] = string(f) + suite.Category = f.GetCategory() + } + } + if first > 0 { + suite.Timestamp = time.UnixMilli(first) + suite.Duration = duration(first, last) + } + suite.RecomputeCounts() + return suite +} + +// groupByTest groups executions of the same test (historyId, falling back to +// testCaseId, fullName, then uuid), each group ordered oldest first, and returns +// the group keys in first-seen order so the output is stable. +func groupByTest(results []result) (map[string][]result, []string) { + groups := map[string][]result{} + var order []string + for _, r := range results { + key := firstNonEmpty(r.HistoryID, r.TestCaseID, r.FullName, r.UUID) + if _, seen := groups[key]; !seen { + order = append(order, key) + } + groups[key] = append(groups[key], r) + } + for _, key := range order { + g := groups[key] + sort.SliceStable(g, func(i, j int) bool { + if g[i].Stop != g[j].Stop { + return g[i].Stop < g[j].Stop + } + return g[i].Start < g[j].Start + }) + } + return groups, order +} + +// isFlaky: the final attempt passed after an earlier one did not, or the +// adapter flagged the result flaky itself. +func isFlaky(attempts []result) bool { + final := attempts[len(attempts)-1] + if final.StatusDetails.Flaky { + return true + } + if mapStatus(final.Status) != domain.StatusPassed { + return false + } + for _, a := range attempts[:len(attempts)-1] { + if mapStatus(a.Status) != domain.StatusPassed { + return true + } + } + return false +} + +func buildCase(r result, dir string) domain.Case { + started := time.UnixMilli(r.Start) + c := domain.Case{ + ID: firstNonEmpty(r.TestCaseID, r.HistoryID, r.UUID), + Name: firstNonEmpty(r.Name, r.FullName, r.UUID), + ClassName: suitePath(r), + Status: mapStatus(r.Status), + Duration: duration(r.Start, r.Stop), + Error: errorText(r.StatusDetails), + Priority: mapSeverity(labelValue(r.Labels, "severity")), + Properties: map[string]string{}, + } + if r.Start > 0 { + c.StartedAt = &started + } + if r.FullName != "" { + c.Properties[propFullName] = r.FullName + } + for _, l := range r.Labels { + c.Labels = append(c.Labels, domain.Label{Name: l.Name, Value: l.Value}) + if l.Name == "tag" && l.Value != "" { + c.Tags = append(c.Tags, l.Value) + } + } + for _, l := range r.Links { + if l.URL != "" { + c.Links = append(c.Links, domain.Link{Type: l.Type, Name: l.Name, URL: l.URL}) + } + } + for _, p := range visibleParams(r.Parameters) { + v := p.Value + if p.Masked { + v = maskedValue + } + c.Properties[propParamPrefix+p.Name] = v + } + c.Steps = flattenSteps(r.Steps, nil, nil) + c.Attachments = resolveAttachments(collectAttachments(r), dir) + return c +} + +// suitePath is the Allure suite hierarchy, which Allure Report shows as nested +// groups. It goes in ClassName because a PathAwareParser returns one Suite. +func suitePath(r result) string { + var parts []string + for _, name := range []string{"parentSuite", "suite", "subSuite"} { + if v := labelValue(r.Labels, name); v != "" { + parts = append(parts, v) + } + } + if len(parts) > 0 { + return strings.Join(parts, suiteSep) + } + return labelValue(r.Labels, "testClass") +} + +func flattenSteps(steps []step, parent *int, out []domain.Step) []domain.Step { + for _, s := range steps { + idx := len(out) + var p *int + if parent != nil { + v := *parent + p = &v + } + out = append(out, domain.Step{ + Name: s.Name, + Status: mapStatus(s.Status), + Duration: duration(s.Start, s.Stop), + Error: s.StatusDetails.Message, + ParentIndex: p, + Parameters: visibleParams(s.Parameters), + }) + out = flattenSteps(s.Steps, &idx, out) + } + return out +} + +// visibleParams applies Allure's parameter modes: hidden parameters are dropped, +// masked ones keep their name but not their value. "excluded" only affects +// Allure's historyId calculation, so the parameter is still shown. +func visibleParams(params []parameter) []domain.Parameter { + var out []domain.Parameter + for _, p := range params { + switch p.Mode { + case "hidden": + continue + case "masked": + out = append(out, domain.Parameter{Name: p.Name, Masked: true}) + default: + out = append(out, domain.Parameter{Name: p.Name, Value: p.Value}) + } + } + return out +} + +func collectAttachments(r result) []attachment { + out := append([]attachment(nil), r.Attachments...) + var walk func([]step) + walk = func(steps []step) { + for _, s := range steps { + out = append(out, s.Attachments...) + walk(s.Steps) + } + } + walk(r.Steps) + return out +} + +// resolveAttachments maps Allure attachments to files in dir. Images, videos and +// Playwright traces go through the artifact upload pass (LocalPath + +// ArtifactKind, gated by --upload-artifacts); other small files travel inline. +// A missing or unsafe source is dropped rather than sent as a dangling reference. +func resolveAttachments(atts []attachment, dir string) []domain.Attachment { + if dir == "" { + return nil + } + var out []domain.Attachment + for _, a := range atts { + if a.Source == "" || a.Source != filepath.Base(a.Source) { + continue // a source must be a plain file name inside the results dir + } + full := filepath.Join(dir, a.Source) + info, err := os.Stat(full) + if err != nil || info.IsDir() { + continue + } + att := domain.Attachment{Name: firstNonEmpty(a.Name, a.Source), MimeType: a.Type} + if kind := artifactKind(a.Type, a.Source); kind != "" { + att.LocalPath, att.ArtifactKind = full, kind + } else { + if info.Size() > maxInlineSize { + continue + } + data, err := os.ReadFile(full) + if err != nil { + continue + } + att.Content = base64.StdEncoding.EncodeToString(data) + } + out = append(out, att) + } + return out +} + +func artifactKind(mime, source string) string { + switch { + case strings.HasPrefix(mime, "image/"): + return domain.ArtifactKindImage + case strings.HasPrefix(mime, "video/"): + return domain.ArtifactKindVideo + case mime == "application/zip" && strings.Contains(strings.ToLower(source), "trace"): + return domain.ArtifactKindTrace + } + return "" +} + +// mapStatus maps Allure's statuses. "broken" means the test could not run +// properly (an exception outside an assertion): an error, not a failure. +func mapStatus(s string) domain.Status { + switch strings.ToLower(s) { + case "passed": + return domain.StatusPassed + case "failed": + return domain.StatusFailed + case "skipped": + return domain.StatusSkipped + default: // broken, unknown, anything new + return domain.StatusError + } +} + +// mapSeverity maps Allure's five severities onto Qualflare's. +func mapSeverity(s string) domain.Severity { + switch strings.ToLower(s) { + case "blocker": + return domain.SeverityCritical + case "critical": + return domain.SeverityCritical + case "normal": + return domain.SeverityMedium + case "minor": + return domain.SeverityLow + case "trivial": + return domain.SeverityInfo + } + return "" +} + +// frameworkForLabel maps an Allure "framework" label onto the framework that +// produced the results, reporting whether it could. Like the CTRF parser it +// returns false rather than guessing, so the suite falls back to generic. +func frameworkForLabel(v string) (domain.Framework, bool) { + switch strings.ToLower(strings.TrimSpace(v)) { + case "pytest", "pytest-bdd": + return domain.FrameworkPython, true + case "junit4", "junit5", "junit-platform", "junit": + return domain.FrameworkJUnit, true + case "testng": + return domain.FrameworkTestNG, true + case "playwright": + return domain.FrameworkPlaywright, true + case "cypress": + return domain.FrameworkCypress, true + case "jest": + return domain.FrameworkJest, true + case "vitest": + return domain.FrameworkVitest, true + case "mocha": + return domain.FrameworkMocha, true + case "cucumberjvm", "cucumber-jvm", "cucumberjs", "cucumber-js", "cucumber": + return domain.FrameworkCucumber, true + case "webdriverio", "wdio": + return domain.FrameworkWebdriverIO, true + case "phpunit": + return domain.FrameworkPHPUnit, true + case "rspec": + return domain.FrameworkRSpec, true + case "karate": + return domain.FrameworkKarate, true + } + return "", false +} + +func errorText(d statusDetails) string { + msg, trace := strings.TrimSpace(d.Message), strings.TrimSpace(d.Trace) + switch { + case msg != "" && trace != "": + return msg + "\n" + trace + case msg != "": + return msg + } + return trace +} + +func labelValue(labels []label, name string) string { + for _, l := range labels { + if l.Name == name { + return l.Value + } + } + return "" +} + +func duration(start, stop int64) time.Duration { + if stop <= start { + return 0 + } + return time.Duration(stop-start) * time.Millisecond +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} diff --git a/internal/adapters/parsers/generic/allure/allure_test.go b/internal/adapters/parsers/generic/allure/allure_test.go new file mode 100644 index 0000000..ff30dbc --- /dev/null +++ b/internal/adapters/parsers/generic/allure/allure_test.go @@ -0,0 +1,282 @@ +package allure + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "qualflare-cli/internal/core/domain" + "qualflare-cli/internal/core/ports" +) + +const fixtureDir = "testdata/allure-results" + +var _ ports.PathAwareParser = (*Parser)(nil) + +func parseFixture(t *testing.T) *domain.Suite { + t.Helper() + suite, err := New().ParsePath(fixtureDir) + if err != nil { + t.Fatalf("ParsePath: %v", err) + } + return suite +} + +func caseNamed(t *testing.T, s *domain.Suite, name string) domain.Case { + t.Helper() + for _, c := range s.Cases { + if c.Name == name { + return c + } + } + t.Fatalf("no case named %q", name) + return domain.Case{} +} + +func TestParser_Contract(t *testing.T) { + p := New() + if p.GetFramework() != domain.FrameworkAllure { + t.Errorf("GetFramework() = %q, want allure", p.GetFramework()) + } + if got := p.SupportedFileExtensions(); len(got) != 1 || got[0] != ".json" { + t.Errorf("SupportedFileExtensions() = %v, want [.json]", got) + } +} + +// Six result files, two of them attempts of one test: five cases. Containers, +// executor.json, categories.json and environment.properties are not tests. +func TestParsePath_OneCasePerTestAndCountsMatch(t *testing.T) { + s := parseFixture(t) + if len(s.Cases) != 5 { + t.Fatalf("got %d cases, want 5", len(s.Cases)) + } + if s.TotalTests != 5 || s.Passed != 2 || s.Failed != 1 || s.Errors != 1 || s.Skipped != 1 { + t.Errorf("counts total=%d passed=%d failed=%d errors=%d skipped=%d, want 5/2/1/1/1", + s.TotalTests, s.Passed, s.Failed, s.Errors, s.Skipped) + } + if s.Flaky != 1 || s.Retries != 1 { + t.Errorf("flaky=%d retries=%d, want 1/1", s.Flaky, s.Retries) + } +} + +// Allure's statuses: broken = the test could not run properly (an error, not an +// assertion failure); unknown has no better home than error either. +func TestParsePath_StatusMapping(t *testing.T) { + s := parseFixture(t) + for name, want := range map[string]domain.Status{ + "test_login_ok": domain.StatusPassed, + "test_logout": domain.StatusFailed, + "test_db_setup": domain.StatusError, + "test_later": domain.StatusSkipped, + "test_search": domain.StatusPassed, + } { + if got := caseNamed(t, s, name).Status; got != want { + t.Errorf("%s: status %q, want %q", name, got, want) + } + } +} + +// Results sharing a historyId are attempts of one test. The latest is the +// result; earlier ones are retries; failed-then-passed is flaky. +func TestParsePath_RetriesBecomeAttempts(t *testing.T) { + c := caseNamed(t, parseFixture(t), "test_search") + if c.RetryCount == nil || *c.RetryCount != 1 { + t.Errorf("RetryCount = %v, want 1", c.RetryCount) + } + if c.IsFlaky == nil || !*c.IsFlaky { + t.Errorf("IsFlaky = %v, want true", c.IsFlaky) + } + if len(c.Attempts) != 2 || c.Attempts[0].Status != domain.StatusFailed || c.Attempts[1].Status != domain.StatusPassed { + t.Fatalf("attempts = %+v, want [failed, passed]", c.Attempts) + } + if c.Attempts[0].Message != "TimeoutError" { + t.Errorf("first attempt message = %q", c.Attempts[0].Message) + } +} + +func TestParsePath_FailureDetails(t *testing.T) { + c := caseNamed(t, parseFixture(t), "test_logout") + if !strings.Contains(c.Error, "expected 200, got 500") || !strings.Contains(c.Error, "test_logout.py") { + t.Errorf("Error should carry message and trace, got %q", c.Error) + } +} + +func TestParsePath_CaseMetadata(t *testing.T) { + c := caseNamed(t, parseFixture(t), "test_login_ok") + + if c.ID != "tc-login" { + t.Errorf("ID = %q, want the testCaseId", c.ID) + } + if c.ClassName != "tests.auth > test_login > TestAuth" { + t.Errorf("ClassName = %q, want the Allure suite path", c.ClassName) + } + if c.Duration != 1500*time.Millisecond { + t.Errorf("Duration = %v", c.Duration) + } + if c.StartedAt == nil || !c.StartedAt.Equal(time.UnixMilli(1760000000000)) { + t.Errorf("StartedAt = %v", c.StartedAt) + } + if c.Priority != domain.SeverityCritical { + t.Errorf("Priority = %q, want critical", c.Priority) + } + if len(c.Tags) != 1 || c.Tags[0] != "smoke" { + t.Errorf("Tags = %v, want [smoke]", c.Tags) + } + if len(c.Links) != 1 || c.Links[0].URL != "https://tracker.example/QF-1" || c.Links[0].Type != "issue" { + t.Errorf("Links = %+v", c.Links) + } + found := false + for _, l := range c.Labels { + if l.Name == "feature" && l.Value == "Login" { + found = true + } + } + if !found { + t.Errorf("labels must be kept verbatim, got %+v", c.Labels) + } +} + +// Allure parameter modes: default and excluded are shown; masked is shown +// without its value; hidden is not shown at all. +func TestParsePath_ParameterModes(t *testing.T) { + p := caseNamed(t, parseFixture(t), "test_login_ok").Properties + if p[propParamPrefix+"browser"] != "chrome" || p[propParamPrefix+"run"] != "3" { + t.Errorf("default/excluded params must be kept: %v", p) + } + if v, ok := p[propParamPrefix+"password"]; !ok || v != maskedValue { + t.Errorf("masked param must be present without its value, got %q (present=%v)", v, ok) + } + if strings.Contains(strings.Join(mapValues(p), " "), "hunter2") { + t.Error("a masked value leaked") + } + if _, ok := p[propParamPrefix+"internal"]; ok { + t.Error("hidden param must not be kept") + } +} + +// Nested steps are flattened depth-first, children pointing at their parent. +func TestParsePath_NestedSteps(t *testing.T) { + steps := caseNamed(t, parseFixture(t), "test_login_ok").Steps + if len(steps) != 3 { + t.Fatalf("got %d steps, want 3", len(steps)) + } + if steps[0].Name != "open login page" || steps[0].ParentIndex != nil { + t.Errorf("step 0 = %+v", steps[0]) + } + if steps[1].Name != "wait for form" || steps[1].ParentIndex == nil || *steps[1].ParentIndex != 0 { + t.Errorf("step 1 must be a child of step 0: %+v", steps[1]) + } + if steps[2].Name != "submit credentials" || steps[2].ParentIndex != nil { + t.Errorf("step 2 = %+v", steps[2]) + } + if len(steps[0].Parameters) != 1 || steps[0].Parameters[0].Value != "/login" { + t.Errorf("step parameters = %+v", steps[0].Parameters) + } +} + +// Attachments resolve against the results directory; a missing file is dropped +// rather than sent as a dangling reference. +func TestParsePath_Attachments(t *testing.T) { + a := caseNamed(t, parseFixture(t), "test_login_ok").Attachments + if len(a) != 1 { + t.Fatalf("got %d attachments, want 1 (the missing one dropped)", len(a)) + } + want, _ := filepath.Abs(filepath.Join(fixtureDir, "aaaaaaaa-attachment.png")) + if a[0].Name != "after login" || a[0].MimeType != "image/png" || a[0].LocalPath != want { + t.Errorf("attachment = %+v", a[0]) + } +} + +// Allure is a FORMAT: the suite records the framework that produced it, and its +// category is that framework's, so the launch is labelled pytest, not allure. +func TestParsePath_SourceFramework(t *testing.T) { + s := parseFixture(t) + if s.Properties[domain.PropSourceFramework] != string(domain.FrameworkPython) { + t.Errorf("source framework = %q, want python", s.Properties[domain.PropSourceFramework]) + } + if s.Category != domain.FrameworkPython.GetCategory() { + t.Errorf("category = %q", s.Category) + } +} + +// An unmappable producer falls back to generic rather than guessing — an +// unknown category would 400 the whole launch server-side. +func TestParse_UnknownProducerIsGeneric(t *testing.T) { + s, err := New().Parse(strings.NewReader(`{"uuid":"u","historyId":"h","name":"t","status":"passed","start":1,"stop":2, + "labels":[{"name":"framework","value":"jasmine"}]}`)) + if err != nil { + t.Fatal(err) + } + if _, ok := s.Properties[domain.PropSourceFramework]; ok { + t.Errorf("no source framework for an unmapped producer, got %v", s.Properties) + } + if s.Category != domain.CategoryGeneric { + t.Errorf("category = %q, want generic", s.Category) + } +} + +func TestParse_SingleResultFile(t *testing.T) { + f, err := os.Open(filepath.Join(fixtureDir, "22222222-result.json")) + if err != nil { + t.Fatal(err) + } + defer f.Close() + s, err := New().Parse(f) + if err != nil { + t.Fatal(err) + } + if len(s.Cases) != 1 || s.Cases[0].Status != domain.StatusFailed { + t.Errorf("got %+v", s.Cases) + } +} + +func TestParsePath_DirectoryWithoutResultsIsAnError(t *testing.T) { + if _, err := New().ParsePath(t.TempDir()); err == nil { + t.Error("an empty directory must be an error, not an empty upload") + } +} + +func TestIsResultsDir(t *testing.T) { + if !IsResultsDir(fixtureDir) { + t.Error("fixture is an allure-results dir") + } + if IsResultsDir(t.TempDir()) { + t.Error("an empty dir is not") + } + if IsResultsDir(filepath.Join(fixtureDir, "22222222-result.json")) { + t.Error("a file is not a directory") + } +} + +func mapValues(m map[string]string) []string { + out := make([]string, 0, len(m)) + for _, v := range m { + out = append(out, v) + } + return out +} + +// An attachment source is a file name inside the results directory, never a +// path: a crafted "../" must not read files from elsewhere on the machine. +func TestParsePath_AttachmentSourceCannotEscapeTheDirectory(t *testing.T) { + dir := t.TempDir() + outside := filepath.Join(filepath.Dir(dir), "secret.txt") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + defer os.Remove(outside) + doc := `{"uuid":"u","historyId":"h","name":"t","status":"passed","start":1,"stop":2, + "attachments":[{"name":"x","source":"../secret.txt","type":"text/plain"}]}` + if err := os.WriteFile(filepath.Join(dir, "u-result.json"), []byte(doc), 0o644); err != nil { + t.Fatal(err) + } + s, err := New().ParsePath(dir) + if err != nil { + t.Fatal(err) + } + if n := len(s.Cases[0].Attachments); n != 0 { + t.Fatalf("a ../ source must be dropped, got %d attachments", n) + } +} diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/11111111-result.json b/internal/adapters/parsers/generic/allure/testdata/allure-results/11111111-result.json new file mode 100644 index 0000000..fe55db6 --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/11111111-result.json @@ -0,0 +1,8 @@ +{"uuid":"11111111","historyId":"h-login","testCaseId":"tc-login","fullName":"tests.auth.test_login.TestAuth#test_login_ok","name":"test_login_ok","status":"passed","stage":"finished","start":1760000000000,"stop":1760000001500, + "labels":[{"name":"parentSuite","value":"tests.auth"},{"name":"suite","value":"test_login"},{"name":"subSuite","value":"TestAuth"},{"name":"framework","value":"pytest"},{"name":"language","value":"cpython3"},{"name":"severity","value":"critical"},{"name":"tag","value":"smoke"},{"name":"feature","value":"Login"},{"name":"testClass","value":"tests.auth.test_login.TestAuth"}], + "links":[{"type":"issue","name":"QF-1","url":"https://tracker.example/QF-1"}], + "parameters":[{"name":"browser","value":"chrome"},{"name":"password","value":"hunter2","mode":"masked"},{"name":"internal","value":"x","mode":"hidden"},{"name":"run","value":"3","excluded":true}], + "steps":[{"name":"open login page","status":"passed","start":1760000000000,"stop":1760000000500,"parameters":[{"name":"url","value":"/login"}], + "steps":[{"name":"wait for form","status":"passed","start":1760000000100,"stop":1760000000300}]}, + {"name":"submit credentials","status":"passed","start":1760000000500,"stop":1760000001400}], + "attachments":[{"name":"after login","source":"aaaaaaaa-attachment.png","type":"image/png"},{"name":"gone","source":"missing-attachment.txt","type":"text/plain"}]} diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/22222222-result.json b/internal/adapters/parsers/generic/allure/testdata/allure-results/22222222-result.json new file mode 100644 index 0000000..6ac6649 --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/22222222-result.json @@ -0,0 +1,3 @@ +{"uuid":"22222222","historyId":"h-logout","fullName":"tests.auth.test_logout#test_logout","name":"test_logout","status":"failed","start":1760000002000,"stop":1760000002250, + "statusDetails":{"message":"AssertionError: expected 200, got 500","trace":"Traceback (most recent call last):\n File \"test_logout.py\", line 9"}, + "labels":[{"name":"parentSuite","value":"tests.auth"},{"name":"suite","value":"test_logout"},{"name":"framework","value":"pytest"}]} diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/33333333-result.json b/internal/adapters/parsers/generic/allure/testdata/allure-results/33333333-result.json new file mode 100644 index 0000000..bf7fba8 --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/33333333-result.json @@ -0,0 +1,3 @@ +{"uuid":"33333333","historyId":"h-db","name":"test_db_setup","status":"broken","start":1760000003000,"stop":1760000003100, + "statusDetails":{"message":"ConnectionRefusedError: [Errno 111]"}, + "labels":[{"name":"suite","value":"test_db"},{"name":"framework","value":"pytest"}]} diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/44444444-result.json b/internal/adapters/parsers/generic/allure/testdata/allure-results/44444444-result.json new file mode 100644 index 0000000..ff92f39 --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/44444444-result.json @@ -0,0 +1,3 @@ +{"uuid":"44444444","historyId":"h-skip","name":"test_later","status":"skipped","start":1760000004000,"stop":1760000004000, + "statusDetails":{"message":"not implemented yet"}, + "labels":[{"name":"suite","value":"test_misc"},{"name":"framework","value":"pytest"}]} diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/55555555-result.json b/internal/adapters/parsers/generic/allure/testdata/allure-results/55555555-result.json new file mode 100644 index 0000000..3e788cc --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/55555555-result.json @@ -0,0 +1,3 @@ +{"uuid":"55555555","historyId":"h-flaky","name":"test_search","status":"failed","start":1760000005000,"stop":1760000005400, + "statusDetails":{"message":"TimeoutError"}, + "labels":[{"name":"suite","value":"test_search"},{"name":"framework","value":"pytest"}]} diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/66666666-result.json b/internal/adapters/parsers/generic/allure/testdata/allure-results/66666666-result.json new file mode 100644 index 0000000..f2dca31 --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/66666666-result.json @@ -0,0 +1,2 @@ +{"uuid":"66666666","historyId":"h-flaky","name":"test_search","status":"passed","start":1760000006000,"stop":1760000006300, + "labels":[{"name":"suite","value":"test_search"},{"name":"framework","value":"pytest"}]} diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/aaaaaaaa-attachment.png b/internal/adapters/parsers/generic/allure/testdata/allure-results/aaaaaaaa-attachment.png new file mode 100644 index 0000000..208b38b --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/aaaaaaaa-attachment.png @@ -0,0 +1,3 @@ +‰PNG + +fake \ No newline at end of file diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/categories.json b/internal/adapters/parsers/generic/allure/testdata/allure-results/categories.json new file mode 100644 index 0000000..14fa5a1 --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/categories.json @@ -0,0 +1 @@ +[{"name":"Product defects","matchedStatuses":["failed"]}] \ No newline at end of file diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/cccccccc-container.json b/internal/adapters/parsers/generic/allure/testdata/allure-results/cccccccc-container.json new file mode 100644 index 0000000..1980897 --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/cccccccc-container.json @@ -0,0 +1 @@ +{"uuid":"cccccccc","name":"db fixture","children":["33333333"],"befores":[{"name":"db","status":"broken","start":1760000002900,"stop":1760000003000}],"afters":[]} diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/environment.properties b/internal/adapters/parsers/generic/allure/testdata/allure-results/environment.properties new file mode 100644 index 0000000..071b25c --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/environment.properties @@ -0,0 +1,2 @@ +Browser=Chrome +Python=3.12 diff --git a/internal/adapters/parsers/generic/allure/testdata/allure-results/executor.json b/internal/adapters/parsers/generic/allure/testdata/allure-results/executor.json new file mode 100644 index 0000000..7466f06 --- /dev/null +++ b/internal/adapters/parsers/generic/allure/testdata/allure-results/executor.json @@ -0,0 +1 @@ +{"name":"GitHub Actions","type":"github","buildName":"ci #42"} \ No newline at end of file diff --git a/internal/adapters/parsers/native/qualflare/qualflare.go b/internal/adapters/parsers/native/qualflare/qualflare.go index 831127b..c4369db 100644 --- a/internal/adapters/parsers/native/qualflare/qualflare.go +++ b/internal/adapters/parsers/native/qualflare/qualflare.go @@ -252,7 +252,7 @@ func buildSuite(collect Collect, sourceDir string) (*domain.Suite, error) { // and until this the launch was labelled "qualflare-json" — the format it // arrived in. if f := domain.Framework(collect.Framework); f.IsValid() && - f != domain.FrameworkQualflareJSON && f != domain.FrameworkCTRF { + f != domain.FrameworkQualflareJSON && f != domain.FrameworkCTRF && f != domain.FrameworkAllure { if suite.Properties == nil { suite.Properties = map[string]string{} } diff --git a/internal/core/domain/models.go b/internal/core/domain/models.go index 338fefc..5b7ea5a 100644 --- a/internal/core/domain/models.go +++ b/internal/core/domain/models.go @@ -24,6 +24,11 @@ const ( // particular can otherwise only arrive as generic JUnit XML. // See internal/adapters/parsers/generic/ctrf. FrameworkCTRF Framework = "ctrf" + // FrameworkAllure ingests an Allure 2 results directory (allure-results/): + // one *-result.json per test execution, written by the Allure adapter of + // whatever framework ran the tests. Like CTRF it is a FORMAT, not a tool — + // the producing framework comes from each result's "framework" label. + FrameworkAllure Framework = "allure" // Unit Testing Frameworks FrameworkPython Framework = "python" @@ -137,6 +142,7 @@ func AllFrameworks() []Framework { return []Framework{ FrameworkJUnit, FrameworkCTRF, + FrameworkAllure, FrameworkPython, FrameworkGolang, FrameworkJest, @@ -195,7 +201,7 @@ const ( // degrades to CategoryGeneric instead of round-tripping verbatim. func (f Framework) GetCategory() FrameworkCategory { switch f { - case FrameworkQualflareJSON, FrameworkCTRF: + case FrameworkQualflareJSON, FrameworkCTRF, FrameworkAllure: // Both are PASSTHROUGH formats: the real per-suite category comes from // the file's own embedded producer identity — qualflare-json from its // `framework` field, CTRF from `results.tool.name` — not from this diff --git a/internal/core/domain/models_test.go b/internal/core/domain/models_test.go index 5f31249..6179137 100644 --- a/internal/core/domain/models_test.go +++ b/internal/core/domain/models_test.go @@ -148,7 +148,8 @@ func TestAllFrameworksAreValidAndCategorised(t *testing.T) { // FrameworkCategory("ctrf") would fail the server's oneof and reject the // whole launch, so this expectation is what pins the safe behaviour. want := FrameworkCategory(f) - if f == FrameworkQualflareJSON || f == FrameworkCTRF { + // Allure is the same case as CTRF: "allure" is not in the server's oneof. + if f == FrameworkQualflareJSON || f == FrameworkCTRF || f == FrameworkAllure { want = CategoryGeneric } // Detox is the third exception, and for the same reason as CTRF rather diff --git a/internal/core/services/report_service.go b/internal/core/services/report_service.go index 602f412..5ef7b2c 100644 --- a/internal/core/services/report_service.go +++ b/internal/core/services/report_service.go @@ -538,7 +538,7 @@ func unwrapPassthrough(f domain.Framework, suites []domain.Suite) domain.Framewo // isPassthroughFormat reports whether a framework value names a TRANSPORT rather // than a producing tool. func isPassthroughFormat(f domain.Framework) bool { - return f == domain.FrameworkQualflareJSON || f == domain.FrameworkCTRF + return f == domain.FrameworkQualflareJSON || f == domain.FrameworkCTRF || f == domain.FrameworkAllure } // producersOf returns the distinct producing frameworks the suites recorded, in a