diff --git a/tests/tasks/uipath-test/playwright_first_mile_e2e.yaml b/tests/tasks/uipath-test/playwright_first_mile_e2e.yaml new file mode 100644 index 0000000000..fe92751caf --- /dev/null +++ b/tests/tasks/uipath-test/playwright_first_mile_e2e.yaml @@ -0,0 +1,94 @@ +# Prerequisites (see PR description): +# - a CLI carrying the external-package commands (npm `latest` on 1.200+) +# - the eval tenant's Test Manager has Playwright support enabled +# - a folder the runner is a member of, with a Cloud Robots - Serverless +# machine, and working serverless Playwright execution +task_id: skill-test-playwright-first-mile-e2e +description: > + Full Playwright first mile: package a suite, upload it, let ingestion + auto-create the test cases, fill a test set by label, run only one Playwright + project, and collect the results. Graded on the saved report rather than + tenant state, and the agent cleans the project up afterwards. +tags: [uipath-test, e2e, mode:operate, lifecycle:setup, feature:test-case] + +run_limits: + turn_timeout: 1200 + +initial_prompt: | + I have a Playwright suite I want to run through Test Manager. Create a + minimal one in ./pw-suite with two passing tests and a config defining the + projects "chromium" and "firefox", then get it running from the Test Manager + project PWE2E (create it if needed) on chromium only — not firefox. + Save the final test report to ./report.json, then delete the project so my + tenant stays clean. + Do NOT ask for approval, confirmation, or feedback. + Do NOT pause between planning and implementation. + +success_criteria: + - type: command_executed + description: "Packaged and uploaded the suite" + tool_name: "Bash" + command_pattern: 'uip\s+(or|orchestrator)\s+packages\s+upload' + min_count: 1 + weight: 1.5 + pass_threshold: 1.0 + + - type: command_executed + description: "Filled the test set by label rather than hand-collected keys" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+testcases?\s+add\s+.*--labels' + min_count: 1 + weight: 1.5 + pass_threshold: 1.0 + + - type: command_executed + description: "Ran the test set scoped to the chromium project" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+testsets?\s+run\s+.*--playwright-projects\s+chromium' + min_count: 1 + weight: 3.0 + pass_threshold: 1.0 + + - type: command_not_executed + description: "Did not widen the run to a second Playwright project" + tool_name: "Bash" + command_pattern: '--playwright-projects\s+\S+\s+\S' + weight: 1.0 + pass_threshold: 1.0 + + - type: file_exists + description: "Saved the final report" + path: "report.json" + weight: 1.5 + pass_threshold: 1.0 + + - type: run_command + description: "Both tests passed — the run reached real results" + command: "grep -qE '\"Passed\":[[:space:]]*2' report.json" + timeout: 30 + expected_exit_code: 0 + weight: 5.0 + pass_threshold: 1.0 + + - type: run_command + description: "Nothing failed or was left without a result" + command: "grep -qE '\"Failed\":[[:space:]]*0' report.json" + timeout: 30 + expected_exit_code: 0 + weight: 2.0 + pass_threshold: 1.0 + + - type: command_executed + description: "Cleaned up the project it created" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+project\s+delete\s+.*PWE2E' + min_count: 1 + weight: 1.0 + pass_threshold: 1.0 + + - type: command_not_executed + description: "Did not use the Studio/RPA link-automation pipeline" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+testcases?\s+link-automation' + weight: 1.0 + pass_threshold: 1.0 diff --git a/tests/tasks/uipath-test/playwright_ingest_and_select_integration.yaml b/tests/tasks/uipath-test/playwright_ingest_and_select_integration.yaml new file mode 100644 index 0000000000..2b85fedc67 --- /dev/null +++ b/tests/tasks/uipath-test/playwright_ingest_and_select_integration.yaml @@ -0,0 +1,91 @@ +# Prerequisites (see PR description): +# - a CLI carrying the external-package commands (npm `latest` on 1.200+) +# - the eval tenant's Test Manager has Playwright support enabled +# - the released CLI registers `testsets playwright-context` on stable +# (UiPath/cli#3336) — this task's assertion reads that command's output +# Does NOT need serverless execution — it stops before running the test set. +task_id: skill-test-playwright-ingest-and-select +description: > + Integration test for the Playwright first mile up to (not including) + execution: package a suite, upload it, let ingestion auto-create the test + cases, fill a test set by the ingestion-applied labels, and confirm the set + resolves to a single Playwright package. Grades the recorded context rather + than tenant state, and the agent cleans the project up afterwards. +tags: [uipath-test, integration, mode:operate, lifecycle:setup, feature:test-case] + +initial_prompt: | + I have a Playwright suite I want to run from Test Manager. Create a minimal + one in ./pw-suite with two passing tests and a config defining the projects + "chromium" and "firefox", get it into the Test Manager project PWINT + (create it if needed), and collect its smoke tests into a test set. + Save what Test Manager reports about that test set's Playwright packaging to + ./context.json, then delete the project so my tenant stays clean. + +success_criteria: + - type: command_executed + description: "Packaged the suite as an external test package" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+pack\s+.*--type\s+playwright' + min_count: 1 + weight: 1.5 + pass_threshold: 1.0 + + - type: command_executed + description: "Uploaded the package to Orchestrator" + tool_name: "Bash" + command_pattern: 'uip\s+(or|orchestrator)\s+packages\s+upload' + min_count: 1 + weight: 1.5 + pass_threshold: 1.0 + + - type: command_executed + description: "Waited for ingestion by listing the project's test cases" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+testcases?\s+list\s+.*--project-key\s+PWINT' + min_count: 1 + weight: 2.0 + pass_threshold: 1.0 + + - type: command_not_executed + description: "Did not filter that list by package name — the filter never matches ingested test cases" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+testcases?\s+list\s+.*--filter' + weight: 1.0 + pass_threshold: 1.0 + + - type: command_executed + description: "Filled the test set by label rather than hand-collected keys" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+testcases?\s+add\s+.*--labels' + min_count: 1 + weight: 2.0 + pass_threshold: 1.0 + + - type: file_exists + description: "Recorded what Test Manager reported about the test set" + path: "context.json" + weight: 1.5 + pass_threshold: 1.0 + + - type: run_command + description: "Test set resolves to a single Playwright package, with its projects listed" + command: "grep -qiE '\"IsPlaywright\"[^,}]*true' context.json && grep -qi 'chromium' context.json" + timeout: 30 + expected_exit_code: 0 + weight: 3.0 + pass_threshold: 1.0 + + - type: command_executed + description: "Cleaned up the project it created" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+project\s+delete\s+.*PWINT' + min_count: 1 + weight: 1.0 + pass_threshold: 1.0 + + - type: command_not_executed + description: "Did not use the Studio/RPA link-automation pipeline" + tool_name: "Bash" + command_pattern: 'uip\s+tm\s+testcases?\s+link-automation' + weight: 1.0 + pass_threshold: 1.0