diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000000..566ada042f143 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,147 @@ +name: Test +permissions: read-all +on: + merge_group: + types: [checks_requested] + push: + branches: + - develop + - release-* + pull_request: + branches: + - develop + - release-* + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true +jobs: + e2e_and_acceptance_coverage: + name: Verify all e2e/acceptance tests are included + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository so that local actions can be used + uses: actions/checkout@v4 + - name: Merge develop and set up dependencies + uses: ./.github/actions/merge-develop-and-set-up-dependencies + - name: Check that all e2e and acceptance test files are captured in wdio.conf.js and core/tests/ci-test-suite-configs + run: python -m scripts.check_tests_are_captured_in_ci + - name: Report failure if failed on oppia/oppia develop branch + if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}} + uses: ./.github/actions/send-webhook-notification + with: + message: "The e2e/acceptance coverage test failed on the upstream develop branch." + webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }} + build: + name: Build the app, and store build files as an artifact + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository so that local actions can be used + uses: actions/checkout@v4 + - name: Merge develop and set up dependencies + uses: ./.github/actions/merge-develop-and-set-up-dependencies + - name: Generate build files + uses: ./.github/actions/generate-build-files + acceptance_test_with_python_installation: + needs: [build] + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + suite: + - name: exploration-creator/preview-math-interactions + short_name: exploration-creator/preview-math-interactions + run1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + run2: [1, 2, 3, 4, 5] + mobile: [false, true] + name: ${{ matrix.mobile && 'M' || 'D' }}-${{ matrix.suite.short_name }} + steps: + - name: Check for broken suites + id: check_skip + # TODO(#22448): Remove once all suites are fixed. + run: | + broken_suites=("exploration-editor/manage-exploration-misconceptions" "exploration-editor/modify-translations-through-modal" "exploration-editor/publish-the-exploration-with-an-interaction" "practice-question-submitter/submit-practice-questions-with-different-interactions-and-difficulties") + for suite in "${broken_suites[@]}"; do + if [[ "${{ matrix.suite.name }}" == "$suite" ]]; then + echo "Suite '${{ matrix.suite.name }}' is marked to be skipped." + echo "SKIP_SUITE=true" >> $GITHUB_OUTPUT + break + fi + done + - name: Checkout repository so that local actions can be used + uses: actions/checkout@v4 + - name: Merge develop and set up dependencies + uses: ./.github/actions/merge-develop-and-set-up-dependencies + - name: Generate build files + uses: ./.github/actions/generate-build-files + - name: Generate modified suite name for artifacts + id: generate_suite_name_for_artifacts + # Replace slashes in the filename with underscores. + run: | + SUITE_NAME=${{ matrix.suite.name }}-${{ matrix.run1 }}-${{ matrix.run2 }}-${{ matrix.mobile }} + echo "MODIFIED_SUITE_NAME=${SUITE_NAME//\//_}" >> $GITHUB_OUTPUT + - name: Run Desktop Acceptance Test ${{ matrix.suite.name }} + # TODO(#22448): Remove once all suites are fixed. + if: ${{ steps.check_skip.outputs.SKIP_SUITE != 'true' }} + run: > + set -o pipefail; + VIDEO_RECORDING_IS_ENABLED=1 + xvfb-run -a --server-args="-screen 0, 1920x1080x24" + python -m scripts.run_acceptance_tests + --suite=${{ matrix.suite.name }} ${{ matrix.mobile && '--mobile' || '' }} + | tee desktop_acceptance_test_${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}.log + - name: Upload Desktop test recordings as Artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: desktop-acceptance-test-video-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}-original + path: /home/runner/work/oppia/oppia_full_stack_test_video_recordings/acceptance + - name: Upload desktop acceptance test logs + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: desktop-acceptance-test-logs-${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }} + path: desktop_acceptance_test_${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}.log + - name: Upload desktop test failure screenshots as artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: desktop-acceptance-test-failure-screenshot-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}} + path: /home/runner/work/oppia/oppia_full_stack_test_failure_screenshots/acceptance + # TODO(#22667): We are running this step twice, as snapshot gets deleted if 2nd attempt passes. + - name: Uploading diff screenshots as an artifact + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: diff-snapshots-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}} + # Note: The path must be kept in sync with the one specified in + # core/tests/puppeteer-acceptance-tests/utilities/common/puppeteer-utils.ts + path: /home/runner/work/oppia/oppia/core/tests/puppeteer-acceptance-tests/diff-snapshots + - name: Upload PNG snapshots + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: png-snapshots-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}-new + path: core/tests/puppeteer-acceptance-tests/specs/**/*.png + + # New job to collect all PNG snapshots into a single artifact + collect_all_screenshots: + name: Collect all screenshots into single artifact + needs: [acceptance_test_with_python_installation] + runs-on: ubuntu-22.04 + if: ${{ always() }} + steps: + - name: Download all PNG snapshot artifacts + uses: actions/download-artifact@v4 + with: + pattern: png-snapshots-*-new + path: temp-screenshots + merge-multiple: true + continue-on-error: true + + - name: Upload all screenshots as single artifact + uses: actions/upload-artifact@v4 + with: + name: all-png-snapshots-combined + path: temp-screenshots + if-no-files-found: warn diff --git a/core/tests/puppeteer-acceptance-tests/utilities/user/exploration-editor.ts b/core/tests/puppeteer-acceptance-tests/utilities/user/exploration-editor.ts index 93b4047422cdc..35c06027bcfb2 100644 --- a/core/tests/puppeteer-acceptance-tests/utilities/user/exploration-editor.ts +++ b/core/tests/puppeteer-acceptance-tests/utilities/user/exploration-editor.ts @@ -1679,9 +1679,7 @@ export class ExplorationEditor extends BaseUser { if (!algebricExpressionEditor) { throw new Error('Algebric expression editor not found.'); } - - await algebricExpressionEditor.click(); - await algebricExpressionEditor.type(solution); + await this.typeInInputField(algebricExpressionEditor, solution); await this.clickOnElementWithSelector(submitAnswerButton);