diff --git a/README.en.md b/README.en.md new file mode 100644 index 0000000..bb51e79 --- /dev/null +++ b/README.en.md @@ -0,0 +1,187 @@ +# 🔌 QA API + pytest-bdd - restful-booker + +[![API Tests](https://github.com/ThomasTDS/qa-api-python/actions/workflows/tests.yml/badge.svg)](https://github.com/ThomasTDS/qa-api-python/actions/workflows/tests.yml) +[![codecov](https://codecov.io/gh/ThomasTDS/qa-api-python/branch/master/graph/badge.svg)](https://codecov.io/gh/ThomasTDS/qa-api-python) +[![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13-blue)](https://github.com/ThomasTDS/qa-api-python/actions/workflows/tests.yml) +[![License](https://img.shields.io/github/license/ThomasTDS/qa-api-python)](LICENSE) + +[Português](README.md) | **English** + +## Description + +This repository contains automated tests for the public **[restful-booker](https://restful-booker.herokuapp.com)** API, using **requests** (plain HTTP calls, no browser), **pytest-bdd (BDD/Gherkin)** and an **API Clients** layer. For API tests, the clients play the same role the Page Object Model plays in UI tests: they wrap the HTTP calls and keep endpoint and header details out of the steps and scenarios. + +The goal is to practice real-world API testing: authentication, full CRUD, the difference between PUT and PATCH, and authorization rules. + +The Gherkin scenarios, the test case matrix and the commit history are written in Portuguese, the project's main language. + +![Test run report generated by pytest-html](docs/assets/report-example.png) + +*Example of a test run report (screenshot). The live, always up-to-date report is at [thomastds.github.io/qa-api-python](https://thomastds.github.io/qa-api-python/).* + +--- + +## Project Structure + +```text +qa-api-python/ +├── .github/ +│ ├── ISSUE_TEMPLATE/ +│ │ └── bug_report.md # Issue template for real bugs +│ ├── workflows/ +│ │ ├── tests.yml # CI pipeline (push, PR and a scheduled daily run) +│ │ └── dependabot-auto-merge.yml # Auto-merge for Dependabot patch/minor PRs +│ └── dependabot.yml # Weekly dependency updates (uv and GitHub Actions) +├── docs/ +│ ├── assets/ # Images used in the README +│ └── test-cases.md # Test case traceability matrix +├── features/ # Gherkin scenarios (.feature) +├── steps/ # pytest-bdd steps (integration tests, hit the real API) +│ ├── test_.py # One module per .feature file, with the steps only it uses +│ ├── common_steps.py # Steps shared by more than one feature (tokens, created booking, 403, 405) +│ └── booking_data.py # Default booking and payloads with invalid fields +├── tests/ +│ └── unit/ # Isolated unit tests for api/ and models/ (requests-mock, no network) and the test case matrix check +├── api/ # API Clients (AuthApiClient and BookingApiClient, built on the shared BaseApiClient) +├── models/ # Pydantic schemas and derived types (shape of the API data) +├── reports/ # HTML report and coverage (coverage.xml) generated on every run (not versioned) +├── conftest.py # Fixtures (per-scenario context) and failure evidence capture +├── pyproject.toml # Dependencies and configuration (pytest, ruff, mypy) +├── uv.lock # Exact versions of every dependency (generated by uv) +├── .pre-commit-config.yaml # Pre-commit hooks (ruff, mypy) +├── codecov.yml # Coverage targets checked by Codecov on PRs +├── LICENSE +├── README.md # Portuguese version of this README +└── README.en.md # This file +``` + +--- + +### Install Dependencies + +The exact versions of every dependency, including transitive ones, are locked in `uv.lock`, generated by [uv](https://docs.astral.sh/uv/). With uv installed (see the [official documentation](https://docs.astral.sh/uv/getting-started/installation/), or `pip install uv`): + +``` +uv sync --extra dev # creates .venv and installs exactly what is in uv.lock +.venv\Scripts\activate # PowerShell / Windows +source .venv/bin/activate # bash / Linux / macOS + +pre-commit install +``` + +When you change a dependency in `pyproject.toml`, run `uv lock` and commit `uv.lock` along with it. CI installs with `uv sync --locked` and fails if the two files are out of sync. + +No browser is needed: these are API tests, so only the `requests` library is used to make the HTTP calls. + +### Run all tests + +``` +pytest +``` + +At the end of the run, an HTML report is generated at `reports/report.html` (not versioned). + +The report for the latest run on `master` is also published at **[thomastds.github.io/qa-api-python](https://thomastds.github.io/qa-api-python/)**, so there is no need to run anything locally or download a CI artifact. + +### Run only the smoke suite + +``` +pytest -m smoke +``` + +Runs only the most critical end-to-end flows (authentication, creation, retrieval, update and deletion; see [docs/test-cases.md](docs/test-cases.md)). Since `pyproject.toml` always writes to the same file, running this after `pytest` **overwrites** `reports/report.html` with just these 5 scenarios. + +### Run only the unit tests + +``` +pytest tests/unit +``` + +Isolated tests for `api/` and `models/`, with the HTTP calls mocked through [`requests-mock`](https://requests-mock.readthedocs.io/). They make no network calls, so they run in milliseconds and do not depend on the public API being up. They complement the scenarios in `steps/`, which are real integration tests against `restful-booker`. + +### Lint and formatting + +``` +mypy . # typecheck +ruff check . # lint +ruff format --check . # formatting, check only +ruff format . # formatting, apply fixes +pip-audit --skip-editable # checks installed dependencies against known vulnerabilities +``` + +CI runs `mypy`, `ruff check`, `ruff format --check` and `pip-audit` in the `unit` job, before any test against the API. Changes with type errors, style problems or a vulnerable dependency fail fast, without spending time on calls to the public API. A vulnerability found by `pip-audit` breaks the build, so nothing can be merged without fixing it or assessing that specific case. + +Every `pytest` run also measures coverage of `api/` and `models/` (the API Clients and schemas code), printed in the terminal and written to `reports/coverage.xml` (not versioned). In CI, this file is sent to [Codecov](https://codecov.io/gh/ThomasTDS/qa-api-python), which keeps the history and provides the badge at the top of this README. + +Codecov also blocks the PR merge (configured in `codecov.yml`) if the new or changed code in the PR (`patch`) is not 100% covered. `codecov.yml` also sets a target for total project coverage (`project`), which may not drop by more than 1 percentage point, but that check is informational only: it is not one of the branch's required checks. + +A pre-commit hook (the `pre-commit` framework, installed with `pre-commit install` after `uv sync`) runs `ruff --fix` and `ruff format` on staged files, and `mypy .` on the whole project, before every commit. Most lint, formatting or type problems are caught locally before they reach CI. + +### Run against another environment + +By default the tests target `https://restful-booker.herokuapp.com` and authenticate with the public credentials from the API documentation (`admin` / `password123`). To run against another environment (for example, a local or staging instance), set these environment variables: + +| Variable | Default | Purpose | +| -------------- | --------------------------------------- | ------------------------------------ | +| `BASE_URL` | `https://restful-booker.herokuapp.com` | API address | +| `API_USERNAME` | `admin` | Username used to get the token | +| `API_PASSWORD` | `password123` | Password used to get the token | + +``` +# PowerShell +$env:BASE_URL="http://localhost:3001"; $env:API_USERNAME="qa"; $env:API_PASSWORD="qa123"; pytest + +# bash +BASE_URL=http://localhost:3001 API_USERNAME=qa API_PASSWORD=qa123 pytest +``` + +--- + +### API under test + +| Method | Route | Purpose | +| ------ | -------------- | --------------------------------------------------- | +| POST | `/auth` | Creates an authentication token | +| GET | `/booking` | Lists booking IDs (accepts filters) | +| GET | `/booking/:id` | Gets a specific booking | +| POST | `/booking` | Creates a booking | +| PUT | `/booking/:id` | Updates a booking (all fields required) | +| PATCH | `/booking/:id` | Partially updates a booking | +| DELETE | `/booking/:id` | Deletes a booking | + +Authentication: `POST /auth` with `{ "username": "admin", "password": "password123" }` (public credentials from the API documentation, configurable through `API_USERNAME` and `API_PASSWORD`) returns a token, sent on `PUT`, `PATCH` and `DELETE` calls as the cookie `token=`. + +**Security note:** `POST /booking` (creation) **does not require authentication**: anyone can create bookings without a token. This is an access control flaw in the demo app itself, and it is covered explicitly in [features/create_booking.feature](features/create_booking.feature) as documented behavior, not as a test bug. + +--- + +### Test Structure and Applied Patterns + +- BDD / Gherkin: clear, readable scenarios in `.feature` files, written in Portuguese (`# language: pt`, with `Funcionalidade`, `Cenário`, `Dado`, `Quando` and `Então`, the Portuguese for Feature, Scenario, Given, When and Then) and run with `pytest-bdd`. +- API Client Objects: `AuthApiClient` and `BookingApiClient` wrap the HTTP calls, the same way Page Objects wrap UI elements. +- Timeout on every HTTP call: both clients inherit from `BaseApiClient`, which applies a 5-second limit to connect and 30 seconds to receive the response. Without it, `requests` can wait forever, and a hung API would leave CI stuck until the GitHub Actions run time limit. +- Test pyramid: besides the BDD scenarios (`steps/`), which are real integration tests against `restful-booker`, `tests/unit/` covers `api/` and `models/` in isolation, with `requests-mock` simulating the HTTP responses: no network, no dependency on the public API being up. +- Schema validation with [Pydantic](https://docs.pydantic.dev/): API responses are validated at runtime against the models in `models/booking.py` (single source of truth), not just typed through annotations. If the API changes the shape of a response, the test fails with a clear message instead of passing silently or breaking further along. The models use Pydantic's strict mode, which does not convert types on its own: a `totalprice` sent as text (`"150"`), a `depositpaid` as `"true"` or a date outside the `YYYY-MM-DD` format fails validation. Extra fields in the response are ignored, because the API belongs to a third party and an extra field is not a defect. +- Coverage of authentication, full CRUD, PUT vs. PATCH, and unauthorized access cases (403). +- Automated HTML report on every run (`reports/report.html`, through `pytest-html`), with the `master` version published at [thomastds.github.io/qa-api-python](https://thomastds.github.io/qa-api-python/) through GitHub Pages. +- Code coverage (`pytest-cov`) of `api/` and `models/` on every run, tracked on [Codecov](https://codecov.io/gh/ThomasTDS/qa-api-python). +- Automatic cleanup: the `context` fixture deletes the booking created in the scenario (with its own cleanup token) at the end of each test, so data does not pile up on the public API. If the deletion fails because of a network or authentication error, pytest shows a warning with the id of the booking left behind, without changing the test result. +- Unique test data: each created booking gets a random suffix in its last name (e.g. `Ciclano-3f9a1c`). Since the public API is shared, including by the parallel CI runs, searching by name finds only the test's own booking. +- Automatic retry for network failures only (`pytest-rerunfailures`, with `--reruns 1` and `only_rerun` in `pyproject.toml`): a test that fails because of a connection error or timeout runs a second time, absorbing the instability of the public demo API. Assertion or schema failures are never retried, so an intermittent bug cannot pass on the second attempt. +- Continuous integration with GitHub Actions, in two jobs: `unit` runs the static checks and the unit tests, with no network, and `integration` starts only if `unit` passes, running the full suite against the public API. A failure in `integration` with `unit` green therefore points to the API or a scenario, not to the clients' code. Both run on every push and pull request to `master`, on Python 3.12 and 3.13, and also daily at 06:00 UTC (see [.github/workflows/tests.yml](.github/workflows/tests.yml)) to catch breakage caused by the public API itself, with the HTML report published as a workflow artifact. +- Dependencies updated automatically by Dependabot (uv and GitHub Actions, weekly; see [.github/dependabot.yml](.github/dependabot.yml)). Patch/minor PRs with green CI are merged automatically ([.github/workflows/dependabot-auto-merge.yml](.github/workflows/dependabot-auto-merge.yml)); major bumps require manual review. +- Audit of known vulnerabilities in the installed dependencies on every CI run, with [`pip-audit`](https://github.com/pypa/pip-audit); finding a vulnerability breaks the build. +- QA traceability: test case matrix in [docs/test-cases.md](docs/test-cases.md), with `@TC-XXX` tags on every `Cenário` (scenario) and a `@smoke` subset (`pytest -m smoke`) covering the most critical end-to-end flows. The `TC-XXX` markers are registered automatically from the `.feature` tags, pytest runs with `--strict-markers` (a misspelled tag breaks collection), and the test [tests/unit/test_traceability.py](tests/unit/test_traceability.py) ensures the matrix lists exactly the TCs in the `.feature` files, with links to the right line of each scenario. Real bugs found are documented as GitHub Issues using the template in [.github/ISSUE_TEMPLATE/bug_report.md](.github/ISSUE_TEMPLATE/bug_report.md). + +--- + +### Workflow + +The `master` branch is protected: every change goes through a Pull Request, and merging is only allowed after the CI checks (`unit (3.12)`, `unit (3.13)`, `integration (3.12)` and `integration (3.13)`) and the Codecov new-code coverage check (`codecov/patch`) pass. Standard flow: + +``` +git checkout -b +# edit, run pytest locally +git push -u origin +# open a PR on GitHub, wait for the checks to pass, merge +``` diff --git a/README.md b/README.md index d81d72b..20f765e 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ [![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13-blue)](https://github.com/ThomasTDS/qa-api-python/actions/workflows/tests.yml) [![License](https://img.shields.io/github/license/ThomasTDS/qa-api-python)](LICENSE) +**Português** | [English](README.en.md) + ## Descrição Este repositório contém testes automatizados da API pública **[restful-booker](https://restful-booker.herokuapp.com)** utilizando **requests** (chamadas HTTP puras, sem navegador), **pytest-bdd (BDD/Gherkin)** e uma camada de **API Clients** que cumpre, para testes de API, o mesmo papel que o Page Object Model cumpre em testes de UI: encapsular as chamadas HTTP e esconder detalhes de endpoint/headers dos steps e dos cenários. @@ -47,7 +49,8 @@ qa-api-python/ ├── .pre-commit-config.yaml # Hooks de pre-commit (ruff, mypy) ├── codecov.yml # Metas de cobertura verificadas pelo Codecov nos PRs ├── LICENSE -└── README.md # Este arquivo +├── README.md # Este arquivo +└── README.en.md # Versão em inglês deste README ``` --- @@ -124,10 +127,10 @@ Por padrão os testes apontam para `https://restful-booker.herokuapp.com` e se a ``` # PowerShell -$env:BASE_URL="http://localhost:3001"; $env:API_USERNAME="usuario"; $env:API_PASSWORD="senha"; pytest +$env:BASE_URL="http://localhost:3001"; $env:API_USERNAME="qa"; $env:API_PASSWORD="qa123"; pytest # bash -BASE_URL=http://localhost:3001 API_USERNAME=usuario API_PASSWORD=senha pytest +BASE_URL=http://localhost:3001 API_USERNAME=qa API_PASSWORD=qa123 pytest ``` --- @@ -166,7 +169,7 @@ Autenticação: `POST /auth` com `{ "username": "admin", "password": "password12 - Integração contínua via GitHub Actions, em dois jobs: `unit` roda as checagens estáticas e os testes unitários, sem rede, e `integration` só começa se o `unit` passar, rodando a suíte completa contra a API pública. Assim, uma falha no `integration` com o `unit` verde aponta para a API ou para um cenário, e não para o código dos clients. Os dois rodam a cada push e pull request para `master`, em Python 3.12 e 3.13, e também diariamente às 06:00 UTC (ver [.github/workflows/tests.yml](.github/workflows/tests.yml)) para detectar quebras causadas pela própria API pública, com o relatório HTML publicado como artifact do workflow. - Dependências atualizadas automaticamente pelo Dependabot (uv e GitHub Actions, semanal — ver [.github/dependabot.yml](.github/dependabot.yml)). PRs de patch/minor com CI verde são mergeados automaticamente ([.github/workflows/dependabot-auto-merge.yml](.github/workflows/dependabot-auto-merge.yml)); bumps de major exigem revisão manual. - Auditoria de vulnerabilidades conhecidas nas dependências instaladas a cada execução do CI, via [`pip-audit`](https://github.com/pypa/pip-audit); encontrar uma vulnerabilidade quebra o build. -- Rastreabilidade de QA: matriz de test cases em [docs/test-cases.md](docs/test-cases.md), com tags `@TC-XXX` em cada `Cenário`. Os marcadores `TC-XXX` são registrados automaticamente a partir das tags dos `.feature`, o pytest roda com `--strict-markers` (uma tag com erro de digitação quebra a coleta), e o teste [tests/unit/test_traceability.py](tests/unit/test_traceability.py) garante que a matriz lista exatamente os TCs dos `.feature`, com links para a linha certa de cada cenário e um subconjunto `@smoke` (`pytest -m smoke`) cobrindo os fluxos ponta-a-ponta mais críticos. Bugs reais encontrados são documentados como GitHub Issues usando o template em [.github/ISSUE_TEMPLATE/bug_report.md](.github/ISSUE_TEMPLATE/bug_report.md). +- Rastreabilidade de QA: matriz de test cases em [docs/test-cases.md](docs/test-cases.md), com tags `@TC-XXX` em cada `Cenário` e um subconjunto `@smoke` (`pytest -m smoke`) cobrindo os fluxos ponta-a-ponta mais críticos. Os marcadores `TC-XXX` são registrados automaticamente a partir das tags dos `.feature`, o pytest roda com `--strict-markers` (uma tag com erro de digitação quebra a coleta), e o teste [tests/unit/test_traceability.py](tests/unit/test_traceability.py) garante que a matriz lista exatamente os TCs dos `.feature`, com links para a linha certa de cada cenário. Bugs reais encontrados são documentados como GitHub Issues usando o template em [.github/ISSUE_TEMPLATE/bug_report.md](.github/ISSUE_TEMPLATE/bug_report.md). --- @@ -175,8 +178,8 @@ Autenticação: `POST /auth` com `{ "username": "admin", "password": "password12 A branch `master` é protegida: toda mudança passa por Pull Request, e o merge só é liberado depois que os checks de CI (`unit (3.12)`, `unit (3.13)`, `integration (3.12)` e `integration (3.13)`) e de cobertura do código novo no Codecov (`codecov/patch`) passarem. Fluxo padrão: ``` -git checkout -b minha-branch +git checkout -b # editar, rodar pytest localmente -git push -u origin minha-branch +git push -u origin # abrir PR no GitHub, aguardar os checks passarem, fazer merge ``` diff --git a/tests/unit/test_readme_sync.py b/tests/unit/test_readme_sync.py new file mode 100644 index 0000000..fb0e6b0 --- /dev/null +++ b/tests/unit/test_readme_sync.py @@ -0,0 +1,71 @@ +import re +from pathlib import Path + +# O README existe em português (README.md, o principal) e em inglês +# (README.en.md). A tradução em si não dá para conferir automaticamente, mas +# a estrutura dá: se alguém atualizar só um dos dois (uma seção nova, um +# comando que mudou, um link novo), estes testes falham. +ROOT = Path(__file__).parents[2] +README_PT = ROOT / "README.md" +README_EN = ROOT / "README.en.md" + +FENCE = "```" +LINK = re.compile(r"\]\(([^)\s]+)\)") + + +def _blocks_and_prose(path: Path) -> tuple[list[list[str]], list[str]]: + blocks: list[list[str]] = [] + prose: list[str] = [] + current: list[str] | None = None + for line in path.read_text(encoding="utf-8").splitlines(): + if line.startswith(FENCE): + if current is None: + current = [] + else: + blocks.append(current) + current = None + elif current is not None: + current.append(line) + else: + prose.append(line) + return blocks, prose + + +def _heading_levels(path: Path) -> list[int]: + _, prose = _blocks_and_prose(path) + return [len(line) - len(line.lstrip("#")) for line in prose if line.startswith("#")] + + +# Comentários (# ...) são traduzidos, então ficam de fora da comparação. O +# resto de cada linha (comandos, nomes de arquivo) precisa ser idêntico. +def _code_without_comments(path: Path) -> list[list[str]]: + blocks, _ = _blocks_and_prose(path) + result = [] + for block in blocks: + lines = [line.split("#", 1)[0].rstrip() if not line.lstrip().startswith("#") else "" for line in block] + result.append([line for line in lines if line]) + return result + + +# Os links de troca de idioma apontam um para o outro, então ficam de fora. +def _links(path: Path) -> list[str]: + _, prose = _blocks_and_prose(path) + links = LINK.findall("\n".join(prose)) + return sorted(link for link in links if link not in ("README.md", "README.en.md")) + + +def test_both_readmes_link_to_each_other() -> None: + assert "](README.en.md)" in README_PT.read_text(encoding="utf-8") + assert "](README.md)" in README_EN.read_text(encoding="utf-8") + + +def test_both_readmes_have_the_same_sections() -> None: + assert _heading_levels(README_PT) == _heading_levels(README_EN) + + +def test_both_readmes_have_the_same_commands() -> None: + assert _code_without_comments(README_PT) == _code_without_comments(README_EN) + + +def test_both_readmes_have_the_same_links() -> None: + assert _links(README_PT) == _links(README_EN)