From ec7791478c3194aefc7e847dd0538b1a8294b06a Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Mon, 3 Aug 2026 16:20:38 +1000 Subject: [PATCH 1/2] Serve CI fixtures from localhost so no gating step is live Closes the exception Copilot found on #4. tests/good-links.html was scanned with fail-on-broken and no continue-on-error, so one gating step depended on github.com, python.org, jupyter.org and docs.python.org all staying reachable and redirect-free. A redirect appearing on any of them turns CI red with nothing wrong in the action. A genuine 200 over genuine HTTP was the reason that step had to stay: every other deterministic fixture either skips the request (ignore-patterns) or fails transport (.invalid), so none of them exercises a successful response. The workflow now starts a small server on 127.0.0.1 and asserts four paths against it end to end -- a clean page exits 0 under fail-on-broken, a 301 is followed and counted as one redirect, a 404 is reported, and the same 404 is silenced by silent-codes. That last pair is what the httpstat.us fixture was for, before the service stopped answering and took the coverage with it. The live scans stay, because real DNS, TLS and cross-host redirects are the one thing a local server cannot cover, but all three are now informational and none can gate. Verified by running the generated serve step under the composite shell and the real checker against the real fixtures: clean 0 broken / 0 redirects, moved 0 / 1, missing 1 / 0, and missing with silent-codes 404 back to 0 / 0. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 125 ++++++++++++++++++++++++++++++++++++++- CHANGELOG.md | 3 + tests/README.md | 7 ++- 3 files changed, 130 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24477e1..e7f480b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,12 +64,16 @@ jobs: run: | echo "Action completed with output: ${{ steps.test-mixed.outputs.broken-links-found }}" - - name: Test with good links only (should pass) + # Informational, like the scan below it. A clean page is asserted against + # the local server instead, so this no longer has to gate on github.com, + # python.org, jupyter.org and docs.python.org all staying redirect-free. + - name: Live scan of known-good links (informational, does not gate) + continue-on-error: true uses: ./ with: html-path: 'tests/good-links.html' fail-on-broken: 'true' - + # Informational only, and expected to report findings: the fixture holds # a deliberately unreachable host and three deliberate redirects. It runs # with fail-on-broken so that the PR-comment path is exercised, and with @@ -84,6 +88,112 @@ jobs: fail-on-broken: 'true' timeout: 30 + # A real 200 over real HTTP was the one thing only a public site could + # give us: every other deterministic fixture either skips the request + # (ignore-patterns) or fails transport (.invalid). Serving it from + # localhost keeps that coverage and removes the dependency, which is what + # lets every gating step in this job be offline. + # + # The routes also make the status-code paths assertable end to end for the + # first time. That is what the httpstat.us fixture was for, before the + # service stopped answering and took the coverage with it. + - name: Serve fixtures from localhost + run: | + mkdir -p ci-server ci-fixtures-clean ci-fixtures-redirect ci-fixtures-missing + + cat > ci-server/serve.py <<'PY' + from http.server import BaseHTTPRequestHandler, HTTPServer + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + path = self.path.split('?')[0] + if path == '/moved': + self.send_response(301) + self.send_header('Location', '/ok') + self.end_headers() + return + body = b'ok' + self.send_response(404 if path == '/missing' else 200) + self.send_header('Content-Type', 'text/html') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + HTTPServer(('127.0.0.1', 8123), Handler).serve_forever() + PY + + nohup python3 ci-server/serve.py >/dev/null 2>&1 & + for i in $(seq 1 40); do + curl -sf http://127.0.0.1:8123/ok >/dev/null && break + sleep 0.25 + done + if ! curl -sf http://127.0.0.1:8123/ok >/dev/null; then + echo "::error::local fixture server did not come up on 127.0.0.1:8123" + exit 1 + fi + echo "local fixture server ready" + + cat > ci-fixtures-clean/clean.html <<'HTML' + + Fine + Also fine + + HTML + + cat > ci-fixtures-redirect/moved.html <<'HTML' + + Moved + + HTML + + cat > ci-fixtures-missing/missing.html <<'HTML' + + Gone + + HTML + + # fail-on-broken is deliberately true: a clean page must exit 0 on its own, + # without waiting for the assertions at the end of the job. + - name: A clean page passes end to end + id: local-clean + uses: ./ + with: + html-path: 'ci-fixtures-clean' + fail-on-broken: 'true' + ai-suggestions: 'false' + timeout: 10 + + - name: A redirect is followed and counted + id: local-redirect + uses: ./ + with: + html-path: 'ci-fixtures-redirect' + fail-on-broken: 'false' + ai-suggestions: 'false' + timeout: 10 + + - name: An error status is reported + id: local-missing + uses: ./ + with: + html-path: 'ci-fixtures-missing' + fail-on-broken: 'false' + ai-suggestions: 'false' + timeout: 10 + + - name: silent-codes suppresses that error status + id: local-silenced + uses: ./ + with: + html-path: 'ci-fixtures-missing' + fail-on-broken: 'true' + ai-suggestions: 'false' + timeout: 10 + silent-codes: '404' + # Status 0 is what the checker reports when a request never reaches the # server. .invalid is reserved by RFC 2606 and can never resolve, so these # two links fail transport on every run, immediately and without contacting @@ -178,6 +288,17 @@ jobs: assert "unreachable found by default" "${{ steps.unreachable-loud.outputs.broken-links-found }}" "true" assert "unreachable silenced by code 0" "${{ steps.unreachable-quiet.outputs.broken-link-count }}" "0" assert "no findings once silenced" "${{ steps.unreachable-quiet.outputs.broken-links-found }}" "false" + assert "clean page has no broken links" "${{ steps.local-clean.outputs.broken-link-count }}" "0" + assert "clean page reports nothing" "${{ steps.local-clean.outputs.broken-links-found }}" "false" + assert "redirect counted" "${{ steps.local-redirect.outputs.redirect-count }}" "1" + assert "redirect is not broken" "${{ steps.local-redirect.outputs.broken-link-count }}" "0" + assert "error status reported" "${{ steps.local-missing.outputs.broken-link-count }}" "1" + assert "error status silenced by 404" "${{ steps.local-silenced.outputs.broken-link-count }}" "0" + assert "nothing found once 404 silenced" "${{ steps.local-silenced.outputs.broken-links-found }}" "false" + + - name: Stop the local fixture server + if: always() + run: pkill -f ci-server/serve.py || true test-empty-directory: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index f450156..42872f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- CI serves its own fixtures from `127.0.0.1`, so no gating step depends on the public internet any more. A clean page, a followed redirect, a reported error status and that status silenced by `silent-codes` are all asserted end to end against responses under our control. The scans of `good-links.html` and `broken-links.html` remain, as informational smoke that cannot turn CI red + ## [1.1.0] - 2026-08-03 ### Added diff --git a/tests/README.md b/tests/README.md index 21d695c..53ac1ae 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,9 +4,9 @@ This directory contains test HTML files and scripts used to validate the link-ch ## What may depend on the network -Most of the HTML fixtures here feed a **live smoke scan** that is informational and does not gate CI. Every *assertion* about the action's behaviour runs offline, against a mocked session or a fixture under `.invalid`. +The HTML fixtures here feed **live smoke scans** that are informational and do not gate CI. Nothing that gates touches the public internet: every gating step runs against a mocked session, a fixture under `.invalid`, or a server on `127.0.0.1` that the workflow starts itself. -One step is a deliberate exception. `good-links.html` is scanned with `fail-on-broken: 'true'` and no `continue-on-error`, so it gates — and its four targets are real sites. If any of them starts redirecting, or is down when CI runs, that step goes red without the action having changed. It is kept because a genuine 200 over genuine HTTP is the one thing a mock cannot give us, but treat a failure there as a fixture problem until you have ruled the action out. Nothing else that gates touches the public internet. +That last one is why the rule below can be absolute. A genuine 200 over genuine HTTP used to be the one thing only a public site could provide, so one gating step had to scan `good-links.html` and depend on four real sites staying redirect-free. The workflow now serves `/ok`, `/moved` and `/missing` from `127.0.0.1`, which covers a clean page, a followed redirect, a reported error status and that same status silenced by `silent-codes` — all deterministically, and all faster than a network round trip. **Do not point a fixture at a status-code service.** `httpstat.us` and `httpbin.org/redirect/3` were both used here and both stopped answering. When that happened the links reported `Status: 0 (Connection Error)` instead of the 404/500/503 they were named for, so the CI step called "test with silent codes" stopped exercising silent codes entirely — and because it carried `continue-on-error: true`, nothing went red to say so. A third-party service that returns a status on demand is a dependency that will fail this way eventually. @@ -18,7 +18,8 @@ Where each kind of coverage belongs: | Bot-blocking domain detection | `test_bot_blocking.py` | Pure substring matching on the URL; the request never has to succeed | | A host that cannot be reached | Workflow-generated fixtures under `.invalid` | RFC 2606 reserves `.invalid`, so it can never resolve — unlike a made-up name under a registrable TLD, which stops testing anything the day somebody buys it | | `ignore-patterns` and output plumbing | Workflow-generated fixtures + explicit assertions | Exact counts stay exact as fixtures are added to this directory | -| Real DNS, TLS and cross-host redirects | The fixtures here, live smoke scan | The one thing a mock genuinely cannot cover. Informational, except the `good-links.html` step noted above | +| A real request end to end — 200, redirect, error status, silenced status | A server on `127.0.0.1` started by the workflow | Exercises the whole path through `requests`, the shell and the outputs, with the response under our control rather than a third party's | +| Real DNS, TLS and cross-host redirects | The fixtures here, live smoke scans | The one thing even a local server cannot cover — so it is kept, and never gates | ## Test Files From 4de9cdd9da71e85b767ead1c27e5d449bf696ba3 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Mon, 3 Aug 2026 16:29:17 +1000 Subject: [PATCH 2/2] Give the fixture server an explicit route table The handler answered 200 to any path except /missing and /moved, so "A clean page passes end to end" only proved that the action requested something and got 200 -- not that it requested the URLs the fixture names. A typo in a generated fixture, or a regression that mangled the URL before the request, would have passed green. Routes are now exhaustive and anything unregistered is a 500 rather than a 404, so that silent-codes: '404' cannot suppress it. With a 404 default a typo in the missing fixture would still have been silenced and that step would still have passed. Verified by injecting a typo into each of the three fixtures in turn: clean goes from 0 broken to 1, redirect from 1 redirect to 0, and missing-with-silent-codes from 0 broken to 1. All three assertions go red where previously the first two would and the third would not. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7f480b..ad5b429 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,12 @@ jobs: cat > ci-server/serve.py <<'PY' from http.server import BaseHTTPRequestHandler, HTTPServer + # Every path the generated fixtures link to, and nothing else. The + # table is exhaustive on purpose: answering 200 to whatever is asked + # would let a typo in a fixture, or a regression that mangled the URL + # before the request, pass as a clean page. + ROUTES = {'/ok': 200, '/ok/two': 200, '/missing': 404} + class Handler(BaseHTTPRequestHandler): def do_GET(self): path = self.path.split('?')[0] @@ -112,8 +118,15 @@ jobs: self.send_header('Location', '/ok') self.end_headers() return + # An unregistered path means the fixtures and this table have + # drifted. 500 rather than 404 so that silent-codes: '404' + # cannot suppress it -- every assertion below goes red on a + # request nobody intended, instead of one of them passing. + status = ROUTES.get(path, 500) body = b'ok' - self.send_response(404 if path == '/missing' else 200) + if status == 500: + body = b'no such fixture route' + self.send_response(status) self.send_header('Content-Type', 'text/html') self.send_header('Content-Length', str(len(body))) self.end_headers()