From 4d7f32fa2e57905497815f22621a36ef93444550 Mon Sep 17 00:00:00 2001 From: Nithilan Kumaran <126580000+sathariels@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:04:57 +0000 Subject: [PATCH 1/5] chore(deps): bump typesafe-sdk to 0.7.4 and GitHub Actions Lock typesafe-sdk at 0.7.4. The pyproject floor stays >=0.7.0: 0.7.1 through 0.7.4 are bug fixes, an optional http2 extra, stricter request checks, and non-finite JSON serialization, with no change to successful System One calls or response adaptation. Pin actions/checkout 7.0.1, actions/setup-python 7.0.0, and hashgraph-online/ai-plugin-scanner-action 1.2.777 by commit SHA. Checkout v5+ and setup-python v6+ run on Node 24. Checkout v7 only changes fork-PR checkout for pull_request_target and workflow_run. setup-python v7 drops the unused pip-install input. The scanner bump keeps plugin_dir, min_score, and fail_on_severity. --- .github/actions/jevcheck/action.yml | 2 +- .github/workflows/ci.yml | 8 ++++---- .github/workflows/jevcheck-example.yml | 4 ++-- .github/workflows/plugin-scan.yml | 4 ++-- examples/jevcheck.yml | 4 ++-- requirements.lock | 9 ++++++--- tests/test_sdk_transport.py | 2 +- 7 files changed, 18 insertions(+), 15 deletions(-) diff --git a/.github/actions/jevcheck/action.yml b/.github/actions/jevcheck/action.yml index 6328997..1b2a205 100644 --- a/.github/actions/jevcheck/action.yml +++ b/.github/actions/jevcheck/action.yml @@ -89,7 +89,7 @@ runs: using: composite steps: - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ inputs.python-version }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53d6aa5..dfbcddd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,10 @@ jobs: matrix: python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - name: Install @@ -29,10 +29,10 @@ jobs: packaging: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - name: Build wheel and sdist diff --git a/.github/workflows/jevcheck-example.yml b/.github/workflows/jevcheck-example.yml index 14d1d7d..95ad3a8 100644 --- a/.github/workflows/jevcheck-example.yml +++ b/.github/workflows/jevcheck-example.yml @@ -17,7 +17,7 @@ jobs: name: eval unchanged replay runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Gate candidate against contract (replay, no API key) @@ -32,7 +32,7 @@ jobs: name: compare unchanged replay runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Compare candidate replay to recorded baseline diff --git a/.github/workflows/plugin-scan.yml b/.github/workflows/plugin-scan.yml index 4af9134..44957b7 100644 --- a/.github/workflows/plugin-scan.yml +++ b/.github/workflows/plugin-scan.yml @@ -6,10 +6,10 @@ jobs: scan: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: hashgraph-online/ai-plugin-scanner-action@caba2e96aa8ad2feb6cf6fca52442b52e22e779f # v1.2.635 + - uses: hashgraph-online/ai-plugin-scanner-action@6f088dedc8f1f035eddd1597de8eacd8262a3862 # v1.2.777 with: plugin_dir: "." min_score: 80 diff --git a/examples/jevcheck.yml b/examples/jevcheck.yml index ad2d7e3..c044c67 100644 --- a/examples/jevcheck.yml +++ b/examples/jevcheck.yml @@ -20,7 +20,7 @@ jobs: name: eval unchanged replay runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Gate candidate against contract (replay, no API key) @@ -35,7 +35,7 @@ jobs: name: compare unchanged replay runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Compare candidate replay to recorded baseline diff --git a/requirements.lock b/requirements.lock index b8fc228..3409a26 100644 --- a/requirements.lock +++ b/requirements.lock @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --extra=dev --no-index --output-file=requirements.lock --strip-extras pyproject.toml +# pip-compile --extra=dev --output-file=requirements.lock --strip-extras pyproject.toml # annotated-types==0.8.0 # via pydantic @@ -21,7 +21,9 @@ idna==3.20 iniconfig==2.3.0 # via pytest packaging==26.3 - # via pytest + # via + # pytest + # typesafe-sdk pluggy==1.6.0 # via pytest pydantic==2.13.5 @@ -42,7 +44,8 @@ truststore==0.10.4 # via # httpcore2 # httpx2 -typesafe-sdk==0.7.0 + # typesafe-sdk +typesafe-sdk==0.7.4 # via jevcheck (pyproject.toml) typing-extensions==4.16.0 # via diff --git a/tests/test_sdk_transport.py b/tests/test_sdk_transport.py index 504931b..6fbecf8 100644 --- a/tests/test_sdk_transport.py +++ b/tests/test_sdk_transport.py @@ -1,4 +1,4 @@ -"""Regression tests against the real typesafe-sdk 0.7.0 with a mocked HTTP transport.""" +"""Regression tests against the locked typesafe-sdk 0.7.4 with a mocked HTTP transport.""" from __future__ import annotations From 28fe1623298121aa7f833ad3de3900eddbbac349 Mon Sep 17 00:00:00 2001 From: Nithilan Kumaran <126580000+sathariels@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:05:31 +0000 Subject: [PATCH 2/5] docs: mark v0.1 recheck P1/P2 findings resolved Verify the four remaining recheck findings against be58041 and the current tree, then record Resolved in be58041 (v0.2.0) on each. Point the README at that recheck as historical rather than open. --- README.md | 2 +- docs/release-readiness-audit-v0.1-recheck.md | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 89fe9b7..523d56a 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ python -m jevcheck compare fixtures/support-triage.json \ v0.2 two-model lock: [`docs/adr-009-two-model-compare.md`](https://github.com/sathariels/jevcheck/blob/main/docs/adr-009-two-model-compare.md). -See [`docs/release-readiness-audit-v0.1.md`](https://github.com/sathariels/jevcheck/blob/main/docs/release-readiness-audit-v0.1.md). Recheck: [`docs/release-readiness-audit-v0.1-recheck.md`](https://github.com/sathariels/jevcheck/blob/main/docs/release-readiness-audit-v0.1-recheck.md). +The v0.1 audit ([`docs/release-readiness-audit-v0.1.md`](https://github.com/sathariels/jevcheck/blob/main/docs/release-readiness-audit-v0.1.md)) and recheck ([`docs/release-readiness-audit-v0.1-recheck.md`](https://github.com/sathariels/jevcheck/blob/main/docs/release-readiness-audit-v0.1-recheck.md)) are historical. Each P1/P2 finding in the recheck is resolved in be58041 (v0.2.0). Release history: [`CHANGELOG.md`](https://github.com/sathariels/jevcheck/blob/main/CHANGELOG.md). Tag and publish steps: [`RELEASING.md`](https://github.com/sathariels/jevcheck/blob/main/RELEASING.md). diff --git a/docs/release-readiness-audit-v0.1-recheck.md b/docs/release-readiness-audit-v0.1-recheck.md index 2939b62..4702119 100644 --- a/docs/release-readiness-audit-v0.1-recheck.md +++ b/docs/release-readiness-audit-v0.1-recheck.md @@ -2,6 +2,8 @@ **Verdict: NOT READY, but substantially improved.** I would not tag exact commit `8b1852d05e952d66d7cee49cc6e49d14a10b1a5f` until the remaining effective-expectation and score-boundary defects are fixed, and the advertised alias opt-in is reconciled with response identity checking. +**Later status:** each remaining P1/P2 finding below is **Resolved in be58041 (v0.2.0)**. The recheck text is unchanged. The status lines were added after that commit was checked against the current tree. + Audited the latest fetched `origin/main`, advancing from `55986d9` to `8b1852d05e952d66d7cee49cc6e49d14a10b1a5f`. No source changes were made. Git status is clean. This is still fixture-versus-model evaluation, not execution of two versions; the updated README now says so explicitly. ## Verification completed @@ -39,6 +41,8 @@ Passing replay exits 0; breaking replay exits 1. Minimum-confidence and Noul pol ### 1. P1 — a baseline-confidence-only expectation is still ineffective +**Status: Resolved in be58041 (v0.2.0).** `require_actionable_resolved_expect()` rejects a baseline-confidence-only expect after defaults when neither `confidence_tolerance` nor `min_confidence` applies, and `evaluate_case` calls that check. The same choice at confidence 0.51 now raises `ValueError` instead of reporting compatible. + A choice contract with `expect: {"q": {"baseline_confidence": 0.95}}`, no defaults, and an actual confidence of **0.51** exits **0**, prints **unchanged**, and declares **compatible**. The response is otherwise coherent: selected `b`, probabilities `a=0.49`, `b=0.51`. `_require_applicable_expect()` treats `baseline_confidence` as sufficient, but `_confidence_diff()` only evaluates a drop when `confidence_tolerance` is also present. Without a floor, choice constraint, or resolved tolerance, nothing is actually tested. The documentation explicitly calls this an effective constraint, making the silent pass misleading. @@ -49,6 +53,8 @@ Suggested fix: validate effective constraints after resolving defaults; reject a ### 2. P2 — score tolerance still mishandles exact decimal boundaries +**Status: Resolved in be58041 (v0.2.0).** `_eval_score` uses `exceeds_tolerance()` with `FLOAT_TOLERANCE_EPS` (`1e-9`). Expected 1.4 versus actual 1.6 at `score_tolerance` 0.2 is unchanged. Same-rounded-level behavior is unchanged. + Expected score **1.4**, actual score **1.6**, `score_tolerance: 0.2` exits **1**, reporting an answer flip. The actual score response has valid legend and normalized probabilities (`1=0.4`, `2=0.6`). Decimal difference is exactly 0.2, which the documented inclusive tolerance should permit. Binary subtraction produces `0.20000000000000018`. Confidence and Noul now use a tolerant comparison; score still uses direct `<=`. The original 2.0/1.5/0.5 score test passes because those values are exactly representable in binary and does not catch this. @@ -59,6 +65,8 @@ Suggested fix: apply the agreed inclusive numerical-boundary policy to score tol ### 3. P2 — allowed aliases cannot accept their legitimate resolved identity +**Status: Resolved in be58041 (v0.2.0).** `require_response_identity()` accepts a nonempty concrete response model for an opted-in floating alias. `jev-preview` with `--allow-unpinned` and response `jev-1.13.0` is compatible. Concrete pins still require exact identity. + With `--candidate-model jev-preview --allow-unpinned`, a matching response whose `model` is `jev-1.13.0` exits **2**, saying it does not match the requested candidate. The official API explicitly returns the versioned ID that answered an alias request. The CLI opt-in passes initial pin validation, but unconditional exact identity matching then rejects the expected API behavior. This was reproduced with replay input, exercising the same `evaluate()` identity check used by live calls; no live alias request was made. [Official model/alias behavior](https://docs.typesafe.ai/models.md). @@ -69,6 +77,8 @@ Suggested fix: explicitly decide how an opted-in alias resolves to a concrete ve ### 4. P2 — malformed/missing score distributions still pass replay validation +**Status: Resolved in be58041 (v0.2.0).** `ScoreAnswer` requires a nonempty `legend` and a nonempty normalized probability map. `adapt_answer()` does not invent either field, so omitted, null, and empty maps fail validation (exit 2) on the replay path. + A score response with `score=2`, `confidence=0.9`, and omitted, null, or empty `probabilities` exits **0** in each case. The corresponding choice defects are fixed, but ScoreAnswer still defaults missing maps to empty, accepts empty maps, and the adapter drops explicit nulls. Missing legends likewise remain optional locally, even though the real SDK response schema requires those fields. The new contract documentation explicitly permits the empty-score-map path by only validating nonempty maps, so this is a remaining validation limitation and API parity gap, not a claim that the new implementation contradicts that particular documentation sentence. Replay can accept data that the SDK live path would reject. @@ -108,4 +118,4 @@ git -C work/repo status --short The evidence archive includes exact installed-CLI command arrays, outputs, synthetic inputs, scripts, and test/build results. The new artifact tests used SDK 0.7.0 and Pydantic 2.13.5. All original source edits were made upstream; this recheck modified no project source. -**Tag decision: no for this exact commit.** Most earlier blockers are genuinely fixed. Address the four remaining findings above, rerun the new reproductions, and the project will be much closer to a defensible fixture-evaluator v0.1.0 release. +**Tag decision: no for this exact commit.** Most earlier blockers are genuinely fixed. Address the four remaining findings above, rerun the new reproductions, and the project will be much closer to a defensible fixture-evaluator v0.1.0 release. Those four findings were later **Resolved in be58041 (v0.2.0)**; see the status line on each. From 5cbcbeea90e377dc810a7233d2c93b01a427d5bd Mon Sep 17 00:00:00 2001 From: Nithilan Kumaran <126580000+sathariels@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:09:33 +0000 Subject: [PATCH 3/5] fix(eval): identity-check the stripped candidate pin require_pinned returns the stripped model, and the CLI uses that value. evaluate() discarded it, so a concrete candidate with surrounding whitespace passed the pin check and then failed equality against the same id. --- src/jevcheck/eval.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jevcheck/eval.py b/src/jevcheck/eval.py index 0c61575..eba4424 100644 --- a/src/jevcheck/eval.py +++ b/src/jevcheck/eval.py @@ -135,7 +135,9 @@ def evaluate( opt_in = contract.allow_unpinned if allow_unpinned is None else allow_unpinned require_pinned(contract.baseline_model, allow_unpinned=opt_in) if candidate_model is not None: - require_pinned(candidate_model, allow_unpinned=opt_in) + # Identity must use the stripped pin. Discarding this return value + # let " jev-1.14 " pass require_pinned and then fail equality. + candidate_model = require_pinned(candidate_model, allow_unpinned=opt_in) results: list[CaseResult] = [] diffs: list[FieldDiff] = [] From c67498a10a18b1d7c7088b055bf190aef57c98b2 Mon Sep 17 00:00:00 2001 From: Nithilan Kumaran <126580000+sathariels@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:09:33 +0000 Subject: [PATCH 4/5] test: cover eval tolerance, expectations, and error paths Add cases for inclusive tolerance edges, several expects on one case, wrong-type and missing answers, identity failures, and alias resolution. Put pytest-cov on the dev extra and print a coverage report in CI without a fail threshold. --- .github/workflows/ci.yml | 2 +- pyproject.toml | 2 +- requirements.lock | 10 +- tests/test_eval.py | 78 ++++++ tests/test_eval_boundaries.py | 507 +++++++++++++++++++++++++++++++++- 5 files changed, 593 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfbcddd..13e7888 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - name: Install run: pip install -e ".[dev]" - name: Test - run: pytest + run: pytest --cov=jevcheck --cov-report=term-missing packaging: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 85d1c6e..dfa6935 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8"] +dev = ["pytest>=8", "pytest-cov>=5"] [project.scripts] jevcheck = "jevcheck.cli:main" diff --git a/requirements.lock b/requirements.lock index 3409a26..0f6cfae 100644 --- a/requirements.lock +++ b/requirements.lock @@ -8,6 +8,8 @@ annotated-types==0.8.0 # via pydantic anyio==4.15.1 # via httpx2 +coverage==7.16.2 + # via pytest-cov h11==0.16.0 # via httpcore2 httpcore2==2.13.0 @@ -25,7 +27,9 @@ packaging==26.3 # pytest # typesafe-sdk pluggy==1.6.0 - # via pytest + # via + # pytest + # pytest-cov pydantic==2.13.5 # via # jevcheck (pyproject.toml) @@ -37,6 +41,10 @@ pydantic-core==2.46.5 pygments==2.21.0 # via pytest pytest==9.1.1 + # via + # jevcheck (pyproject.toml) + # pytest-cov +pytest-cov==7.1.0 # via jevcheck (pyproject.toml) tenacity==9.1.4 # via typesafe-sdk diff --git a/tests/test_eval.py b/tests/test_eval.py index 3719710..443b686 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -132,3 +132,81 @@ def test_missing_answer_is_flip(contract: Contract) -> None: report = evaluate(contract, lambda case: replay[case.id], candidate_model="jev-1.14") assert report.results[0].outcome is Outcome.ANSWER_FLIP assert any(diff.detail == "missing answer" for diff in report.results[0].diffs) + + +def test_flip_outranks_regression_across_expectations(contract: Contract) -> None: + """One case can carry a flip, a regression, and an unchanged field. The case counts once, as a flip.""" + replay = { + "ticket-001": response( + "jev-1.14", + intent=choice("general", 0.93), + billing=noul(0.61), + urgency=score(2.0, 0.91), + ), + "ticket-002": response("jev-1.14", intent=choice("general", 0.88)), + "ticket-003": response( + "jev-1.14", + intent=choice("billing", 0.92), + urgency=score(2.0, 0.91), + ), + } + report = evaluate(contract, lambda case: replay[case.id], candidate_model="jev-1.14") + by_field = {diff.field: diff.outcome for diff in report.results[0].diffs} + assert by_field == { + "intent": Outcome.ANSWER_FLIP, + "billing": Outcome.CONFIDENCE_REGRESSION, + "urgency": Outcome.UNCHANGED, + } + assert report.results[0].outcome is Outcome.ANSWER_FLIP + assert report.answer_flips == 1 + assert report.confidence_regressions == 0 + assert report.unchanged == 2 + assert "billing→general" in report.summary() + assert "unchanged" not in report.summary().split("behavioral flips:")[1].split("breaking")[0] + + +def test_regression_when_only_one_expectation_drops(contract: Contract) -> None: + replay = { + "ticket-001": response( + "jev-1.14", + intent=choice("billing", 0.80), + billing=noul(0.95), + urgency=score(1.9, 0.9), + ), + "ticket-002": response("jev-1.14", intent=choice("general", 0.88)), + "ticket-003": response( + "jev-1.14", + intent=choice("billing", 0.92), + urgency=score(2.0, 0.91), + ), + } + report = evaluate(contract, lambda case: replay[case.id], candidate_model="jev-1.14") + by_field = {diff.field: diff.outcome for diff in report.results[0].diffs} + assert by_field["intent"] is Outcome.CONFIDENCE_REGRESSION + assert by_field["billing"] is Outcome.UNCHANGED + assert by_field["urgency"] is Outcome.UNCHANGED + assert report.results[0].outcome is Outcome.CONFIDENCE_REGRESSION + assert report.confidence_regressions == 1 + assert report.answer_flips == 0 + + +def test_missing_one_of_several_answers_is_flip(contract: Contract) -> None: + replay = { + "ticket-001": response( + "jev-1.14", + intent=choice("billing", 0.93), + urgency=score(1.9, 0.9), + ), + "ticket-002": response("jev-1.14", intent=choice("general", 0.88)), + "ticket-003": response( + "jev-1.14", + intent=choice("billing", 0.92), + urgency=score(2.0, 0.91), + ), + } + report = evaluate(contract, lambda case: replay[case.id], candidate_model="jev-1.14") + missing = next(diff for diff in report.results[0].diffs if diff.field == "billing") + assert missing.outcome is Outcome.ANSWER_FLIP + assert missing.detail == "missing answer" + assert missing.expected == "yes" + assert report.results[0].outcome is Outcome.ANSWER_FLIP diff --git a/tests/test_eval_boundaries.py b/tests/test_eval_boundaries.py index 6604bf8..d4429ec 100644 --- a/tests/test_eval_boundaries.py +++ b/tests/test_eval_boundaries.py @@ -3,9 +3,16 @@ import pytest from jevcheck.answers import JevResponse, ScoreAnswer -from jevcheck.contract import Case, Contract, ContractDefaults -from jevcheck.eval import Outcome, evaluate, evaluate_case, nearest_level -from jevcheck.pinning import ModelIdentityError +from jevcheck.contract import Case, Contract, ContractDefaults, FieldExpect +from jevcheck.eval import ( + Outcome, + evaluate, + evaluate_case, + evaluate_field, + exceeds_tolerance, + nearest_level, +) +from jevcheck.pinning import ModelIdentityError, UnpinnedModelError from tests.helpers import choice, noul, response, score @@ -192,3 +199,497 @@ def test_evaluate_does_not_report_requested_model_when_response_differs() -> Non lambda case: JevResponse(model="other", answers={"q": choice("a", 0.9)}), candidate_model="jev-1.14", ) + + +class _BareResponse: + def __init__(self, model: object, answers: dict) -> None: + self.model = model + self.answers = answers + + +def _floor_contract(*, min_confidence: float) -> Contract: + return Contract.model_validate( + { + "baseline_model": "jev-1.13", + "cases": [ + { + "id": "c1", + "state": "s", + "questions": { + "q": {"type": "choice", "criteria": {"a": None, "b": None}} + }, + "expect": { + "q": { + "choice": "a", + "baseline_confidence": min_confidence, + "confidence_tolerance": 0.5, + "min_confidence": min_confidence, + } + }, + } + ], + } + ) + + +def test_exceeds_tolerance_keeps_exact_and_epsilon_boundaries() -> None: + """Inclusive tolerance: binary overshoot of a decimal tenth stays inside; past 1e-9 does not.""" + assert (0.8 - 0.7) > 0.1 + assert exceeds_tolerance(0.8 - 0.7, 0.1) is False + assert exceeds_tolerance(0.1, 0.1) is False + assert exceeds_tolerance(0.1 + 1e-9, 0.1) is False + assert exceeds_tolerance(0.1 + 2e-9, 0.1) is True + assert exceeds_tolerance(0.0, 0.0) is False + assert exceeds_tolerance(-0.4, 0.1) is False + + +def test_confidence_increase_is_not_a_drop() -> None: + contract = _choice_contract(baseline_confidence=0.5, confidence_tolerance=0.1) + report = evaluate( + contract, + lambda case: response("jev-1.14", q=choice("a", 0.95)), + candidate_model="jev-1.14", + ) + assert report.results[0].outcome is Outcome.UNCHANGED + + +def test_zero_tolerance_accepts_exact_confidence_and_rejects_a_drop() -> None: + exact = _choice_contract(baseline_confidence=0.8, confidence_tolerance=0.0) + passed = evaluate( + exact, + lambda case: response("jev-1.14", q=choice("a", 0.8)), + candidate_model="jev-1.14", + ) + assert passed.results[0].outcome is Outcome.UNCHANGED + dropped = evaluate( + exact, + lambda case: response("jev-1.14", q=choice("a", 0.799)), + candidate_model="jev-1.14", + ) + assert dropped.results[0].outcome is Outcome.CONFIDENCE_REGRESSION + + +def test_min_confidence_boundary_is_inclusive() -> None: + contract = Contract.model_validate( + { + "baseline_model": "jev-1.13", + "cases": [ + { + "id": "c1", + "state": "s", + "questions": { + "q": {"type": "choice", "criteria": {"a": None, "b": None}} + }, + "expect": {"q": {"choice": "a", "min_confidence": 0.8}}, + } + ], + } + ) + at_floor = evaluate( + contract, + lambda case: response("jev-1.14", q=choice("a", 0.8)), + candidate_model="jev-1.14", + ) + assert at_floor.results[0].outcome is Outcome.UNCHANGED + below = evaluate( + contract, + lambda case: response("jev-1.14", q=choice("a", 0.7)), + candidate_model="jev-1.14", + ) + assert below.results[0].outcome is Outcome.CONFIDENCE_REGRESSION + assert below.results[0].diffs[0].detail == "0.70 < min 0.80" + + +def test_choice_constraints_short_circuit_on_flip() -> None: + """A label change is a flip even when confidence would also miss the floor.""" + contract = _floor_contract(min_confidence=0.9) + report = evaluate( + contract, + lambda case: response("jev-1.14", q=choice("b", 0.4, other="a")), + candidate_model="jev-1.14", + ) + diff = report.results[0].diffs[0] + assert diff.outcome is Outcome.ANSWER_FLIP + assert diff.detail == "a→b" + assert report.confidence_regressions == 0 + + +def test_noul_drift_just_past_decimal_boundary_fails() -> None: + contract = _noul_contract() + report = evaluate( + contract, + lambda case: response("jev-1.14", q=noul(0.699999)), + candidate_model="jev-1.14", + ) + assert report.results[0].outcome is Outcome.CONFIDENCE_REGRESSION + + +def test_noul_upward_drift_past_tolerance_is_regression() -> None: + contract = Contract.model_validate( + { + "baseline_model": "jev-1.13", + "defaults": {"noul_true_threshold": 0.5}, + "cases": [ + { + "id": "c1", + "state": "s", + "questions": {"q": {"type": "noul", "instructions": "yes?"}}, + "expect": {"q": {"noul_true": True, "noul": 0.5, "noul_tolerance": 0.1}}, + } + ], + } + ) + report = evaluate( + contract, + lambda case: response("jev-1.14", q=noul(0.7)), + candidate_model="jev-1.14", + ) + assert report.results[0].outcome is Outcome.CONFIDENCE_REGRESSION + assert report.results[0].diffs[0].detail == "0.50→0.70" + + +def test_noul_threshold_boundary_and_min_noul_floor() -> None: + contract = Contract.model_validate( + { + "baseline_model": "jev-1.13", + "defaults": {"noul_true_threshold": 0.5}, + "cases": [ + { + "id": "c1", + "state": "s", + "questions": {"q": {"type": "noul", "instructions": "yes?"}}, + "expect": { + "q": { + "noul_true": True, + "noul": 0.8, + "noul_tolerance": 0.2, + "min_noul": 0.8, + } + }, + } + ], + } + ) + at_floor = evaluate( + contract, + lambda case: response("jev-1.14", q=noul(0.8)), + candidate_model="jev-1.14", + ) + assert at_floor.results[0].outcome is Outcome.UNCHANGED + below_floor = evaluate( + contract, + lambda case: response("jev-1.14", q=noul(0.7)), + candidate_model="jev-1.14", + ) + assert below_floor.results[0].outcome is Outcome.CONFIDENCE_REGRESSION + assert "min_noul" in below_floor.results[0].diffs[0].detail + + polarity = Contract.model_validate( + { + "baseline_model": "jev-1.13", + "defaults": {"noul_true_threshold": 0.5}, + "cases": [ + { + "id": "c1", + "state": "s", + "questions": {"q": {"type": "noul", "instructions": "yes?"}}, + "expect": {"q": {"noul_true": True, "noul": 0.5, "noul_tolerance": 0.1}}, + } + ], + } + ) + on_threshold = evaluate( + polarity, + lambda case: response("jev-1.14", q=noul(0.5)), + candidate_model="jev-1.14", + ) + assert on_threshold.results[0].outcome is Outcome.UNCHANGED + under_threshold = evaluate( + polarity, + lambda case: response("jev-1.14", q=noul(0.499999)), + candidate_model="jev-1.14", + ) + assert under_threshold.results[0].outcome is Outcome.ANSWER_FLIP + assert "yes→no" in under_threshold.results[0].diffs[0].detail + + +def test_score_inside_tolerance_still_checks_confidence() -> None: + """Level-flip suppression does not skip the confidence floor.""" + contract = _score_contract(expected=1.4, tolerance=0.2) + # _score_contract pins min_confidence at 0.0; override with a real floor. + contract = contract.model_copy( + update={ + "cases": [ + contract.cases[0].model_copy( + update={ + "expect": { + "q": contract.cases[0].expect["q"].model_copy( + update={"min_confidence": 0.9} + ) + } + } + ) + ] + } + ) + actual = ScoreAnswer( + score=1.6, + confidence=0.4, + legend={"1": "mid", "2": "hi"}, + probabilities={"1": 0.4, "2": 0.6}, + ) + report = evaluate( + contract, + lambda case: response("jev-1.14", q=actual), + candidate_model="jev-1.14", + ) + assert nearest_level(1.4) != nearest_level(1.6) + assert report.results[0].outcome is Outcome.CONFIDENCE_REGRESSION + + +def test_score_tolerance_zero_exact_match_and_level_boundary() -> None: + contract = _score_contract(expected=1.5, tolerance=0.0) + exact = evaluate( + contract, + lambda case: response("jev-1.14", q=score(1.5, 0.9)), + candidate_model="jev-1.14", + ) + assert exact.results[0].outcome is Outcome.UNCHANGED + low = 1.5 - 1e-6 + assert nearest_level(1.5) != nearest_level(low) + flipped = evaluate( + contract, + lambda case: response("jev-1.14", q=score(low, 0.9)), + candidate_model="jev-1.14", + ) + assert flipped.results[0].outcome is Outcome.ANSWER_FLIP + + +def test_wrong_answer_type_is_flip_for_each_kind() -> None: + choice_report = evaluate( + _choice_contract(), + lambda case: response("jev-1.14", q=noul(0.9)), + candidate_model="jev-1.14", + ) + assert choice_report.results[0].diffs[0].detail == "choice→noul" + noul_report = evaluate( + _noul_contract(), + lambda case: response("jev-1.14", q=choice("a", 0.9)), + candidate_model="jev-1.14", + ) + assert noul_report.results[0].diffs[0].detail == "noul→choice" + score_report = evaluate( + _score_contract(expected=1.4, tolerance=0.2), + lambda case: response("jev-1.14", q=choice("a", 0.9)), + candidate_model="jev-1.14", + ) + assert score_report.results[0].diffs[0].detail == "score→choice" + assert score_report.answer_flips == 1 + + +def test_unsupported_question_type_is_flip() -> None: + diff = evaluate_field( + case_id="c1", + field="q", + question=object(), + expect=FieldExpect(min_confidence=0.0), + defaults=ContractDefaults(), + actual=choice("a", 0.9), + ) + assert diff.outcome is Outcome.ANSWER_FLIP + assert diff.detail.startswith("unsupported question type ") + + +def test_null_and_blank_response_models_are_identity_errors() -> None: + contract = _choice_contract() + answers = {"q": choice("a", 0.9)} + with pytest.raises(ModelIdentityError, match="null"): + evaluate( + contract, + lambda case: _BareResponse(None, answers), + candidate_model="jev-1.14", + ) + with pytest.raises(ModelIdentityError, match="nonempty"): + evaluate( + contract, + lambda case: _BareResponse(" ", answers), + candidate_model="jev-1.14", + ) + with pytest.raises(ModelIdentityError, match="nonempty"): + evaluate( + contract, + lambda case: _BareResponse("", answers), + candidate_model="jev-1.14", + ) + + +def test_padded_concrete_candidate_matches_stripped_pin() -> None: + contract = _choice_contract() + report = evaluate( + contract, + lambda case: response("jev-1.14", q=choice("a", 0.9)), + candidate_model=" jev-1.14 ", + ) + assert report.breaking is False + assert report.candidate_model == "jev-1.14" + + +def test_unpinned_candidate_and_fetch_errors() -> None: + contract = _choice_contract() + with pytest.raises(UnpinnedModelError): + evaluate( + contract, + lambda case: response("jev-latest", q=choice("a", 0.9)), + candidate_model="jev-preview", + ) + + def fetch(case: object) -> JevResponse: + raise RuntimeError("transport down") + + with pytest.raises(RuntimeError, match="transport down"): + evaluate(contract, fetch, candidate_model="jev-1.14") + + +def test_noul_polarity_is_inferred_from_expected_float() -> None: + """noul_true may be omitted; the expected float is compared with the threshold.""" + def contract(expected: float) -> Contract: + return Contract.model_validate( + { + "baseline_model": "jev-1.13", + "defaults": {"noul_true_threshold": 0.5}, + "cases": [ + { + "id": "c1", + "state": "s", + "questions": {"q": {"type": "noul", "instructions": "yes?"}}, + "expect": {"q": {"noul": expected, "noul_tolerance": 0.0}}, + } + ], + } + ) + + on_threshold = evaluate( + contract(0.5), + lambda case: response("jev-1.14", q=noul(0.5)), + candidate_model="jev-1.14", + ) + assert on_threshold.results[0].outcome is Outcome.UNCHANGED + below = evaluate( + contract(0.5), + lambda case: response("jev-1.14", q=noul(0.2)), + candidate_model="jev-1.14", + ) + assert below.results[0].outcome is Outcome.ANSWER_FLIP + assert "yes→no" in below.results[0].diffs[0].detail + no_side = evaluate( + contract(0.49), + lambda case: response("jev-1.14", q=noul(0.8)), + candidate_model="jev-1.14", + ) + assert no_side.results[0].outcome is Outcome.ANSWER_FLIP + assert "no→yes" in no_side.results[0].diffs[0].detail + + +def test_missing_answer_labels_noul_and_score() -> None: + noul_contract = Contract.model_validate( + { + "baseline_model": "jev-1.13", + "defaults": {"noul_true_threshold": 0.5}, + "cases": [ + { + "id": "c1", + "state": "s", + "questions": {"q": {"type": "noul", "instructions": "yes?"}}, + "expect": {"q": {"noul": 0.2, "noul_tolerance": 0.1}}, + } + ], + } + ) + missing_noul = evaluate( + noul_contract, + lambda case: JevResponse(model="jev-1.14", answers={}), + candidate_model="jev-1.14", + ) + assert missing_noul.results[0].diffs[0].detail == "missing answer" + assert missing_noul.results[0].diffs[0].expected == "no" + + missing_score = evaluate( + _score_contract(expected=1.4, tolerance=0.2), + lambda case: JevResponse(model="jev-1.14", answers={}), + candidate_model="jev-1.14", + ) + assert missing_score.results[0].diffs[0].expected == "1.40" + assert missing_score.results[0].outcome is Outcome.ANSWER_FLIP + + +def test_missing_answer_without_a_kind_label() -> None: + contract = Contract.model_validate( + { + "baseline_model": "jev-1.13", + "cases": [ + { + "id": "c1", + "state": "s", + "questions": { + "q": {"type": "choice", "criteria": {"a": None, "b": None}} + }, + "expect": {"q": {"min_confidence": 0.5}}, + } + ], + } + ) + report = evaluate( + contract, + lambda case: JevResponse(model="jev-1.14", answers={}), + candidate_model="jev-1.14", + ) + assert report.results[0].diffs[0].detail == "missing answer" + assert report.results[0].diffs[0].expected is None + assert report.results[0].outcome is Outcome.ANSWER_FLIP + + +def test_alias_cases_that_resolve_to_different_models_are_listed() -> None: + contract = Contract.model_validate( + { + "baseline_model": "jev-1.13", + "allow_unpinned": True, + "cases": [ + { + "id": "c1", + "state": "s", + "questions": { + "q": {"type": "choice", "criteria": {"a": None, "b": None}} + }, + "expect": {"q": {"choice": "a", "min_confidence": 0.0}}, + }, + { + "id": "c2", + "state": "s", + "questions": { + "q": {"type": "choice", "criteria": {"a": None, "b": None}} + }, + "expect": {"q": {"choice": "a", "min_confidence": 0.0}}, + }, + ], + } + ) + models = {"c1": "jev-1.13.0", "c2": "jev-1.14.0"} + report = evaluate( + contract, + lambda case: response(models[case.id], q=choice("a", 0.9)), + candidate_model="jev-preview", + allow_unpinned=True, + ) + assert report.breaking is False + assert report.candidate_model == "jev-1.13.0, jev-1.14.0" + + +def test_replay_without_candidate_skips_identity() -> None: + contract = _choice_contract() + report = evaluate( + contract, + lambda case: response("other-model", q=choice("a", 0.9)), + ) + assert report.candidate_model is None + assert report.breaking is False + assert "(replay)" in report.summary() From 671a6993a9a57e43c01f562c98410b471e5a1bb3 Mon Sep 17 00:00:00 2001 From: Nithilan Kumaran <126580000+sathariels@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:10:34 +0000 Subject: [PATCH 5/5] chore(release): prepare 0.2.2 Bump the package, module, and Action default to 0.2.2. Document @v0.2.2 refs and the changelog. Keep the in-repo example workflow on published jevcheck 0.2.1 until the post-merge PyPI upload. --- .github/actions/jevcheck/action.yml | 2 +- .github/workflows/jevcheck-example.yml | 5 ++++ CHANGELOG.md | 19 ++++++++++++ README.md | 10 +++---- RELEASING.md | 41 ++++++++++++++------------ docs/tutorial.md | 6 ++-- examples/README.md | 4 +-- examples/jevcheck.yml | 6 ++-- pyproject.toml | 2 +- src/jevcheck/__init__.py | 2 +- tests/test_cli.py | 4 +-- 11 files changed, 64 insertions(+), 37 deletions(-) diff --git a/.github/actions/jevcheck/action.yml b/.github/actions/jevcheck/action.yml index 1b2a205..16eabef 100644 --- a/.github/actions/jevcheck/action.yml +++ b/.github/actions/jevcheck/action.yml @@ -77,7 +77,7 @@ inputs: jevcheck-version: description: PyPI version pin installed as `jevcheck==`. required: false - default: "0.2.1" + default: "0.2.2" working-directory: description: > Directory to run the CLI from. Contract, replay, and `out` paths are diff --git a/.github/workflows/jevcheck-example.yml b/.github/workflows/jevcheck-example.yml index 95ad3a8..f2c907d 100644 --- a/.github/workflows/jevcheck-example.yml +++ b/.github/workflows/jevcheck-example.yml @@ -23,6 +23,9 @@ jobs: - name: Gate candidate against contract (replay, no API key) uses: ./.github/actions/jevcheck with: + # Action default is 0.2.2, which is not on PyPI until the post-merge + # upload. Install the published package so this check stays green. + jevcheck-version: "0.2.1" contract: fixtures/support-triage.json command: eval candidate-model: jev-1.14 @@ -38,6 +41,8 @@ jobs: - name: Compare candidate replay to recorded baseline uses: ./.github/actions/jevcheck with: + # Same published pin as the eval job. Drop this input after 0.2.2 is on PyPI. + jevcheck-version: "0.2.1" contract: fixtures/support-triage.json command: compare from: fixtures/replay-baseline.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 321c448..b86ad57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.2.2] - 2026-10-13 + +### Added + +- Eval tests for tolerance edges, several expectations on one case, wrong-type and missing answers, identity failures, and alias resolution. +- `pytest-cov` on the `dev` extra. CI prints a coverage report and does not fail on a coverage threshold. + +### Changed + +- `requirements.lock` pins `typesafe-sdk==0.7.4`. The `pyproject.toml` floor stays `typesafe-sdk>=0.7.0`. Releases 0.7.1 through 0.7.4 are bug fixes, an optional `http2` extra, stricter request validation, and non-finite number serialization. They do not change a successful System One call or jevcheck's response adaptation. +- GitHub Actions stay pinned by commit SHA: `actions/checkout` 7.0.1, `actions/setup-python` 7.0.0, and `hashgraph-online/ai-plugin-scanner-action` 1.2.777 (from 4.2.2, 5.6.0, and 1.2.635). Checkout v5+ and setup-python v6+ run on Node 24 and need Actions runner v2.327.1 or newer. Checkout v7 refuses fork-PR checkouts for `pull_request_target` and `workflow_run`; these workflows do not use those events. `persist-credentials` and `python-version` are unchanged. setup-python v7 removes the unused `pip-install` input. The scanner bump is a patch; `plugin_dir`, `min_score`, and `fail_on_severity` are unchanged. +- Package version is `0.2.2` in `pyproject.toml`, `jevcheck.__version__`, and the Action input `jevcheck-version` (default `0.2.2`). Documented Action refs pin `@v0.2.2`. `@v0.2.1` remains the first tag that contains the Action. +- The v0.1 recheck's four P1/P2 findings are marked resolved in be58041 (v0.2.0). + +### Fixed + +- `evaluate()` identity-checks the stripped candidate returned by `require_pinned`. A concrete name with surrounding whitespace no longer passes the pin check and then fails against the same model id. + ## [0.2.1] - 2026-09-30 Library source since [0.2.0] is the version string only. CLI commands, exit codes, and eval rules are unchanged. @@ -57,6 +75,7 @@ Library source since [0.2.0] is the version string only. CLI commands, exit code - A baseline-confidence-only expect needs a floor or tolerance after defaults. An opted-in floating alias accepts a concrete resolved response model. Replay and adapter paths require nonempty normalized score probabilities and a legend. - The sdist ships tests and fixtures. +[0.2.2]: https://github.com/sathariels/jevcheck/compare/v0.2.1...v0.2.2 [0.2.1]: https://github.com/sathariels/jevcheck/compare/v0.2.0...v0.2.1 [0.2.0]: https://github.com/sathariels/jevcheck/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/sathariels/jevcheck/releases/tag/v0.1.0 diff --git a/README.md b/README.md index 523d56a..84afd2a 100644 --- a/README.md +++ b/README.md @@ -122,10 +122,10 @@ A `Gate` helper exists for auto / ask-human / reject thresholds. It is optional Reuse jevcheck from other repos as a [composite action](https://github.com/sathariels/jevcheck/blob/main/.github/actions/jevcheck/action.yml). The action sets up Python, installs `jevcheck` from PyPI, and runs `eval` (default), `compare`, or `record`. A nonzero CLI exit fails the job (exit 1 is a breaking contract). -Composite actions live in a subdirectory, so the `uses:` ref must point at a commit or tag that contains `.github/actions/jevcheck`. Pin `@v0.2.1`, the first tag that includes this action: +Composite actions live in a subdirectory, so the `uses:` ref must point at a commit or tag that contains `.github/actions/jevcheck`. Pin `@v0.2.2`. `@v0.2.1` is the first tag that includes this action: ```yaml -- uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.1 +- uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.2 with: contract: contracts/support.json command: compare @@ -134,7 +134,7 @@ Composite actions live in a subdirectory, so the `uses:` ref must point at a com answers: fixtures/candidate-replay.json ``` -`v0.2.0` does not contain `.github/actions/jevcheck`. Pin `@v0.2.1` (or a later tag) for the Action. +`v0.2.0` does not contain `.github/actions/jevcheck`. Pin `@v0.2.2` (or a later tag). `@v0.2.1` is the first tag that includes the Action. This repo’s [example workflow](https://github.com/sathariels/jevcheck/blob/main/.github/workflows/jevcheck-example.yml) is the green CI proof: `eval` (and `compare`) on `fixtures/support-triage.json` + `fixtures/replay-unchanged.json`. No `TYPESAFE_API_KEY`. A breaking replay (`fixtures/replay-breaking.json`) exits 1; do not mark a job that uses it as a required check. @@ -153,7 +153,7 @@ This repo’s [example workflow](https://github.com/sathariels/jevcheck/blob/mai | `out` | — | `record --out` (required when `command` is `record`) | | `allow-unpinned` | `false` | `--allow-unpinned` | | `python-version` | `3.12` | `actions/setup-python` | -| `jevcheck-version` | `0.2.1` | `pip install jevcheck==…` | +| `jevcheck-version` | `0.2.2` | `pip install jevcheck==…` | | `working-directory` | — | `cd` before the CLI; paths are relative to it | `candidate-model` and `to` are aliases for the same candidate id. If both are set they must match. `from` and `from-model` stay mutually exclusive, same as the CLI. @@ -163,7 +163,7 @@ This repo’s [example workflow](https://github.com/sathariels/jevcheck/blob/mai Omit `answers` (and use `from-model` instead of `from`) only when the job has a TypeSafe key: ```yaml -- uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.1 +- uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.2 env: TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} with: diff --git a/RELEASING.md b/RELEASING.md index e1fc7d4..11beec2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -4,9 +4,11 @@ Tag, GitHub Release, and PyPI upload happen after the release pull request is me There is no publish workflow. `.github/workflows/` contains `ci.yml`, `jevcheck-example.yml`, `pr-triage.yml`, and `plugin-scan.yml` only. PyPI upload is a manual `twine upload` from the tag. -`v0.2.1` is the first tag that contains `.github/actions/jevcheck`. Documented `uses:` refs pin `@v0.2.1`. The composite action installs `jevcheck==0.2.1` from PyPI, so that install succeeds only after the upload below. +`v0.2.1` is on PyPI and is the first tag that contains `.github/actions/jevcheck`. Documented `uses:` refs for this release pin `@v0.2.2`. The composite action's default install is `jevcheck==0.2.2`, so that install succeeds only after the upload below. -## Post-merge steps for 0.2.1 +`.github/workflows/jevcheck-example.yml` passes `jevcheck-version: "0.2.1"` so the in-repo proof installs the published package and stays green before the upload. `ci.yml` installs the checkout and does not use PyPI. + +## Post-merge steps for 0.2.2 1. Merge the release pull request into `main`. 2. Tag that merge commit and push the tag: @@ -14,23 +16,23 @@ There is no publish workflow. `.github/workflows/` contains `ci.yml`, `jevcheck- ```bash git switch main git pull origin main - # HEAD must be the merge commit of the 0.2.1 release PR. - git tag -a v0.2.1 -m "v0.2.1 — GitHub Action, tutorial, and packaging metadata" - git push origin v0.2.1 + # HEAD must be the merge commit of the 0.2.2 release PR. + git tag -a v0.2.2 -m "v0.2.2 — dependency pins, eval tests, and audit status" + git push origin v0.2.2 ``` -3. Create the GitHub Release `v0.2.1` from that tag. Title: +3. Create the GitHub Release `v0.2.2` from that tag. Title: ```text - v0.2.1 — GitHub Action, tutorial, and packaging metadata + v0.2.2 — dependency pins, eval tests, and audit status ``` - Notes: use the [v0.2.1 release notes](#github-release-notes-for-v021) below. Paste them as the release body. They use `pip install jevcheck==0.2.1` with no "after PyPI publish" heading. + Notes: use the [v0.2.2 release notes](#github-release-notes-for-v022) below. Paste them as the release body. They use `pip install jevcheck==0.2.2` with no "after PyPI publish" heading. 4. From a clean checkout of the tag, build and upload: ```bash - git switch --detach v0.2.1 + git switch --detach v0.2.2 python3 -m venv .venv-release source .venv-release/bin/activate python -m pip install --upgrade build twine @@ -41,33 +43,34 @@ There is no publish workflow. `.github/workflows/` contains `ci.yml`, `jevcheck- `twine check` should pass with no deprecation warnings. `twine upload` uses the owner's PyPI credentials for the `jevcheck` project. -5. After the upload, `.github/workflows/jevcheck-example.yml` on `main` can install `jevcheck==0.2.1`. Until then, that workflow's PyPI install fails. `ci.yml` does not install from PyPI. +5. After the upload, remove `jevcheck-version: "0.2.1"` from `.github/workflows/jevcheck-example.yml` so that workflow uses the Action default `0.2.2`. -If the tag date is not 2026-09-30, update the `[0.2.1]` heading in `CHANGELOG.md` on a follow-up commit. The compare link at the bottom of the changelog resolves once `v0.2.1` exists. +If the tag date is not 2026-10-13, update the `[0.2.2]` heading in `CHANGELOG.md` on a follow-up commit. The compare link at the bottom of the changelog resolves once `v0.2.2` exists. -## GitHub release notes for v0.2.1 +## GitHub release notes for v0.2.2 Paste the following as the release body: ~~~~markdown ## Highlights -- Composite GitHub Action `.github/actions/jevcheck`. Pin `sathariels/jevcheck/.github/actions/jevcheck@v0.2.1`. `v0.2.0` does not contain the Action. -- First-run tutorial (`docs/tutorial.md`) and an offline `examples/` pack. Replay eval and compare need no API key. -- Optional jevtriage pull-request workflow, skipped when `TYPESAFE_API_KEY` is unset. -- SPDX `license = "MIT"`, Python 3.13 and 3.14 in classifiers and CI, changelog, and Homepage / Source / Issues / Changelog URLs. -- CLI commands, exit codes, and eval rules match 0.2.0. +- `typesafe-sdk` lock is 0.7.4. The install floor stays `typesafe-sdk>=0.7.0`. +- GitHub Actions pins: `actions/checkout` 7.0.1, `actions/setup-python` 7.0.0, and `hashgraph-online/ai-plugin-scanner-action` 1.2.777, still by commit SHA. +- Eval coverage for tolerance edges, multiple expectations, and error paths. CI prints a coverage report. +- `evaluate()` accepts a concrete candidate pin that has surrounding whitespace. +- The v0.1 recheck P1/P2 findings are marked resolved in be58041 (v0.2.0). +- Pin `sathariels/jevcheck/.github/actions/jevcheck@v0.2.2`. `@v0.2.1` is the first tag that contains the Action. ## Install ``` -pip install jevcheck==0.2.1 +pip install jevcheck==0.2.2 ``` ## Action ```yaml -- uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.1 +- uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.2 with: contract: contracts/support.json command: eval diff --git a/docs/tutorial.md b/docs/tutorial.md index e3461a8..e494454 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -17,7 +17,7 @@ pip install -e ".[dev]" Or install the published package and copy the [examples pack](../examples/README.md) into your own repo: ```bash -pip install "jevcheck==0.2.1" +pip install "jevcheck==0.2.2" ``` `jevcheck` and `python -m jevcheck` run the same CLI. Use the module form when the `jevcheck` script is not on your `PATH`. @@ -152,7 +152,7 @@ Other repos can call the [composite action](../.github/actions/jevcheck/action.y Copy [`examples/jevcheck.yml`](../examples/jevcheck.yml) to `.github/workflows/jevcheck.yml`. It runs fixture-only `eval` and `compare` and does not set `TYPESAFE_API_KEY`. The breaking replay stays local so the workflow can remain a green check. ```yaml -- uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.1 +- uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.2 with: contract: examples/support.json command: compare @@ -161,7 +161,7 @@ Copy [`examples/jevcheck.yml`](../examples/jevcheck.yml) to `.github/workflows/j answers: examples/replay-unchanged.json ``` -Pin `@v0.2.1`, the first tag that contains `.github/actions/jevcheck`. `v0.2.0` does not include the action. This repo's in-tree proof is [`.github/workflows/jevcheck-example.yml`](../.github/workflows/jevcheck-example.yml), which points `uses:` at `./.github/actions/jevcheck` and the fuller `fixtures/` set. +Pin `@v0.2.2`. `@v0.2.1` is the first tag that contains `.github/actions/jevcheck`. `v0.2.0` does not include the action. This repo's in-tree proof is [`.github/workflows/jevcheck-example.yml`](../.github/workflows/jevcheck-example.yml), which points `uses:` at `./.github/actions/jevcheck` and the fuller `fixtures/` set. It installs published `jevcheck==0.2.1` until 0.2.2 is on PyPI. ## 7. Optional PR gate (jevtriage) diff --git a/examples/README.md b/examples/README.md index 0254435..55fb582 100644 --- a/examples/README.md +++ b/examples/README.md @@ -66,6 +66,6 @@ Exit 1. Copy `jevcheck.yml` to `.github/workflows/jevcheck.yml`. Leave this directory at the repo root so the workflow paths (`examples/support.json`, `examples/replay-unchanged.json`, `examples/replay-baseline.json`) resolve. If you move the JSON files, change those inputs. -The workflow calls `sathariels/jevcheck/.github/actions/jevcheck@v0.2.1`, installs `jevcheck==0.2.1` from PyPI, and runs `eval` plus `compare` on the unchanged replay. It does not set `TYPESAFE_API_KEY`. The breaking replay is omitted because exit 1 fails the job. +The workflow calls `sathariels/jevcheck/.github/actions/jevcheck@v0.2.2`, installs `jevcheck==0.2.2` from PyPI, and runs `eval` plus `compare` on the unchanged replay. It does not set `TYPESAFE_API_KEY`. The breaking replay is omitted because exit 1 fails the job. -Pin `@v0.2.1`, the first tag that contains `.github/actions/jevcheck`. `v0.2.0` does not include the action. +Pin `@v0.2.2`. `@v0.2.1` is the first tag that contains `.github/actions/jevcheck`. `v0.2.0` does not include the action. diff --git a/examples/jevcheck.yml b/examples/jevcheck.yml index c044c67..21a3c44 100644 --- a/examples/jevcheck.yml +++ b/examples/jevcheck.yml @@ -2,7 +2,7 @@ # Copy this file to .github/workflows/jevcheck.yml. # Paths assume examples/ stays at the repository root. # A breaking replay exits 1; keep that command local so this workflow can stay green. -# Pin @v0.2.1, the first tag that contains .github/actions/jevcheck. +# Pin @v0.2.2. @v0.2.1 is the first tag that contains .github/actions/jevcheck. # v0.2.0 does not include the action. name: jevcheck @@ -24,7 +24,7 @@ jobs: with: persist-credentials: false - name: Gate candidate against contract (replay, no API key) - uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.1 + uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.2 with: contract: examples/support.json command: eval @@ -39,7 +39,7 @@ jobs: with: persist-credentials: false - name: Compare candidate replay to recorded baseline - uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.1 + uses: sathariels/jevcheck/.github/actions/jevcheck@v0.2.2 with: contract: examples/support.json command: compare diff --git a/pyproject.toml b/pyproject.toml index dfa6935..8254830 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "jevcheck" -version = "0.2.1" +version = "0.2.2" description = "pytest for Jev — pin production behavioral contracts and eval candidate model upgrades" readme = "README.md" license = "MIT" diff --git a/src/jevcheck/__init__.py b/src/jevcheck/__init__.py index 6484692..e5ae7c1 100644 --- a/src/jevcheck/__init__.py +++ b/src/jevcheck/__init__.py @@ -21,7 +21,7 @@ require_response_identity, ) -__version__ = "0.2.1" +__version__ = "0.2.2" __all__ = [ "AUTH_ENV", diff --git a/tests/test_cli.py b/tests/test_cli.py index d6002d4..b5e40c7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,8 +9,8 @@ from tests.helpers import FIXTURES -def test_package_version_is_0_2_1() -> None: - assert jevcheck.__version__ == "0.2.1" +def test_package_version_is_0_2_2() -> None: + assert jevcheck.__version__ == "0.2.2" def test_cli_compatible_exit_zero() -> None: