Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions corpus/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,28 @@ validator. See `corpus/semantic_verdict.py`'s module docstring and
No row in this corpus was changed to `any_of` by this change; `expect`
values remain chris's to set, same as any other row edit (see "## Versioning"
above).

## Version history

This corpus has no separate numeric version constant: the **corpus version
IS this file's git commit sha** (`git show <sha>:corpus/corpus.py`), which
is what a scoring run's `corpus_version` provenance field (see
`corpus/semantic_verdict.py:build_verdict`) records. Each entry below is a
commit (or PR) that changed a row's `expect` and moved the pass/fail bar —
"bumping the corpus version" means landing that commit; there is nothing
else to increment.

- **CHAOS-5620** — added the `any_of` shape and `corpus/expect_schema.py`
(`SCHEMA_VERSION`). No row's `expect` changed.
- **CHAOS-5721** — six rows whose scalar `expect="refuse"` had been
overtaken by the D27/D28/D33 servable-kind admissions
(`basis-discovered-incidents`, `basis-discovered-repo-count`,
`cv-org-count-projects`, `basis-grouped-metric-by-repo`,
`basis-grouped-pr-by-project`, `pos-grouped-per-phrasing`) rescored to
`any_of {serve with the row's own family, refuse}`.
`cv-scoped-projects-by-team-bounded`'s existing `any_of` had its `refuse`
alternative dropped: the role-aware declared-kind terminal (acr #544
`74d8fc4c`) removed the terminal error that alternative had certified as
a match. The prior scoring series for these seven rows stays in git
history, never overwritten; every future run against them scores under
this new expect.
42 changes: 26 additions & 16 deletions corpus/corpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,10 @@
note="carryover=chaos4632 NEGATIVE for anchor (possessive-grammar trap)", expect="serve"),
dict(id="pos-grouped-per-phrasing", text="Show me open incidents per repository.",
family="grouped_cohort_status", variant="grouped_members", member_kind="incident", group_kind="repository",
note="carryover=chaos4632; CHAOS-4926 ACCEPTANCE ROW #2 / ROW-10 INCIDENT SHAPE (expect refuse basis=member_kind_unservable)",
expect="refuse"),
note="carryover=chaos4632; CHAOS-4926 ACCEPTANCE ROW #2 / ROW-10 INCIDENT SHAPE; CHAOS-5721: rescored "
"any_of (D27/D28/D33 servable-kind admissions overtook the scalar refuse; corpus version bump, "
"old scoring series kept as history) -- was expect refuse basis=member_kind_unservable",
expect={"any_of": [{"outcome": "serve", "answer": {"family": "grouped_cohort_status"}}, {"outcome": "refuse"}]}),
dict(id="neg-explicit-comparison", text="Compare the acr project to the ask-dev project over the last 90 days.",
family="explicit_comparison", variant="explicit_set", member_kind=None, group_kind=None,
note="carryover=chaos4632; temporal=bounded_window; known pre-existing crash class per lane-rig-advance-13; "
Expand Down Expand Up @@ -83,30 +85,35 @@
dict(id="cv-org-count-projects", text="How many projects does the organization have in total?",
family="discovered_cohort_ranking", variant="discovered_kind", member_kind="project", group_kind=None,
note="org-wide count on a SERVABLE kind (project); contrasts with basis-discovered-repo-count (unservable); "
"D24: refuses stably post-CHAOS-5637 despite this SERVABLE design note -- revisit if organization_scope "
"member-kind widening (CHAOS-5641/D27) lands",
expect="refuse"),
"D24: refused stably post-CHAOS-5637 despite this SERVABLE design note; CHAOS-5721: rescored any_of "
"now that organization_scope member-kind widening (CHAOS-5641/D27, plus D28/D33) landed -- corpus "
"version bump, old scoring series kept as history",
expect={"any_of": [{"outcome": "serve", "answer": {"family": "discovered_cohort_ranking"}}, {"outcome": "refuse"}]}),

# --- C. Unservable member-kind basis -- CHAOS-4926 class (I6-legal, cohort-wire-contract-illegal) ---
# servableCohortKinds = {team, project} only (acr/internal/contextfabric/graphrank/cohort_kind.go:104-107).
# Every row below is a LEGAL frame (I6 satisfied: group_kind != member_kind where applicable) whose
# declared member_kind is outside the v1 cohort wire contract -- expect basis=member_kind_unservable.
dict(id="basis-discovered-repo-count", text="How many repositories are there across the organization?",
family="discovered_cohort_ranking", variant="discovered_kind", member_kind="repository", group_kind=None,
note="CHAOS-4926 class, 3rd instance; analog of recorded-13 C7 (org-wide repository count); expect refuse basis=member_kind_unservable",
expect="refuse"),
note="CHAOS-4926 class, 3rd instance; analog of recorded-13 C7 (org-wide repository count); CHAOS-5721: "
"rescored any_of (D27/D28/D33 overtook the scalar refuse; corpus version bump, old scoring series "
"kept as history) -- was expect refuse basis=member_kind_unservable",
expect={"any_of": [{"outcome": "serve", "answer": {"family": "discovered_cohort_ranking"}}, {"outcome": "refuse"}]}),
dict(id="basis-grouped-pr-by-project", text="Show me open pull requests per project.",
family="grouped_cohort_status", variant="grouped_members", member_kind="pull_request", group_kind="project",
note="CHAOS-4926 class; expect refuse basis=member_kind_unservable",
expect="refuse"),
note="CHAOS-4926 class; CHAOS-5721: rescored any_of (D27/D28/D33 overtook the scalar refuse; corpus "
"version bump, old scoring series kept as history) -- was expect refuse basis=member_kind_unservable",
expect={"any_of": [{"outcome": "serve", "answer": {"family": "grouped_cohort_status"}}, {"outcome": "refuse"}]}),
dict(id="basis-grouped-deployment-by-team", text="Show me deployments per team.",
family="grouped_cohort_status", variant="grouped_members", member_kind="deployment", group_kind="team",
note="CHAOS-4926 class; expect refuse basis=member_kind_unservable",
expect="refuse"),
dict(id="basis-discovered-incidents", text="Which incidents need the most attention right now?",
family="discovered_cohort_ranking", variant="discovered_kind", member_kind="incident", group_kind=None,
note="CHAOS-4926 class; expect refuse basis=member_kind_unservable",
expect="refuse"),
note="CHAOS-4926 class; CHAOS-5721: rescored any_of (D27/D28/D33 overtook the scalar refuse; corpus "
"version bump, old scoring series kept as history) -- was expect refuse basis=member_kind_unservable",
expect={"any_of": [{"outcome": "serve", "answer": {"family": "discovered_cohort_ranking"}}, {"outcome": "refuse"}]}),
dict(id="basis-scoped-workitems-by-project", text="What work items does the Dev Health Ops project have?",
family="scoped_cohort_status", variant="children_of_scope", member_kind="work_item", group_kind=None,
note="CHAOS-4926 class; anchor=Dev Health Ops/project; expect refuse basis=member_kind_unservable",
Expand All @@ -117,18 +124,21 @@
expect="refuse"),
dict(id="basis-grouped-metric-by-repo", text="Show me metrics per repository.",
family="grouped_cohort_status", variant="grouped_members", member_kind="metric", group_kind="repository",
note="CHAOS-4926 class; expect refuse basis=member_kind_unservable",
expect="refuse"),
note="CHAOS-4926 class; CHAOS-5721: rescored any_of (D27/D28/D33 overtook the scalar refuse; corpus "
"version bump, old scoring series kept as history) -- was expect refuse basis=member_kind_unservable",
expect={"any_of": [{"outcome": "serve", "answer": {"family": "grouped_cohort_status"}}, {"outcome": "refuse"}]}),

# --- D. Additional servable coverage -- goal/temporal breadth on servable kinds (team/project) ---
dict(id="cv-discovered-project-behind", text="Which projects are behind schedule?",
family="discovered_cohort_ranking", variant="discovered_kind", member_kind="project", group_kind=None,
note="SERVABLE; goal=rank_or_survey", expect="serve"),
dict(id="cv-scoped-projects-by-team-bounded", text="Which projects has the fullchaos team shipped in the last 30 days?",
family="scoped_cohort_status", variant="children_of_scope", member_kind="project", group_kind=None,
note="SERVABLE; anchor=fullchaos/team; temporal=bounded_window; D24: 2/3 reps unserved despite this "
"SERVABLE design note -- not yet stable enough for a scalar serve",
expect={"any_of": [{"outcome": "serve", "answer": {"family": "scoped_cohort_status"}}, {"outcome": "refuse"}]}),
note="SERVABLE; anchor=fullchaos/team; temporal=bounded_window; CHAOS-5721: refuse branch dropped now "
"that the role-aware declared-kind terminal landed (acr #544 74d8fc4c) -- the refuse branch had "
"certified a role-blind terminal error as agree; corpus version bump, old scoring series kept as "
"history",
expect={"any_of": [{"outcome": "serve", "answer": {"family": "scoped_cohort_status"}}]}),
dict(id="cv-grouped-allocate", text="How is engineering investment allocated across each team's projects?",
family="grouped_cohort_status", variant="grouped_members", member_kind="project", group_kind="team",
note="SERVABLE; goal=allocate_investment", expect="serve"),
Expand Down
112 changes: 103 additions & 9 deletions corpus/test_corpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -439,15 +439,15 @@ def mutated(mutate):
# "## Ownership"); this pin catches a silent edit to any of the 12, it
# does not judge whether any value is correct -- a pin that fails is
# reported, never "fixed" by updating the pin to match.
#
# CHAOS-5721: five of the original 12 (pos-grouped-per-phrasing,
# basis-discovered-repo-count, basis-grouped-pr-by-project,
# basis-discovered-incidents, basis-grouped-metric-by-repo) moved OFF this
# scalar pin -- rescored any_of, now pinned by PINNED_5721_ANY_OF below.
PINNED_REFUSE_DECLINE_EXPECT = {
"pos-grouped-per-phrasing": "refuse",
"basis-discovered-repo-count": "refuse",
"basis-grouped-pr-by-project": "refuse",
"basis-grouped-deployment-by-team": "refuse",
"basis-discovered-incidents": "refuse",
"basis-scoped-workitems-by-project": "refuse",
"basis-discovered-documents": "refuse",
"basis-grouped-metric-by-repo": "refuse",
"neg-nonexistent-team": "decline",
"neg-nonexistent-project": "decline",
"neg-nonexistent-repo-scope": "decline",
Expand All @@ -460,10 +460,12 @@ def mutated(mutate):
# into CHAOS-5597's 12 above so each pin's own origin/history stays legible --
# CHAOS-5597 is the first tranche, D24 the second, and drift between either
# tranche and the rest of the corpus is still caught (see the reverse check).
#
# CHAOS-5721: cv-org-count-projects moved OFF this scalar pin -- rescored
# any_of, now pinned by PINNED_5721_ANY_OF below.
PINNED_D24_REFUSE_DECLINE_EXPECT = {
"cv-c3-grouped-explain-change": "refuse",
"cv-b5-org-health": "refuse",
"cv-org-count-projects": "refuse",
"neg-open-question": "decline",
"neg-open-vague": "decline",
}
Expand Down Expand Up @@ -496,8 +498,8 @@ def _self_test_pin_guard_fires():
rejects a drifted pin -- a changed value, a pinned row deleted
outright, and a row drifting into (or out of) the refuse/decline set
without being added to (or removed from) the pin -- and that the
canonical shape (both the CHAOS-5597 12 and the D24 5) is still
accepted (CHAOS-5610, CHAOS-D24)."""
canonical shape (the current CHAOS-5597 and D24 scalar tranches, post
CHAOS-5721's any_of moves) is still accepted (CHAOS-5610, CHAOS-D24)."""
canonical = [{"id": row_id, "expect": expect}
for row_id, expect in {**PINNED_REFUSE_DECLINE_EXPECT,
**PINNED_D24_REFUSE_DECLINE_EXPECT}.items()]
Expand Down Expand Up @@ -566,10 +568,99 @@ def _stub_wrongly_accepts(_corpus):
"by the accepted-check, never let its own accept-message satisfy a later "
"reason-check")

# GREEN control: the canonical 12, untouched.
# GREEN control: the canonical scalar tranches, untouched.
_check_pinned_refuse_decline_expect(copy.deepcopy(canonical))


# CHAOS-5721: `any_of` for the six rows whose scalar `refuse` was overtaken
# by the D27/D28/D33 servable-kind admissions, plus the seventh row's
# existing any_of with its refuse branch dropped now that the role-aware
# declared-kind terminal (acr #544 74d8fc4c) removed the terminal error
# that branch had certified. Pinned by BRANCH SHAPE (outcome, and
# for a serve branch its answer family) -- never by question text -- so a
# silent edit to any of the seven rows' accepted alternatives is caught
# here, not "fixed" by updating the pin to match (corpus/README.md
# "## Ownership": row expectations are chris's call).
PINNED_5721_ANY_OF = {
"pos-grouped-per-phrasing": frozenset({("serve", "grouped_cohort_status"), ("refuse", None)}),
"cv-org-count-projects": frozenset({("serve", "discovered_cohort_ranking"), ("refuse", None)}),
"basis-discovered-repo-count": frozenset({("serve", "discovered_cohort_ranking"), ("refuse", None)}),
"basis-grouped-pr-by-project": frozenset({("serve", "grouped_cohort_status"), ("refuse", None)}),
"basis-discovered-incidents": frozenset({("serve", "discovered_cohort_ranking"), ("refuse", None)}),
"basis-grouped-metric-by-repo": frozenset({("serve", "grouped_cohort_status"), ("refuse", None)}),
"cv-scoped-projects-by-team-bounded": frozenset({("serve", "scoped_cohort_status")}),
}


def _branch_key(branch):
outcome = branch.get("outcome")
family = (branch.get("answer") or {}).get("family") if outcome == "serve" else None
return (outcome, family)


def _check_pinned_5721_any_of(corpus):
by_id = {row["id"]: row for row in corpus}
for row_id, want in PINNED_5721_ANY_OF.items():
_require(row_id in by_id,
f"pinned any_of row {row_id!r} (CHAOS-5721) no longer exists in CORPUS")
expect = by_id[row_id].get("expect")
_require(isinstance(expect, dict) and isinstance(expect.get("any_of"), list),
f"CORPUS[{row_id!r}].expect is no longer an any_of declaration (CHAOS-5721): got {expect!r} -- "
"row expectations are chris's call (corpus/README.md ## Ownership); report, do not silently "
"update the pin")
branches = expect["any_of"]
got = frozenset(_branch_key(b) for b in branches)
_require(len(branches) == len(want) and got == want,
f"CORPUS[{row_id!r}].expect.any_of alternatives changed: pinned {sorted(want)}, "
f"got {sorted(got)} (branch count pinned {len(want)}, got {len(branches)}) -- "
"row expectations are chris's call (corpus/README.md ## Ownership); report, do not silently "
"update the pin")


def _self_test_5721_any_of_pin_guard_fires():
"""RED CONTROLs: prove `_check_pinned_5721_any_of` rejects a drifted
branch set, a pinned row reverted to scalar, and a pinned row deleted
outright -- and that the canonical seven-row shape is still accepted."""
canonical = [
{"id": row_id, "expect": {"any_of": [
({"outcome": "serve", "answer": {"family": family}} if outcome == "serve"
else {"outcome": outcome})
for outcome, family in sorted(branches)
]}}
for row_id, branches in PINNED_5721_ANY_OF.items()
]

def mutated(mutate):
corpus = copy.deepcopy(canonical)
mutate(corpus)
return corpus

def _serve_branch(corpus, row_index):
return next(b for b in corpus[row_index]["expect"]["any_of"] if b["outcome"] == "serve")

cases = [
(mutated(lambda c: c[0]["expect"]["any_of"].pop()), "alternatives changed"),
(mutated(lambda c: _serve_branch(c, 0)["answer"].__setitem__("family", "explicit_comparison")),
"alternatives changed"),
(mutated(lambda c: c[0].__setitem__("expect", "refuse")), "no longer an any_of declaration"),
(mutated(lambda c: c.pop(0)), "no longer exists"),
]
for bad_corpus, must_mention in cases:
accepted = False
reason = None
try:
_check_pinned_5721_any_of(bad_corpus)
accepted = True
except CorpusValidationError as exc:
reason = str(exc)
_require(not accepted, f"RED CONTROL FAILED: drifted CHAOS-5721 pin accepted: {bad_corpus!r}")
_require(must_mention in reason,
f"RED CONTROL FAILED: drifted CHAOS-5721 pin rejected for the wrong reason: {reason}")

# GREEN control: the canonical seven-row shape, untouched.
_check_pinned_5721_any_of(copy.deepcopy(canonical))


# D24: every corpus row must now declare an `expect` except the two named
# below, which are intentionally left unset -- both hit an unresolved 5-turn
# clarification loop with no served text at all on the 2026-09-12 yardstick
Expand Down Expand Up @@ -633,6 +724,7 @@ def main():
_self_test_guard_fires()
_self_test_baseline_guard_fires()
_self_test_pin_guard_fires()
_self_test_5721_any_of_pin_guard_fires()

corpus = corpus_module.CORPUS
_require(isinstance(corpus, list) and corpus, "CORPUS must be a nonempty list")
Expand All @@ -653,6 +745,7 @@ def main():
sys.exit(1)

_check_pinned_refuse_decline_expect(corpus)
_check_pinned_5721_any_of(corpus)
_self_test_every_row_scored_guard_fires()
_check_every_row_scored_except_named(corpus)

Expand All @@ -661,6 +754,7 @@ def main():
print(f"PASS: {len(corpus)} corpus rows validated")
print(f"PASS: {len(PINNED_REFUSE_DECLINE_EXPECT)} CHAOS-5597 + {len(PINNED_D24_REFUSE_DECLINE_EXPECT)} D24 "
"refuse/decline expect values pinned and matched")
print(f"PASS: {len(PINNED_5721_ANY_OF)} CHAOS-5721 any_of row branch sets pinned and matched")
print(f"PASS: every corpus row scored except {sorted(UNSCORED_BY_DESIGN)} (CHAOS-5660)")
print(f"PASS: baseline sha256 pin matched, {baseline_rows} baseline rows shape-checked "
f"({dropped} historical-only, not in the current corpus)")
Expand Down