diff --git a/core/verification/baseline.py b/core/verification/baseline.py index 90de9ee..c07b36b 100644 --- a/core/verification/baseline.py +++ b/core/verification/baseline.py @@ -20,6 +20,14 @@ The verifier ids are those of the invariant definitions given (INV-SEC-001 for groups and rules, INV-SEC-003 for databases); a definition set without them makes the item UNSUPPORTED, never PASS. +**Which definition (C-216, P8b-2).** The item uses the definition the run's definitions hold for its +invariant, the highest version if several are present; its evidence (``observations``) records that +``invariant_id``, ``invariant_version`` and ``registry``, and a security item evaluates ``PIP(p)`` +under that definition's registry (version 1 -> registry-v1.0, version 2 -> registry-v1.1), so a +baseline item and its invariant item never use different rules. Before P8b-2 the lookup was fixed +at version 1, so under registry v1.1 every security baseline item was UNSUPPORTED +``invariant.unknown`` (found by the external review of PR #31). + Standard library and ``core.domain`` only. """ @@ -30,9 +38,11 @@ from typing import Any, Final from core.domain.enums import ResourceSupport, VerificationResult +from core.domain.invariant import Invariant from core.domain.registry_v1 import FUNC_RELEVANT_UNSUPPORTED from core.domain.resource import Resource from core.domain.verification import VerificationReason +from core.verification.dispatch import registry_of from core.verification.inputs import VerificationInputs from core.verification.outcome import CLAUSE_IPV4, CLAUSE_IPV6, combine from core.verification.public_ingress import GroupStatus, analyse @@ -66,21 +76,41 @@ class BaselineOutcome: verifier_version: str -def _verifier(inputs: VerificationInputs, invariant_id: str) -> tuple[str, str] | None: - definition = inputs.invariant(invariant_id, 1) - return None if definition is None else (definition.verifier_id, definition.verifier_version) +def _definition(inputs: VerificationInputs, invariant_id: str) -> Invariant | None: + """The run's definition of ``invariant_id``: the highest version present, or ``None``.""" + found = [d for d in inputs.invariants if d.invariant_id == invariant_id] + return max(found, key=lambda d: d.version) if found else None + + +def baseline_definition(inputs: VerificationInputs, kind: str) -> Invariant | None: + """The definition a baseline item of this resource type belongs to (C-216), if any.""" + if kind in _SECURITY_TYPES: + return _definition(inputs, "INV-SEC-001") + if kind == DB_INSTANCE_TYPE: + return _definition(inputs, "INV-SEC-003") + return None def baseline_verifier(inputs: VerificationInputs, kind: str) -> tuple[str, str] | None: """The verifier a baseline item of this resource type is attributed to, or ``None`` when the definitions do not have the invariant it belongs to.""" - if kind in _SECURITY_TYPES: - return _verifier(inputs, "INV-SEC-001") - if kind == DB_INSTANCE_TYPE: - return _verifier(inputs, "INV-SEC-003") + if kind in _SECURITY_TYPES or kind == DB_INSTANCE_TYPE: + definition = baseline_definition(inputs, kind) + return None if definition is None else (definition.verifier_id, definition.verifier_version) return BASELINE_VERIFIER +def _named(definition: Invariant | None) -> dict[str, Any]: + """What the evidence records about the definition the item used (C-216).""" + if definition is None: + return {} + return { + "invariant_id": definition.invariant_id, + "invariant_version": definition.version, + "registry": registry_of(definition), + } + + def _worst(values: list[VerificationResult]) -> VerificationResult: """FAIL if any value FAILs, otherwise the worst of UNSUPPORTED, UNKNOWN, PASS.""" return combine({str(index): value for index, value in enumerate(values)}) if values else P @@ -93,7 +123,7 @@ def _contributing_groups(resource: Resource) -> list[str]: def _security( - resource: Resource, inputs: VerificationInputs, ids: tuple[str, str] + resource: Resource, inputs: VerificationInputs, ids: tuple[str, str], definition: Invariant ) -> BaselineOutcome: groups = _contributing_groups(resource) predicates: dict[str, dict[str, str]] = {} @@ -105,6 +135,7 @@ def _security( inputs.candidate_resources, parent_resources=inputs.parent_resources, identity=inputs.identity, + registry=registry_of(definition), ) statuses: list[GroupStatus] = [analysis.groups[g] for g in groups if g in analysis.groups] shown: dict[str, str] = {} @@ -122,7 +153,9 @@ def _security( reason = reasons[0] elif result is U: reason = R.RULE_INDETERMINATE - observations = _base(resource) | {"groups": groups, "predicates": predicates} + observations = ( + _base(resource) | _named(definition) | {"groups": groups, "predicates": predicates} + ) return BaselineOutcome(result, reason, observations, (), True, *ids) @@ -148,21 +181,23 @@ def verify_baseline(address: str, inputs: VerificationInputs) -> BaselineOutcome return BaselineOutcome( S, R.INVARIANT_UNKNOWN, _base(resource), (), True, *BASELINE_VERIFIER ) + definition = baseline_definition(inputs, kind) + named = _base(resource) | _named(definition) if kind in FUNC_RELEVANT_UNSUPPORTED: - return BaselineOutcome(S, R.TYPE_RELEVANT_UNSUPPORTED, _base(resource), (), True, *ids) + return BaselineOutcome(S, R.TYPE_RELEVANT_UNSUPPORTED, named, (), True, *ids) if resource.support_status is ResourceSupport.UNKNOWN: - return BaselineOutcome(U, R.RESOURCE_INDETERMINATE, _base(resource), (), True, *ids) - if kind in _SECURITY_TYPES or kind == DB_INSTANCE_TYPE: + return BaselineOutcome(U, R.RESOURCE_INDETERMINATE, named, (), True, *ids) + if definition is not None: if resource.support_status is ResourceSupport.UNSUPPORTED: - return BaselineOutcome(S, R.RULE_UNSUPPORTED, _base(resource), (), True, *ids) + return BaselineOutcome(S, R.RULE_UNSUPPORTED, named, (), True, *ids) if kind in _SECURITY_TYPES: - return _security(resource, inputs, ids) - return _database(resource, ids) + return _security(resource, inputs, ids, definition) + return _database(resource, ids, definition) note = f"registry-v1.0 declares no baseline predicate for {kind}" return BaselineOutcome(P, None, _base(resource), (note,), False, *ids) -def _database(resource: Resource, ids: tuple[str, str]) -> BaselineOutcome: +def _database(resource: Resource, ids: tuple[str, str], definition: Invariant) -> BaselineOutcome: result, shown = instance_state(resource) reasons: Mapping[VerificationResult, VerificationReason | None] = { P: None, @@ -171,5 +206,10 @@ def _database(resource: Resource, ids: tuple[str, str]) -> BaselineOutcome: S: R.RULE_UNSUPPORTED, } return BaselineOutcome( - result, reasons[result], _base(resource) | {"value": shown}, (), True, *ids + result, + reasons[result], + _base(resource) | _named(definition) | {"value": shown}, + (), + True, + *ids, ) diff --git a/docs/BUILD_SEQUENCE.md b/docs/BUILD_SEQUENCE.md index 0444226..37fe9f0 100644 --- a/docs/BUILD_SEQUENCE.md +++ b/docs/BUILD_SEQUENCE.md @@ -32,7 +32,7 @@ The build is split into 18 work packages (N-06 in `DECISIONS_REGISTER.md`). Ever | 20 | Dec 3–6 (built 12 Nov 2026) | P7b patch validation: the C-188 patch meaning, the §13 validator, AllowedChangeScope and the operation check, `LLMEvaluationRequest` (C-188 to C-195) → P7 gate, M4 | | 21 | — | ~~P9 AWS integration (free route) → M6~~ removed by C-71 (advisor, 11 Nov 2026): AWS stays fully offline, no deployment, no AWS account, no live calibration; milestone M6 (AWS Demonstration) is removed | | 22 | Dec 12–14 (built 12 Nov 2026) | P8a experiment harness: conditions A, B, C, C′ (C-198), open targets (C-196), the TerraPreserve loader with hash pins and label blinding, trials, records and artifacts, the dev-split harness check (C-196 to C-208) | -| 23 | Dec 15–18 | P8b metrics and reports → P8 gate, M5 | +| 23 | Dec 15–18 (built 13 Nov 2026) | P8b metrics and reports: scoring (the only label reader), the extractor metrics-draft-1, the ExperimentReport, docs/P8_DEV_PILOT.md (C-212 to C-215) → P8 gate, M5 | | 24–28 | Dec 19–Jan 2 | P10 hardening, protocol freeze + pilot, LLM patch generation, main run, QA report | | 29 | Jan 3–4 | TerraPreserve v1.1 (real LLM patches) | diff --git a/docs/DECISIONS_REGISTER.md b/docs/DECISIONS_REGISTER.md index 99ff585..37f5e1d 100644 --- a/docs/DECISIONS_REGISTER.md +++ b/docs/DECISIONS_REGISTER.md @@ -211,15 +211,21 @@ This register records every conflict found between the 15 `CloudSpartanX_*` spec | C-197 | P8a: N-05 amended | Condition A's rule for scenarios without a repair target applies to **every scenario whose `target.kind` is `change_request`** (in v1.0: F1, F3, F5, F6, F7), not to a list of families. N-05 points here. | Decided (authors, 12 Nov 2026) | | C-198 | P8a: the conditions for P8 (C-20, C-22, N-05) | A, B, C and C′. All four use the same verifiers, the same orchestrator and the same promotion controller (RV-001); they differ only in this: **A, Stateless:** evidence identity; verification work = the gate items (STRUCTURAL, CONSISTENCY) and the repair target's invariant item, or the gate items only for `change_request` scenarios (C-197); required set = the gate items and the repair target only; no ledger use, no carry-forward. **B, Stateful Full:** evidence identity; verification work = the gate items, every invariant on the current state and every RESOURCE baseline item of the impact work set; required set as `stateful_full_policy` (every invariant on the state, carry-forward off). **C, DAIL:** evidence identity; `DailScopePolicy` unchanged; `dail_policy` (carry-forward on). **C′, DAIL without identity evidence (H3, C-20):** `AddressOnlyIdentity` (C-127), `DailScopePolicy`, as C. | Decided (authors, 12 Nov 2026) | | C-199 | P8a: P7a, P7b, P7 gate and M4 status | The user pasted the verdicts on PR #28 and PR #29 (C-28). `docs/PHASE_GATES.md`: "P7a: passed with limits (verdict on PR #28)", "P7b: passed with limits (verdict on PR #29)", "P7 gate (Doc 15 §14.2) and milestone M4 (Doc 15 §24): passed with limits (verdict on PR #29)"; the limits of both verdicts are listed there as open items with the package that closes each (P8 or P10). | Decided (authors, 12 Nov 2026) | -| C-200 | P8a-1: trial unit and trajectory | A trial is one TerraPreserve target scenario under one condition (repetition 1; fixed patches are deterministic). Its trajectory evaluates steps 1..n in order, each as a FIXED_PATCH candidate against that trial's current trusted state. After any decision other than PROMOTE at step i < n, steps i+1..n are not evaluated and are recorded as TRUNCATED (early termination; the paper reports it and runs a common-prefix sensitivity analysis in P8b). | Proposed (P8a) | -| C-201 | P8a-2: admission (C-99) | A TerraPreserve step enters DAIL only if its recomputed `content_hash` equals the frozen manifest's value; every v1.0 step passed `terraform validate` 1.15.8 at D6 gate 2, and that is recorded as its admission evidence. DAIL does not run Terraform in P8a. | Proposed (P8a) | -| C-202 | P8a-3: isolation | Each trial has its own fresh store (its own SQLite file and artifact directory) and its own lineage id. Nothing is shared between trials or conditions except the read-only scenario files. No experiment lineage is ever the authoritative lineage (Doc 13 §20). | Proposed (P8a) | -| C-203 | P8a-4: registry | Every condition uses invariant registry **v1.1** (INV-SEC-001/002 version 2). TerraPreserve's `protected_invariants` (registry v1.0 ids) map to the v1.1 definitions; P6d showed the v1.0 labels are identical under v1.1. Every record that carries an invariant version also carries the mapping "stored invariant_registry_version 2 = registry 1.1" (C-180). | Proposed (P8a) | -| C-204 | P8a-5: where the run lives | Run output goes under `$DAIL_EXPERIMENT_ROOT` (outside the repository, never committed). The TerraPreserve root comes from `$TERRAPRESERVE_ROOT`. | Proposed (P8a) | +| C-200 | P8a-1: trial unit and trajectory | A trial is one TerraPreserve target scenario under one condition (repetition 1; fixed patches are deterministic). Its trajectory evaluates steps 1..n in order, each as a FIXED_PATCH candidate against that trial's current trusted state. After any decision other than PROMOTE at step i < n, steps i+1..n are not evaluated and are recorded as TRUNCATED (early termination; the paper reports it and runs a common-prefix sensitivity analysis in P8b). | Decided (authors, 13 Nov 2026): accepted as built after external review | +| C-201 | P8a-2: admission (C-99) | A TerraPreserve step enters DAIL only if its recomputed `content_hash` equals the frozen manifest's value; every v1.0 step passed `terraform validate` 1.15.8 at D6 gate 2, and that is recorded as its admission evidence. DAIL does not run Terraform in P8a. | Decided (authors, 13 Nov 2026): accepted as built after external review | +| C-202 | P8a-3: isolation | Each trial has its own fresh store (its own SQLite file and artifact directory) and its own lineage id. Nothing is shared between trials or conditions except the read-only scenario files. No experiment lineage is ever the authoritative lineage (Doc 13 §20). | Decided (authors, 13 Nov 2026): accepted as built after external review | +| C-203 | P8a-4: registry | Every condition uses invariant registry **v1.1** (INV-SEC-001/002 version 2). TerraPreserve's `protected_invariants` (registry v1.0 ids) map to the v1.1 definitions; P6d showed the v1.0 labels are identical under v1.1. Every record that carries an invariant version also carries the mapping "stored invariant_registry_version 2 = registry 1.1" (C-180). | Decided (authors, 13 Nov 2026): accepted as built after external review | +| C-204 | P8a-5: where the run lives | Run output goes under `$DAIL_EXPERIMENT_ROOT` (outside the repository, never committed). The TerraPreserve root comes from `$TERRAPRESERVE_ROOT`. | Decided (authors, 13 Nov 2026): accepted as built after external review | | C-205 | P8a: condition A's required set and C-41 | The work package asks for a policy that requires only the gate items and the repair target, but C-40/C-41 forbid a promotion from dropping an invariant of the current state (`build_proofs`). **User's decision (12 Nov 2026): A's lineage tracks only what A requires.** A's baseline registers only the repair target's definition (and only the bindings of that invariant); the policy constructor `stateless_policy` sets `explicit_required_set = True`, so the required set is exactly the gate items and `extra_required` (no RESOURCE baseline items, no other invariant of the state), with carry-forward off. A current state that holds an invariant outside the explicit set is an ESCALATE (`POLICY_REGISTRY_REGRESSION`, C-41), so nothing is dropped silently. C-41 is unchanged. A's S0 records cover only that set. | Decided (user, 12 Nov 2026) | | C-206 | P8a: condition A on `change_request` scenarios needs a baseline with no invariant | `TrustedState.establish_baseline` refuses a baseline with no invariant proof (cited as Doc 06 §24), and A tracks no invariant for a `change_request` scenario (C-197, C-205). **User's decision (12 Nov 2026): an explicit opt-in.** `TrustedState.establish_baseline(..., allow_no_invariants=False)` and `BaselineRequest.allow_no_invariants = False`: only with the flag may the set be empty; every existing caller keeps the "at least one" rule. Only the experiment harness sets it, for A on `change_request`. Recorded as a reading of Doc 06 §24 for the reviewer: the spec says the baseline establishes "its protected invariants" and does not say the set is non-empty. | Decided (user, 12 Nov 2026) | -| C-207 | P8a: readings made while building the harness | (1) **Condition items.** A work item that the condition's scope policy adds for an invariant impact did not list (B's unaffected invariants; A's target when impact did not list it) needs a `ReasonCode`, and C-138's vocabulary is closed (a test pins it) with no code for "the condition re-verifies it"; the stand-in is `SCOPE_TYPE_CHANGED` (`scope_policy.CONDITION_ITEM_REASON`). Nothing reads a work item's reason after selection (the verification request does not carry it); the records mark such items `origin = condition`. Its scope is the definition's scope (resources, else resource types); the verifiers do not read it. Question for the reviewer: a new reason code through a C-138 row instead? (2) **Registry history.** A fresh store must hold version 1 of INV-SEC-001/002 before version 2 (Doc 06 §4.2, no gaps), so each trial store registers the registry-v1.0 definitions of the ids it tracks first; the baseline registers v1.1. (3) **Automatic promotion.** `tests/unit/test_config.py::test_real_config_loads_for_all_four_environments` pins `promotion.automatic_enabled = false` in every environment of `config/`, so the harness does not change `config/`: it loads a copy of `config/` with `base/promotion.toml` replaced by the reviewed file `experiments/config/promotion.toml` (`automatic_enabled = true`, a versioned file, never an environment variable or override), environment `experiment`. Its SHA-256 is pinned in every definition. Without it every PROMOTE would be an ESCALATE. (4) **Layout.** §41's `trials//{baseline/ ...}` gets an `attempts//` level, so that a re-run never overwrites the original (§47); each attempt has its own `store/dail.sqlite`. The lifecycle is `trials//events.jsonl` (append-only). (5) **Time.** Everything DAIL hashes uses a logical clock (2026-01-01T00:00:00Z plus one second per call), so a re-run stores the same records; wall-clock times and durations are only in `timing.json` and the events. (6) **Random ids.** The identity and impact reports draw a random `report_id` (the orchestrator does not pass one); the stored copies leave it out and keep the content hash. (7) **Outcomes.** A construction failure is recorded as a REJECT with reason `CONSTRUCTION_FAILED` (DAIL stores no decision, C-170 (3)). A baseline that DAIL refuses (C-171) leaves the trial COMPLETED with every step TRUNCATED. An open target that DAIL's own parent state does not hold VIOLATED is refused by `check_open_targets` (C-196) and the trial is FAILED as a HARNESS_FAILURE. A stale decision records no open target. A FAILED trial keeps its baseline artifacts and no transition in its result. (8) **Lifecycle (§7).** A trial interrupted before it executed may go INITIALIZED → ABORTED; an ABORTED trial is re-run as a new attempt (ABORTED → INITIALIZED); a FAILED trial is not re-run. (9) **Loader.** The manifest is parsed whole (its bytes are pinned) but only the `dev` list is indexed; `template_id` is passed (not the source lineage id); the admission evidence names D6 gate 2 only for the pinned v1.0 manifest. (10) **Conditions.** B's "every invariant on the current state" is every definition passed (B's lineage holds all four); A evaluates with the four definitions (graph, impact, verification inputs) but tracks and verifies only its required set. A baseline `state_hash` covers the lineage (C-33), so S0 consistency across conditions is checked on `resource_set_hash`. | Proposed (P8a) | +| C-207 | P8a: readings made while building the harness | (1) **Condition items.** A work item that the condition's scope policy adds for an invariant impact did not list (B's unaffected invariants; A's target when impact did not list it) needs a `ReasonCode`, and C-138's vocabulary is closed (a test pins it) with no code for "the condition re-verifies it"; the stand-in is `SCOPE_TYPE_CHANGED` (`scope_policy.CONDITION_ITEM_REASON`). Nothing reads a work item's reason after selection (the verification request does not carry it); the records mark such items `origin = condition`. Its scope is the definition's scope (resources, else resource types); the verifiers do not read it. Question for the reviewer: a new reason code through a C-138 row instead? (2) **Registry history.** A fresh store must hold version 1 of INV-SEC-001/002 before version 2 (Doc 06 §4.2, no gaps), so each trial store registers the registry-v1.0 definitions of the ids it tracks first; the baseline registers v1.1. (3) **Automatic promotion.** `tests/unit/test_config.py::test_real_config_loads_for_all_four_environments` pins `promotion.automatic_enabled = false` in every environment of `config/`, so the harness does not change `config/`: it loads a copy of `config/` with `base/promotion.toml` replaced by the reviewed file `experiments/config/promotion.toml` (`automatic_enabled = true`, a versioned file, never an environment variable or override), environment `experiment`. Its SHA-256 is pinned in every definition. Without it every PROMOTE would be an ESCALATE. (4) **Layout.** §41's `trials//{baseline/ ...}` gets an `attempts//` level, so that a re-run never overwrites the original (§47); each attempt has its own `store/dail.sqlite`. The lifecycle is `trials//events.jsonl` (append-only). (5) **Time.** Everything DAIL hashes uses a logical clock (2026-01-01T00:00:00Z plus one second per call), so a re-run stores the same records; wall-clock times and durations are only in `timing.json` and the events. (6) **Random ids.** The identity and impact reports draw a random `report_id` (the orchestrator does not pass one); the stored copies leave it out and keep the content hash. (7) **Outcomes.** A construction failure is recorded as a REJECT with reason `CONSTRUCTION_FAILED` (DAIL stores no decision, C-170 (3)). A baseline that DAIL refuses (C-171) leaves the trial COMPLETED with every step TRUNCATED. An open target that DAIL's own parent state does not hold VIOLATED is refused by `check_open_targets` (C-196) and the trial is FAILED as a HARNESS_FAILURE. A stale decision records no open target. A FAILED trial keeps its baseline artifacts and no transition in its result. (8) **Lifecycle (§7).** A trial interrupted before it executed may go INITIALIZED → ABORTED; an ABORTED trial is re-run as a new attempt (ABORTED → INITIALIZED); a FAILED trial is not re-run. (9) **Loader.** The manifest is parsed whole (its bytes are pinned) but only the `dev` list is indexed; `template_id` is passed (not the source lineage id); the admission evidence names D6 gate 2 only for the pinned v1.0 manifest. (10) **Conditions.** B's "every invariant on the current state" is every definition passed (B's lineage holds all four); A evaluates with the four definitions (graph, impact, verification inputs) but tracks and verifies only its required set. A baseline `state_hash` covers the lineage (C-33), so S0 consistency across conditions is checked on `resource_set_hash`. | Decided (authors, 13 Nov 2026): accepted as built after external review | | C-208 | P8a: the identity-strategy injection point and the P3b test that forbade it | The work package asks for the narrowest injection point in the application layer for condition C′ (`AddressOnlyIdentity`, C-127). `tests/unit/test_identity_strategy.py::test_no_configuration_key_or_environment_variable_selects_a_strategy` rejected any mention of `identity_strategy`, `ablation` or `address-only` in `core/application/*.py`. **User's decision (12 Nov 2026): keep the injection point and amend the test, keeping its intent.** `EvaluationOrchestrator(..., identity_strategy=DEFAULT_STRATEGY)`; `config/` and `scripts/` (and `core/application/config.py`) are still scanned for those words exactly as before; the rest of `core/application` is checked by a new test, `test_the_application_layer_reads_no_strategy_switch`: no module names `AddressOnlyIdentity` or `ADDRESS_ONLY`, none reads `os.environ` or `getenv` (the configuration loader excepted), no `get(...)` or `[...]` lookup uses a key that names an identity, a strategy or the ablation, and the orchestrator's default is `EvidenceIdentity`. Two mutants (a strategy read from a configuration key, and from the environment) are caught. Only `experiments/` names the ablation. | Decided (user, 12 Nov 2026) | | C-209 | P8a dev run: under C and C′ the open target of C-196 rejects every F8 trajectory at step 1 | Found in the P8a harness check (no label read). C-196 says an open target "still needs a fresh verification result ... a missing result blocks", and its reason is that "without this rule every condition rejects F8 at step 1". In the dev run, on all 28 F8 targets, C and C′ REJECT step 1 with `INSUFFICIENT_EVIDENCE` on `INVARIANT:INV-SEC-001:2`: the middle step does not touch INV-SEC-001's footprint, so impact does not list it, `DailScopePolicy` (unchanged, C-198) owes no work item for it, and a VIOLATED reference cannot be carried forward (C-159), so no fresh result exists and the rule blocks. A and B verify the target on every step and promote these steps (A 84 of 84 F8 transitions, B 63 of 77). So for C and C′ the rule does not reach its stated purpose, and F8 measures only step 1 there. Nothing was changed (no tuning to the dev run). Options for the authors include: let an open target that the candidate does not affect stay VIOLATED with the parent's evidence (a carry of a VIOLATED reference, which the commit path already supports through `InvariantProof.carry_forward`); or make the harness add the open target's item to every condition's work set. The same mechanism also rejects a candidate whose parent holds a non-target invariant VIOLATED that the candidate does not touch (1 transition each under C and C′). **Closed by C-210 (authors, 12 Nov 2026).** | Decided (authors, 12 Nov 2026): see C-210 | | C-210 | P8a-2: DAIL always re-checks every invariant that is not PROTECTED in the parent (closes C-209) | DAIL's verification work for a candidate is the impact work set plus one invariant item for every invariant whose reference in the parent trusted state is not PROTECTED (VIOLATED or UNCERTAIN) and that impact did not already list. This is the paper's required set R(C_t) = G_gate ∪ U_t ∪ I_t ∪ B, where U_t is the set of non-PROTECTED references (C-159 already requires a fresh result for each of them; the work set did not schedule one). It applies to conditions C and C′ (both use `DailScopePolicy`). These items count as verification work. A VIOLATED reference is never carried forward (C-159 unchanged). **Built:** `DailScopePolicy` version `dail-scope-2` adds the U_t items (reason `NOT_PROTECTED_IN_PARENT`, BLOCKING, sorted, never a second item for an invariant impact listed). `VerificationScopePolicy.select` takes the parent state's invariant references as a keyword `parent_refs` (default none): the narrowest input that carries the statuses, which the orchestrator already holds; the impact report does not carry them and a scope policy does not read the database. Impact, verification, identity and the promotion decision are unchanged; with every parent reference PROTECTED the work set is impact's, unchanged. | Decided (authors, 12 Nov 2026) | | C-211 | P8a-2: two reason codes for work a scope policy adds (amends C-138's vocabulary; answers C-207 (1)) | `ReasonCode.CONDITION_SCOPE` (the condition's scope policy requires this check: B's unaffected invariants, A's target when impact did not list it) and `ReasonCode.NOT_PROTECTED_IN_PARENT` (the U_t items of C-210), in a fifth group `POLICY_CODES`. They are valid only on invariant work items, and an impact report that carries one is refused (impact never gives them). The `SCOPE_TYPE_CHANGED` stand-in of C-207 (1) is gone; the experiment records keep `origin = condition` and add the item's reason (`selected_reason`). `tests/unit/test_impact_domain.py` lists the two new codes and the new group explicitly (stricter). `StatefulFullScopePolicy` and `StatelessScopePolicy` are now version 2 (their items' reason changed). | Decided (authors, 12 Nov 2026) | +| C-212 | P8b-1: metric definitions | The metrics of the P8b work package Section 4, exactly, versioned `metrics-draft-1`. They are a draft for the dev pilot; P10 freezes the final definitions before the test split is opened. | Proposed (P8b) | +| C-213 | P8b-2: identity truth for H3 | Identity truth comes from a hand-written table in DAIL, `experiments/scoring/identity_truth.json`: one row per TerraPreserve operator that moves, replaces, deletes or creates a resource, giving the expected change type at the operator's target address, with the sentence of `D4_OPERATORS.md` it comes from. An operator whose identity truth the catalogue does not state gets no row, and its pairs are excluded and counted. The table is written from the catalogue's text, never from DAIL's output. | Proposed (P8b) | +| C-214 | P8b-3: statistics on dev are descriptive only | Counts, proportions with numerator and denominator, Wilson 95% intervals, and per-cluster summaries (cluster = `template_id` for the controlled tier, `repo_lineage_id` for the real-world tier). No hypothesis test, no p-value, no "significant". Inference is for the test split after the protocol freeze (P10). | Proposed (P8b) | +| C-215 | P8b: readings made while building scoring, the extractor and the report | (1) **Evidence-transfer error** (item 10, not defined in the work package): a true replacement that DAIL treats as the same object (neither REPLACED nor UNCERTAIN) in a transition where some required invariant was CARRIED_FORWARD. (2) **Appearing and vanishing resources** for the identity truth are those declared (a top-level `resource` block in the step's `.tf` files) in step i and not in step i−1, or the reverse, restricted to the addresses the operator's mutation rows name; the first version used "every row has no `before`", which wrongly made an existing group that only gains an attribute a CREATE (found on `op_f3_db_ingress_peer_safe` before the report was written; a test now covers it). Scoring reads the step declarations, not DAIL's output. (3) **Excluded operators**: `op_f5_count_each_safe` and the four F1/F4 unsafe operators whose sentence adds an ingress "on the group" without saying whether it is a new resource. (4) **Repair success** is per scenario, so it is reported on all transitions only, not on a common prefix. (5) **MetricRecord** carries Appendix B's fields plus `scope`, `stratum` and `wilson95`; a denominator of 0 gives `value = null`; the Wilson interval is clamped to [0, 1] (floating rounding gave -0.000 for k = 0; the first generated report of run 2 was deleted and regenerated after that fix, before anything referenced it). (6) **The 20-trial re-run** is a new experiment, `p8b-repro-check`, on the same code, compared with run 2 on (scenario, condition, step): a new attempt of a COMPLETED trial is not a transition Doc 13 §7 allows, and no stored record is touched. The 4 extra F8 are those with most steps among the F8 not already chosen, ties by lowest scenario id. (7) **The dev report** `docs/P8_DEV_PILOT.json` (the ExperimentReport, about 1 MB) and its rendering `docs/P8_DEV_PILOT.md` are committed; a test renders the JSON and compares the file. (8) **Agreement** (item 8) uses B's verification results: at S0 the baseline record's results, at every evaluated step the results of every invariant item B ran. | Proposed (P8b) | +| C-216 | P8b-2: baseline items under registry v1.1 (the version-1 lookup in `core/verification/baseline.py`) | Baseline items (C-148) use the definition that the run's definitions hold for the invariant they belong to (INV-SEC-001 for security groups and rules, INV-SEC-003 for database instances). If more than one version is present, the highest is used. The item's evidence records that id and version, and the §3.3 clause it applies is that definition's registry (version 1 → registry-v1.0, version 2 → registry-v1.1), so a baseline item and its invariant item can never use different rules. If no definition is present, the item stays UNSUPPORTED `invariant.unknown`, as now. Found by the external review of PR #31; it affected every run under registry v1.1. | Decided (authors, 13 Nov 2026) | +| C-217 | P8b-2: readings made in the fix round | (1) **An unproven required check.** DAIL records a required check that is not proven with no proof source (only VERIFIED and CARRIED_FORWARD are sources). The P8b extractor read a fresh result only for VERIFIED checks, so the UNKNOWN and UNSUPPORTED rates (run 2: 0 for every condition) and the new non-PASS table missed every failing or uncertain check, and the guard could not fire on real data; the hand-built record set had marked every check VERIFIED. Found while writing the non-PASS table; the extractor now takes the fresh result of every check that is not CARRIED_FORWARD, the record set records sources as DAIL does, and a mutant that restores the old rule fails 3 tests. (2) **Regenerated report.** The first report of run 3 was written by commit a34e25a's extractor with that rule; it was deleted and regenerated after the fix, before anything referenced it (as for C-215 (5)). The extractor version stays `metrics-draft-1` because the metric definitions (C-212) did not change. (3) **Evidence fields** of a baseline item: `observations.invariant_id`, `invariant_version` and `registry` (C-216). (4) **Guard on real data:** on run 2's records the guard flags 144 baseline items (B 32 groups and 14 rules, C 32 and 14, C' 32 and 20); on run 3's, none. | Proposed (P8b-2) | diff --git a/docs/P8_DEV_PILOT.json b/docs/P8_DEV_PILOT.json new file mode 100644 index 0000000..ff93c15 --- /dev/null +++ b/docs/P8_DEV_PILOT.json @@ -0,0 +1,37565 @@ +{ + "artifact_refs": [ + "definition/experiment.json", + "896 trial result records under trials/" + ], + "condition_summaries": [ + { + "all_transitions": 280, + "condition": "A", + "evaluated_transitions": 280, + "truncated_transitions": 0 + }, + { + "all_transitions": 280, + "condition": "B", + "evaluated_transitions": 273, + "truncated_transitions": 7 + }, + { + "all_transitions": 280, + "condition": "C", + "evaluated_transitions": 273, + "truncated_transitions": 7 + }, + { + "all_transitions": 280, + "condition": "C_PRIME", + "evaluated_transitions": 273, + "truncated_transitions": 7 + } + ], + "conditions": [ + "A", + "B", + "C", + "C_PRIME" + ], + "confusion": { + "INV-FUNC-001": [ + { + "count": 61, + "dail": "FAIL", + "oracle_truth": "FAIL" + }, + { + "count": 434, + "dail": "PASS", + "oracle_truth": "PASS" + }, + { + "count": 2, + "dail": "UNSUPPORTED", + "oracle_truth": "PASS" + } + ], + "INV-SEC-001": [ + { + "count": 119, + "dail": "FAIL", + "oracle_truth": "FAIL" + }, + { + "count": 376, + "dail": "PASS", + "oracle_truth": "PASS" + }, + { + "count": 2, + "dail": "UNSUPPORTED", + "oracle_truth": "PASS" + } + ], + "INV-SEC-002": [ + { + "count": 37, + "dail": "FAIL", + "oracle_truth": "FAIL" + }, + { + "count": 458, + "dail": "PASS", + "oracle_truth": "PASS" + }, + { + "count": 2, + "dail": "UNSUPPORTED", + "oracle_truth": "PASS" + } + ], + "INV-SEC-003": [ + { + "count": 35, + "dail": "FAIL", + "oracle_truth": "FAIL" + }, + { + "count": 462, + "dail": "PASS", + "oracle_truth": "PASS" + } + ] + }, + "definitions_held": [ + "INV-FUNC-001", + "INV-SEC-001", + "INV-SEC-002", + "INV-SEC-003" + ], + "excluded_count": 0, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "experiment_name": "p8b-dev-run-3", + "experiment_version": "1", + "extractor_version": "metrics-draft-1", + "failures": [], + "included_count": 896, + "lists": { + "clusters": [ + { + "cluster": "tpl-01", + "condition": "A", + "evaluated": 5, + "promoted": 4, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-02", + "condition": "A", + "evaluated": 6, + "promoted": 6, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-03", + "condition": "A", + "evaluated": 5, + "promoted": 5, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-06", + "condition": "A", + "evaluated": 8, + "promoted": 8, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 3 + }, + { + "cluster": "tpl-08", + "condition": "A", + "evaluated": 4, + "promoted": 3, + "safe": 1, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-09", + "condition": "A", + "evaluated": 6, + "promoted": 6, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 3 + }, + { + "cluster": "tpl-10", + "condition": "A", + "evaluated": 7, + "promoted": 7, + "safe": 6, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 1 + }, + { + "cluster": "tpl-11", + "condition": "A", + "evaluated": 8, + "promoted": 8, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 3 + }, + { + "cluster": "tpl-14", + "condition": "A", + "evaluated": 5, + "promoted": 4, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-15", + "condition": "A", + "evaluated": 7, + "promoted": 7, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 3 + }, + { + "cluster": "tpl-16", + "condition": "A", + "evaluated": 9, + "promoted": 8, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 3 + }, + { + "cluster": "tpl-19", + "condition": "A", + "evaluated": 7, + "promoted": 7, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 3 + }, + { + "cluster": "tpl-21", + "condition": "A", + "evaluated": 6, + "promoted": 6, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 3 + }, + { + "cluster": "tpl-22", + "condition": "A", + "evaluated": 9, + "promoted": 9, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 4 + }, + { + "cluster": "tpl-24", + "condition": "A", + "evaluated": 6, + "promoted": 6, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-25", + "condition": "A", + "evaluated": 7, + "promoted": 7, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-26", + "condition": "A", + "evaluated": 5, + "promoted": 5, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-28", + "condition": "A", + "evaluated": 4, + "promoted": 3, + "safe": 1, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-29", + "condition": "A", + "evaluated": 6, + "promoted": 5, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-30", + "condition": "A", + "evaluated": 7, + "promoted": 7, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-31", + "condition": "A", + "evaluated": 4, + "promoted": 3, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 1 + }, + { + "cluster": "tpl-32", + "condition": "A", + "evaluated": 5, + "promoted": 5, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "tpl-33", + "condition": "A", + "evaluated": 5, + "promoted": 4, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 2 + }, + { + "cluster": "lin-000071997470", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000190086027", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000204981091", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000217549106", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000229872486", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000239222450", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000248612047", + "condition": "A", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000297431998", + "condition": "A", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000331017763", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000342718779", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000369695830", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000373209616", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000379545777", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000413228433", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000424321374", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000439694431", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000445395033", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000486196334", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000500019072", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000507616966", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000518452613", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000551625593", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000567767911", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000571703667", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000579171915", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000587734769", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000599720488", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000609289453", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000618306157", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000624434439", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000625926227", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000629046278", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000632074016", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000635474331", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000655245703", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000658586339", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000666589002", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000667312922", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000681399111", + "condition": "A", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000694931289", + "condition": "A", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000730242067", + "condition": "A", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000737712009", + "condition": "A", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000742918458", + "condition": "A", + "evaluated": 4, + "promoted": 4, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 2 + }, + { + "cluster": "lin-000744144248", + "condition": "A", + "evaluated": 2, + "promoted": 2, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000756264433", + "condition": "A", + "evaluated": 2, + "promoted": 2, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000773216125", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000788147459", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000795021596", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000795223927", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000799593584", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000813142218", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000838092263", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000840480004", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000844428778", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000845889776", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000849709897", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000849828058", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000852344138", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000863109001", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000863549786", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000882762150", + "condition": "A", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000907471682", + "condition": "A", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000910302341", + "condition": "A", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000912707779", + "condition": "A", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000941060246", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000949225941", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000951597420", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000958802708", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000961118787", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000961290495", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000962008774", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000964060252", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000967983742", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000975393827", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000978961077", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000979990883", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000983671103", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000985939483", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000986578999", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-000998307733", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001001134954", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001006913302", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001022568295", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001031040906", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001031910861", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001034019508", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001035187878", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001058669091", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001061193057", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001066128255", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001066965801", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001067900351", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001067983175", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001071162848", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001071963417", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001074711129", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001077405237", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001077792647", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001084654910", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001087734205", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001097239042", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001097780303", + "condition": "A", + "evaluated": 1, + "promoted": 1, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "lin-001100904098", + "condition": "A", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001108255200", + "condition": "A", + "evaluated": 2, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001109067234", + "condition": "A", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001110841825", + "condition": "A", + "evaluated": 4, + "promoted": 4, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 2 + }, + { + "cluster": "lin-001117805595", + "condition": "A", + "evaluated": 3, + "promoted": 3, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 2 + }, + { + "cluster": "lin-001119287227", + "condition": "A", + "evaluated": 4, + "promoted": 4, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 2 + }, + { + "cluster": "lin-001122985569", + "condition": "A", + "evaluated": 2, + "promoted": 2, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 1 + }, + { + "cluster": "tpl-01", + "condition": "B", + "evaluated": 5, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-02", + "condition": "B", + "evaluated": 6, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-03", + "condition": "B", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-06", + "condition": "B", + "evaluated": 8, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-08", + "condition": "B", + "evaluated": 4, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-09", + "condition": "B", + "evaluated": 6, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-10", + "condition": "B", + "evaluated": 7, + "promoted": 6, + "safe": 6, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-11", + "condition": "B", + "evaluated": 7, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-14", + "condition": "B", + "evaluated": 5, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-15", + "condition": "B", + "evaluated": 7, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-16", + "condition": "B", + "evaluated": 9, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-19", + "condition": "B", + "evaluated": 7, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-21", + "condition": "B", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-22", + "condition": "B", + "evaluated": 8, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-24", + "condition": "B", + "evaluated": 6, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-25", + "condition": "B", + "evaluated": 7, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-26", + "condition": "B", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-28", + "condition": "B", + "evaluated": 4, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-29", + "condition": "B", + "evaluated": 6, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-30", + "condition": "B", + "evaluated": 7, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-31", + "condition": "B", + "evaluated": 4, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-32", + "condition": "B", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-33", + "condition": "B", + "evaluated": 5, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000071997470", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000190086027", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000204981091", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000217549106", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000229872486", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000239222450", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000248612047", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000297431998", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000331017763", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000342718779", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000369695830", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000373209616", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000379545777", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000413228433", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000424321374", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000439694431", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000445395033", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000486196334", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000500019072", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 1, + "safe_not_promoted": 1, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000507616966", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000518452613", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000551625593", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000567767911", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000571703667", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000579171915", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000587734769", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 1, + "safe_not_promoted": 1, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000599720488", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000609289453", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000618306157", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000624434439", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000625926227", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000629046278", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000632074016", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000635474331", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000655245703", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000658586339", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000666589002", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000667312922", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000681399111", + "condition": "B", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000694931289", + "condition": "B", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000730242067", + "condition": "B", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000737712009", + "condition": "B", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000742918458", + "condition": "B", + "evaluated": 3, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000744144248", + "condition": "B", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000756264433", + "condition": "B", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000773216125", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000788147459", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000795021596", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000795223927", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000799593584", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000813142218", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000838092263", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000840480004", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000844428778", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000845889776", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000849709897", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000849828058", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000852344138", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000863109001", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000863549786", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000882762150", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000907471682", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000910302341", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000912707779", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000941060246", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000949225941", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000951597420", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000958802708", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000961118787", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000961290495", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000962008774", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000964060252", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000967983742", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000975393827", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000978961077", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000979990883", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000983671103", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000985939483", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000986578999", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000998307733", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001001134954", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001006913302", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001022568295", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001031040906", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001031910861", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001034019508", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001035187878", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001058669091", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001061193057", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001066128255", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001066965801", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001067900351", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001067983175", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001071162848", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001071963417", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001074711129", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001077405237", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001077792647", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001084654910", + "condition": "B", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001087734205", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001097239042", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001097780303", + "condition": "B", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001100904098", + "condition": "B", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001108255200", + "condition": "B", + "evaluated": 2, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001109067234", + "condition": "B", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001110841825", + "condition": "B", + "evaluated": 3, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001117805595", + "condition": "B", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001119287227", + "condition": "B", + "evaluated": 3, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001122985569", + "condition": "B", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-01", + "condition": "C", + "evaluated": 5, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-02", + "condition": "C", + "evaluated": 6, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-03", + "condition": "C", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-06", + "condition": "C", + "evaluated": 8, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-08", + "condition": "C", + "evaluated": 4, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-09", + "condition": "C", + "evaluated": 6, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-10", + "condition": "C", + "evaluated": 7, + "promoted": 6, + "safe": 6, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-11", + "condition": "C", + "evaluated": 7, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-14", + "condition": "C", + "evaluated": 5, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-15", + "condition": "C", + "evaluated": 7, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-16", + "condition": "C", + "evaluated": 9, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-19", + "condition": "C", + "evaluated": 7, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-21", + "condition": "C", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-22", + "condition": "C", + "evaluated": 8, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-24", + "condition": "C", + "evaluated": 6, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-25", + "condition": "C", + "evaluated": 7, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-26", + "condition": "C", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-28", + "condition": "C", + "evaluated": 4, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-29", + "condition": "C", + "evaluated": 6, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-30", + "condition": "C", + "evaluated": 7, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-31", + "condition": "C", + "evaluated": 4, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-32", + "condition": "C", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-33", + "condition": "C", + "evaluated": 5, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000071997470", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000190086027", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000204981091", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000217549106", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000229872486", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000239222450", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000248612047", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000297431998", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000331017763", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000342718779", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000369695830", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000373209616", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000379545777", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000413228433", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000424321374", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000439694431", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000445395033", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000486196334", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000500019072", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 1, + "safe_not_promoted": 1, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000507616966", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000518452613", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000551625593", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000567767911", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000571703667", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000579171915", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000587734769", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 1, + "safe_not_promoted": 1, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000599720488", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000609289453", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000618306157", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000624434439", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000625926227", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000629046278", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000632074016", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000635474331", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000655245703", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000658586339", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000666589002", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000667312922", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000681399111", + "condition": "C", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000694931289", + "condition": "C", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000730242067", + "condition": "C", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000737712009", + "condition": "C", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000742918458", + "condition": "C", + "evaluated": 3, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000744144248", + "condition": "C", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000756264433", + "condition": "C", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000773216125", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000788147459", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000795021596", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000795223927", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000799593584", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000813142218", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000838092263", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000840480004", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000844428778", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000845889776", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000849709897", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000849828058", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000852344138", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000863109001", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000863549786", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000882762150", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000907471682", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000910302341", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000912707779", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000941060246", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000949225941", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000951597420", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000958802708", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000961118787", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000961290495", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000962008774", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000964060252", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000967983742", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000975393827", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000978961077", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000979990883", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000983671103", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000985939483", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000986578999", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000998307733", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001001134954", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001006913302", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001022568295", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001031040906", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001031910861", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001034019508", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001035187878", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001058669091", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001061193057", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001066128255", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001066965801", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001067900351", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001067983175", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001071162848", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001071963417", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001074711129", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001077405237", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001077792647", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001084654910", + "condition": "C", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001087734205", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001097239042", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001097780303", + "condition": "C", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001100904098", + "condition": "C", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001108255200", + "condition": "C", + "evaluated": 2, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001109067234", + "condition": "C", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001110841825", + "condition": "C", + "evaluated": 3, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001117805595", + "condition": "C", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001119287227", + "condition": "C", + "evaluated": 3, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001122985569", + "condition": "C", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-01", + "condition": "C_PRIME", + "evaluated": 5, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-02", + "condition": "C_PRIME", + "evaluated": 6, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-03", + "condition": "C_PRIME", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-06", + "condition": "C_PRIME", + "evaluated": 8, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-08", + "condition": "C_PRIME", + "evaluated": 4, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-09", + "condition": "C_PRIME", + "evaluated": 6, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-10", + "condition": "C_PRIME", + "evaluated": 7, + "promoted": 6, + "safe": 6, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-11", + "condition": "C_PRIME", + "evaluated": 7, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-14", + "condition": "C_PRIME", + "evaluated": 5, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-15", + "condition": "C_PRIME", + "evaluated": 7, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-16", + "condition": "C_PRIME", + "evaluated": 9, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-19", + "condition": "C_PRIME", + "evaluated": 7, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-21", + "condition": "C_PRIME", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-22", + "condition": "C_PRIME", + "evaluated": 8, + "promoted": 4, + "safe": 5, + "safe_not_promoted": 1, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-24", + "condition": "C_PRIME", + "evaluated": 6, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-25", + "condition": "C_PRIME", + "evaluated": 7, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-26", + "condition": "C_PRIME", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-28", + "condition": "C_PRIME", + "evaluated": 4, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-29", + "condition": "C_PRIME", + "evaluated": 6, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-30", + "condition": "C_PRIME", + "evaluated": 7, + "promoted": 5, + "safe": 5, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-31", + "condition": "C_PRIME", + "evaluated": 4, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-32", + "condition": "C_PRIME", + "evaluated": 5, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "tpl-33", + "condition": "C_PRIME", + "evaluated": 5, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "controlled", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000071997470", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000190086027", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000204981091", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000217549106", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000229872486", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000239222450", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000248612047", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000297431998", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000331017763", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000342718779", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000369695830", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000373209616", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000379545777", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000413228433", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000424321374", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000439694431", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000445395033", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000486196334", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000500019072", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 1, + "safe_not_promoted": 1, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000507616966", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000518452613", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000551625593", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000567767911", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000571703667", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000579171915", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000587734769", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 1, + "safe_not_promoted": 1, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000599720488", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000609289453", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000618306157", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000624434439", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000625926227", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000629046278", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000632074016", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000635474331", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000655245703", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000658586339", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000666589002", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000667312922", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000681399111", + "condition": "C_PRIME", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000694931289", + "condition": "C_PRIME", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000730242067", + "condition": "C_PRIME", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000737712009", + "condition": "C_PRIME", + "evaluated": 4, + "promoted": 4, + "safe": 4, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000742918458", + "condition": "C_PRIME", + "evaluated": 3, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000744144248", + "condition": "C_PRIME", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000756264433", + "condition": "C_PRIME", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000773216125", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000788147459", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000795021596", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000795223927", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000799593584", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000813142218", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000838092263", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000840480004", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000844428778", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000845889776", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000849709897", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000849828058", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000852344138", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000863109001", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000863549786", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000882762150", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000907471682", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000910302341", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000912707779", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000941060246", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000949225941", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000951597420", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000958802708", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000961118787", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000961290495", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000962008774", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000964060252", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000967983742", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000975393827", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000978961077", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000979990883", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000983671103", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000985939483", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000986578999", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-000998307733", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001001134954", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001006913302", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001022568295", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001031040906", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001031910861", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001034019508", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 1, + "safe_not_promoted": 1, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001035187878", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001058669091", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001061193057", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001066128255", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001066965801", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001067900351", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001067983175", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001071162848", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001071963417", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001074711129", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001077405237", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001077792647", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001084654910", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001087734205", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001097239042", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001097780303", + "condition": "C_PRIME", + "evaluated": 1, + "promoted": 0, + "safe": 0, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001100904098", + "condition": "C_PRIME", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001108255200", + "condition": "C_PRIME", + "evaluated": 2, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001109067234", + "condition": "C_PRIME", + "evaluated": 3, + "promoted": 3, + "safe": 3, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001110841825", + "condition": "C_PRIME", + "evaluated": 3, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001117805595", + "condition": "C_PRIME", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001119287227", + "condition": "C_PRIME", + "evaluated": 3, + "promoted": 2, + "safe": 2, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + }, + { + "cluster": "lin-001122985569", + "condition": "C_PRIME", + "evaluated": 2, + "promoted": 1, + "safe": 1, + "safe_not_promoted": 0, + "tier": "real_world", + "unsafe_promoted": 0 + } + ], + "disagreements": [ + { + "dail": "UNSUPPORTED", + "invariant_id": "INV-FUNC-001", + "oracle_truth": "PASS", + "scenario_id": "tp-v1-F5-0319", + "step": 0 + }, + { + "dail": "UNSUPPORTED", + "invariant_id": "INV-FUNC-001", + "oracle_truth": "PASS", + "scenario_id": "tp-v1-F5-0319", + "step": 1 + }, + { + "dail": "UNSUPPORTED", + "invariant_id": "INV-SEC-001", + "oracle_truth": "PASS", + "scenario_id": "tp-v1-F6-0350", + "step": 0 + }, + { + "dail": "UNSUPPORTED", + "invariant_id": "INV-SEC-002", + "oracle_truth": "PASS", + "scenario_id": "tp-v1-F6-0350", + "step": 0 + }, + { + "dail": "UNSUPPORTED", + "invariant_id": "INV-SEC-001", + "oracle_truth": "PASS", + "scenario_id": "tp-v1-F6-0350", + "step": 1 + }, + { + "dail": "UNSUPPORTED", + "invariant_id": "INV-SEC-002", + "oracle_truth": "PASS", + "scenario_id": "tp-v1-F6-0350", + "step": 1 + } + ], + "identity_errors": [ + { + "condition": "C", + "dail_pairs": [ + [ + "aws_instance.acme_app", + "aws_instance.acme_app", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_instance.acme_app", + "old": "aws_instance.acme_app", + "operator": "op_f5_force_replacement_safe", + "scenario_id": "tp-v1-F5-0107", + "step": 1 + }, + { + "condition": "C", + "dail_pairs": [ + [ + "aws_instance.prod_app_server", + "aws_instance.prod_app_server", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_instance.prod_app_server", + "old": "aws_instance.prod_app_server", + "operator": "op_f5_force_replacement_safe", + "scenario_id": "tp-v1-F5-0114", + "step": 1 + }, + { + "condition": "C", + "dail_pairs": [ + [ + "aws_instance.tp_app", + "aws_instance.tp_app", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_instance.tp_app", + "old": "aws_instance.tp_app", + "operator": "op_f5_force_replacement_safe", + "scenario_id": "tp-v1-F5-0315", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_subnet.app", + null, + "DELETED", + false + ], + [ + null, + "aws_subnet.app_app", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_subnet.app_app", + "old": "aws_subnet.app", + "operator": "op_f5_restructure_moved_safe", + "scenario_id": "tp-v1-F5-0106", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_instance.acme_app", + "aws_instance.acme_app", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_instance.acme_app", + "old": "aws_instance.acme_app", + "operator": "op_f5_force_replacement_safe", + "scenario_id": "tp-v1-F5-0107", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_subnet.acme_app_subnet", + null, + "DELETED", + false + ], + [ + null, + "aws_subnet.acme_app_subnet_app", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_subnet.acme_app_subnet_app", + "old": "aws_subnet.acme_app_subnet", + "operator": "op_f5_restructure_moved_safe", + "scenario_id": "tp-v1-F5-0112", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group_rule.storage_access_from_app_5432", + null, + "DELETED", + false + ], + [ + null, + "aws_security_group_rule.storage_access_from_app_5432_v2", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_security_group_rule.storage_access_from_app_5432_v2", + "old": "aws_security_group_rule.storage_access_from_app_5432", + "operator": "op_f5_db_rule_moved_safe", + "scenario_id": "tp-v1-F5-0113", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_instance.prod_app_server", + "aws_instance.prod_app_server", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_instance.prod_app_server", + "old": "aws_instance.prod_app_server", + "operator": "op_f5_force_replacement_safe", + "scenario_id": "tp-v1-F5-0114", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.sg_db", + "aws_security_group.sg_db", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.sg_db", + "old": "aws_security_group.sg_db", + "operator": "op_f5_db_sg_replacement_unsafe", + "scenario_id": "tp-v1-F5-0118", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.prod_db_sg", + "aws_security_group.prod_db_sg", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.prod_db_sg", + "old": "aws_security_group.prod_db_sg", + "operator": "op_f5_db_sg_replacement_unsafe", + "scenario_id": "tp-v1-F5-0119", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.backend_sg", + "aws_security_group.backend_sg", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.backend_sg", + "old": "aws_security_group.backend_sg", + "operator": "op_f5_db_sg_replacement_unsafe", + "scenario_id": "tp-v1-F5-0120", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.storage_access", + "aws_security_group.storage_access", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.storage_access", + "old": "aws_security_group.storage_access", + "operator": "op_f5_db_sg_replacement_unsafe", + "scenario_id": "tp-v1-F5-0123", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group_rule.pg_from_app_5432", + null, + "DELETED", + false + ], + [ + null, + "aws_security_group_rule.pg_from_app_5432_v2", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_security_group_rule.pg_from_app_5432_v2", + "old": "aws_security_group_rule.pg_from_app_5432", + "operator": "op_f5_db_rule_moved_unsafe", + "scenario_id": "tp-v1-F5-0125", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.storage_access", + "aws_security_group.storage_access", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.storage_access", + "old": "aws_security_group.storage_access", + "operator": "op_f5_db_sg_replacement_unsafe", + "scenario_id": "tp-v1-F5-0126", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.sg_db", + "aws_security_group.sg_db", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.sg_db", + "old": "aws_security_group.sg_db", + "operator": "op_f5_db_sg_replacement_unsafe", + "scenario_id": "tp-v1-F5-0127", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group_rule.data_secgroup_from_app_5432", + null, + "DELETED", + false + ], + [ + null, + "aws_security_group_rule.data_secgroup_from_app_5432_v2", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_security_group_rule.data_secgroup_from_app_5432_v2", + "old": "aws_security_group_rule.data_secgroup_from_app_5432", + "operator": "op_f5_db_rule_moved_unsafe", + "scenario_id": "tp-v1-F5-0128", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_subnet.tp_app_subnet", + null, + "DELETED", + false + ], + [ + null, + "aws_subnet.tp_app_subnet_app", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_subnet.tp_app_subnet_app", + "old": "aws_subnet.tp_app_subnet", + "operator": "op_f5_restructure_moved_safe", + "scenario_id": "tp-v1-F5-0313", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group_rule.tp_db_ingress_postgres", + null, + "DELETED", + false + ], + [ + null, + "aws_security_group_rule.tp_db_ingress_postgres_v2", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_security_group_rule.tp_db_ingress_postgres_v2", + "old": "aws_security_group_rule.tp_db_ingress_postgres", + "operator": "op_f5_db_rule_moved_safe", + "scenario_id": "tp-v1-F5-0314", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_instance.tp_app", + "aws_instance.tp_app", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_instance.tp_app", + "old": "aws_instance.tp_app", + "operator": "op_f5_force_replacement_safe", + "scenario_id": "tp-v1-F5-0315", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.tp_db_sg", + "aws_security_group.tp_db_sg", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.tp_db_sg", + "old": "aws_security_group.tp_db_sg", + "operator": "op_f5_db_sg_replacement_safe", + "scenario_id": "tp-v1-F5-0316", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_subnet.tp_app_subnet", + null, + "DELETED", + false + ], + [ + null, + "aws_subnet.tp_app_subnet_app", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_subnet.tp_app_subnet_app", + "old": "aws_subnet.tp_app_subnet", + "operator": "op_f5_restructure_moved_safe", + "scenario_id": "tp-v1-F5-0319", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.tp_db_sg", + "aws_security_group.tp_db_sg", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.tp_db_sg", + "old": "aws_security_group.tp_db_sg", + "operator": "op_f5_db_sg_replacement_unsafe", + "scenario_id": "tp-v1-F5-0326", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group_rule.tp_db_ingress_postgres", + null, + "DELETED", + false + ], + [ + null, + "aws_security_group_rule.tp_db_ingress_postgres_v2", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_security_group_rule.tp_db_ingress_postgres_v2", + "old": "aws_security_group_rule.tp_db_ingress_postgres", + "operator": "op_f5_db_rule_moved_unsafe", + "scenario_id": "tp-v1-F5-0328", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.tp_db_sg", + "aws_security_group.tp_db_sg", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.tp_db_sg", + "old": "aws_security_group.tp_db_sg", + "operator": "op_f5_db_sg_replacement_unsafe", + "scenario_id": "tp-v1-F5-0329", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group_rule.tp_db_ingress_postgres", + null, + "DELETED", + false + ], + [ + null, + "aws_security_group_rule.tp_db_ingress_postgres_v2", + "CREATED", + false + ] + ], + "effect": "MOVE", + "new": "aws_security_group_rule.tp_db_ingress_postgres_v2", + "old": "aws_security_group_rule.tp_db_ingress_postgres", + "operator": "op_f5_db_rule_moved_unsafe", + "scenario_id": "tp-v1-F5-0331", + "step": 1 + }, + { + "condition": "C_PRIME", + "dail_pairs": [ + [ + "aws_security_group.tp_db_sg", + "aws_security_group.tp_db_sg", + "MODIFIED", + false + ] + ], + "effect": "REPLACE", + "new": "aws_security_group.tp_db_sg", + "old": "aws_security_group.tp_db_sg", + "operator": "op_f5_db_sg_replacement_unsafe", + "scenario_id": "tp-v1-F5-0332", + "step": 1 + } + ], + "identity_excluded": [ + { + "address": "aws_db_instance.storage_db", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0015", + "step": 1 + }, + { + "address": "aws_security_group.storage_access", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0015", + "step": 1 + }, + { + "address": "aws_security_group.web", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0016", + "step": 1 + }, + { + "address": "aws_vpc.net", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0016", + "step": 1 + }, + { + "address": "aws_security_group.app", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0019", + "step": 1 + }, + { + "address": "aws_vpc.main", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0019", + "step": 1 + }, + { + "address": "aws_db_instance.pg", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0021", + "step": 1 + }, + { + "address": "aws_security_group.pg", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0021", + "step": 1 + }, + { + "address": "aws_security_group.sg_app", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0022", + "step": 1 + }, + { + "address": "aws_vpc.vpc_main", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0022", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0223", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0223", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_app_sg_ingress_ssh", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0224", + "step": 1 + }, + { + "address": "aws_vpc.tp_vpc", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0224", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0226", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0226", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_app_sg_ingress_ssh", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0227", + "step": 1 + }, + { + "address": "aws_vpc.tp_vpc", + "condition": "C", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0227", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0232", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0232", + "step": 1 + }, + { + "address": "aws_security_group.web_secgroup", + "condition": "C", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0093", + "step": 1 + }, + { + "address": "aws_security_group_rule.data_secgroup_public_5432", + "condition": "C", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0093", + "step": 1 + }, + { + "address": "aws_db_instance.acme_db", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0094", + "step": 1 + }, + { + "address": "aws_security_group_rule.acme_db_sg_ingress_db_public", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0094", + "step": 1 + }, + { + "address": "aws_db_instance.pg", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0097", + "step": 1 + }, + { + "address": "aws_security_group_rule.pg_ingress_db_public", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0097", + "step": 1 + }, + { + "address": "aws_security_group_rule.data_secgroup_public_5432", + "condition": "C", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0099", + "step": 1 + }, + { + "address": "aws_security_group_rule.web_secgroup_ingress_ssh", + "condition": "C", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0099", + "step": 1 + }, + { + "address": "aws_db_instance.backend_database", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0100", + "step": 1 + }, + { + "address": "aws_security_group_rule.backend_sg_ingress_db_public", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0100", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_app_sg_ingress_ssh", + "condition": "C", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0301", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_ingress_postgres", + "condition": "C", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0301", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0302", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0302", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_app_sg_ingress_ssh", + "condition": "C", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0304", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_ingress_postgres", + "condition": "C", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0304", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0305", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0305", + "step": 1 + }, + { + "address": "aws_instance.prod_app_server", + "condition": "C", + "operator": "op_f5_count_each_safe", + "reason": "adds or removes `count = 1` on `aws_instance`: the catalogue does not state whether the instance keeps its identity (an implicit count move) or is replaced.", + "scenario_id": "tp-v1-F5-0110", + "step": 1 + }, + { + "address": "aws_instance.tp_app", + "condition": "C", + "operator": "op_f5_count_each_safe", + "reason": "adds or removes `count = 1` on `aws_instance`: the catalogue does not state whether the instance keeps its identity (an implicit count move) or is replaced.", + "scenario_id": "tp-v1-F5-0317", + "step": 1 + }, + { + "address": "aws_db_instance.storage_db", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0015", + "step": 1 + }, + { + "address": "aws_security_group.storage_access", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0015", + "step": 1 + }, + { + "address": "aws_security_group.web", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0016", + "step": 1 + }, + { + "address": "aws_vpc.net", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0016", + "step": 1 + }, + { + "address": "aws_security_group.app", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0019", + "step": 1 + }, + { + "address": "aws_vpc.main", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0019", + "step": 1 + }, + { + "address": "aws_db_instance.pg", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0021", + "step": 1 + }, + { + "address": "aws_security_group.pg", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0021", + "step": 1 + }, + { + "address": "aws_security_group.sg_app", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0022", + "step": 1 + }, + { + "address": "aws_vpc.vpc_main", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0022", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0223", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0223", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_app_sg_ingress_ssh", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0224", + "step": 1 + }, + { + "address": "aws_vpc.tp_vpc", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0224", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0226", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0226", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_app_sg_ingress_ssh", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0227", + "step": 1 + }, + { + "address": "aws_vpc.tp_vpc", + "condition": "C_PRIME", + "operator": "op_f1_vpc_dns_toggle_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0227", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0232", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C_PRIME", + "operator": "op_f1_storage_expand_unsafe", + "reason": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "scenario_id": "tp-v1-F1-0232", + "step": 1 + }, + { + "address": "aws_security_group.web_secgroup", + "condition": "C_PRIME", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0093", + "step": 1 + }, + { + "address": "aws_security_group_rule.data_secgroup_public_5432", + "condition": "C_PRIME", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0093", + "step": 1 + }, + { + "address": "aws_db_instance.acme_db", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0094", + "step": 1 + }, + { + "address": "aws_security_group_rule.acme_db_sg_ingress_db_public", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0094", + "step": 1 + }, + { + "address": "aws_db_instance.pg", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0097", + "step": 1 + }, + { + "address": "aws_security_group_rule.pg_ingress_db_public", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0097", + "step": 1 + }, + { + "address": "aws_security_group_rule.data_secgroup_public_5432", + "condition": "C_PRIME", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0099", + "step": 1 + }, + { + "address": "aws_security_group_rule.web_secgroup_ingress_ssh", + "condition": "C_PRIME", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0099", + "step": 1 + }, + { + "address": "aws_db_instance.backend_database", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0100", + "step": 1 + }, + { + "address": "aws_security_group_rule.backend_sg_ingress_db_public", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0100", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_app_sg_ingress_ssh", + "condition": "C_PRIME", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0301", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_ingress_postgres", + "condition": "C_PRIME", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0301", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0302", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0302", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_app_sg_ingress_ssh", + "condition": "C_PRIME", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0304", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_ingress_postgres", + "condition": "C_PRIME", + "operator": "op_f4_repair_db_port_open_ssh_unsafe", + "reason": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0304", + "step": 1 + }, + { + "address": "aws_db_instance.tp_db", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0305", + "step": 1 + }, + { + "address": "aws_security_group_rule.tp_db_sg_ingress_db_public", + "condition": "C_PRIME", + "operator": "op_f4_repair_rds_public_open_db_unsafe", + "reason": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created.", + "scenario_id": "tp-v1-F4-0305", + "step": 1 + }, + { + "address": "aws_instance.prod_app_server", + "condition": "C_PRIME", + "operator": "op_f5_count_each_safe", + "reason": "adds or removes `count = 1` on `aws_instance`: the catalogue does not state whether the instance keeps its identity (an implicit count move) or is replaced.", + "scenario_id": "tp-v1-F5-0110", + "step": 1 + }, + { + "address": "aws_instance.tp_app", + "condition": "C_PRIME", + "operator": "op_f5_count_each_safe", + "reason": "adds or removes `count = 1` on `aws_instance`: the catalogue does not state whether the instance keeps its identity (an implicit count move) or is replaced.", + "scenario_id": "tp-v1-F5-0317", + "step": 1 + } + ], + "non_pass_checks": [ + { + "condition": "A", + "count": 60, + "kind": "INVARIANT:INV-SEC-001", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "A", + "count": 5, + "kind": "INVARIANT:INV-SEC-002", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "A", + "count": 5, + "kind": "INVARIANT:INV-SEC-003", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.rds_public_flag" + }, + { + "condition": "B", + "count": 58, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "path.missing", + "result": "FAIL", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "B", + "count": 3, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "port.mismatch", + "result": "FAIL", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "B", + "count": 1, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "route.unsupported", + "result": "UNSUPPORTED", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "B", + "count": 74, + "kind": "INVARIANT:INV-SEC-001", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "B", + "count": 1, + "kind": "INVARIANT:INV-SEC-001", + "reason": "rule.unsupported", + "result": "UNSUPPORTED", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "B", + "count": 19, + "kind": "INVARIANT:INV-SEC-002", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "B", + "count": 1, + "kind": "INVARIANT:INV-SEC-002", + "reason": "rule.unsupported", + "result": "UNSUPPORTED", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "B", + "count": 14, + "kind": "INVARIANT:INV-SEC-003", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.rds_public_flag" + }, + { + "condition": "B", + "count": 5, + "kind": "RESOURCE:aws_db_instance", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.rds_public_flag" + }, + { + "condition": "B", + "count": 9, + "kind": "RESOURCE:aws_security_group", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "B", + "count": 13, + "kind": "RESOURCE:aws_security_group_rule", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C", + "count": 58, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "path.missing", + "result": "FAIL", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "C", + "count": 3, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "port.mismatch", + "result": "FAIL", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "C", + "count": 1, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "route.unsupported", + "result": "UNSUPPORTED", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "C", + "count": 74, + "kind": "INVARIANT:INV-SEC-001", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C", + "count": 1, + "kind": "INVARIANT:INV-SEC-001", + "reason": "rule.unsupported", + "result": "UNSUPPORTED", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C", + "count": 19, + "kind": "INVARIANT:INV-SEC-002", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C", + "count": 1, + "kind": "INVARIANT:INV-SEC-002", + "reason": "rule.unsupported", + "result": "UNSUPPORTED", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C", + "count": 14, + "kind": "INVARIANT:INV-SEC-003", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.rds_public_flag" + }, + { + "condition": "C", + "count": 5, + "kind": "RESOURCE:aws_db_instance", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.rds_public_flag" + }, + { + "condition": "C", + "count": 9, + "kind": "RESOURCE:aws_security_group", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C", + "count": 13, + "kind": "RESOURCE:aws_security_group_rule", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C_PRIME", + "count": 2, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "endpoint.absent", + "result": "FAIL", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "C_PRIME", + "count": 58, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "path.missing", + "result": "FAIL", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "C_PRIME", + "count": 3, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "port.mismatch", + "result": "FAIL", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "C_PRIME", + "count": 1, + "kind": "INVARIANT:INV-FUNC-001", + "reason": "route.unsupported", + "result": "UNSUPPORTED", + "verifier_id": "dail.verify.reachability_cfg" + }, + { + "condition": "C_PRIME", + "count": 74, + "kind": "INVARIANT:INV-SEC-001", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C_PRIME", + "count": 1, + "kind": "INVARIANT:INV-SEC-001", + "reason": "rule.unsupported", + "result": "UNSUPPORTED", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C_PRIME", + "count": 19, + "kind": "INVARIANT:INV-SEC-002", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C_PRIME", + "count": 1, + "kind": "INVARIANT:INV-SEC-002", + "reason": "rule.unsupported", + "result": "UNSUPPORTED", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C_PRIME", + "count": 14, + "kind": "INVARIANT:INV-SEC-003", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.rds_public_flag" + }, + { + "condition": "C_PRIME", + "count": 5, + "kind": "RESOURCE:aws_db_instance", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.rds_public_flag" + }, + { + "condition": "C_PRIME", + "count": 9, + "kind": "RESOURCE:aws_security_group", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + }, + { + "condition": "C_PRIME", + "count": 13, + "kind": "RESOURCE:aws_security_group_rule", + "reason": "predicate.violated", + "result": "FAIL", + "verifier_id": "dail.verify.public_ingress" + } + ], + "prefix_sizes": { + "common_prefix_B_C": 273, + "common_prefix_all": 273 + }, + "truth_changes_outside_v": [], + "unsafe_promotions_c": [], + "work_difference_f5": { + "aws_security_group_rule": { + "difference": 6, + "transitions": 6, + "work_c": 34, + "work_c_prime": 40 + }, + "aws_subnet": { + "difference": 0, + "transitions": 4, + "work_c": 12, + "work_c_prime": 12 + } + } + }, + "preface": [ + "This report supersedes the one generated from run 2 (`p8a-dev-harness-check-2`, ec2b4fa8...).", + "Run 2 was affected by a DAIL defect found by the external review of PR #31: under registry v1.1 every", + "baseline item of a created security group or rule was UNSUPPORTED `invariant.unknown` and never", + "evaluated (`core/verification/baseline.py` looked the definition up at version 1). The fix is", + "commit a34e25a (C-216); this report is generated from run 3 (`p8b-dev-run-3`), run on that commit.", + "Run 2 is kept unchanged under the experiment root." + ], + "primary_metrics": [ + { + "condition_id": "A", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2d70c7d4-e9c2-579b-bcaf-edd28ac0a4db", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.023303960351823035 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "53ed4beb-a564-59b8-b586-111ea1cee125", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "78bc9e2b-76ea-5a52-8fcd-327a36194665", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9d67f6f1-da60-5b8e-af80-2579a894ef4f", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5bc73323-5d0e-565e-baf9-3648aa53e07b", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c84c29a8-4956-5b19-9a58-46c43c7e9489", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5556579e-ca6f-57a0-8f08-6334c0a476d3", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2e25c128-9968-578b-9866-08601390d731", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e03516ca-25d3-5102-ae0d-a90b2094bf24", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "A", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "63e81577-9cab-5f9d-9fa0-9235fc09f09a", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "A", + "denominator": 81, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0ac760fa-6a45-54e9-a1f6-5202aeb478cb", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.045278085432415204 + ] + }, + { + "condition_id": "A", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1d8a4068-c86c-5709-8e45-4c09935b6429", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.023303960351823035 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "592b6bfe-7fcf-5325-bf21-3d09124986f8", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "537fec14-8aa0-53b1-8a9b-7feb1c842111", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "be968ac3-3f6b-543f-b73d-4d5ce2c90bf9", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "64f36a01-de30-5f35-a134-e10ca434d40f", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "baf3038b-09f2-5a58-84e5-b45a0f3b83cd", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9e231ad4-c4a2-51ec-b5c8-c0f9a7e66555", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "38d0eafa-d201-51b8-bcb3-af9fdac88170", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "360d81eb-f022-5be4-8e5b-1c259adf11df", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "A", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "21736db7-ac0a-54e2-84ed-df9493daf542", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "A", + "denominator": 81, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f4376ca4-7015-5ce5-989d-4f2aafffab07", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.045278085432415204 + ] + }, + { + "condition_id": "B", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "609d4d44-1baf-51ab-ae35-941120db3a17", + "metric_name": "false_rejection_rate", + "numerator": 2, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.012422360248447204, + "wilson95": [ + 0.0034132960972489516, + 0.04415640437005673 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5fd50f38-adde-529f-90b9-1962650cef09", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39f46722-5166-50d0-9f72-c0b38ddb6336", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c2f9cb6d-2eaf-577d-ad33-6f64530fd3b4", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "433fcfc9-59f4-59da-a0de-cf2269153f3d", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b667ee87-6481-5054-abc4-4cb851fbae81", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a073d7ee-c737-5abb-9273-c0a9d22b94a7", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "52df29db-79ab-581e-9b72-412a8a60ff22", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "33d99fa8-ab5d-595f-813d-b99121ca9c3a", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "B", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a92e1d3b-9e6b-519c-b08a-598454662e0a", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "B", + "denominator": 81, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6bb16fdf-acf5-534e-82ae-4145c2fcb26c", + "metric_name": "false_rejection_rate", + "numerator": 2, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.024691358024691357, + "wilson95": [ + 0.006797587423664028, + 0.08562725922196524 + ] + }, + { + "condition_id": "B", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "09e4b73b-0d84-5262-bf45-f7eb97a59093", + "metric_name": "false_rejection_rate", + "numerator": 2, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.012422360248447204, + "wilson95": [ + 0.0034132960972489516, + 0.04415640437005673 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4795b689-8379-53d1-b417-257beb5b004a", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5ae4b856-da33-5286-b689-7ead955a1424", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "05559dac-4d71-5423-9db2-745fc1648e02", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b82c4c00-f775-501e-910f-26face388f32", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "166f8883-93ad-5ae0-836a-521a02ceb9ec", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4fde29fd-1594-59b0-bf9c-a39ec46cccd8", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "88e5c5d7-ce04-55c7-ae7e-8c90eaf9879b", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0650105-179d-56fb-b688-4261a39b239f", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "B", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7131fce8-4fa8-5020-949d-45825141c8e7", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "B", + "denominator": 81, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ea7a2dfe-325b-5979-98f5-5a79b94110f5", + "metric_name": "false_rejection_rate", + "numerator": 2, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.024691358024691357, + "wilson95": [ + 0.006797587423664028, + 0.08562725922196524 + ] + }, + { + "condition_id": "C", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e5ab3e45-3111-5be6-aec5-0093ab332802", + "metric_name": "false_rejection_rate", + "numerator": 2, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.012422360248447204, + "wilson95": [ + 0.0034132960972489516, + 0.04415640437005673 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a11f55e3-d226-5e98-b5ba-7b63028b3e2e", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "44372249-e139-5277-8bc7-ffb9b9c23e79", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c748921f-57c9-5b6c-9fb2-05f1d1a2453b", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cf11b743-48f1-59bc-9cb8-5c7ae303d317", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e4933748-2385-5ead-b139-a6b7465296a2", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "32a4e32b-97e2-510e-96c0-361dd808fd25", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f357219a-0266-5fe7-b946-0d7b9578d8ef", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6120eb15-bb88-54f4-a1fb-65449ea7f521", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "C", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bb11039b-91b5-5a1b-99b3-d5f87209c36f", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "C", + "denominator": 81, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fdebcf56-2646-5a03-aa8b-1203f2cf846f", + "metric_name": "false_rejection_rate", + "numerator": 2, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.024691358024691357, + "wilson95": [ + 0.006797587423664028, + 0.08562725922196524 + ] + }, + { + "condition_id": "C", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "adfe05be-db82-5258-b7eb-6f306c5d632a", + "metric_name": "false_rejection_rate", + "numerator": 2, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.012422360248447204, + "wilson95": [ + 0.0034132960972489516, + 0.04415640437005673 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6ca36a27-24cd-50c8-aed8-ff40f010e266", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "246b276e-6605-5cf0-bf2f-7722a9fa6ded", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0f274d43-d7f7-597a-b0fc-142bddf853e5", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0ab3fcb4-740e-504d-b525-8e5f6c2e8908", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4a7cd005-a9cc-5e8c-8e1e-09681d39b02d", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "399c54b0-c952-53d5-b977-e34c6bfde823", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4831e565-c6f2-5133-8575-d0febe1401b5", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "53f60563-d7d0-585f-b3f1-3b2342ca4ab7", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "C", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "385b6717-6f1d-5728-b726-30a0769eba09", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "C", + "denominator": 81, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2968fee3-1f46-5dd1-8264-951d29ac3b95", + "metric_name": "false_rejection_rate", + "numerator": 2, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.024691358024691357, + "wilson95": [ + 0.006797587423664028, + 0.08562725922196524 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "22e3e7da-2d8a-52db-9ace-34cba41d27ec", + "metric_name": "false_rejection_rate", + "numerator": 4, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.024844720496894408, + "wilson95": [ + 0.00970308506388464, + 0.062132355518903705 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "99eaf4b5-d661-5441-8073-0380d8871f56", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "150fbe4f-09ae-554e-9480-bf23d73f114c", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cfac300d-449f-5448-8fd3-5683758bb1b2", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "144438c5-99ec-553a-8ebb-f54318352cc8", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1025874e-272c-5367-ad97-feb44d5cc3c3", + "metric_name": "false_rejection_rate", + "numerator": 3, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.21428571428571427, + "wilson95": [ + 0.07571386712693184, + 0.47589230586600295 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "323876f4-b849-573f-982b-7889dcb9a3e7", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d5573b0a-4281-563e-9711-c79877cacc26", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "38cde4f0-f634-5d2f-9baa-0117bfe37e50", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9b5cb3af-159b-5947-9f16-407104d2777b", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0125, + "wilson95": [ + 0.0022099755204522745, + 0.06746270077668665 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 81, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "847924be-b94b-5568-804e-e746ddab684d", + "metric_name": "false_rejection_rate", + "numerator": 3, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.037037037037037035, + "wilson95": [ + 0.012675114183298107, + 0.10332311306893818 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9ae2ede1-6c50-5eec-a14d-e50b78e6979e", + "metric_name": "false_rejection_rate", + "numerator": 4, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.024844720496894408, + "wilson95": [ + 0.00970308506388464, + 0.062132355518903705 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8f0c2d82-6445-5980-95e3-eaed60634e4e", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "21b5be41-4d02-55ed-a92c-ae27149e0236", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "71d113f0-d996-5649-85af-a0498b45ad6d", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7a879dde-c89e-58e6-8766-50f18616bcfc", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "33d38351-fb2e-55cb-9f31-139fecfe6060", + "metric_name": "false_rejection_rate", + "numerator": 3, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.21428571428571427, + "wilson95": [ + 0.07571386712693184, + 0.47589230586600295 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8ddff1a8-9dc0-52ed-ac1e-2345a0634d54", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.07142857142857142, + "wilson95": [ + 0.012722215247890939, + 0.31468704424151117 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b0107082-4766-56d5-a78b-dbcec37dece5", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8c5a217a-1ec1-5376-b1f1-3acd972cdda7", + "metric_name": "false_rejection_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "825e0291-4ef7-5720-bd3c-8b98a4335882", + "metric_name": "false_rejection_rate", + "numerator": 1, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0125, + "wilson95": [ + 0.0022099755204522745, + 0.06746270077668665 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 81, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "78751b52-5e07-5bdb-a8ec-b1af34cc1469", + "metric_name": "false_rejection_rate", + "numerator": 3, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.037037037037037035, + "wilson95": [ + 0.012675114183298107, + 0.10332311306893818 + ] + }, + { + "condition_id": "A", + "denominator": 98, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5c608c56-019d-50e9-b308-24217e4626e9", + "metric_name": "missed_regressions_per_regression", + "numerator": 98, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9622800098783192, + 0.9999999999999999 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "59b3f866-a5eb-54b7-b572-5af15af0e3a1", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 0, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "75e8785c-ca6f-576a-8e80-60d8836d34f3", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": null, + "wilson95": [ + null, + null + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "17e73c4d-a0e4-5b7c-8a8e-c73166c848fb", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ed48cfa2-5538-5cca-bf4b-92e30325d68c", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eb8803aa-849b-5712-b8de-f82c386543a7", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e97771b8-2e8f-57cb-823a-2a9c73eda0a2", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bf08735b-6203-5880-928b-47ea02aac55b", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bd1aa34e-2a34-571a-a931-8f569cbfdf7f", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 50, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1ff13336-e749-5229-821c-3f836ec0a140", + "metric_name": "missed_regressions_per_regression", + "numerator": 50, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9286524008666414, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e12b8dfd-cca3-5500-87c8-34aeb03eb935", + "metric_name": "missed_regressions_per_regression", + "numerator": 48, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9258998703338824, + 0.9999999999999999 + ] + }, + { + "condition_id": "A", + "denominator": 98, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a8f0ef87-7278-542b-a571-ef28b0c7c6d7", + "metric_name": "missed_regressions_per_regression", + "numerator": 98, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9622800098783192, + 0.9999999999999999 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3112ce1f-0a58-51bc-9abd-123dd4249d22", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 0, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "acd81ec1-b76b-52cb-be40-d194b849fe29", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": null, + "wilson95": [ + null, + null + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1525b25d-c14e-5773-9e3c-8c4a212cd40f", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9cd5ff2-42c3-5a0d-9470-10ec952a699d", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c7c9e99c-9285-513e-9bef-d694f2951c13", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5d40d826-a303-5d2e-bd79-f8b71b8edb36", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3b699f77-86fa-5419-96aa-0dd318951616", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2f190b48-f9a3-5ce1-b137-9107f12991d7", + "metric_name": "missed_regressions_per_regression", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 50, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7d77edb7-a66b-5e76-ad2a-de1576918af8", + "metric_name": "missed_regressions_per_regression", + "numerator": 50, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9286524008666414, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e104faf8-c3fb-5f93-9bbc-4c39d0e68ad3", + "metric_name": "missed_regressions_per_regression", + "numerator": 48, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9258998703338824, + 0.9999999999999999 + ] + }, + { + "condition_id": "B", + "denominator": 98, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "65eaa1c4-db22-5202-9c43-6e51f942c131", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.037719990121680605 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "37c15ffb-d1bd-5b95-a84f-04f93f3c75ce", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 0, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6f9c84b6-8336-512a-beb9-47c81fd5565e", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": null, + "wilson95": [ + null, + null + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "48cb9092-0521-5d27-8b96-1685b4147a7b", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "245b91cc-8108-5698-a285-26c4aaad2522", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c083566a-ce13-553a-a00d-a2cd46b3fe71", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6fd32ad3-81d0-5777-9b56-c356cf05abf0", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eff01aa2-acbc-53d7-8911-a512a8d5394a", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e3164a13-b77c-5527-b883-30ed5856ef7c", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 50, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7cae2446-a9f2-52d4-817c-5c2eebb3a0df", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07134759913335872 + ] + }, + { + "condition_id": "B", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4ac80dfd-0a0d-5cfd-bdab-54a4db9016d9", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07410012966611748 + ] + }, + { + "condition_id": "B", + "denominator": 98, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "42b95fd4-24ed-5e7c-a345-b9e87c9836c9", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.037719990121680605 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5c5468b0-29e6-53e8-a8d5-35f60774bc84", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 0, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4c8e5d57-8f7d-507b-9751-a2a36b7f0532", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": null, + "wilson95": [ + null, + null + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eb824e19-186e-5f4b-b4fb-fb60c0aae8dc", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c491cca9-af62-5a68-9992-628fb5ac69ca", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "97344466-7948-5ded-9ce0-91faadcbaa42", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "427745c1-5d1c-5a5c-931f-fa21abf9cff1", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c124951c-6a7d-5136-8346-34f6454658ff", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d91c2731-d375-584a-a72a-ab17074e2ec3", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 50, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "436ae345-786b-5f95-9d7a-947c83226d48", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07134759913335872 + ] + }, + { + "condition_id": "B", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e6df1d09-84da-562f-864d-4473c5cc8b9e", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07410012966611748 + ] + }, + { + "condition_id": "C", + "denominator": 98, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5af19fa5-5f42-54c9-8026-1d832d3dae72", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.037719990121680605 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "12b4884b-50d0-5712-8716-666f9eaa5274", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 0, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "29d19dc8-4874-518d-b6c6-c94fbd8764da", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": null, + "wilson95": [ + null, + null + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3b5ebaa5-c60d-5b51-b74e-7f6e2ab0642a", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "58104306-9974-5e99-82a3-92ff181fdc24", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "21f6a143-3f5d-5fc4-bc92-b7af7931ea83", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f6cb49e1-88f8-5a4e-9af2-5eabbbc250d7", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c15939c1-5e7d-5736-8acf-bebe45c66454", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ea6a7e28-ac19-54cd-b04c-f1390228b145", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 50, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "970d4fc1-c8dd-59d7-99df-383e0cc95b39", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07134759913335872 + ] + }, + { + "condition_id": "C", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "183f8cb2-abb1-5989-9090-6f8f99ece10c", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07410012966611748 + ] + }, + { + "condition_id": "C", + "denominator": 98, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "223d1da3-bc81-5487-84d5-e0d7d390b5f3", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.037719990121680605 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4719c392-ec41-5a10-9205-1a89d39032a3", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 0, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "abb96ad0-cd03-5169-8b90-0aaa13e4d8e4", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": null, + "wilson95": [ + null, + null + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e268f318-15ac-5fdb-bd6c-47dd93313570", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b7529a38-f1bc-5f01-b42e-4399afd11da4", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fd0d999c-81b3-57b4-80fa-b562a20bcc32", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fbcf04dd-d27d-5102-8811-8bec62c23f07", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6c65180c-c1b8-544f-a81d-5bbe012a7091", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b22d933b-df4e-5854-bd02-2b8245f818db", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 50, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "76ae14b6-5344-5244-9232-7ff676e183a9", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07134759913335872 + ] + }, + { + "condition_id": "C", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "79049512-9389-5d4f-8e6c-2ce08f0829a2", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07410012966611748 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 98, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4b935b14-ce28-51bf-a161-1d6510dab155", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.037719990121680605 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b3952c40-5bd9-50f7-a0b7-3f8d991a5325", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 0, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cdc2dad8-6699-5e2b-82a6-625be15d8265", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": null, + "wilson95": [ + null, + null + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "45fabfe1-5f8a-5142-80f2-b8f0132be5bf", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8957a34a-6b25-5075-9619-fb3ea0290718", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "65263a17-e7c5-545c-b287-53427650cfd6", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "06fcf765-d08c-51f8-a81a-0d81980c7d70", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f8e42041-e392-58aa-8ef0-5e5aa5b02c30", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "91c692f1-9542-541c-bd0c-d79ae3e1403e", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 50, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "999a5b26-149d-5314-bba4-554ae9398910", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07134759913335872 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1affe270-245e-58d7-85e0-25a7f66c292e", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07410012966611748 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 98, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "46be6fb9-ea52-5c43-a4ba-e03deeddaf3e", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.037719990121680605 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d082262d-b132-59a1-b0c6-a4e1ee893795", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 0, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "019c04c9-25e2-5232-b972-c30579d821e6", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": null, + "wilson95": [ + null, + null + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "78f2a775-0366-5bf1-a908-b95c252c72cc", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ef5f93fa-529c-5150-9c01-ee45eaefd544", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f8c5ade7-487a-5fc1-8422-3f9ec81cf97a", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c132fbbe-266f-5094-8fe2-d54169ff8794", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "939740e4-1b24-5938-a831-11142fd70ee1", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6c409c51-b53e-5d07-8700-c870836df49b", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 50, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "73603b2d-ce09-5f54-a579-41053152048e", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07134759913335872 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "faa31b3f-6f76-5c11-a232-3fc533a1e67b", + "metric_name": "missed_regressions_per_regression", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.07410012966611748 + ] + }, + { + "condition_id": "A", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0bb9791a-fa87-5a1a-9aa4-423082e08b8e", + "metric_name": "regressions", + "numerator": 98, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.35, + "wilson95": [ + 0.2965047188389011, + 0.4075554267472878 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "90d9267e-441c-5053-8ec9-fee9d7885635", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "68893e7c-5a42-515c-9cd3-8d7421719fea", + "metric_name": "regressions", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d643a8e5-6355-5c1f-89e5-954150c63b41", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0f9e54ac-eb38-5173-af6f-7bd7f0c2c9c5", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "551c0467-bdd9-52a9-817a-ff709372743f", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "df3de6cb-6583-532c-a5eb-da65d26b515c", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "73d921b5-d7fc-5d48-8161-a4c2c1c119e5", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "93d881d6-bcd9-574b-8d17-0aef74781384", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.16666666666666666, + "wilson95": [ + 0.10195742148970746, + 0.2605303974533991 + ] + }, + { + "condition_id": "A", + "denominator": 141, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "87e538e5-de92-5a5d-a440-caca5196ef23", + "metric_name": "regressions", + "numerator": 50, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.3546099290780142, + "wilson95": [ + 0.2804614644937273, + 0.4364704116843788 + ] + }, + { + "condition_id": "A", + "denominator": 139, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8ebab1ef-f861-55ea-8102-2fa2af5b86ad", + "metric_name": "regressions", + "numerator": 48, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.34532374100719426, + "wilson95": [ + 0.2713990184612785, + 0.4275679316454339 + ] + }, + { + "condition_id": "A", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3b4db512-441d-55e4-b869-fa483acbad49", + "metric_name": "regressions", + "numerator": 98, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.358974358974359, + "wilson95": [ + 0.30439040061997985, + 0.4174720677833828 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "71e4d70d-57fd-59a6-80ed-9efa2058559a", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d3627e8e-74ae-59d2-b18d-2a8d3cbd1953", + "metric_name": "regressions", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c90c215c-4705-508e-ae31-4907433ac49f", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d994201d-0006-5727-8ca6-3f78312e59c4", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "33608369-e225-5804-9b5e-45a3a98a12e7", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8bae8c6d-4318-550d-8921-0319c13471fb", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c23e3a4a-7909-5484-8d8b-fb1ad6a0fe81", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "74d2b21a-06bc-5d01-b87f-0c2111e54396", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "A", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "177bd9b5-e9da-5aea-90cf-87d477a06412", + "metric_name": "regressions", + "numerator": 50, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.36231884057971014, + "wilson95": [ + 0.28685664689873536, + 0.44523860708453833 + ] + }, + { + "condition_id": "A", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "69cda930-063c-53e4-a300-79cc512ac70d", + "metric_name": "regressions", + "numerator": 48, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.35555555555555557, + "wilson95": [ + 0.2798294544129023, + 0.4392746206143827 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "32596b53-dee2-5c8a-a5c0-307d3f67e19c", + "metric_name": "regressions", + "numerator": 98, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.358974358974359, + "wilson95": [ + 0.30439040061997985, + 0.4174720677833828 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6500a580-45ce-5de7-b44e-a0599bb5bb2a", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7171e639-691a-5d1a-8c58-1f5faa50d240", + "metric_name": "regressions", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4bb00393-d25e-5636-94a0-658475badf64", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "80157a02-79a5-553d-aba2-f36af03b18b7", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3c4cc271-96d5-5748-ac2b-b72001e09b68", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "af56a82e-b40e-54f7-8ea5-6f5f085c56f3", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "287050b3-33d1-56fb-8d55-bdb5194408a8", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3de26e12-19a7-5f87-8b5a-2a6be90f29c7", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5a0e0843-cd1c-5208-8b35-6575ec7f6a97", + "metric_name": "regressions", + "numerator": 50, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.36231884057971014, + "wilson95": [ + 0.28685664689873536, + 0.44523860708453833 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6633a329-e1a0-5a5d-9626-5e74f12cd8df", + "metric_name": "regressions", + "numerator": 48, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.35555555555555557, + "wilson95": [ + 0.2798294544129023, + 0.4392746206143827 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e3576a1b-4fcc-5975-b735-c91db6981ac1", + "metric_name": "regressions", + "numerator": 98, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.358974358974359, + "wilson95": [ + 0.30439040061997985, + 0.4174720677833828 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "69169678-2f55-50c1-8b57-f5c3ee31281c", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "43e3cbac-52c7-5ab1-8671-0c60962b24b8", + "metric_name": "regressions", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "529dedf0-eae3-5a68-a3a3-05a609fd0266", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9cbf870-d8ab-5e9b-8622-94339429c2b9", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "782280ec-7acf-581e-add5-5106ba847ad1", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1dfbeea6-3426-522d-ac2a-bd206fb2d2ed", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4d413409-b445-52e4-8c62-941cd28cc8bf", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d9572d29-897d-515c-a7f5-bfe3eafefa33", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "302c4fb0-0efc-55c4-b9c7-a745f81639d2", + "metric_name": "regressions", + "numerator": 50, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.36231884057971014, + "wilson95": [ + 0.28685664689873536, + 0.44523860708453833 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fe57e7d6-d98c-5608-a6b7-a322dbbb65ac", + "metric_name": "regressions", + "numerator": 48, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.35555555555555557, + "wilson95": [ + 0.2798294544129023, + 0.4392746206143827 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dec1d644-ef30-5770-91d0-d5a79f619b7e", + "metric_name": "regressions", + "numerator": 98, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.358974358974359, + "wilson95": [ + 0.30439040061997985, + 0.4174720677833828 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "08e65fb9-7ade-5488-8bf9-3fb9b0258ffc", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0e0060c1-1f97-5caa-8bdc-4cad7f2c6faf", + "metric_name": "regressions", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "207bcc8b-6297-5530-8e7c-2b919dad7c9d", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "31e878a5-dc3c-5d9c-ad65-b4d0e518c8ff", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6d891caf-0383-5c31-acd9-06d799ed38cc", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "82163e06-cf74-5e1b-88e5-fc740b0a47c8", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7d8aed20-c4c9-517f-94f5-31d9407b475b", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2583eac0-311a-5e53-975b-01dd9aef7c2f", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a502a634-3c5f-545f-ba95-833d3daf474e", + "metric_name": "regressions", + "numerator": 50, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.36231884057971014, + "wilson95": [ + 0.28685664689873536, + 0.44523860708453833 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f96c73c4-ce1c-540d-9d7c-58b201f45b06", + "metric_name": "regressions", + "numerator": 48, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.35555555555555557, + "wilson95": [ + 0.2798294544129023, + 0.4392746206143827 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e4bd1555-0d9e-58a6-b8e8-772d787bbeff", + "metric_name": "regressions", + "numerator": 98, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.358974358974359, + "wilson95": [ + 0.30439040061997985, + 0.4174720677833828 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6256590c-74dd-5675-9df0-2c73a5132a71", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7af33759-9f0f-5332-b8fd-f268329d66ed", + "metric_name": "regressions", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "df1a39ef-0ff2-5ecf-82bd-1f5d392fc452", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f6499ca7-587d-5239-b7f6-e36fcff80d54", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "735b3e69-dd09-5109-b499-31ccbbb3f85f", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "945ef48a-27e1-5f91-9040-a6a6a57850eb", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "25e7fa2e-8d4f-5d79-b6fa-fd86003673fd", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7f18bbf1-b90c-5d30-84ba-adee1279388e", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6577c55a-d8d0-5526-b184-ffc8715006b8", + "metric_name": "regressions", + "numerator": 50, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.36231884057971014, + "wilson95": [ + 0.28685664689873536, + 0.44523860708453833 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1e0f301a-f711-5212-ab29-8405bf214bc6", + "metric_name": "regressions", + "numerator": 48, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.35555555555555557, + "wilson95": [ + 0.2798294544129023, + 0.4392746206143827 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0900ec8a-97cf-575e-ac48-b218d669c145", + "metric_name": "regressions", + "numerator": 98, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.358974358974359, + "wilson95": [ + 0.30439040061997985, + 0.4174720677833828 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9e89e579-93af-545f-bf4b-4f47c5901f5f", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d4c93b87-ef1f-5c93-bed9-0b17b8162806", + "metric_name": "regressions", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9a300676-3ca3-5ab9-8d17-98381baf8081", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cbc72957-2ee3-5f57-9729-86cab87e73f0", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ebbc2d17-06f0-5902-8576-45f71a1d04ed", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "98a149de-5e04-5c3a-b18a-578d5c9d4ef7", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "19ca8dc6-df8b-565e-b985-440ce175c34b", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c33288cc-5f6f-5c10-9c18-c732d9d60386", + "metric_name": "regressions", + "numerator": 14, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ddc0c37b-ea6e-57ae-87f8-69c2047f81f8", + "metric_name": "regressions", + "numerator": 50, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.36231884057971014, + "wilson95": [ + 0.28685664689873536, + 0.44523860708453833 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "014cf347-24ca-5a5a-9681-171ace93bb21", + "metric_name": "regressions", + "numerator": 48, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.35555555555555557, + "wilson95": [ + 0.2798294544129023, + 0.4392746206143827 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "841e8dac-71b1-55a9-af9a-5e615549df85", + "metric_name": "regressions", + "numerator": 98, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.358974358974359, + "wilson95": [ + 0.30439040061997985, + 0.4174720677833828 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "56c9ccda-8d1e-58ff-80f7-96b1fbbcdf5f", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0c2078b6-ba80-512a-94fc-91b4823c877b", + "metric_name": "regressions", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "524e6758-4b23-54ae-8d4c-e76d1054e1cc", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0d1e4770-3301-5602-9349-6df24c93b020", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ce07bd12-5aae-53cd-86d8-37d7f2b841ab", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "49040ffb-8b0b-5bf5-ac1e-bdc08414ab12", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ac4a2665-ade3-50a8-86f3-22690e23e08f", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ef72a596-4019-59b8-9cfe-387bf4d7344d", + "metric_name": "regressions", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f23b8775-7127-51d6-a46d-813b7e41e5f0", + "metric_name": "regressions", + "numerator": 50, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.36231884057971014, + "wilson95": [ + 0.28685664689873536, + 0.44523860708453833 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "09154eb6-7525-518e-8ee3-af5ba46d193c", + "metric_name": "regressions", + "numerator": 48, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.35555555555555557, + "wilson95": [ + 0.2798294544129023, + 0.4392746206143827 + ] + }, + { + "condition_id": "A", + "denominator": 140, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a0577025-d0e5-5d1a-bbd6-8bb4c8fd4cce", + "metric_name": "repair_success", + "numerator": 70, + "scope": "all", + "stratum": "kind=change_request", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.4182898409476934, + 0.5817101590523066 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "41271403-ecfc-5337-9b43-fff041b8cea7", + "metric_name": "repair_success", + "numerator": 42, + "scope": "all", + "stratum": "kind=repair", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.3954393376854136, + 0.6045606623145864 + ] + }, + { + "condition_id": "B", + "denominator": 140, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3f53b5b7-0bf1-59c0-9fe8-d119ab976ea7", + "metric_name": "repair_success", + "numerator": 68, + "scope": "all", + "stratum": "kind=change_request", + "trial_id": null, + "unit": "proportion", + "value": 0.4857142857142857, + "wilson95": [ + 0.4044181106648916, + 0.5677734950599487 + ] + }, + { + "condition_id": "B", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2863794b-6e19-5252-a9fd-d1a80b5ebec6", + "metric_name": "repair_success", + "numerator": 42, + "scope": "all", + "stratum": "kind=repair", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.3954393376854136, + 0.6045606623145864 + ] + }, + { + "condition_id": "C", + "denominator": 140, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a45e39ea-bf57-52d2-8857-36e84251aeb5", + "metric_name": "repair_success", + "numerator": 68, + "scope": "all", + "stratum": "kind=change_request", + "trial_id": null, + "unit": "proportion", + "value": 0.4857142857142857, + "wilson95": [ + 0.4044181106648916, + 0.5677734950599487 + ] + }, + { + "condition_id": "C", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7d56fb68-f5ca-5ace-b688-584aa8dfc3ff", + "metric_name": "repair_success", + "numerator": 42, + "scope": "all", + "stratum": "kind=repair", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.3954393376854136, + 0.6045606623145864 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 140, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1c79e63b-c382-5928-b859-512676a76438", + "metric_name": "repair_success", + "numerator": 66, + "scope": "all", + "stratum": "kind=change_request", + "trial_id": null, + "unit": "proportion", + "value": 0.4714285714285714, + "wilson95": [ + 0.3906113916175145, + 0.5537718198321663 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1c0a5eca-2075-560b-a9df-ef354bf2f30c", + "metric_name": "repair_success", + "numerator": 42, + "scope": "all", + "stratum": "kind=repair", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.3954393376854136, + 0.6045606623145864 + ] + }, + { + "condition_id": "A", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5fb5b506-10b5-56fc-83eb-6150423b1ede", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 105, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.375, + "wilson95": [ + 0.32034591399099716, + 0.4330375406641603 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "54d0b480-7b41-5481-9409-8a278448900b", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b797f52f-4071-5959-b28a-0aaaddd6f692", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c4e143e8-9c50-55b4-83e4-71a30567b84a", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b8ae60cf-495d-577c-be61-992dc8c56b3c", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4f4b697a-b5b0-5884-a0c1-615507d663de", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b7385c3d-d1ab-5aa4-abde-42a629bced73", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "62d8ea28-c22d-565a-8b5f-d1eb2cadd650", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5cbbef2e-d213-5b1e-9b44-7c258da2da61", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 21, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.25, + "wilson95": [ + 0.16972312954861019, + 0.35214273465871976 + ] + }, + { + "condition_id": "A", + "denominator": 141, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a8bde19a-2521-58c7-9658-151c814bbe40", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 53, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.375886524822695, + "wilson95": [ + 0.30023051207778817, + 0.45812596758644863 + ] + }, + { + "condition_id": "A", + "denominator": 139, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "db75d087-593d-5c84-bc72-c221651cff91", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 52, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.37410071942446044, + "wilson95": [ + 0.2980606005928367, + 0.4569124983312315 + ] + }, + { + "condition_id": "A", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "84f8184a-98e1-56dc-95a5-946188346c6b", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 98, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.358974358974359, + "wilson95": [ + 0.30439040061997985, + 0.4174720677833828 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "51940cfe-c151-51c8-a4d1-64443ba79f95", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0e8b4a5f-05e3-5504-964a-760bc66287f8", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fe2815f7-a8e3-5a68-877e-1b51aeb69023", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c46b3d98-2246-58fa-b107-b229f473bfdb", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39889533-25ae-565b-bcd1-a0893009ce3d", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f62b7358-4662-5658-a07a-23081707ab63", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "51b0991e-a691-5070-a3c9-db3f0f0e2edc", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4aefbc85-aa28-584b-97ba-5c516c0be4a1", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "A", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0a3194ac-c687-5bcd-9fb5-352bef512ad9", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 50, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.36231884057971014, + "wilson95": [ + 0.28685664689873536, + 0.44523860708453833 + ] + }, + { + "condition_id": "A", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f0580340-4254-564e-a163-c3bf6a1b642b", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 48, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.35555555555555557, + "wilson95": [ + 0.2798294544129023, + 0.4392746206143827 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5b592eb7-05c4-5244-895f-6dcbe633675d", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9723be38-48b1-5abf-ac29-bd321fe4201d", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "00078987-351d-5455-b888-879e8d7a6429", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b726ec78-1700-5501-bf8b-1f696e0026a6", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e16e19ca-a9c7-5adb-a7e1-7adf9528749a", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1ddae7f0-de7f-5a53-a32a-688aba268814", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1b76ffd5-95c0-5a70-a381-3bbdc0530d01", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0cf06e0f-07c9-5bff-a890-38b125d7308e", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4babd6ff-f2bd-5259-869e-479e98b8d56f", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f65908ec-9db6-5863-b22c-0773a9c2f893", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d443bda0-687f-5040-95bb-a564c0a26ff7", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "908883cb-6cb2-54fa-94a2-b7f61ebcebb4", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2725fb44-af3e-5c20-8923-9e41e484cc0a", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0e00ab30-008a-54b4-b72d-c6676f83fdee", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7ebff252-1c89-5ee8-9dc9-f9c2152f2b5c", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2bc1fc19-9928-5a42-8dbe-755808e3ad98", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5d96cd9e-d0a5-5084-b707-689557354b56", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6249a6ef-a39c-52b7-9f31-b13a7e26d1f7", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c8c0be61-3997-58bd-a81d-9138a9e200be", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e20bcca7-a75d-5b40-8d1d-a4be63de4401", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "194d4994-d85b-524e-a0a5-09a0e79b2e1b", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7dce71f3-8ae4-5f5f-8b03-6d9682633be6", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3fa86bec-0478-5d30-bff9-fff49b160de4", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ec74de25-2cc4-5c79-8b0f-073c70b2e67e", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c7701584-58c8-5fe9-8430-30c49f6c322a", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8db3d805-65bb-5a85-b603-6a8dafae3b1f", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "95654efe-0934-5844-91a1-d6f0fcb4cdbe", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bc9a0ddc-0821-5428-8c0a-c4bc478e8826", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "06579b5c-3dce-528d-ae2a-fb75accaf87d", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fca3bcdb-c485-55e2-9d82-42ec332e8ffd", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "659f5b77-e756-5402-b703-cbc2311f96db", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ffba8c81-7b51-5a46-9a43-bf23b61db1d2", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "273d4e0d-c7c9-5502-9d9c-7e6ea17d2602", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1380df98-a79e-58ba-b721-d0a1902e44cb", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dd018902-3c4d-597a-877f-e393725199cd", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9c4f6e5f-9585-50b7-ab56-377b61efda26", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b212e390-23c6-545e-b035-d229a3f984e3", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c97daa11-f21d-50d0-af00-0e90a7c71e04", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2ab58cac-1b8b-563e-9231-e203e4d96763", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c6200d57-c35c-5627-afdd-7caa44d595ae", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9835c4db-2e93-59b1-b7b3-19cff3b3ef2a", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a730f074-62d5-5152-b6ef-700397a76d38", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c055c28e-b17c-5386-86c2-938ad1da989b", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e0b4ad15-e65f-530c-9f5e-f24c4f2a5d0e", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d660f53e-efb7-5d42-a430-0baec8a20b74", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "153c7540-f67a-59ae-90f0-859e258289f4", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1f533c3c-4f52-54d4-89ab-5ddeed60604c", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7ebd19fd-cb92-5c4b-a0a0-324aabc05cec", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b3c837a8-2af0-5412-bee0-7ebe674bfb4b", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b559fb37-4150-56ee-a855-09f8428aea15", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1c9d9348-b8a9-5522-b928-a6d90c34ba1e", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ec065fac-456d-5e2b-9888-51feec90553c", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9fec80b1-b06c-573b-a634-4314201f96ef", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d037d360-f323-5575-a9ed-c19807f263a4", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9e6e0855-6321-55b1-93eb-8be49d3e41ab", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "19c217f9-a1b1-56fa-a4df-5a569a558c31", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1b67119a-ca2e-5221-b5de-8774f6ec213c", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bf76e166-d616-5167-a98a-92111172fa82", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5f989a77-7393-5efd-8f42-a57acb6b1bd4", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7bed8949-752a-57d6-bfee-a553f730db2f", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0b5ecf20-ebbe-5b0e-8287-778c3ca10ddc", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5859ced5-60dd-5a49-a3ba-97db5a5d8f7b", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "572c8853-ec22-51f4-8001-3b1948bcc4bd", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "524e9a59-8e87-5abe-9228-4351103f1851", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a6ded5fb-94cd-59a7-9182-f277bbe0d1b0", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2218587c-e298-55f9-b0dd-61ceb61d34ae", + "metric_name": "unsafe_promotions_per_evaluated", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "A", + "denominator": 266, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d1d11a17-2284-50f0-8cd1-274034355951", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 105, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.39473684210526316, + "wilson95": [ + 0.3378958239649705, + 0.45457490961043906 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cb48d667-855d-5ef8-b084-1df636ed5da7", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e6324e4f-ab28-5257-a52a-78551a966ad0", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fbd46a4e-ef47-5988-8a45-eefcb8623fd0", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a7af4e97-21db-5397-a6c7-5e239e38798b", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "175be42f-8b20-5d11-ad48-55a93cf1870e", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cbdf4b70-248c-5c02-b876-e1b315db63c2", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "97881b9f-47c8-5ae6-a1dc-883b537e5fa8", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b2452f1f-aae9-5922-aa63-def6600cf253", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 21, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.25, + "wilson95": [ + 0.16972312954861019, + 0.35214273465871976 + ] + }, + { + "condition_id": "A", + "denominator": 133, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "997c316f-1342-58be-85c2-73577125b3de", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 53, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.39849624060150374, + "wilson95": [ + 0.3192666242835357, + 0.48342475132231333 + ] + }, + { + "condition_id": "A", + "denominator": 133, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f78b26a3-f5a0-56ff-994c-6828be27cea5", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 52, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.39097744360902253, + "wilson95": [ + 0.31222241073029255, + 0.4758535112167305 + ] + }, + { + "condition_id": "A", + "denominator": 259, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c51be7b7-b934-50de-ab66-b2f3a8808075", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 98, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.3783783783783784, + "wilson95": [ + 0.32149794542226806, + 0.43881383993052325 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "367627a2-e151-5109-bc45-593203d8dce0", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "16a4d09a-db62-5cc0-a449-af3d39b363ec", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b36587f5-87f3-5daf-b91d-5ee4b8ce8058", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "85b45e43-8cfc-5e1b-bc01-d8fed9b4e7ba", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2adfb099-0f9b-56a8-ad9c-1eba7ea4956f", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a91e57d5-53dd-5fd0-9ad6-551a72277d9b", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6a657e32-e5a3-5967-ae3e-0485e0725e99", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fbdcc83b-3fea-55f5-a6bc-1b6cf741665c", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "A", + "denominator": 130, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bbfe7549-9f82-597b-8f41-f99e11220c41", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 50, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.38461538461538464, + "wilson95": [ + 0.3054393077426831, + 0.47041489868680114 + ] + }, + { + "condition_id": "A", + "denominator": 129, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7f080136-fe7a-5d59-abe1-aff5e7bfe550", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 48, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.37209302325581395, + "wilson95": [ + 0.2935118520317434, + 0.4580717251665278 + ] + }, + { + "condition_id": "B", + "denominator": 159, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bd6e02eb-255e-5d93-86d0-b10689059074", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.023590176902824132 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cdd274e2-d337-5547-b546-d320eddfa7a6", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f1f65c44-6ce0-56cb-9900-d3f440a40520", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "17b65988-fe2a-5bff-a7d5-123808caada7", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9fb8edde-7079-5d2e-b788-7420f8211de0", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "23ec0df9-fc8a-57d9-b6b5-1907277df893", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "B", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cf5a961b-e207-56a0-9eed-7173f0f8c100", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4c3650a3-81e6-54fe-8b71-d7370b6b8478", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "46fd20d2-e642-5932-ba44-378d5995ddd4", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "B", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "862b2617-8f12-5881-b354-d4410b62822c", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "B", + "denominator": 79, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2496e3a5-487c-57f6-897c-6d4c15cdfaff", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04637121165392266 + ] + }, + { + "condition_id": "B", + "denominator": 159, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e25ddff7-1fa9-5b22-82e0-432982d0f68b", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.023590176902824132 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "743b7505-b8bd-5b63-8580-8a49d4d7ccef", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f06ced1d-4323-59de-ac80-b30c88ebf986", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8bb46f28-91b2-54e8-bb57-494deab8f46f", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eacbe336-4369-5f42-9145-df90bae647f2", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "98ee9f7c-7457-5188-8a77-a37bd0b2081d", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "B", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39954334-dc24-53f9-92f4-6afeb536b6c7", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "B", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "be78dd4c-caab-5f2b-890f-dd56b32e191e", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "B", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0e73ebfd-aaa6-53c8-9653-df575f6c63e5", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "B", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8ba78cbf-7043-50ba-9973-00b78e3623b4", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "B", + "denominator": 79, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8efbe841-42d6-5ae8-82de-b32c9317c3a6", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04637121165392266 + ] + }, + { + "condition_id": "C", + "denominator": 159, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "626eba1a-58f1-5223-a1db-3d2a1bcdfab7", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.023590176902824132 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8165be92-eb44-5d91-bb6f-8de37724921a", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "66a1ef8b-6727-5333-ae85-f16df4350f88", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cd86d414-fbfc-5c60-9fe6-e169e1861bc3", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "018dea1b-4054-5ef6-8412-2a2260fe552f", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "446b8f80-953e-59d5-b64d-b0a7301ad580", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "C", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "94fa226a-224d-502f-ac15-cb3a0cffcb88", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6301fd19-9c17-5c3f-ae27-7a79fc4d765e", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "73e47152-cd53-547d-99b7-f5b4b3df7dc4", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "C", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "688bec53-07a1-551f-8f85-c1e9641be35e", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "C", + "denominator": 79, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "308aa219-5255-50ca-8021-10f13b2dc2e5", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04637121165392266 + ] + }, + { + "condition_id": "C", + "denominator": 159, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6ed9e018-a479-5510-80cb-750e992f2040", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.023590176902824132 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cfdb096c-a048-5c54-9fac-e81d4dbb50c4", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cd4f1bc5-6c81-53b7-b8fc-7a66577fd3ed", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0c47772-73af-516a-9224-77e3fe382852", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "327c627d-fa0a-5b51-b6bd-c1bb2f8a3784", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b3a80910-11be-56fe-922d-3c5a1663909e", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "C", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2f6f2076-6b05-5fe9-a267-1b0dae7433c4", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4f5873d6-8765-5ff6-af12-ccb20b10405d", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7f4ac6b8-cd96-5479-9767-8aa286ca79a6", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "C", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "46237284-0a36-5575-9d06-2dca86c703e8", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "C", + "denominator": 79, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ecc79fe3-6d1e-5179-af73-38c4b3dc4cef", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04637121165392266 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 157, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "148ec510-b42a-5300-be7b-189df9393c67", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.023883511433308864 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2885ef8f-ba1e-5feb-8a1c-1dbc4f699fb7", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bf1e5a40-feae-53b4-b7ae-465649c54d21", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ce553aae-48d9-57b0-997d-d0ea5eb29597", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "27dfdc40-4146-5488-b548-126242de627f", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 11, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8a0b115e-74dc-5d4c-9425-3c37af9c86b0", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.2588329669680317 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "786cfa90-c1ca-554a-b2e7-91bcf48567dc", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a9aceece-5f16-520a-92f5-99df2237e7ae", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a9ae2dfd-f27d-5166-849d-c4e28a0dca2c", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 79, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a4b8c1ac-ac9d-5410-92c7-32fbe4d5f89a", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04637121165392266 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 78, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "110c2867-4eca-5fe3-a461-58e57e00fae6", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04693780971219429 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 157, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "64794ded-cc83-59b8-bf07-00ac2ec1df59", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.023883511433308864 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e9532621-5126-5883-850f-a76aded38928", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b42dc861-d5b1-5eb9-b2c8-8c47d054c526", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "429209c3-ac7c-56f4-a352-5ea5d7c99477", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7dd8b7c8-46e6-5f8b-bb85-92658c4c87b4", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 11, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "90abcee5-3e43-5225-95bb-f1ad40b785df", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.2588329669680317 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 13, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ed25ad99-783d-577f-95d0-bc05a9b851be", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.22809537235419838 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b659eaa7-56fc-5fd4-bf18-2ebb14f740a1", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.2153108027376358 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 63, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "89a94cc9-d430-5d95-bbce-e0ad54971229", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 3.469446951953614e-18, + 0.05747119958885172 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 79, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "831825c1-cca8-518f-98d0-f250b9572dd0", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04637121165392266 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 78, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "27cf3266-c2e3-528b-a739-0bc50090598c", + "metric_name": "unsafe_promotions_per_promotion", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04693780971219429 + ] + } + ], + "reproducibility_summary": { + "extractor": "metrics-draft-1; a second run of `extract` must write byte-identical records (it fails otherwise)", + "rerun_20_trials": "p8b-repro-check-3 (id 6764aa91-5745-548a-80a8-b46ed52b030a): 20 scenarios (one per family x class by lowest scenario id, plus 4 F8 with most steps) x 4 conditions = 80 trials, 136 transitions in common with this experiment; decision or status differences 0; invariant result differences 0 (python -m experiments.report compare)", + "rerun_20_trials_scenarios": "tp-v1-F1-0001, tp-v1-F1-0014, tp-v1-F2-0027, tp-v1-F2-0040, tp-v1-F3-0053, tp-v1-F3-0066, tp-v1-F4-0079, tp-v1-F4-0092, tp-v1-F5-0105, tp-v1-F5-0118, tp-v1-F6-0131, tp-v1-F6-0144, tp-v1-F7-0157, tp-v1-F7-0170, tp-v1-F8-0183, tp-v1-F8-0185, tp-v1-F8-0188, tp-v1-F8-0191, tp-v1-F8-0196, tp-v1-F8-0204" + }, + "secondary_metrics": [ + { + "condition_id": "A", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "838d130a-701c-5624-b2ba-9c2638d4bda4", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 8.673617379884035e-19, + 0.013533818620629409 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a336be67-2689-5621-9cce-5e94ab96dbed", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "93584960-340a-55ad-9c26-5a3bb9fa1830", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "be5a04e7-336a-5490-9ae3-569b0b1f3817", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0f81c46c-4bce-5198-a15f-0dd902f33b29", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e4f9a7b2-18d1-5cbc-958e-fd695382243b", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "165c227f-9ad7-58a7-98ef-f42997f45efd", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bf57c1da-3e53-58c5-af98-a887e97231da", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bd3edf0d-8fa1-580a-8e5b-65ec30ab0b59", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 141, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f0fd60a8-a05f-508e-b289-e8f125959f8f", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.026521818075925647 + ] + }, + { + "condition_id": "A", + "denominator": 139, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fb6c9d33-4e4b-5048-8a30-1bf595bd64e0", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.026893164298442428 + ] + }, + { + "condition_id": "A", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f223f32f-c53b-597d-9886-3c7b507df0fa", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8baf486c-9511-5e84-be3d-198b3e9de2f9", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6fd22c55-b455-51e4-9c14-fee0658eced7", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2514f079-a71b-515b-b562-dcaf2c1d5a23", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8bfab0c0-e0bc-5d36-a3cb-9035b840d6ae", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "76ab04f1-f427-538c-be82-63c271a701a8", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5107f499-67fc-54f7-a2b5-64a13b79d6c9", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a7427f1e-09f8-5372-8beb-568aed9db782", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "22cefd01-e9be-5c3e-9988-8bb746b905ac", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "A", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "36d31cea-38a5-5a31-a6e8-2e0fcbbc10a9", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "A", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "86fee8ee-4884-5882-8367-50ff7f471e81", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ae1df260-f13a-5d88-82cd-3bd9814e6ce3", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2ff220cf-bed2-5eca-9bfa-5d12b85a6974", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "31c08fa1-3035-52b2-8c48-1bf4009b9c67", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5408e483-c1bf-5752-88b0-1b3cd249616e", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e50c3c73-6799-5235-b4f7-079c2e4a28b6", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2ddbf435-c8fa-53c2-be2b-27a2ceff82da", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "783d7de4-d736-5807-8749-dbb7f63b13f7", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4b296874-a91e-573e-a69e-689c6c532b2d", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8fda6b98-42b3-52e5-b26d-2f103f473e8c", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b8ea4192-8120-505e-8310-a1ece41da3e3", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e1365dd4-9095-5b50-bc93-857c697f1d8c", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "49ce121c-2ba7-52cf-af49-60c38cc0c7ce", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0519773-f80b-53cb-a2ce-f5bdc105dae6", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "05233010-b835-5880-addb-29bc5cf2ee4c", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f3f8a488-938f-5088-8a51-5f0b3f160484", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5150935d-cdfc-5948-aa7d-d8df0fc4a330", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "51616cf6-e029-5e18-95ca-7a5ee670b422", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3bac316b-d258-59b0-8cac-d4c076fdefec", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1ec1968b-516e-5050-a53c-235c7e5140e1", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "74c273fc-e792-5fe1-926f-40be14d8b094", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8e139ed1-5ea4-57d6-9c97-c358640d49d4", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0efbbe33-35f2-513d-a4c9-a982151a3b31", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d73fa1fd-92e6-5e5a-b80f-64208a4cfcd5", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "63e11ea3-fc5a-5984-a803-a2b1014e89b1", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ff752946-740c-5369-9674-8546c4481a1d", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "94d8ae5b-a287-598a-9e87-bf9c4c5557b0", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4afb6da9-dc8c-53e7-bd77-ee34be7444b3", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7e3452be-1fd0-5f9c-843d-d991bdc854e1", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "53e5ad70-c720-57c7-95a5-30cf668e2ec9", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "668f0e0c-997c-596c-8660-0379b35dec44", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0c11113-69ea-55ed-9643-3d277c6470b9", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6c11f1ca-aa7c-5f86-b84d-a95d8cc1546e", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b153a317-210d-54d0-9ff6-d47e35a24d48", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e3a69df8-6f6c-527b-9d87-37b32c56559c", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d95870dd-574c-5aa8-904c-7b6b0049bc5a", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6da89ec4-3920-5fde-9e33-0f8583cb1d8c", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fdb57f41-975c-558e-b768-dadf70a4be66", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dfdcba7a-da66-56b8-885a-29d2ba0402d4", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "672a2632-1947-5623-823c-9af17dfa6d90", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1b55a253-1ecf-50af-9784-7de1d7d74556", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1dc08028-78de-5587-b734-83aaf89b49ce", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "979292f4-a894-54ce-83c7-4b86e98945f8", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b05da295-88e8-5cf2-bd2b-b5e901357ade", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c4f06546-972c-5d94-870e-5f5d6c316be1", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9bddb829-03ad-5493-a51c-10551836ecf5", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ea5e7857-cc9b-5d53-8d2f-01c19ff1a970", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3266b177-3b7c-5a15-94f5-4495498530e0", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1b4a1df2-d309-51a3-80aa-28050568c5a3", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3f250a56-e636-5ec9-94d3-14900e728be0", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ca116b70-c981-5900-ba88-5c680bca842c", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dd2181ef-f936-5ba5-858e-d71b26dac562", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "574fd83c-f030-50de-a24d-f867989f3a00", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dd63dd4a-88ff-5b7d-9797-c0377af6e8fd", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ae208e0d-8faf-51ba-b067-0737c11eda32", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f301a88f-9191-5e4f-9cc5-a8f9d82466af", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "12f562ee-c81b-55f8-984d-db3f42599f61", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "84ad9184-fc10-5c5f-b8fc-cfc678d58d2d", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "83562f53-8ef7-5ade-a0e6-9546653a308b", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1dad1136-5396-51f1-8dd7-33d316340063", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2c141bfe-a13c-5fe5-96ff-16da674365a8", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "167f6020-77e6-53fe-a147-a7b6e8c283d3", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d4943de1-9b86-54d9-87e8-168ca80c8f32", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7d812ede-5def-54fb-a8b3-7cfa348287b5", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "527a91f1-add8-5f56-823c-89853e9c29fc", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d73d458c-6f63-5d3d-bbfd-099cdf72f5c2", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "239a8943-6071-5893-a11f-58defeea268a", + "metric_name": "escalate_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d9a51b87-0015-5555-b0b1-8266cce49c76", + "metric_name": "evidence_transfer_errors", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "count", + "value": 3, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b14fe451-6d59-56fe-9955-1dbc215a29b5", + "metric_name": "evidence_transfer_errors", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "count", + "value": 13, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "13ed536b-0acc-59ec-a12d-7fe65a3480b5", + "metric_name": "identity_error_rate", + "numerator": 3, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0375, + "wilson95": [ + 0.012834568919911402, + 0.10454720090045115 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0eae5a5c-a103-52ef-98ac-c43a1334e06d", + "metric_name": "identity_error_rate", + "numerator": 23, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.2875, + "wilson95": [ + 0.19986924292269914, + 0.3946034621298997 + ] + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7de159ef-99d2-5d71-a0b9-b026b8169fdc", + "metric_name": "identity_excluded_pairs", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "count", + "value": 40, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eb0a1a74-41de-528d-af35-f0f22c9cd1cc", + "metric_name": "identity_excluded_pairs", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "count", + "value": 40, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c840f81e-26ac-577d-a31c-1a335775b2b6", + "metric_name": "identity_uncertain_share", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 80, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "50d39be8-9aae-5b9b-b8e1-0c446b426acc", + "metric_name": "identity_uncertain_share", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0458181295355271 + ] + }, + { + "condition_id": "C", + "denominator": 627, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1089e699-1ff9-5fca-8543-dfc923544700", + "metric_name": "impact_precision", + "numerator": 462, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.7368421052631579, + "wilson95": [ + 0.7010072286786366, + 0.7697925195738273 + ] + }, + { + "condition_id": "C", + "denominator": 61, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ba239ada-d07f-57f1-93c1-738cd31d08c0", + "metric_name": "impact_precision", + "numerator": 34, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5573770491803278, + "wilson95": [ + 0.4330336769314388, + 0.6749219447851712 + ] + }, + { + "condition_id": "C", + "denominator": 74, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9c596d4a-d684-52ea-9b2a-2f09d458bc72", + "metric_name": "impact_precision", + "numerator": 57, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.7702702702702703, + "wilson95": [ + 0.6625368552184813, + 0.8513281288915803 + ] + }, + { + "condition_id": "C", + "denominator": 72, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4165a63d-789e-5599-b5cf-17f4665aee61", + "metric_name": "impact_precision", + "numerator": 60, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.8333333333333334, + "wilson95": [ + 0.7308929306946451, + 0.9020062875743832 + ] + }, + { + "condition_id": "C", + "denominator": 85, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4d5abc8c-4d70-5537-8e4e-c1640c7a283e", + "metric_name": "impact_precision", + "numerator": 76, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.8941176470588236, + "wilson95": [ + 0.8108648139957895, + 0.9432875928801936 + ] + }, + { + "condition_id": "C", + "denominator": 66, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b0e1a2a2-bb51-5f4c-bf69-853fd8896f33", + "metric_name": "impact_precision", + "numerator": 48, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.7272727272727273, + "wilson95": [ + 0.6095777422224589, + 0.8199665500441133 + ] + }, + { + "condition_id": "C", + "denominator": 46, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "acbd2af9-bb25-5e47-8912-ecd83b03231f", + "metric_name": "impact_precision", + "numerator": 44, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.9565217391304348, + "wilson95": [ + 0.8546774097378094, + 0.9879945544014435 + ] + }, + { + "condition_id": "C", + "denominator": 78, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "08cf5af6-7fc3-5225-afe6-42c86cc30d3a", + "metric_name": "impact_precision", + "numerator": 71, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.9102564102564102, + "wilson95": [ + 0.8261527543939476, + 0.9558469914832263 + ] + }, + { + "condition_id": "C", + "denominator": 145, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "84f3ef0f-1a9d-567d-99b6-554c5063e0fa", + "metric_name": "impact_precision", + "numerator": 72, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.496551724137931, + "wilson95": [ + 0.41631658272010663, + 0.5769648591044815 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 627, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0a083055-58cf-5b77-8eda-d976c8fe3bc6", + "metric_name": "impact_precision", + "numerator": 462, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.7368421052631579, + "wilson95": [ + 0.7010072286786366, + 0.7697925195738273 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 61, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "44984216-692c-5fc4-94da-bfc2ce177028", + "metric_name": "impact_precision", + "numerator": 34, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5573770491803278, + "wilson95": [ + 0.4330336769314388, + 0.6749219447851712 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 74, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d8089873-4a93-58a7-a7d5-128f37140389", + "metric_name": "impact_precision", + "numerator": 57, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.7702702702702703, + "wilson95": [ + 0.6625368552184813, + 0.8513281288915803 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 72, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2313207e-8efe-58cb-b829-4ac33e437d48", + "metric_name": "impact_precision", + "numerator": 60, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.8333333333333334, + "wilson95": [ + 0.7308929306946451, + 0.9020062875743832 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 85, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3c7969e9-560a-5327-b7aa-4da9f6308db7", + "metric_name": "impact_precision", + "numerator": 76, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.8941176470588236, + "wilson95": [ + 0.8108648139957895, + 0.9432875928801936 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 66, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e85484c3-1ddd-55ce-8ef0-c6d15fc332b7", + "metric_name": "impact_precision", + "numerator": 48, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.7272727272727273, + "wilson95": [ + 0.6095777422224589, + 0.8199665500441133 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 46, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "27fe9c61-7e6a-50d5-9ecb-4d37e2294acd", + "metric_name": "impact_precision", + "numerator": 44, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.9565217391304348, + "wilson95": [ + 0.8546774097378094, + 0.9879945544014435 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 78, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "70167607-3dfd-5899-92aa-65a55aa06dc6", + "metric_name": "impact_precision", + "numerator": 71, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.9102564102564102, + "wilson95": [ + 0.8261527543939476, + 0.9558469914832263 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 145, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "37a403d6-efcc-576c-9037-95071718ca5c", + "metric_name": "impact_precision", + "numerator": 72, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.496551724137931, + "wilson95": [ + 0.41631658272010663, + 0.5769648591044815 + ] + }, + { + "condition_id": "C", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3f1cb402-84bc-5457-ab7f-800c9eda8e57", + "metric_name": "impact_recall", + "numerator": 462, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9917537206104003, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 34, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "68f2770a-de6b-56b9-bcf4-92b3c4efcd16", + "metric_name": "impact_recall", + "numerator": 34, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8984854458466761, + 0.9999999999999999 + ] + }, + { + "condition_id": "C", + "denominator": 57, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5de4369b-1b8a-5853-bdac-4ed1e3413c1c", + "metric_name": "impact_recall", + "numerator": 57, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9368611651470209, + 0.9999999999999998 + ] + }, + { + "condition_id": "C", + "denominator": 60, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "796ccc6e-1aa9-5d2d-bbba-15d295450bb3", + "metric_name": "impact_recall", + "numerator": 60, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.93982814785791, + 0.9999999999999998 + ] + }, + { + "condition_id": "C", + "denominator": 76, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "668ff28d-acaa-5866-9a5d-d4d42f447771", + "metric_name": "impact_recall", + "numerator": 76, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9518864149348628, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eca9d60b-1b25-5d8e-9a74-f03cc49db5af", + "metric_name": "impact_recall", + "numerator": 48, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9258998703338824, + 0.9999999999999999 + ] + }, + { + "condition_id": "C", + "denominator": 44, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ddb17b3b-6a6b-5e0e-b902-9465f4fafa7c", + "metric_name": "impact_recall", + "numerator": 44, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9197043962415192, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 71, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "77531ee6-876c-521c-b349-74d7af142930", + "metric_name": "impact_recall", + "numerator": 71, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9486720478031097, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 72, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7fcec70c-3a30-55b1-ac19-3daa23011644", + "metric_name": "impact_recall", + "numerator": 72, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9493488274035422, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "934ee362-c037-586f-a428-ce1868a30faf", + "metric_name": "impact_recall", + "numerator": 462, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9917537206104003, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 34, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "52d36085-9b73-5ee4-a2d2-a251003f71bd", + "metric_name": "impact_recall", + "numerator": 34, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8984854458466761, + 0.9999999999999999 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 57, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8c63ede9-bb08-5488-94a6-03f2604a41f0", + "metric_name": "impact_recall", + "numerator": 57, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9368611651470209, + 0.9999999999999998 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 60, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3bcd8605-0c30-5a21-a174-be65cee33745", + "metric_name": "impact_recall", + "numerator": 60, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.93982814785791, + 0.9999999999999998 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 76, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fefeb7be-d287-539e-b304-7326fe91b58d", + "metric_name": "impact_recall", + "numerator": 76, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9518864149348628, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 48, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fe16cbf8-52b2-5aaa-96f3-513651eff2e3", + "metric_name": "impact_recall", + "numerator": 48, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9258998703338824, + 0.9999999999999999 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 44, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fcd5d531-bada-5cd7-8dcc-03c2559da480", + "metric_name": "impact_recall", + "numerator": 44, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9197043962415192, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 71, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "df8bbbba-a71f-5d71-94da-b62bf303d835", + "metric_name": "impact_recall", + "numerator": 71, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9486720478031097, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 72, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8a460ed7-b099-5e7f-bd4a-c4e965901f8a", + "metric_name": "impact_recall", + "numerator": 72, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9493488274035422, + 1.0 + ] + }, + { + "condition_id": "A", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d595b81d-1aa1-551a-9bf0-2e1f8f9efa81", + "metric_name": "promote_rate", + "numerator": 266, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.95, + "wilson95": [ + 0.9178339854797117, + 0.9699855777617219 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "feb1a613-f517-54cf-a16b-23f4974bac56", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "09d0c709-01c8-50e0-a44f-cd384bd62085", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8a1e0336-c1b5-5a93-8118-2fb05a2294f2", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "06bdbc7b-ca12-51d8-92a1-6af7a1e6e1f3", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bce964ed-5d24-5cff-8844-327d2072d4ac", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "98a8fda2-144e-5a69-88c3-9112dc040e19", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a22fbf94-f5f2-5733-9a54-e56036606ad8", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8c97e9fc-e9d8-5581-9f66-4be3d0530d7f", + "metric_name": "promote_rate", + "numerator": 84, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.95626827158534, + 0.9999999999999999 + ] + }, + { + "condition_id": "A", + "denominator": 141, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ae1e9c1a-539a-592b-bd0f-d6b566e4a7fa", + "metric_name": "promote_rate", + "numerator": 133, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.9432624113475178, + "wilson95": [ + 0.8920396844298962, + 0.9709728881978293 + ] + }, + { + "condition_id": "A", + "denominator": 139, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1e0165b9-fc3f-566b-9440-7061a30d5737", + "metric_name": "promote_rate", + "numerator": 133, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.9568345323741008, + "wilson95": [ + 0.9090285840124278, + 0.9800690284630961 + ] + }, + { + "condition_id": "A", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "29cd3e27-2f78-5262-9107-8cddf89342ed", + "metric_name": "promote_rate", + "numerator": 259, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.9487179487179487, + "wilson95": [ + 0.9157732291903499, + 0.9692098258898599 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fd2bef1f-e91b-56bd-8954-dae21c67ea42", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bf94d9d9-8eec-5f04-a48c-fffb7ece839a", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4a7455ea-2466-5cd1-9a03-69fbae8b2697", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8ef72f0f-fc07-5cd2-a0a0-7450bd1d548d", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "953e91ae-ab04-518e-935e-97d28c8b369f", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5da43099-292f-590c-855d-ccbbd267247e", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0e418b0f-1a9f-5c22-9af7-a25c7719e948", + "metric_name": "promote_rate", + "numerator": 28, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.8793566952341543, + 0.9999999999999998 + ] + }, + { + "condition_id": "A", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "aed73661-8949-5d2f-8907-dab491f452c2", + "metric_name": "promote_rate", + "numerator": 77, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9524815747175658, + 0.9999999999999999 + ] + }, + { + "condition_id": "A", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5e66168e-3350-5264-a390-e24953ba45e0", + "metric_name": "promote_rate", + "numerator": 130, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.9420289855072463, + "wilson95": [ + 0.889779664644606, + 0.9703355725669888 + ] + }, + { + "condition_id": "A", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e29465b9-3906-5410-8a22-23dd8b335bf3", + "metric_name": "promote_rate", + "numerator": 129, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.9555555555555556, + "wilson95": [ + 0.906428641545762, + 0.9794738910604933 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e7926d64-552f-598e-af63-9ffc8b07cc94", + "metric_name": "promote_rate", + "numerator": 159, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.5824175824175825, + "wilson95": [ + 0.5231700911333642, + 0.6393778169425929 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ee826a49-bb57-541e-8097-3266ebd26768", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fd62f116-ff2b-5d4f-ab13-51e0d9cc870e", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0677e2f4-9823-58f1-abba-2a29fd53d438", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "595c5955-a231-502b-bf44-31f418055641", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c71afd47-e5a1-5d23-abba-db7e5617457c", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "db1e709a-502d-5618-946e-cc0eed3e05d5", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "72b661e9-d594-5c4a-906c-feb60e2aaf3c", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b130fab2-c963-5611-a942-ac9f02048ee4", + "metric_name": "promote_rate", + "numerator": 63, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.8181818181818182, + "wilson95": [ + 0.7176371712788386, + 0.8884874671777944 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "19617209-8781-59f4-bc36-04a78e806cf5", + "metric_name": "promote_rate", + "numerator": 80, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.5797101449275363, + "wilson95": [ + 0.49629083327975365, + 0.6588119144141406 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eb4b6194-b650-556c-b7da-39ecb8474766", + "metric_name": "promote_rate", + "numerator": 79, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.5851851851851851, + "wilson95": [ + 0.5008418839530941, + 0.6648146871846607 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9a05d7cb-432a-58d9-ba6a-a631adcbf619", + "metric_name": "promote_rate", + "numerator": 159, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.5824175824175825, + "wilson95": [ + 0.5231700911333642, + 0.6393778169425929 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "04cd6268-685c-574f-ac63-37befd169b91", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "72d76164-c018-57cd-a705-480e8d49eef0", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4f7c2bda-3353-5e15-85fe-710de8ecb346", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0505feaf-4973-5deb-8271-e17dda6f1677", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a17d1fb7-d966-57cf-88bd-c8d078aee61f", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5f31e6af-4123-5b47-98a2-21826181030d", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4aac5305-91f0-5d13-9322-ff24c89da84f", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f322b3df-2daa-5367-af1c-b68f395eac5c", + "metric_name": "promote_rate", + "numerator": 63, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.8181818181818182, + "wilson95": [ + 0.7176371712788386, + 0.8884874671777944 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3b879763-a7ea-58ed-9fe3-45e60d4a686e", + "metric_name": "promote_rate", + "numerator": 80, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.5797101449275363, + "wilson95": [ + 0.49629083327975365, + 0.6588119144141406 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "725d5a8b-a94d-5935-a67f-b2bed84053d9", + "metric_name": "promote_rate", + "numerator": 79, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.5851851851851851, + "wilson95": [ + 0.5008418839530941, + 0.6648146871846607 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1a3834fe-eb4b-5bec-ba3b-4c5e21d1cc27", + "metric_name": "promote_rate", + "numerator": 159, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.5824175824175825, + "wilson95": [ + 0.5231700911333642, + 0.6393778169425929 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3352977e-0556-574a-bf59-f6c1dcdc0d28", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0ebed180-f725-5303-9132-439ec6cc5793", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0ee8c794-8545-54c0-851b-871fc098568e", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e0cae3e5-562e-5357-855c-7d9999448a15", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "84c1cc91-8b2b-5817-a0b5-a011adc90249", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "73acab43-bb8f-52e6-a43e-00ae27625a09", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4e95c42a-4127-5550-990a-bb1783922335", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1328b4e8-1333-5b33-a4ac-a0722984f815", + "metric_name": "promote_rate", + "numerator": 63, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.8181818181818182, + "wilson95": [ + 0.7176371712788386, + 0.8884874671777944 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8383ac90-1c0e-5193-86b8-cc2cbd0776c0", + "metric_name": "promote_rate", + "numerator": 80, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.5797101449275363, + "wilson95": [ + 0.49629083327975365, + 0.6588119144141406 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b14f0dd6-0c8a-5c7b-86e3-7653e44553ab", + "metric_name": "promote_rate", + "numerator": 79, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.5851851851851851, + "wilson95": [ + 0.5008418839530941, + 0.6648146871846607 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "578de2e5-9120-5ed6-a8b6-d59da08c58ca", + "metric_name": "promote_rate", + "numerator": 159, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.5824175824175825, + "wilson95": [ + 0.5231700911333642, + 0.6393778169425929 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0f28334-00bb-51a8-abf7-9863e7e1bd42", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2975972e-e416-5230-b7bb-55f71568afa5", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "02e3d4df-10b0-50b4-af0f-9e54ed3f4d55", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e8256b3b-9c10-5edf-9b9f-95b8b9861b66", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0daf8922-0043-5918-9529-38ee22b0d23a", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c1fbefb7-98d4-5fcf-99d6-dabd88e73972", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "959241ad-abca-5968-a737-39accae831d6", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1fdf9101-a3c6-52e7-a659-3a568250a815", + "metric_name": "promote_rate", + "numerator": 63, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.8181818181818182, + "wilson95": [ + 0.7176371712788386, + 0.8884874671777944 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2c9927e7-ab84-5e07-a619-a1af9407c97d", + "metric_name": "promote_rate", + "numerator": 80, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.5797101449275363, + "wilson95": [ + 0.49629083327975365, + 0.6588119144141406 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f5d63513-a735-5fae-9cbc-b23c6e0620e3", + "metric_name": "promote_rate", + "numerator": 79, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.5851851851851851, + "wilson95": [ + 0.5008418839530941, + 0.6648146871846607 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bf6b9f76-5ab3-562c-b498-105687c187ed", + "metric_name": "promote_rate", + "numerator": 157, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.575091575091575, + "wilson95": [ + 0.5158100246716592, + 0.6322891804642127 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5bd0e07d-6f60-5721-b731-116e6c120db7", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a6182d6c-fa97-5018-b386-c6cb4f68678e", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "90fce288-f6d3-5173-8a67-313d22a70c4c", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4ff0b0d6-f251-5a6d-8239-9cea8c7efbb6", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "704e795d-e905-507c-bbda-bfa6979286be", + "metric_name": "promote_rate", + "numerator": 11, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.39285714285714285, + "wilson95": [ + 0.2356568654157959, + 0.5759095570340281 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "09523368-2f30-56d6-a210-5df4da1ffa32", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d906e859-3cfe-5679-97df-0eb51b9edb86", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "134213e9-0b49-5c66-a174-a84da2431810", + "metric_name": "promote_rate", + "numerator": 63, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.8181818181818182, + "wilson95": [ + 0.7176371712788386, + 0.8884874671777944 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "992c6db4-5e1e-5fd6-b122-646f06931687", + "metric_name": "promote_rate", + "numerator": 79, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.572463768115942, + "wilson95": [ + 0.48906212405330773, + 0.6519403738502323 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "89d8aa21-0853-588a-938e-17c2a648f9c6", + "metric_name": "promote_rate", + "numerator": 78, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.5777777777777777, + "wilson95": [ + 0.4934416129305894, + 0.6578100389777954 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "115e3ad8-b92f-5fe7-81f5-9b8157530202", + "metric_name": "promote_rate", + "numerator": 157, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.575091575091575, + "wilson95": [ + 0.5158100246716592, + 0.6322891804642127 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "70019e67-b374-5c80-b4ab-00a6d559547d", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c70846eb-62fa-5515-be5e-831ddfea2b00", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b486001b-3151-5275-8034-b0e61d237f7d", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "da5ea742-e7be-5722-9c5e-0817338700e6", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2aa45d41-50f7-50c6-9934-a805272d3a54", + "metric_name": "promote_rate", + "numerator": 11, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.39285714285714285, + "wilson95": [ + 0.2356568654157959, + 0.5759095570340281 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9c12dd46-ef5b-5349-a4d3-7f6a5d77c91f", + "metric_name": "promote_rate", + "numerator": 13, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.4642857142857143, + "wilson95": [ + 0.2953156998279538, + 0.6418731076553209 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3e67ab55-57e8-50d0-b897-ab743caa2a62", + "metric_name": "promote_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "34acd67f-8441-5ff3-a062-7607e3b624f8", + "metric_name": "promote_rate", + "numerator": 63, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.8181818181818182, + "wilson95": [ + 0.7176371712788386, + 0.8884874671777944 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "00368472-d2db-5a83-85ad-24830bacaa8a", + "metric_name": "promote_rate", + "numerator": 79, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.572463768115942, + "wilson95": [ + 0.48906212405330773, + 0.6519403738502323 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1c24a0fe-5641-5ef8-bc0d-775751f8bc01", + "metric_name": "promote_rate", + "numerator": 78, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.5777777777777777, + "wilson95": [ + 0.4934416129305894, + 0.6578100389777954 + ] + }, + { + "condition_id": "A", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b05af2af-dde8-5542-a3cf-0bc164ea8c7b", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.05, + "wilson95": [ + 0.030014422238278143, + 0.08216601452028834 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c9a4011d-a020-5398-9b97-f1591cfef684", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "61b43ec5-4197-545b-84b4-9190f085e122", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4bb91784-1455-5116-95bc-1d61c2cd2f57", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3cf9a9e7-9ea7-528e-8ac8-727495b609a1", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9bdaeb9c-cff3-525b-b9ef-55a6d2d25d1a", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "494eb2a4-2d03-596b-9ab5-39ebb4d87bf9", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7ad1d0a0-944f-55d2-8f5e-1c8cabd4dd1e", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ca48ebce-9626-52a8-b900-f7bf1661a765", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 141, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d96d7294-130f-5d7a-9de6-b0866574a5fe", + "metric_name": "reject_rate", + "numerator": 8, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.05673758865248227, + "wilson95": [ + 0.02902711180217068, + 0.10796031557010383 + ] + }, + { + "condition_id": "A", + "denominator": 139, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a982c5a8-84bd-57ea-9cf2-da73a2da6116", + "metric_name": "reject_rate", + "numerator": 6, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.04316546762589928, + "wilson95": [ + 0.01993097153690398, + 0.09097141598757219 + ] + }, + { + "condition_id": "A", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "806e8aba-7d2a-5fe4-b5a4-e8d0cc33c106", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.05128205128205128, + "wilson95": [ + 0.03079017411014, + 0.08422677080965014 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dc459634-f506-571f-833d-388df156a700", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ca6317cd-1781-5c32-9d38-df7f7f9c5e4f", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "afea6798-d751-5623-9eb2-07e4bc58930f", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f81596f2-0148-5445-b0f8-64412b7374c9", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9e68ef2d-ab3a-5cd2-b925-37dd77c773e8", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "501169ba-7638-51f9-9441-931944ac6b00", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9f6c091-4a2e-5175-bdb2-0c5d832ab1c2", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "addb44b2-c039-5591-bcbb-f60c3cd2f045", + "metric_name": "reject_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "A", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2a87b9af-873d-5380-8811-5bfedb90546b", + "metric_name": "reject_rate", + "numerator": 8, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.057971014492753624, + "wilson95": [ + 0.02966442743301114, + 0.11022033535539394 + ] + }, + { + "condition_id": "A", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "83fbc95e-afed-5984-96cb-983fc3f4ce63", + "metric_name": "reject_rate", + "numerator": 6, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.044444444444444446, + "wilson95": [ + 0.02052610893950682, + 0.0935713584542381 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c03529a3-4f0b-563b-ba3e-8c9258207aea", + "metric_name": "reject_rate", + "numerator": 114, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.4175824175824176, + "wilson95": [ + 0.36062218305740723, + 0.476829908866636 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "26d5ce77-2525-53b4-ae6b-99cfab57bbdf", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "215518b8-2d58-5902-931a-e54738fd2260", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7f767fff-52b0-536e-934b-dbd9ca4a0fd5", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39c0a5da-0582-5984-b00a-327e9bc0c72c", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cd040795-212c-5a21-a893-147a6a547af2", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "18308e6c-1648-5507-bf9c-a2475df7af2b", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "57f5e680-9b88-5ed7-a68d-332f680fee06", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b8997d7c-b672-5c92-8105-95ec2c221add", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d5910076-4bf2-57e6-80f0-ea7f4eabfc44", + "metric_name": "reject_rate", + "numerator": 58, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.42028985507246375, + "wilson95": [ + 0.34118808558585945, + 0.5037091667202463 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6dbeed53-ca23-58ac-8848-e03282e24a03", + "metric_name": "reject_rate", + "numerator": 56, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.4148148148148148, + "wilson95": [ + 0.3351853128153392, + 0.4991581160469058 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "455bd4a5-c0a8-5d76-920c-901121a4ff37", + "metric_name": "reject_rate", + "numerator": 114, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.4175824175824176, + "wilson95": [ + 0.36062218305740723, + 0.476829908866636 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7b85f419-687c-5cc1-82ed-574fc1e032a2", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8fc8bb3e-e45a-5bb4-919c-90b29ffc918b", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "372f71a8-a61b-5d85-ac58-4e4de1fb59b8", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9e4c855a-fe97-5e72-a7f9-758b43eafd2b", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "60a1be05-2d6f-590e-a694-1c0e110847f2", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "59c5de23-cc6a-5a0e-9ebc-587b9e874e54", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5cf17c69-dc09-53c0-9a0f-9f34bab21861", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9b2c15f-f709-53c2-bd04-a688daf28451", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e0362c3a-9534-5850-b6ef-1a795dd83d75", + "metric_name": "reject_rate", + "numerator": 58, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.42028985507246375, + "wilson95": [ + 0.34118808558585945, + 0.5037091667202463 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "be61e2f2-acfb-522b-9f5c-4bfb42eab912", + "metric_name": "reject_rate", + "numerator": 56, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.4148148148148148, + "wilson95": [ + 0.3351853128153392, + 0.4991581160469058 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "460a0c89-125d-575b-8110-50d1ef8927a0", + "metric_name": "reject_rate", + "numerator": 114, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.4175824175824176, + "wilson95": [ + 0.36062218305740723, + 0.476829908866636 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cc599011-c486-5ca5-8641-a03b4ca74bef", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e8c04c93-82cc-5ac8-8ff9-79fc1daea5cb", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3be4d39f-ba82-5d61-8fa0-be7af15adf8f", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0f2ab7f-098d-5ebc-b8d8-070122c97b08", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d90c8c92-d4d3-5d66-bfbf-5f4ef627d30c", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4ea40588-1711-55b3-a989-d84a1f91e969", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6f027a18-d4e6-5061-8b3c-9a26ccd642d9", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "32330e0d-c119-5909-b1b8-aeafa8557411", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "caa4c2d5-5f51-56dd-bda2-9b906b3d836d", + "metric_name": "reject_rate", + "numerator": 58, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.42028985507246375, + "wilson95": [ + 0.34118808558585945, + 0.5037091667202463 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3638f3ee-a723-5f0c-aff7-5f9f350cbb4a", + "metric_name": "reject_rate", + "numerator": 56, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.4148148148148148, + "wilson95": [ + 0.3351853128153392, + 0.4991581160469058 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e4145e88-36b5-53d6-9b8c-5e28c1cb04bc", + "metric_name": "reject_rate", + "numerator": 114, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.4175824175824176, + "wilson95": [ + 0.36062218305740723, + 0.476829908866636 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b73c8174-ae6b-59ed-a720-fa9697835cc3", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4b8f9901-158d-5c5a-8187-7db9dc3cda5a", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6ea2539e-81f9-52fd-8340-2c61389a47e3", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "79fab614-2e7b-5565-ae40-b5738df8d244", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2b2990fa-44c3-5921-84a0-348d5e5097b5", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0f458183-8be1-572f-a7b7-8b16a0838340", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c72a75b0-49ae-5cc4-801e-c8cfb11f35d0", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cb59e4b5-d364-5467-9bba-7278f250c3fb", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6b145c20-1311-5664-b898-a1ca991739b8", + "metric_name": "reject_rate", + "numerator": 58, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.42028985507246375, + "wilson95": [ + 0.34118808558585945, + 0.5037091667202463 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6fd51627-3022-5821-8843-5528b2f3916e", + "metric_name": "reject_rate", + "numerator": 56, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.4148148148148148, + "wilson95": [ + 0.3351853128153392, + 0.4991581160469058 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c3137f35-c769-5cac-b173-6401fcf5d201", + "metric_name": "reject_rate", + "numerator": 116, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.4249084249084249, + "wilson95": [ + 0.3677108195357873, + 0.48418997532834085 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "54e99832-619a-5f58-b09a-a4b161bfa0f2", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d02fda22-35ab-5f7a-9912-c65929d64377", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "22afe6d2-46d7-58d0-84d6-676708dc3436", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9115d9a5-b68b-50b9-b247-c67d1310ceb5", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "135511bc-0d69-57c8-90ae-159439d82dbe", + "metric_name": "reject_rate", + "numerator": 17, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.6071428571428571, + "wilson95": [ + 0.42409044296597176, + 0.764343134584204 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d0ba2290-0c01-5472-9a8a-624d811c9bb8", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "79dc5972-c719-5081-9687-dbadb77cef1e", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4a4406ae-8b87-5952-a9be-1781839836f0", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5de80cc8-b405-532e-90a7-2154982e657c", + "metric_name": "reject_rate", + "numerator": 59, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.427536231884058, + "wilson95": [ + 0.3480596261497677, + 0.5109378759466922 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a36b6f81-a065-5d75-8816-aea1711d311d", + "metric_name": "reject_rate", + "numerator": 57, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.4222222222222222, + "wilson95": [ + 0.34218996102220445, + 0.5065583870694105 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6b5a24f3-85db-5c6e-8376-c6d36e6f8c32", + "metric_name": "reject_rate", + "numerator": 116, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.4249084249084249, + "wilson95": [ + 0.3677108195357873, + 0.48418997532834085 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2fb138dc-3362-53cf-b5df-cde6f4739f5c", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1edcb63d-77ad-5775-b231-828d6c245342", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8bd2ed8d-6cab-5b23-af17-5b29bfa3b7d5", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d6b56d2c-e1a5-57af-a3c0-8c3ad7675c10", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f338c6ce-276d-5353-abdc-0ec2bd0c266e", + "metric_name": "reject_rate", + "numerator": 17, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.6071428571428571, + "wilson95": [ + 0.42409044296597176, + 0.764343134584204 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0046452f-9a82-523f-98b5-77da68f4c4c9", + "metric_name": "reject_rate", + "numerator": 15, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.5357142857142857, + "wilson95": [ + 0.358126892344679, + 0.7046843001720462 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "06f2b3ff-480b-5784-8b2d-148af79d9557", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.5, + "wilson95": [ + 0.32633127457293465, + 0.6736687254270654 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f17ec7ea-ccad-5dbf-a0ac-cc956b52504e", + "metric_name": "reject_rate", + "numerator": 14, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.18181818181818182, + "wilson95": [ + 0.11151253282220572, + 0.28236282872116136 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "16c31a10-841b-5b4a-b5b9-4099f8b21707", + "metric_name": "reject_rate", + "numerator": 59, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.427536231884058, + "wilson95": [ + 0.3480596261497677, + 0.5109378759466922 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "afbfcd5b-f962-545d-a594-57d7f3efa4c2", + "metric_name": "reject_rate", + "numerator": 57, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.4222222222222222, + "wilson95": [ + 0.34218996102220445, + 0.5065583870694105 + ] + }, + { + "condition_id": "A", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "75333124-01c6-5378-9bfe-b275360d31da", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 8.673617379884035e-19, + 0.013533818620629409 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0c3ecce-34ae-55f3-a225-3075252c3bba", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eea75a95-83bc-53b3-9f85-c2860484da9a", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5b1a5717-bcfb-52ce-8553-8ca01d600b6d", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7f807305-3475-5e79-9e8a-bd9348b08b15", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "20fe68b2-cae8-561c-b01a-a25254706ea9", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "901fc4e3-6f9c-5c7a-aa9e-e224dd36e2b6", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "de3a8501-e160-5c22-9e2c-23ce820163df", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cc90cc29-08f9-526d-80ef-77bb669c9feb", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 141, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9e81d71-6dfc-5807-b3d4-ed3ea81948b8", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.026521818075925647 + ] + }, + { + "condition_id": "A", + "denominator": 139, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7b9568db-abe5-5724-8145-724b85e4fdcf", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.026893164298442428 + ] + }, + { + "condition_id": "A", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b319e022-8240-5dc9-b191-84032b8a4bed", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b2390f43-49c4-5a69-97eb-3b13615c41b7", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7f156eec-888b-5565-8be4-d51633363fe4", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "718965cf-5d51-50ae-8a54-0ace887fc029", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ab618a55-f7a7-5b7a-8753-ebcc44b85813", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8787a437-d099-5c97-aba1-fdda5bd9372f", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "097c32c8-7c19-59f8-875e-5650b66e6317", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e63b8d5c-6b99-536c-847f-b6bfdc6ae36d", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "A", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "33d5c757-9209-5f46-a937-61bc73136052", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "A", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f9ed86fd-d3a1-5a11-8336-b36219512cc3", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "A", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b44cf4ad-741f-5c18-90df-6e5fb25f62c8", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eb36755f-4877-5066-b7bd-51ee38caae83", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f3503ecc-3d9c-568b-865d-8759f51fae1c", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "faa6e4bf-f5c2-5456-b1ec-f893deab2627", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d4531611-a5c8-57ab-9075-6be024d05a10", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5d689dc3-142c-5f32-b11b-73a448079ab2", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "58983171-442c-56df-9a11-436ff56eb615", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4fbe22cf-bb99-5aae-bd3a-0d5e4d2c1ef4", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4662003c-eb57-5a49-9c97-2f9c3b547363", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fb1083cd-d413-5c3c-bcb2-f99e12be88d3", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d9330f4e-6910-5c1a-b057-6db9bcc795b1", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "72425101-4bf3-5a88-b4c3-37954a10db1e", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "B", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "621d6295-ba2e-5779-ac3b-7c8f5022e286", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c217677b-18a0-52ab-9dce-88d95a57b91d", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e98e2ec9-84c8-5b7a-8efb-31fe2859ed17", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "789b84a0-9ef6-5fa2-a284-27b60fcd2b4d", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7302b911-abce-5018-9cc9-ab436044765f", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "08e34a56-58d9-5ecb-9e72-764a17fc9c6f", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "453cb0c6-a849-5633-8f4c-6f424adf333b", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2c535309-f130-5ea7-aae2-cc806b161cf7", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "B", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "82af1d5a-23e0-5015-abcd-d6402d2c331f", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "B", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2eb5ef98-791b-58a9-bb33-0a80b8f75966", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "B", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3e39ca1d-dc26-532a-98d4-ded103fd5d53", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bc832f3c-2582-54f0-baf5-2620115b2381", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "585ca295-432a-50bc-9d44-637b41a1893d", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "393855af-c135-5906-a387-0a308d3da6df", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5ecd7536-c26c-5d0b-8613-e864c3bc4027", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2f913769-680d-5940-9535-0ac8fd709aee", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "50dea897-e21b-5b52-a746-979d336f0281", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "527ea958-aecb-58c9-abea-655964e62704", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "143ac359-52ff-53de-86f5-b34bd9d60adb", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39f5c490-e160-57f7-9d29-6ef99c433ba8", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b421c61f-1a66-5cea-b120-6ea675300aa6", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9f9f3ab5-2802-532a-8059-f0889456b037", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d01af584-b788-5764-a5a8-46508399d185", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cb8d7ea4-07cd-5b25-9878-95aac058a757", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "37a124b2-1aa4-54c4-a2ed-b5ed7b98a8aa", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1742e67a-c138-5b46-a491-8d3c7b9b39ad", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "693f9602-6263-5fe2-a7b0-c87859daf2a8", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ea21a0b1-4ad1-5b6d-ad6a-30254292d6f9", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "08b3618e-7818-5817-bc1e-0da17f35b761", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b0a85472-fe86-5d42-b2af-1e1dad7e2eb8", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "aeede1d2-c67e-5717-b033-98765c2f6ded", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e71b25a1-0d91-5036-9008-8f75d3865006", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "896e9571-43ec-5e10-b29c-6ebb8d48818f", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e6878164-986e-52dc-8b1d-f59625b5e9de", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "24f70f5a-390c-58a9-928b-4fcda4908b81", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "813ce116-e23b-5099-875e-d95a7129262b", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f864fa8f-d884-5800-8c7a-e2085ec650c6", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9623334f-036d-5dd0-aee8-7e2517169ad6", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "58d63d2b-25aa-57d1-a746-94ef363ddd50", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4177e7da-4183-5559-81ea-b2b27c51840e", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a086bd57-a214-5bc9-942c-65d0a41bc33b", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9c00a86e-a977-53a1-bc70-1d9b962936ed", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "947d61a1-74cc-5ef6-810f-758aac887b0f", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "639e61bb-ae3f-5898-8be5-e33e69f69a0d", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 273, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d5a9cbad-0584-54c2-86f8-3aa86731d56f", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.013876024339194723 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d75d14f6-ecc9-59cf-83f9-ce8889d80256", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "49fb6e3f-b212-5ac5-973a-67dfa3a59fd6", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f9f564b7-cb77-5835-b3c5-2a34d64e4fdb", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "777f4c18-3d35-5252-99fc-b2fe554840b8", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "45268fbd-eaa5-50d1-8019-4992af632099", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "104bdd36-3f36-5e66-ac17-b631b4ae7eea", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 28, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dc175998-56f0-51bd-8e67-3d7ca0f87058", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.3877787807814457e-17, + 0.12064330476584563 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 77, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "445837c7-de92-53b0-a7f6-f20f4774bf9d", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04751842528243408 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 138, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4ba27290-bdaa-561b-895a-e11d2b7ab145", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.027082764465572957 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 135, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "06cf2f0b-cb7d-58b9-9ade-bc1b4b1d7068", + "metric_name": "retry_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02766795201752491 + ] + }, + { + "condition_id": "A", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d094a5bf-c74c-5de8-bd29-f9726b9cc5cb", + "metric_name": "truncated_transitions", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 8.673617379884035e-19, + 0.013533818620629409 + ] + }, + { + "condition_id": "B", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e723d5ff-0f9e-5cde-b137-30782c14005e", + "metric_name": "truncated_transitions", + "numerator": 7, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.025, + "wilson95": [ + 0.012161668350597052, + 0.050695459339000895 + ] + }, + { + "condition_id": "C", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d52b2146-cb4f-5ec1-a443-d7428c04fef3", + "metric_name": "truncated_transitions", + "numerator": 7, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.025, + "wilson95": [ + 0.012161668350597052, + 0.050695459339000895 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 280, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "651dca77-5f61-5c0d-8ff4-c93ffcdabe25", + "metric_name": "truncated_transitions", + "numerator": 7, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.025, + "wilson95": [ + 0.012161668350597052, + 0.050695459339000895 + ] + }, + { + "condition_id": "C", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a4c10ab5-c97c-50c5-90f2-ac32255c8b16", + "metric_name": "truth_change_coverage", + "numerator": 161, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.976696039648177, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f9f0e7a0-c832-580e-9b94-9a8509bac049", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "80f8d065-3fa7-5c17-9bac-d9774299b591", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3da05810-4205-559b-8daa-109a1acd7147", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 42, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ab9137db-0d46-5042-9a29-26a041665ee1", + "metric_name": "truth_change_coverage", + "numerator": 42, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.916201209134296, + 0.9999999999999999 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b66cfdb3-e3a4-54d8-a34f-ce1c131724ed", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b4a33756-5c83-5747-9979-384f2809e2ea", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e7b88bc6-66a0-52e6-a78a-cc6bbdbee811", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C", + "denominator": 35, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fdfac0a4-bce1-5c95-8701-96fd13bf68ec", + "metric_name": "truth_change_coverage", + "numerator": 35, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9010990076755959, + 0.9999999999999999 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 161, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "27124854-334c-573c-a55d-96d043e9b248", + "metric_name": "truth_change_coverage", + "numerator": 161, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.976696039648177, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0be01c1a-3f62-5594-bc4b-ac7f7f27bf91", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "afdb3639-d786-5c4a-a0cc-4644c5baf2bd", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6db256e2-41e0-5660-8cd9-5c0bcaef2b96", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 42, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "db51493b-a676-536e-b574-4efaf474b877", + "metric_name": "truth_change_coverage", + "numerator": 42, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.916201209134296, + 0.9999999999999999 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6550c853-2c20-5a9f-b5b6-e031ab911dba", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fff043f0-be91-5b70-86ae-8abc32fe3073", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 14, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "49b6ac6a-66d6-5fdb-80b5-7a2d2b368643", + "metric_name": "truth_change_coverage", + "numerator": 14, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.7846891972623643, + 1.0 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 35, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bd332269-c090-52db-8c7a-0c821baef307", + "metric_name": "truth_change_coverage", + "numerator": 35, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 1.0, + "wilson95": [ + 0.9010990076755959, + 0.9999999999999999 + ] + }, + { + "condition_id": "A", + "denominator": 700, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0b3708a6-f9fe-5ed8-9c5c-3a7dc763fabe", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.005457846753060833 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b5a7988c-a64a-5264-a9ed-9f4d1971b1ee", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "61b95b59-e037-554f-b43c-0730534da5ca", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8e5c03d7-a591-58c6-948b-5af4a6ecf07b", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b4e5ec39-788d-5b42-aa6a-b96c06de456f", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "51c9929a-fe0c-57b8-9f7e-f763da8a0148", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ae0ea494-6c16-52b7-954c-dad7238217db", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4db5417b-1cfb-5b4c-8eee-82a662e05408", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 252, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e427a750-2e57-5952-b9e6-d9e8fce27382", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 8.673617379884035e-19, + 0.015014997328429099 + ] + }, + { + "condition_id": "A", + "denominator": 351, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eadee6ca-2e2e-5531-9cc8-d7ec571d0b72", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 8.673617379884035e-19, + 0.010825845529609503 + ] + }, + { + "condition_id": "A", + "denominator": 349, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b6007b94-01d9-5e09-850b-a5cfac74de9d", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.01088720932492875 + ] + }, + { + "condition_id": "A", + "denominator": 679, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "261b782a-42b2-5765-b42d-fc872c1877a3", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 4.336808689942018e-19, + 0.005625696523067804 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e0635085-80c5-5efd-aaaf-ae8e77dd7023", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b82da6ff-a878-5af0-b529-08316f34e6a1", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "02430e64-f6e3-5fa9-8754-b13a2117872a", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7cc125b5-6976-55f3-88aa-11386519f250", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "60cf5c60-4b37-5591-b009-cc33b4de2a0e", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "89baf2af-e273-5067-a15e-5fa0998a65f7", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a6114601-3452-5909-95fd-24057f84b95a", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 231, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "936b13e1-24eb-5f52-82f4-9983f6f5f43f", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.016357668871522173 + ] + }, + { + "condition_id": "A", + "denominator": 342, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b54f3b5c-fbae-5bad-a4a0-596df0a379ad", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.011107571757860812 + ] + }, + { + "condition_id": "A", + "denominator": 337, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3a4e8cee-1250-5ce6-99ce-bd756bcefbd5", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 8.673617379884035e-19, + 0.011270515136232284 + ] + }, + { + "condition_id": "B", + "denominator": 1704, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "84ab8ed9-e28f-51e0-8161-17764309494c", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.002249306456904229 + ] + }, + { + "condition_id": "B", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c2769643-2b28-5b1e-b5be-77c2c4ee501b", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "B", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "10666791-3fb3-57ea-8fee-3959876ee72c", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "B", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8f2371d1-100e-5eb8-ab80-92de8ae56675", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "B", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "804eab15-1d2f-535a-bee6-0b15bb6fb995", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "B", + "denominator": 172, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e89b26da-87fd-5982-87a6-d9802dd627ee", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0218461496307948 + ] + }, + { + "condition_id": "B", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "56d652fd-439e-5371-bf90-0c64ad4ecd0f", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.01875332680605788 + ] + }, + { + "condition_id": "B", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a83d684d-9b0e-5d4a-b677-36fb2def9b0a", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "B", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d8bc9ebb-0ea9-585f-aadc-ba7b5424d029", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "B", + "denominator": 860, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "723c83b1-dc99-58aa-bd7d-e82568332a37", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0044469488949262035 + ] + }, + { + "condition_id": "B", + "denominator": 844, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "577f2286-6ae9-57b8-be87-40250d28cb67", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.00453086928072308 + ] + }, + { + "condition_id": "B", + "denominator": 1704, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "46fab954-08c4-5607-808b-7e7506c10267", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.002249306456904229 + ] + }, + { + "condition_id": "B", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ddbaa983-fe11-5c98-87e0-4fedb8fa8df7", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "B", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f5006e3e-d85c-5ebe-afb9-2f38330befe1", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "B", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9756c8b2-8b14-5e41-a348-8299568de64d", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "B", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6522660b-863f-51ae-98fa-eb68c40146e0", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "B", + "denominator": 172, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2a40fdb9-4301-570e-9ea3-a2ec98b8d20d", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0218461496307948 + ] + }, + { + "condition_id": "B", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1eabf4a0-811e-5515-8032-2b7b607fd3dd", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.01875332680605788 + ] + }, + { + "condition_id": "B", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c57e9a04-da02-5231-839c-92936311eeb6", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "B", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8b493a72-ba53-5c8c-921d-278357e609a9", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "B", + "denominator": 860, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fd971dae-4b66-52ae-acf7-511ff1969aec", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0044469488949262035 + ] + }, + { + "condition_id": "B", + "denominator": 844, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5ee74fca-709d-5946-a44d-6ab7fb1f3508", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.00453086928072308 + ] + }, + { + "condition_id": "C", + "denominator": 1704, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ba37e3e5-7bee-5d59-b6d1-04bf80aaae9f", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.002249306456904229 + ] + }, + { + "condition_id": "C", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "80bfb620-0c4c-572c-94af-c30dbe0f6bb8", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "C", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "665a3133-c36b-5e25-a870-58aad458b156", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "C", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "35a14bfd-0291-5b2f-ad37-65cf156b85b7", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "C", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4dea92c1-ff48-5767-9b29-6c706a7c3cf3", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "C", + "denominator": 172, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "daa711c0-2e1c-5075-8bb8-c5116bfa28ee", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0218461496307948 + ] + }, + { + "condition_id": "C", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ce344a69-aa30-56b4-93df-f0e451ccb930", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.01875332680605788 + ] + }, + { + "condition_id": "C", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3230315c-ad2c-5f2e-8099-d6554b8d47ab", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "C", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "13b22a03-77db-5b16-b66a-f42727667c1a", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "C", + "denominator": 860, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ae7417f8-6e4b-584b-bbb0-17a630abca44", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0044469488949262035 + ] + }, + { + "condition_id": "C", + "denominator": 844, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b77bfaed-b47e-55a8-8a4a-4b7e0852e1db", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.00453086928072308 + ] + }, + { + "condition_id": "C", + "denominator": 1704, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ffa5b61a-0013-51b1-88b8-de8e594927fc", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.002249306456904229 + ] + }, + { + "condition_id": "C", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "babfc529-e82f-545d-afe4-b55b81ad98ba", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "C", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8a63669c-8603-52e1-95e1-4b03ee3aa56d", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "C", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9a9d5a6c-3587-5e5e-962f-754e1e0748c7", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "C", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1ffce4fa-7e3b-5ea5-91d3-be778add0eb6", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "C", + "denominator": 172, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "21ac8071-4277-5575-b579-d3d9faa703bd", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0218461496307948 + ] + }, + { + "condition_id": "C", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "680e3652-4da8-5d1c-96ec-e5bc5df0ecae", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.01875332680605788 + ] + }, + { + "condition_id": "C", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9236991-5f47-5f32-bb50-7e348472acf3", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "C", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "28586ccb-dc65-581c-bcbc-b8d03df4a9a3", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "C", + "denominator": 860, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "257fcb56-4927-5b85-9f4b-94b23e6caa28", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0044469488949262035 + ] + }, + { + "condition_id": "C", + "denominator": 844, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7f9d1c9b-ab1e-5d84-a2a5-3a33194a646b", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.00453086928072308 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 1716, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fbb4745b-938f-5517-bc47-d8cdb0c6f97c", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0022336121745362724 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "075c7030-f7fe-5bf6-ab75-08379a94d807", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "084dfdd0-01e7-518d-aeb1-26f3fe712460", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "76b633fc-b600-5e2e-b442-2e677480da5f", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0dd43035-ea6b-5757-8470-8746d8fc60cf", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 184, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ee5f3952-aeb7-5571-9237-55a126307ec4", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02045053762258643 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "75af7771-4c90-5ad2-807b-4fcd46584f42", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.01875332680605788 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4e20b981-d271-5043-90eb-3152d4631f6f", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3ddd417a-669a-5617-b012-a8be5ab9468f", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 866, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "00d43d9b-6add-5c00-a5eb-ecafd872b797", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.004416274692059705 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 850, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "068aca1d-e6c8-51e6-a96a-8f5b1d790778", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.004499030564760651 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 1716, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9162fbb7-bdaf-5987-8f79-8aeb1fbc779d", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0022336121745362724 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c897a483-6e50-5303-a722-ec563953f86e", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d7f30ac7-8e8a-5da3-afc2-c53619cf590f", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7ff7f1a1-d2a6-509f-a48b-b3ba0b33d5a7", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "221e474d-dbed-5204-8bb5-e9444fc08880", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 184, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "80d7ce82-85f4-5090-9319-5a47bade99b2", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02045053762258643 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "85a0b101-21b2-5e59-b571-e10adb3a588f", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.01875332680605788 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bde4f401-bd2d-5f2e-9c47-5d292d23aead", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c6edc7bd-de06-519e-9140-4e0ccf3064d4", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 866, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "15587d5d-223c-56af-81e8-64faa0b8a778", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.004416274692059705 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 850, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "70bbfbd4-ce8c-5196-b361-52db2c8a19a4", + "metric_name": "unknown_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.004499030564760651 + ] + }, + { + "condition_id": "A", + "denominator": 700, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f7c04a81-60bc-5205-9030-c97271943a2c", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.005457846753060833 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dfb29ec8-0041-5f06-8910-977a4400a427", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bb845fcd-6a03-53a5-864b-a64a18f94cc0", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e52129ac-d3c2-56c9-8adc-2ef07fe5844d", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0f6bcda5-5a3f-52eb-8322-facce689d3ce", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8e81f623-f8b8-50bf-a200-155a9cf81da7", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "67287044-d554-57a9-aa1b-123f1bf2f1d5", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0e3ff4d1-d305-53ce-ac47-8599f47212d3", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 252, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "860af00f-ecf1-53a1-897a-6811978a0519", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 8.673617379884035e-19, + 0.015014997328429099 + ] + }, + { + "condition_id": "A", + "denominator": 351, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "59e0d388-013c-5248-a885-fa7bd1a84e05", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 8.673617379884035e-19, + 0.010825845529609503 + ] + }, + { + "condition_id": "A", + "denominator": 349, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ec4031d6-2734-5f19-aa01-1ef72dc7af45", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.01088720932492875 + ] + }, + { + "condition_id": "A", + "denominator": 679, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "321f3d09-66cd-5850-be49-faaca6b181c2", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 4.336808689942018e-19, + 0.005625696523067804 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "63d0aecd-d102-553f-a40d-446bf0db0345", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "219df2e5-88b4-5dbb-98ae-2ec497ba7e15", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c534d8fb-7926-5ece-a20d-327d4a8ada10", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 84, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ed10127b-7b76-56dd-bc37-c3a9acbf60ba", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.04373172841465989 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5e1e166e-9513-5713-9050-49c7f709afd4", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d76e7d6e-7fc8-55b1-a6f4-60ea0311e43c", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 56, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "989cfd3f-8f6b-5bbd-887c-7f21c63610ca", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 6.938893903907228e-18, + 0.06419393671876342 + ] + }, + { + "condition_id": "A", + "denominator": 231, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8a131d7d-7536-5ba0-b3d1-7dc248da0422", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.016357668871522173 + ] + }, + { + "condition_id": "A", + "denominator": 342, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "89d45bfb-eed0-5fff-b3cc-fbf9cf0c7971", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.011107571757860812 + ] + }, + { + "condition_id": "A", + "denominator": 337, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "44337759-4437-5ac6-b4e7-569dc22e2f3a", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 8.673617379884035e-19, + 0.011270515136232284 + ] + }, + { + "condition_id": "B", + "denominator": 1704, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "715b7377-fbfa-59dd-b050-0b8a31f8aab7", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0017605633802816902, + "wilson95": [ + 0.000598927078061313, + 0.005163586046248182 + ] + }, + { + "condition_id": "B", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bab9aff4-edea-53ee-ad48-389256026646", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "B", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "73fa2268-f483-5361-9c48-464c1ba7340f", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "B", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d6730bdb-3678-524c-acd6-79bca3ee9ecd", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "B", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "98b9c12c-7da1-57cb-9d63-d05269885e4f", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "B", + "denominator": 172, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9b07971a-84ac-5c78-ac6c-e61740946fe1", + "metric_name": "unsupported_rate", + "numerator": 1, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.005813953488372093, + "wilson95": [ + 0.0010270438601417495, + 0.03219298775169032 + ] + }, + { + "condition_id": "B", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "792bf871-25ea-525f-88b1-011e427c32e7", + "metric_name": "unsupported_rate", + "numerator": 2, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.009950248756218905, + "wilson95": [ + 0.002732971824604618, + 0.03554765196043719 + ] + }, + { + "condition_id": "B", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1234047d-d3e0-5151-9c4f-c0f068bdfa20", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "B", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dbb6fc4e-130c-54ba-ab20-4114554c6194", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "B", + "denominator": 860, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a1e70330-eb3e-5df3-8977-f919af58ed75", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0044469488949262035 + ] + }, + { + "condition_id": "B", + "denominator": 844, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4175005b-4984-5463-b21a-499f67b8c128", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0035545023696682463, + "wilson95": [ + 0.0012095720969449962, + 0.010398091951924602 + ] + }, + { + "condition_id": "B", + "denominator": 1704, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5dcc8896-d7fa-5aba-887a-da8d9720f51a", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0017605633802816902, + "wilson95": [ + 0.000598927078061313, + 0.005163586046248182 + ] + }, + { + "condition_id": "B", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5c9f8256-c551-5f4a-8262-6226795b6102", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "B", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ef5afc4c-7dbc-5d57-84b7-cc51353c0056", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "B", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1a981fb1-f793-5397-8351-68a2c7bd41e1", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "B", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "170d7d8e-0049-5d1c-912e-68bcb3b36ab5", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "B", + "denominator": 172, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8551c490-b1bd-5ab4-9551-c80570afd6ab", + "metric_name": "unsupported_rate", + "numerator": 1, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.005813953488372093, + "wilson95": [ + 0.0010270438601417495, + 0.03219298775169032 + ] + }, + { + "condition_id": "B", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "41bd4841-6bdb-58da-9762-42c2dbed3666", + "metric_name": "unsupported_rate", + "numerator": 2, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.009950248756218905, + "wilson95": [ + 0.002732971824604618, + 0.03554765196043719 + ] + }, + { + "condition_id": "B", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fc835682-b773-506e-a878-e78c1501e26d", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "B", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4f5daaad-cdc8-55e2-93f3-747d20e43d34", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "B", + "denominator": 860, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cdccfb31-0d4d-50f3-9d09-6d0bff0f31d4", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0044469488949262035 + ] + }, + { + "condition_id": "B", + "denominator": 844, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "568e61f1-d655-5a5e-8766-b32520e78100", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0035545023696682463, + "wilson95": [ + 0.0012095720969449962, + 0.010398091951924602 + ] + }, + { + "condition_id": "C", + "denominator": 1704, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8d25e17c-0cc7-5bc4-9562-f69fc3852279", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0017605633802816902, + "wilson95": [ + 0.000598927078061313, + 0.005163586046248182 + ] + }, + { + "condition_id": "C", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "af91b428-a93b-5553-9d4e-bdf75b4bedeb", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "C", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5d560d5f-bc38-5a5d-aafd-c0f31c032d70", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "C", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fc4c126a-2084-5f18-92ea-0e999a5fc782", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "C", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "32f0e91e-df02-5cd6-9921-dc3f090f6814", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "C", + "denominator": 172, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7eeabef4-9732-5dd1-91e8-ecdae9ae26af", + "metric_name": "unsupported_rate", + "numerator": 1, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.005813953488372093, + "wilson95": [ + 0.0010270438601417495, + 0.03219298775169032 + ] + }, + { + "condition_id": "C", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0fd22e2-1cb9-5a07-bd93-59420171490a", + "metric_name": "unsupported_rate", + "numerator": 2, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.009950248756218905, + "wilson95": [ + 0.002732971824604618, + 0.03554765196043719 + ] + }, + { + "condition_id": "C", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "87325fce-f875-5200-8db7-15e8b5d351b6", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "C", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "afc96665-798d-57fd-ad65-cdda4284875f", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "C", + "denominator": 860, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bf28e8a1-72fd-5701-9c9b-c0bf5f70cc07", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0044469488949262035 + ] + }, + { + "condition_id": "C", + "denominator": 844, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fbdedebb-6d6e-5e8e-a350-3a6c68c94b90", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0035545023696682463, + "wilson95": [ + 0.0012095720969449962, + 0.010398091951924602 + ] + }, + { + "condition_id": "C", + "denominator": 1704, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c70b3b89-629a-58fe-905a-5caf6d4efcfe", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0017605633802816902, + "wilson95": [ + 0.000598927078061313, + 0.005163586046248182 + ] + }, + { + "condition_id": "C", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "15a3c501-a097-5223-980e-ce9c9dc20f86", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "C", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "25273191-87ce-50a1-8a89-4428bd723fbb", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "C", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "da310285-568c-5a09-ab88-a38cceb7f657", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "C", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a896174f-50be-5fb1-b214-d7153e9315e3", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "C", + "denominator": 172, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9ce76653-e651-510c-803b-6462106a78ce", + "metric_name": "unsupported_rate", + "numerator": 1, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.005813953488372093, + "wilson95": [ + 0.0010270438601417495, + 0.03219298775169032 + ] + }, + { + "condition_id": "C", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "01b75682-9fc6-54e0-a4c5-001de5ee2a05", + "metric_name": "unsupported_rate", + "numerator": 2, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.009950248756218905, + "wilson95": [ + 0.002732971824604618, + 0.03554765196043719 + ] + }, + { + "condition_id": "C", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cb8f352d-ffa8-509f-bcab-d4dcab9f7b8a", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "C", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dc8f7686-a310-528f-b33a-2d1ef9f4b4db", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "C", + "denominator": 860, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "def0eca6-8f2e-528d-aa4f-627b8197ddf0", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.0044469488949262035 + ] + }, + { + "condition_id": "C", + "denominator": 844, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "45c3542f-83e1-52cf-9118-1a1161fdb11d", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0035545023696682463, + "wilson95": [ + 0.0012095720969449962, + 0.010398091951924602 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 1716, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fea6facb-2f2e-5dc8-a565-b68ee0d04e01", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0017482517482517483, + "wilson95": [ + 0.0005947375493696922, + 0.005127568288891977 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5281e070-a870-5dda-b8b4-92376c84433a", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f05aa71c-fb46-5138-b493-6f726d766639", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0125642-4f8f-534f-9403-c977bae0b3da", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2e4bffd6-2806-57bb-b981-ed30b1e0093a", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 184, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6cdc4d3b-0868-5a6a-bc0b-349badaefded", + "metric_name": "unsupported_rate", + "numerator": 1, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.005434782608695652, + "wilson95": [ + 0.0009600176669975514, + 0.030137796720560765 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5bd5e62b-932a-5672-b68c-75911c0669ef", + "metric_name": "unsupported_rate", + "numerator": 2, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.009950248756218905, + "wilson95": [ + 0.002732971824604618, + 0.03554765196043719 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "783a4f08-cf10-5900-ab81-5a8fbffac420", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9e5246bd-5fe3-53d2-b286-aa8e77e8390a", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 866, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f9fb0e51-c7de-5436-8213-50c44c1cc75c", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.004416274692059705 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 850, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "667c1cba-d9dd-53fd-8481-f1bd773eeeac", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0035294117647058825, + "wilson95": [ + 0.0012010288844499055, + 0.010325067346912433 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 1716, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d3f0743d-62a2-5008-9759-94f07368344b", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "proportion", + "value": 0.0017482517482517483, + "wilson95": [ + 0.0005947375493696922, + 0.005127568288891977 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 173, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2494bc4d-9031-50d4-bb96-ea3e19d146b0", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.02172261440451652 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 168, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f7090f00-fde7-5262-b5a8-60ea7227bd39", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.022354668349868052 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 178, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "059772bc-a0fb-5a85-86e5-3a8d674ddf37", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 1.734723475976807e-18, + 0.021125318976251826 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 176, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a0267a94-8406-5004-81e6-8bcf61c86c9a", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021360251667687728 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 184, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6cb065d4-9da0-5d63-96bf-b5cdf699f853", + "metric_name": "unsupported_rate", + "numerator": 1, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "proportion", + "value": 0.005434782608695652, + "wilson95": [ + 0.0009600176669975514, + 0.030137796720560765 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 201, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "44194b0b-60e2-54a7-a3eb-a5bb305efefd", + "metric_name": "unsupported_rate", + "numerator": 2, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "proportion", + "value": 0.009950248756218905, + "wilson95": [ + 0.002732971824604618, + 0.03554765196043719 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 174, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4c04a595-d8d5-5637-a5f8-a83742cbf1ca", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.021600468451887907 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 462, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8c878456-a64b-53ee-a04a-ba4ed5cc2665", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.008246279389599652 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 866, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cfc8cd41-7ce2-56c8-968a-6ed578a99ab9", + "metric_name": "unsupported_rate", + "numerator": 0, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "proportion", + "value": 0.0, + "wilson95": [ + 0.0, + 0.004416274692059705 + ] + }, + { + "condition_id": "C_PRIME", + "denominator": 850, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2f68380b-af10-588b-b53c-608647f529cb", + "metric_name": "unsupported_rate", + "numerator": 3, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "proportion", + "value": 0.0035294117647058825, + "wilson95": [ + 0.0012010288844499055, + 0.010325067346912433 + ] + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5db1f113-8c4a-5069-b8ab-2f94aeae984d", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d01d0330-5a4f-5bee-93f3-bdd1868a2eee", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "57e70e8e-7ef4-58c0-853e-5f5892f8a025", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ee8cde27-600c-513d-b558-01f83d679cb5", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a9474b19-98ef-5e9e-b489-1877f394fb03", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "783b37b7-1387-5f94-9f7c-65756738910d", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "781162d5-0a4d-5d19-850a-a02e3e71193a", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "963ecb9f-ca92-5048-b07d-c5d53d6f9e14", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "97fe6461-1bdb-5692-a8d0-983dadaa4a10", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d2e33605-44a3-5aeb-b10a-6d8996528ca4", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "36cc1b63-d4b8-521f-9241-62b2282d82bb", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4f42c039-cf7d-5371-8679-e57f78be8f39", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3d679f55-e651-5fab-8fa8-eba4a61c4daf", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f79a3e0f-84bd-541d-a900-4b9e590e3e1c", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2d1bb96d-f322-5919-affc-b6d9821543b5", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "392d42f6-ae45-527f-8f5d-82b93222d919", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "db80872b-77b1-5dce-90b9-a0bd359b1b1b", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bca0cc39-f338-51cf-9204-e37fa379f4f8", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9000043-1fa4-51ff-a874-1896e996c68b", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "49f58d0e-96f1-58fb-8177-a0292785a112", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "da9cc5fa-f67e-5edc-86ce-596e882b4d8e", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0dace119-8701-5709-aea7-5d8a9a855440", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8cd858ef-927d-5e3e-b200-e3f33001bdff", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3aa671d1-524d-5f42-bcb7-d1f80bb301ca", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 5, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9d98d97d-c938-5a5d-b565-2ff892a345cb", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d1ae5f1c-6d38-56fc-b3e7-c87cf4eec544", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "91122e30-365c-546a-ae20-3151412d8bb8", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 8, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c7c41a48-23ce-5efc-a658-e89757311a47", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 4, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "34bff710-7761-502e-9db1-d503d36cf66d", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 23, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b44486f7-96d7-5eba-ab6e-bf2602db900f", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fbaf5bd9-4b88-5495-85f6-538ac1638c0e", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f0f8a84a-f5f0-59ab-ac95-34115fc8d393", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1bea4395-6122-5940-89f8-fcd0f610d11f", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f8cc3dff-75db-5749-96ae-160a6177e923", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0be245e4-ae32-5d3b-ac37-8d90c669faac", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 5, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2da9b5bc-50b9-5b82-87ec-c9b7bc34e4bc", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c771c4e9-4f4f-5320-a870-3fb7ecafb5b5", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b1276dba-0d1b-525b-be77-dece4c48eb7e", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 8, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6ae2085f-2c00-5f9e-a6ca-805ff3c0fc74", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 4, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "23c0f776-50b2-5ff0-a120-c280e8366e2a", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 23, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "392efa96-5c53-51e8-8453-4835fc056bbe", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "99b9bcd5-80c7-50b1-8327-3763a71be5bd", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "25970ee0-a176-5517-b16d-aef2a013dde7", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b649eda1-25b7-5c04-849c-a14434592063", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b5873f2b-b90b-516c-b281-3a637658a974", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d0bb4a5f-7d62-5425-8d08-00a3a8ec3e26", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8e985dca-3d2a-5949-ac91-4c25471ee191", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4915e47b-7677-5099-944d-664aa45ea361", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "149d6c39-b29b-5de7-b190-90cb17fcc9d9", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 8, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "205a6dd5-fa66-56c0-97d8-9e33a3c6dcb7", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 4, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cd1ae8b0-c932-5a98-9c89-3d4a69f7841a", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 23, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "45864b02-7419-5206-851d-87babd6bbbe4", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5d7fd512-3275-5bae-a146-6196ae0166f3", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d42e6f32-d354-5053-a122-1178c7c4875b", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1299d8d1-af7b-5425-8394-49379117c200", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0de0d1cf-4db9-5022-bca2-13da08238fad", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "98a7bf12-5abb-5fd5-80ca-c9e3e3337fd8", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c91acbd8-2de2-5f76-a26a-786b0673a73a", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0a02569c-775d-5c72-94f2-84ddac099ece", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bcfe2e4d-2fe6-5f82-9554-5b89f96a8bb6", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 8, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a2fe7051-d87f-5c44-99bf-461a27f5bef1", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 4, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "95189453-0401-50c0-9626-80f4d6ccedf0", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 23, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "772b759f-c613-5c30-8d66-0a2553d46574", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d94f9539-1b8c-57de-9acb-41accaa49434", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "27c2cdd9-b944-5097-89bd-679263f02f08", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "94ad9f27-6f03-56c2-8567-c2ddbaa7e5f7", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7b9a9bdd-79fd-566b-87c0-3b39b8c749bc", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 62, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9b4c8e9d-7b02-54a9-af18-79cb0a06dad9", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "69948b4c-11d7-5bad-9d40-80b92a2fd82c", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4fdf57fd-0723-5620-b346-f2cc05c9c588", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "aeb99620-cd15-515f-a401-60d5f3d2d91e", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 8, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c22ed737-6c48-571a-b9ff-94c5b0685297", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8df9db1b-c0a6-5700-9466-255d9a2468b4", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 23, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "de099a72-d5e6-52c7-b7f4-5b0de658ee56", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f8a41b15-ef4f-59f0-9edc-e289e434ffdc", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4d5e0ab9-4e14-5992-a7c2-11a329ea584f", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 31, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fe4847db-da5f-5877-a637-6426b012697f", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 31, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "28275386-97a6-5b5a-8ad7-2608812e4b2e", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 62, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ede84983-cfe0-5a98-8a5f-1fb9fac15edf", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1a50bdc7-943e-568e-9488-27139d9c8841", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2ffa3252-0bed-54e0-a30a-75cea3098c06", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "697c2b32-4c1a-50df-a85b-2421a8207223", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 8, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6352cbaa-621d-5b9e-b172-0967b025e99d", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "760eaa57-89a1-5dd5-ba91-ccf168133f30", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 23, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dd83cb96-e87b-5aae-a20e-a50635ae04fb", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cb5e6b5c-b43d-580b-9c6d-2bf841a416c9", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "93dfd960-99a1-54bc-b415-7b5db12cb06a", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 31, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f3e76673-391d-5863-b368-35e8d2689c19", + "metric_name": "work_baseline_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 31, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ce8e76f9-839c-5aee-a5f4-9cabf2356fef", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f401a594-0a57-5340-a4ac-eee0bff2f263", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6a42a637-0698-531e-abd3-04b5970fe414", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "caea956f-449e-5fa0-8c2e-e1669acea58b", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7a5f6961-603b-575d-bc4b-309bebc35ae0", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9056c7ce-26a2-590e-815c-1a650bcec1fb", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5268d64a-ade9-5e7c-a4d1-029afd327f2d", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "be969184-44f1-5a86-bdce-103c86a875ee", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1d81b8d5-b1af-5279-a904-100a461a4800", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "15fd8d71-6f9e-552d-945b-9d895f87cfbb", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "150ca997-0603-5908-b2bf-60b94637e5b5", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c55a15e9-07b3-5059-86dd-84926279cb71", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "94ba4edf-0054-5bb5-9062-9017f983ceb3", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "42caf01d-700b-57c2-b67e-2338e0e3cac2", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2f89b782-2b5b-54e3-92e6-edb59b88da8e", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "36de5ab0-0bdc-50a9-bfe9-bb9c1d469536", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2e7c2f94-a210-56be-8158-d14da673811f", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6cbeb2d9-ec01-5083-a5a0-7a2507fc5b2b", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c98c60ae-6a66-560f-a055-3037fd3b98f5", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2b58f0cf-6150-58a2-860e-7ce3594a7435", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a1394674-615a-5c16-9996-8e7cda614368", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "58195192-f5cc-5016-9840-62ea7c662b13", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ebed9a3a-eabd-585c-bc04-b31fa193bff9", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c48e04bc-8489-50f6-9668-259569d3e441", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e6a52f65-1973-5fc3-afca-be0a777389e5", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eb254a5d-997e-5e7a-a1ca-8bd64a61cfe4", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "06b64194-079c-5239-a3f6-df59b8b419c5", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4a87de21-f574-557b-8bf2-85cda3b0b237", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d7f13f18-afc4-5681-a48a-77f364d76bbf", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "59432843-1289-5ebc-9d7c-861a4da58951", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0204033c-2b1d-5e1f-a707-282540addda1", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "309c848c-1ab9-50d7-8cda-00cd2ad4b48b", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 4, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "01180172-24a4-5f41-9a96-75726cf8d82b", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "892ec68b-0dfe-5dfe-866a-25377fa1a30f", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0876112-7f63-5b03-8689-ec89c922377a", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "25c4fffa-1b02-5dff-9c16-3866ecaf2552", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d1607bbe-a31c-51d8-81d9-e1bc3fe9ec20", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b2b796ff-369d-5839-a42e-54907ad71d8d", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fd1094ac-01d3-5b55-962d-0797bf31f1a1", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fc947aea-40b4-5e39-b1e4-c3ce7b909cd0", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d3b548c5-601e-5557-a136-ce201521605d", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f83f7026-b271-51f0-bfd4-d88bd036f326", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b51853b6-91e4-5dfd-94d6-f5e631eb6797", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 4, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fd0da38f-075c-550b-b3e9-2621309abf35", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b2eb4935-8096-5498-a88c-7530c1f663b4", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "566bc504-e359-5c3d-b344-b9c8db9167af", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6418a138-7b45-5dfd-957e-fb4d71bc9022", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0ab833ea-fe83-50b4-b12b-33fccc0a6f06", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "76deaef4-7148-5314-9884-85d93ceedceb", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5e836b08-0048-5a94-a1b8-1a15405e5eaf", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f3b89fd5-9fa4-5b62-8ab1-3ad5c2d5676b", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5f8fe813-5819-5144-b472-ab1b36eca6fe", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3bed56fd-9647-59f3-91eb-90d56f4d8cd4", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "12b50c27-eb52-52c4-bc4f-ad38e5187d61", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 4, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5eba4cb4-ee2c-5147-bbdb-ee9f4ee996ff", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a0c93503-4e36-50c9-bbf6-633b1b51395d", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2d615534-b890-56ce-a5e8-efbeb183c9a0", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f926dcc7-cb5b-504e-8591-d4a1aafb2df8", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "98d09f7a-4511-531f-9fe5-755e2f125305", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e31e170e-9fa5-5799-a96c-673535da95e8", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e21ce043-35c6-56b6-a182-6d6772855c82", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2f3239e8-b855-59f0-8de7-b68933bd7977", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3806e5a1-c638-5010-8093-cabd74fe2f83", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3da6a40b-21be-5da5-8b7f-ffb97adf2c7c", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "78ab0978-5858-5de0-bedc-8e80da3dd8c5", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 4, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5931ec63-0c62-5b4c-bcbd-b5f5fbc7d52e", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a5e873d8-3edd-5b5a-9455-b0efef2dd953", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 16, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "84727199-5f7b-52c9-8b72-523a50b2af3a", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d4838455-2ca8-50d9-bd68-034757391e7a", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6228aaae-5fce-504b-8c8a-9d23cbccb320", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c1054acb-5d9b-5219-8fcb-84b30d723740", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5f4ac521-8576-587b-95de-7a1ff705e652", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "799bf71f-6871-5e71-abc2-2d13bb449d7c", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1cb0a266-5bfa-535a-bf31-1877f154492d", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a416e834-125a-57ca-a790-de601b4da3cc", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e3c29173-6013-51ed-bcb3-515ce5b2d2ee", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 7, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "314118ab-de56-5fbf-a722-c03cebdbbddc", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 9, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "24a8bf5f-0beb-5d4c-9cab-45aca36f5559", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 16, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3fed2d89-37ca-567b-8461-707fafe16176", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "01ba36a1-736a-5502-b739-9c0890c04a7a", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e32721f3-9b41-53fc-95e4-9f616308e23c", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "439005fb-a767-5415-942a-642778e7d713", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6dacd3a6-03bd-5081-adb2-c6ab87bcd049", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39174882-5482-5a41-8bb6-35e86d5fe408", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 10, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "16ea7076-eddc-54de-80c4-8736825e5525", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9e48a632-811e-53a8-aa67-9f4f423846a7", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ed112dc0-5cc3-5e29-a3ef-eb4b3837925c", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 7, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3773248e-361a-5371-84c2-c2f65d2ce6d2", + "metric_name": "work_baseline_not_applicable", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 9, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cb76318e-48f4-52b1-a879-dcfaa63bd854", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 560, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9521f0b5-e904-5c0e-9c91-c0395c6f50cd", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ea945102-282a-52ac-b458-d1b6144423c8", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "27a61294-428e-5f18-a57d-964342804238", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ce49e2ac-b0f6-5e6a-a088-f9ae643e734d", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5d422075-0050-5f21-80e3-7104693bfe36", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8800301b-6067-557e-a5b3-337b11cfcae9", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "70163e80-6b9e-5065-a32d-fcad4666d86d", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "295d2e0e-6e21-5a82-8a94-f6d02519fef2", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 168, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5cdf2a26-9061-5969-ba76-84c5498a0c25", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 282, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4bf92f12-a9f3-52bb-8f64-6c2e55d787ed", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 278, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f891efdc-08eb-5b3f-b262-939dca46e3e8", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 546, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8ab38878-e471-5ada-888f-1a37da1c73f7", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3b3b76a4-b25c-5d57-853b-25a17eaeb4a0", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6c1c487f-f1b9-5b1f-a4c3-d69224033eb2", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f21c8431-9822-52e1-bfa2-fc30aaa09677", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "85cc7752-2d5b-5d4a-a050-5a7be00ebaf7", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "63596809-df00-5015-b191-544e6d317db8", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bdaadaf5-9ba7-5ecd-b0ce-fb902777890d", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9b7c12d6-7492-50c2-9c0a-84d5ce8f6f66", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 154, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "188133cd-93ef-5287-b5c1-17d96fddcbbe", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 276, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fff7ec86-292a-58ad-8f58-eaf176a7ab9f", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 270, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "acc23bed-3cba-5cce-bb5d-0d0f1d7a0283", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 546, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "92e70501-e6f4-548a-ad63-bf2b080176aa", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9d7fff45-e6fc-5df4-8cbf-b7a5aa20d07d", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1f4b53e7-6d9c-5172-badc-f99059dbd570", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f5eb0631-15df-56f6-9ff1-01e4d3e0747e", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2a0ffdc7-0d77-57d8-974f-4c915e791cab", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9a25bb72-bf06-5d76-8bed-297adf76b7a1", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "256ad02f-eca3-5783-8137-3a5b1307a527", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3a5efb63-c2db-5e21-bf92-325befe8e54b", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 154, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a0e44479-d82b-56f5-b2ec-86d6544f16b7", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 276, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b40f0cc8-d4e4-5944-a594-9bffcb10587d", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 270, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bbd600a7-3ae1-5135-9023-160a18fcd8ac", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 546, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7a940dc7-a958-56ec-a567-18f5356585b4", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7e6a2cb9-76a2-5ab2-970e-2667f2702ab4", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a4d63898-b63e-5f5f-ac9d-6d53e3e09ae6", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a212012a-d011-59de-8ac2-728c77f52f23", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e745783e-a61b-5fd6-beff-a470cc4eaeec", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "21517b03-a8d8-50c6-853f-8d8a37c29163", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5f9454d8-f41a-5878-addd-007e55e435cc", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "85b576c8-0b83-55d8-b92a-12f86029ed08", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 154, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "29b870e0-6909-51c4-a5d8-b888444ec0a9", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 276, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d76877e5-e7f9-5772-a087-8f9aee1d3aa3", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 270, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d744b523-ca3e-55d9-af9b-509c2ea17d4d", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 546, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "babfb2ec-7eed-567f-aef4-20d973636a49", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "96963420-fbbe-51f0-92ff-78224054969b", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6798e4c8-af76-5fc7-8332-bffdf37c0419", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0db54d0e-2f31-5648-b487-0f909fd4c9a7", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bca3ffc3-db41-5a53-ae96-4e9fe8fcf22b", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f8377ca7-92b4-5ad1-a451-1d265ac5d9c3", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0beef1de-b972-5b9d-a63e-d6102d2e2a6c", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b467cb25-8942-5d4e-8429-cb9174bccb22", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 154, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "451043a4-8422-5404-8465-c4e61f8f0b7e", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 276, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "656796bd-3f9f-5ab7-9870-0a11a18edd21", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 270, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0e219535-75fd-5278-a1ba-065338968776", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 546, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "83e81d0f-e398-5f54-82e6-b4acb4a039a9", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e7fa7d42-d51b-5f90-97e2-aabe365a63c6", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a21ada60-33e9-5c4f-8360-7cc87143000d", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e2ae5211-e59b-55f4-b121-9a8d5496e32b", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a1246089-d15f-5da0-aa6a-c22042e489c9", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b0d11f49-b55c-5e0b-b0c2-24e25cc84034", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7a679598-a08f-5f78-b2c7-e55205d583cc", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ec4228e6-c588-545a-949a-b57babe0e6f9", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 154, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ff10210c-3f6a-531d-8026-d0338da02209", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 276, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "337f298b-8159-5e42-b686-fc0541d8a661", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 270, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6220f4da-b98b-5ee7-953e-078312cc887e", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 546, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "21b2bae1-470d-5904-95ef-f48c1e11abfa", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "90caae8a-dd31-5f96-a051-61c3691053d3", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "30f60258-00c3-5bf7-a894-92ec77bab858", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "17a8614f-c93d-52bc-af08-ef97d735e246", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e9d2b326-e41f-5aa6-8053-3b16071aa9d5", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "997a586d-cf83-59ab-ae5c-c1e3fb8d240d", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7e93261d-ac67-58ca-90df-4d2f6cf14056", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c2c90587-f176-53e0-8293-a699abafd086", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 154, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2415f05f-33db-57d4-8e42-faaaad8351c6", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 276, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0bed27eb-0469-5600-9c24-a68909e79934", + "metric_name": "work_gate", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 270, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "61471b4e-ee12-5851-8a46-2821388fcfcb", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 546, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b4deede4-71cf-57dd-aa55-5bcbf4c89dc4", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b83e8d64-058d-5f34-9260-8f7c6ec27d17", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b072186d-9174-5726-b332-e26cbe6f112e", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "470e7171-a169-557a-bb3c-4f62ac9a8079", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "883bf541-12cb-54cd-8cb2-0eb309b0af3b", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a9dd2e43-2f51-5648-a1bc-b854652bb9d9", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "89cc7da1-1a74-5828-ad12-1d4306c0c657", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7c1f591f-82a1-5d82-a7c7-e6340c67cb80", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 154, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6ec3c46d-8cf3-509a-834b-98a913fbbc07", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 276, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "57a3a2dd-d527-58b1-b946-a2380bcf1691", + "metric_name": "work_gate", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 270, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "69422930-0f3c-5286-bb71-ab884ad0985b", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 52, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ee62a4d8-ece2-5343-b3ee-9dbbf2c6af14", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "85c32ce7-c188-5e40-a2bd-3a2a0c61987b", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e379bebf-513c-5b3f-8720-471e4531c049", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d2b2dc6c-d78e-520c-8583-5db5749a529c", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4e4fa37b-ec91-500c-a02d-53a847b37ae2", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "859a6649-7074-56c5-b138-3ea49ef81c68", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f816e5e0-6e5d-5cc5-a235-fa324698621b", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2da696a6-5a43-5b75-b7c7-9df862a64ebd", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 52, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b1ff4ab5-3713-5ce0-a967-01cb2b0528a3", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 25, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4fba0089-1a8c-52b8-a407-6a4945d1db11", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 27, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bad03df5-6c23-5de0-ad90-c2fc16c2b5b2", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 52, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8cf86b6e-5151-56ea-8fd4-b0f4778eddf5", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4c676fe0-2c44-5945-bcca-93fc63aaa9a8", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fe60c83e-020e-50ba-8d19-27a2c0f5eb9e", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "422cd456-6944-50b5-9016-c91dd1047b8f", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7ed1988f-2009-5238-8b6c-0a0432a1358e", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "106bbbfb-4a0d-55b0-b236-cc722f59e2b4", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "473d18a7-c990-57a4-badb-edf108ec4d7d", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4efaa5dc-05f4-5825-83b8-2facb94b096a", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 52, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "06508014-69ab-5ad9-9371-ac76060c0775", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 25, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e70f6697-146c-53e6-85c3-a3ba1dc43533", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 27, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "21f813a7-04a7-5535-a5fa-9b2bbed8ca91", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 465, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2374928e-483c-53fa-86b4-b30a58dcca1d", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 51, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5345e16a-c3b1-5147-bc0d-49f741b7c890", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 38, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dadc5233-749b-5571-bf69-ed9a53190bc9", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 40, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "867837ec-8831-5255-8d1b-6c0001222303", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 27, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ab87906f-fe11-5e6e-922a-3ad09a84256a", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 46, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "498dd304-c6c1-5665-99b7-72241a1c7752", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 66, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dff1ddf0-901c-5c89-a711-0782e0d08cf1", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 34, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ca723f89-ec37-5e95-90c5-b16d1e408359", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 163, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "47d655df-c1fb-562e-9d3e-b7d829bbac08", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 241, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c071d7fa-f72d-5e9f-932e-c9f1e3825dfc", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 224, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "07d7e767-094f-525f-9af9-df4847916f1c", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 465, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "77d593b6-76fd-56e9-ae64-5bac728a5854", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 51, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3a7cb0a6-43e0-53a7-b9b3-cf92274b9805", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 38, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "65876107-5468-589f-8941-bb3938d4e52f", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 40, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b234c33b-03e5-5e5b-bdbb-7fbaafbfc16a", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 27, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "25a86085-69da-5ee0-a5d2-cd71a7d85df9", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 46, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e18161b5-f5fb-59e0-a09e-af2f8da14b73", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 66, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ca05d6ec-cd6e-5894-bbe8-2a958d5bec64", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 34, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "31b573b7-2810-5450-bd82-3346f83ed05a", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 163, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2c86d434-2d8f-5848-b9bb-5376d26760a3", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 241, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1eefd702-ddcb-5370-9bce-3d98824fc4ce", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 224, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "760e8923-b911-5eac-8e6f-28bb5bbf3ef9", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2ead0724-6ef7-5689-96ca-e8d4a527ea0e", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d4aee980-d1ca-53fd-8631-87977db10492", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2dff8342-d384-5ab6-b8aa-796216bc3eee", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c1efd155-7093-55f0-977e-fedbe0996838", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7d92fe23-76aa-5321-8839-86c380852be2", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fe946390-8086-5e01-ab7b-8f2ead315c71", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "19443491-32bf-558f-bc2d-df07a5414516", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "94601d71-f136-5531-8c2d-b15d0da1217b", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "feea165f-ef51-57c9-b9d1-8da30f181c0e", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e3ab704e-de1b-5032-887e-c4aec7bdd614", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cf78322e-54d1-5ea4-8e39-6ca215ce8bf1", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ab669fa7-1ef8-50db-83b5-654919f6b4f7", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "19e5748c-cf83-5cb0-bd91-9aaa9783c480", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "906a8074-a1a6-5334-b934-03d5775b2146", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e9d53e40-2556-5e01-83be-5810a2f4970a", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0f091d2-962c-5ca2-9236-1b346d441bb2", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5df472a9-2d4e-5fb3-9ad3-49b93c219150", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bdd6103c-0f6c-5a7c-820b-a6795e43ef2a", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "363b2706-7df1-591c-a9c0-ac8f9aa678c6", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fc1a3c7e-e853-52aa-b59b-4cf62d183308", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "53eb1e9b-5975-5418-a4fe-d875b0a662b9", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cce25160-d1af-51d5-b254-c21b8c87b208", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "46475e1d-ba65-5779-b6a7-656427faca1c", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "abb88bae-bb6d-590d-b4c0-3f2c2ee8955b", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2ffd00dd-be4b-5c71-a36f-684c09fdae6f", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c1c4b5f2-29e4-51cb-9d07-3b88cdb6afca", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a3fab4e3-1e81-5716-b2dd-4680b3d3e35e", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "850af266-a7df-5d34-a5ae-a32bd0fe7bbb", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "05a3c322-cf96-5f6c-a06f-904ca79ab81a", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39e2519d-c76b-5588-8f53-950c757a25d1", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8ba26005-c7ec-5727-b81d-daf95851a505", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9a2d5080-4fb6-5575-9e8d-e5bdd9e1f54b", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1996fd45-a687-5702-ac23-531ff3d4c45a", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "24e3ecf5-67a9-577a-8300-e9d048ac256b", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3409704d-9c94-5976-925e-e6a6a7224088", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7103a8ed-3d44-5d99-8cc1-d5ac71762c72", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0b32c83c-47a6-588d-bae8-16da4a1c2c19", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "216dc627-e47d-5ffa-b5c3-46a986d3c8d0", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f2972be7-9ce8-5dd6-bb18-25bb17aeb082", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e5ac132a-a20c-5c32-9f27-878d3e161688", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "664c6a3d-f759-59ac-bfa4-f12dcd9434d0", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eff172e1-b918-5a80-a3bb-d4a74ac5e76f", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "115ae381-733a-5edb-95f3-f20bcef3f2e0", + "metric_name": "work_invariant_condition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "95f7cd35-c45f-54ed-87cc-318b4fdf2abb", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 88, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f452ee5d-fb13-5582-9ccb-bf095caa7f12", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "aeab4b04-1cb8-53fe-8414-eba3e1a9635a", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "879875af-a375-5011-93f8-4842da43bd50", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "59782a95-53a7-5249-85d0-33f7539cfc84", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "16068e80-3588-5853-84d9-122e50cc95fe", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3ec4f891-ea2e-51a5-b303-cf03f4e5c69b", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "57f7e84f-a8f3-5a90-9849-1bfb2d700f99", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5488fdf1-f80c-5cf0-8f46-6edd21abc654", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 32, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "19d0bfcf-a414-5d84-a1b5-6e72ab3caf68", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 44, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4384e68e-e74d-5bc5-9cde-42195c6b941c", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 44, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0f2a6b32-0e9f-58f6-917c-bed5352dba79", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 81, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "893b1fdd-894b-5930-8571-62b4e889c28a", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5e51256f-29f1-54fb-b517-58d457b0020b", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3ffcd14e-1055-51e6-8a05-87bf538f587c", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6f6ce441-eef0-5338-b2f2-e5b6e0597ae1", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 28, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c382a270-74ee-5a29-ac60-685e20d71c23", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3a773d92-0b86-5f88-aeb1-d85862bc0642", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4b7a8e86-4bbb-5973-b665-378cbe739875", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c3b8aa8d-6ed6-5318-afe9-97bb0486d753", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 25, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "93fb589d-3f8c-5e8c-9089-27f454b5367a", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 41, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c327cc9f-1aa7-5adc-9bf7-3e1609ed03dc", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 40, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1e3565e2-eef5-5ebb-b51d-2179777b9a09", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 627, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b29182f5-96ce-508b-a856-6fbf38aea693", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 61, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "035de2f8-3f95-5c10-9525-b768dc00a1d7", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 74, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "353fb073-3110-5fd8-b53e-11d59a9d123c", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 72, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0e6e0b0c-a312-52b4-a201-5af50e024085", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 85, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "51d14928-4957-51a4-adee-6af62c1fed51", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 66, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "df40daaf-22d8-5d8e-a093-b6bc4b2674e6", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 46, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "caebe038-a07f-59b0-bc9e-a91798400e20", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 78, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1c0cac12-ebb7-5db4-982d-f03b0883e612", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 145, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9da34d6f-e073-570c-8595-6b6e20449b80", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 311, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "612381d2-9f10-5063-b81d-8b4f5a75abbc", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 316, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8a6bf9b4-0544-5206-9fda-52011b8b3eeb", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 627, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0d5ea8d2-32ce-5136-9ec6-6376b0c5b9dc", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 61, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c3d6b5ad-dc15-583f-a5d3-12c46eba4bf9", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 74, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3c1216a2-edec-5c82-b997-58ab3e27a15d", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 72, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "91a4175e-5263-5361-9348-d0b5c8421370", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 85, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "685e993e-fa06-5e6c-a061-998874b462f2", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 66, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b07ea066-8eb9-5f3d-84f6-37ff05a5f4ba", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 46, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ba4b13a4-c4d1-5857-abac-e62c290e92ff", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 78, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ff2a7f00-adbd-578a-84e9-402597c0d48a", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 145, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0c125280-f10a-5000-b168-89f5cecbb999", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 311, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1ba7ba9e-b944-5766-b923-00ab2863405c", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 316, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a891186f-60c0-5e52-9b4f-90f578a4c42e", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 627, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "aeaaf6b6-5bad-5817-aaca-92404bb1ca42", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 61, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "de4d5ea4-14b4-596c-b5df-2754943e6b42", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 74, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "345d8e25-bfc9-59d3-a48d-f50d6861ed37", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 72, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "80569807-bf61-50e0-a05c-21c0ddeb9985", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 85, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c6020d9e-cf52-5ebd-82f0-94a6e0564090", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 66, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ee62dfa1-0a9a-55bb-ad53-d2bf30ece7c5", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 46, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a55e3ac9-fd64-5356-8dda-9745883ed69b", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 78, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0a9b5ebf-efeb-55cd-aeb1-a92a90664eeb", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 145, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "16e1b20b-c5d4-51d3-9460-09d054d69248", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 311, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3dbfa264-9e3e-5db1-af78-22c601974a60", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 316, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a392fa5e-d109-5757-93a0-ec9318bc96f5", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 627, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c9bf908f-e948-5735-a9f0-de54aa35de60", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 61, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5f1fbb48-3ba7-5f25-8ef1-2de0a5116dcc", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 74, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4e9b6c21-c669-59fe-848e-21b99352cae0", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 72, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cf135fe7-6dd4-5f1e-b857-75a1c4a6dccc", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 85, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a96728d6-b1f6-515a-8c10-f34b008ef4d7", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 66, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "02d75064-9b20-5023-b156-d19ee30fc0b4", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 46, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9e1ac7a9-bd73-5d0f-80e3-7e35aa5a1d86", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 78, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bcc19854-53de-5443-81b2-75ce192159f3", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 145, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e6000884-7629-5b6d-9f4e-ba4477a13df6", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 311, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7aeee936-d58f-55a9-9d6a-14edcf5243a5", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 316, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e738a2a3-db48-5920-b4bc-4a6762375001", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 627, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8486ac94-46b0-5ae2-a908-306d5226d149", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 61, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "79af99d9-eeb5-5074-a483-c27c386ffd61", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 74, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "20327eb1-a7c0-561b-a5f0-cd0d669837f8", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 72, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "16a94940-014b-5d82-870b-324aaf1cfea9", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 85, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b82c9319-bc00-5e01-a7ae-7c8760b80476", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 66, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c7a602a6-90a9-5eed-ac2b-63bef5504805", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 46, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1ae9dcc6-3d02-56b8-8d7b-4baed8eaf055", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 78, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a084837d-bccd-5fb9-8b3e-2c38b3748fb3", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 145, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bbf553a4-7457-5fa8-9767-7721ee7ad30a", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 311, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fb026fa8-0568-505b-ac91-dfa19ad0bf34", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 316, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "270d62db-05fc-56b0-be0c-e8eeba6dd7b6", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 627, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "05c6277c-7b14-52c0-95dd-08fb0d044934", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 61, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7ab1c939-5aaa-5ce1-b151-e6e75bc5c31f", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 74, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fbbd8a01-69f6-5dcd-b8af-a8589791dc90", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 72, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eab6bf81-ea81-50d4-aa70-5fcd135747ee", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 85, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ba19a75b-b4a1-5e92-9c6f-4c4eb0213154", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 66, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39f544b2-63c7-5285-870c-06bc32227f87", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 46, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9c092ddd-a9b1-5437-9fb3-ca9f8f8640a5", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 78, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4800ab2a-42d7-5f6d-ae24-3e05b4687557", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 145, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5500e445-c81a-50d0-b23c-fdea778eded8", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 311, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "51e1b28f-426d-5310-a4ed-1d4e564e2a69", + "metric_name": "work_invariant_impact", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 316, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c680c93c-65bf-5112-becf-976de4b54fef", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c1eb0c18-eb5f-5aa0-8003-0dd0782cb5db", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "da6cda85-1b41-5d10-85a0-ed6c8615e166", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "72898024-dae4-5ac9-aa87-4c9eb7bb8164", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "821bb7d8-7d7e-5312-939f-505534145570", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "08e65c84-4cb3-59c7-8c69-29d07f31c969", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "22a2f57b-c4db-5a19-b24f-ba18d7800e20", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6d335ca2-9a0c-5f8c-b9e0-2178f6c38349", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "55ee7a46-362d-52a0-a13a-0a6c81a37589", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dd7a9033-4bf8-5ec4-a81a-047c4774c6dc", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2acbe434-1605-5a3f-a15a-4e60d4b9c48f", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7a3c9af0-516b-5b0d-a667-667db891f87f", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8cf86839-a6e6-5134-8f38-930622d77532", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "43e572cf-1e19-5d0f-b667-95fd925c2c1f", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1f68552e-9d8d-5c1d-bed8-554a91df5eb3", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c84bf36d-ae6f-5329-abd8-f087d31847e8", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a03c3371-fef5-5199-92ea-2db8f975b8ce", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a6d39922-c39e-52af-b5bd-014e3c305516", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "da42eac2-6aef-5618-982e-3a056a67f39e", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "066c4b29-c098-5471-b4c1-ee59838cb5f3", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7857698a-d163-5179-b0f1-d36480c0a36c", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "78739b4a-c257-5c73-afa5-ed35d228d96b", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "51241a71-15d5-51f6-a1f9-4f61cf724159", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "78e56262-9bcc-52bb-952f-d72f9a56c04a", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "09424610-6e87-501d-8fed-19d0d75cb1a1", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fd474286-1ef1-5186-a9b4-ba4aac4de7a3", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f2c5214e-677f-5fd7-9681-9b934fcc463a", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b83672d9-2fe2-57c6-8d53-3647eb448dd8", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0436e534-ccc6-53fb-b1bb-3a631557a729", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3d604966-e1b6-5ff9-8214-d3db89d36524", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c2eb9ceb-3ad3-5542-8ee4-fbe507103115", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c30742fc-34da-584a-8b05-575496a56450", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "84f9b966-738e-5708-8531-c2d7ae406334", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2fda9616-148e-5ae7-b810-8c042af2046f", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2776c142-fb2a-55c7-a49f-cad3c24a8766", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "212d9871-4c84-50ad-aef7-3754365afe2f", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6d257f0e-0ee8-5007-9900-e8eaba52e531", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7a7f625f-9d62-526f-b4d0-68240da69837", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0b451f05-6d9f-53cc-a88e-dcdf560d42fd", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2a29c88f-c54e-55c8-98e9-6627915bc735", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "653fec9f-c475-5987-84ab-a9d5f156c7a0", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2e6a6202-e96f-5240-bb98-21d55088ac73", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7478727f-dce1-5073-a581-b734c62a4d20", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "75cc4ced-9b3f-50cd-b960-63f300b54afc", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bb15e2b7-340e-52e0-ae51-8fb539240597", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 54, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6deecc03-3564-5016-b201-72c56bdc02bb", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c5febcbf-2a22-5c94-8c92-d80e7d7a1dcd", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "aab0d508-bb8a-5171-bb24-8643716ed243", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39a3e190-8cce-507d-93ec-fece525acf11", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c5151fde-1fc7-5df6-8e5a-031ceb8dcb18", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eaf01959-c15b-5d9d-a7b4-11ae7b8ee722", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 2, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c401b54d-8ef3-509d-9405-5d6b68c76729", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4ebdfa3b-4376-518d-9de6-14004df9b892", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 52, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f73cfb95-0239-5e0d-a32a-ff72e5a16ea5", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 25, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b7944324-a46e-55da-a98e-2c5cd0586ecb", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 29, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bd2e0d73-20f9-5095-b4c4-7384b9631a8d", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 54, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "17adfaab-aa93-5901-9c2e-fa338192eb35", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "596ee934-6b53-54a1-be48-986493257c94", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f8e13294-b72b-594a-9ece-bdc473b378d2", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f9af3b48-f7f6-5237-8315-b858e0695ea1", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "35c685d3-cba1-5e44-b0d6-a49e02fe9c2b", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0cd17ae-66ec-5e68-89ab-e29ce0b77577", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 2, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "41caff27-2400-5ad0-864b-fc5b485d4506", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "907bb1df-8ab4-5fc4-bbb4-4cd5ac340dde", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 52, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e605a2a2-4af1-5be5-9e12-b2895e3391db", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 25, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e07e17b2-35be-57b2-bacb-ca640679ec50", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 29, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "441ccef8-3d17-5287-9465-c936937eae8b", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 54, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "42c6035c-68bb-5f9b-afc8-68439e7bf849", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ac8e0d57-db81-57cb-b63e-343afd984dd6", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "74940858-1590-53f8-a076-6070ae6899cb", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "59f715a7-f55f-5d3d-b388-12046b650b11", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d89b1d4c-2ea9-5219-a441-72065472f66c", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c543430d-fb6b-599d-8081-964aec2b2aae", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 2, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8b11c972-f3ac-5b03-a05c-4e17ffe024bb", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2a4760bb-b3ab-56df-8f78-6fa5da4202af", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 52, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2a076f1f-d558-53a6-84cb-d0d88cd1a5d7", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 25, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b183c2a5-6338-5a0d-9318-596ac86a7187", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 29, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9ae6f7d1-0ea5-5ca8-a766-f3bc7f849e48", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 54, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "aefadcef-f269-5494-8aa8-35b83f337a83", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bcb461ff-04ee-5fa4-ad21-6890a6002647", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "279bdb1a-0c81-51f3-a2f1-f442e06e6503", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "18295989-faf1-5a02-bc04-13d010d99072", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "76e010bf-b9e9-50b4-99a9-33c0a867a852", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "81786db4-1a1c-5ad5-95f8-a490f24d3697", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 2, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e844aa50-0df8-58db-8493-867d21be9ab6", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "026fe898-8415-5815-96b4-9a6e6078e443", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 52, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "98243f70-300a-56c7-a620-980b024c1d6c", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 25, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a235d801-8a3c-5bbb-a367-a02bf8051a77", + "metric_name": "work_invariant_not_protected", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 29, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e8edde28-1f1b-5f20-9521-aaa0827bbcb9", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.5, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8d0dc088-2564-5844-8a9c-690f30448e58", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fd2096b9-232c-5289-9725-8efc1e1c80a0", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2c41b095-b2b4-5abb-8b48-e24f4d76c422", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8a92bbb4-9047-57e8-aed1-24eb8800ca97", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "36a4a6ca-c895-5fe7-ab91-bdbb4fa8852c", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e485e5bb-13f2-55f9-879c-a341fb4c4bed", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0f64d3cd-3074-59e7-aaa2-e1f06b7214d3", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8bcc869e-b229-5b8a-817a-c5facad2c1bd", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bf52093c-f4f7-59ff-962e-b52c70684860", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.4893617021276597, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b808462d-0aed-54dd-84c5-c590227d393a", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.5107913669064748, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2c3ec41d-cc75-5019-925a-a9b6918f3928", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.4871794871794872, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "443fb334-dc04-5a2a-a556-9f0a3d91074f", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2fa4e4f4-7af8-59ef-a77a-157bbd9ad273", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6287d336-a24f-5524-bd13-b13a721b1623", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a1b0e8d5-fb75-52d5-b564-d9355344fe8a", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b291b6f3-5e6b-5643-8ea6-174b65e3c6e5", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "622b87d2-d0aa-5174-923f-d7a19bf900c9", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0340b687-fac9-58ff-8951-974d528eaa81", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d6fe86c8-c4e7-539b-aa60-180a91f4fd0b", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fa5048b9-fd2a-5d05-a4cf-bc0859286093", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.4782608695652173, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2d3df48e-a365-526c-8ebe-8aea7a4b375c", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.4962962962962965, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "98d2ca2c-1e84-5828-954a-6978bfee2a49", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.205128205128205, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c0fec800-dc36-58cc-ab25-38d4425f0f23", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.178571428571429, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6205f3d0-a694-5604-80cd-9a37cad4a9cf", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ff4feeac-0253-58aa-bc09-36fb9e01a3ca", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.357142857142857, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4da30522-acec-5a84-8d5b-6a103eae3314", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.285714285714286, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d6ca925c-243a-5090-8111-699da9a679fb", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.142857142857143, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "321a23e9-a675-5ac6-9353-7d641a91d620", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.821428571428571, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4612e508-f719-5e79-8338-d8ffcd9ea11f", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.214285714285714, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "832381db-75b6-5271-8844-1b8121db29a8", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5bbe4ed3-81e7-57d0-af63-f38b5050d241", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.202898550724638, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "797a03ac-99df-576d-b100-da69810b483a", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.207407407407407, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bd0ee78c-c05b-54d7-ad9b-6bca33274ac0", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.205128205128205, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "47a68d08-be21-583d-a8c9-7df779298690", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.178571428571429, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7a8ced56-363c-56d2-b053-033c92a3cad4", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "91c1529a-c68a-5f5c-abea-72792733c240", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.357142857142857, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fba3ca71-518e-599a-9645-a54b038ee688", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.285714285714286, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0441f4a5-1daa-537e-935d-63a600d7b824", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.142857142857143, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7cfce7a3-0591-51b0-84e2-f4eed429c4df", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.821428571428571, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "04d06d00-8485-57c8-a949-5255c859b519", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.214285714285714, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "18e9c5db-89a6-5c9f-b3d8-474c5aa2ea8c", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9b350072-ba5c-5eff-8ce8-d5e502de1024", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.202898550724638, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b5b766fa-634b-572f-b550-4fcade97cb0f", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.207407407407407, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8b7c4422-01eb-5e0e-802e-e959d6d5470b", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.6996336996337, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fb207408-67be-5968-addd-9ec93aa65b48", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.357142857142857, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c5f7912d-1146-5cc0-b159-e61ff94039c6", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.642857142857143, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9304312-0cbf-5156-aad0-ac886e0a5216", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.928571428571429, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5e8af39e-80a4-5010-b970-3d4e79721de9", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.321428571428571, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f9275b50-7fc2-592e-b03c-56bd1c16cc78", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5a27811d-3073-5f64-8564-453d28512203", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.535714285714286, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1212bb20-51a9-5ef4-a387-a77ace765ae8", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fe0b055d-305a-5a3c-ab69-7d57316b00eb", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.558441558441558, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "79af36e8-80e5-5064-a9c5-54f98dc3ec21", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.63768115942029, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "27afc8ea-03c1-5a50-b9ac-983ae1897caf", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.762962962962963, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9cda908d-752d-5ef7-8274-e6f8c9f8c203", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.6996336996337, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5a5b7791-a6ad-53c8-bd50-5ad2d8c489ed", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.357142857142857, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a8074282-bf96-51b2-8c65-23706f917415", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.642857142857143, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1d0598a8-a183-5fc0-b8f7-7599e7568a88", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.928571428571429, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2b1d59c5-ac5f-5fca-bf4f-907376a38d8b", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.321428571428571, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5cf1af0b-0d40-5015-a2ef-315e06ed7c02", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a846d1cb-79ca-5156-9aa7-34f8b237e4c8", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.535714285714286, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "583034d7-89c7-5fd2-81de-2d50a07eeda2", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "358d8eb2-f939-50c0-a99f-be2a116a3262", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.558441558441558, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3a40a14c-7514-541f-a502-208ba10682ec", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.63768115942029, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fa510c12-c369-58b1-a6fd-82aa4ed57125", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.762962962962963, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "489795e3-847e-568b-9cb1-8cefe87e282d", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.721611721611722, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c6af8f55-acae-5e03-a318-79b279b6b175", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.357142857142857, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d4ed38c0-6692-5502-9211-9e9fa072f071", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.642857142857143, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4d12d0a9-3cc8-5723-b44d-7e017b6a0141", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.928571428571429, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f8cf03bc-6b74-53f0-ab55-acd7e0df346b", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.321428571428571, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c5c5fe82-adb3-5eed-85e4-7a3985a26de5", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.714285714285714, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "42c183d0-4e24-5e37-b1ee-064e81a68412", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.535714285714286, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e0a6e60b-2aa2-5066-a172-83affc0fddfe", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "efaddc76-ae81-5967-bf4c-47d415065507", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.558441558441558, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4a515575-f53c-564d-af75-d2b4b7bfa116", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.659420289855072, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "76f6f3a9-5675-5c6f-aa73-0dbdcfc199ef", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.785185185185185, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0c7888bb-1ded-5050-823e-b1f3e2a85dbd", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.721611721611722, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "012e4607-4248-5578-8e6a-90c68edd59b7", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.357142857142857, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4868dae0-78f8-5574-a4d6-ca6369260b5b", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.642857142857143, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0751cfe4-73cf-5ee9-be51-cb9c6e4cfa76", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.928571428571429, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d870090d-97df-51c4-ab57-e8f93b8dc4ed", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.321428571428571, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "85d31978-31de-568d-813f-2f07f2bc5d68", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.714285714285714, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2ae4ebf8-6c16-50ee-896b-b331c0310dcb", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.535714285714286, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7e5f04d7-6b91-54a3-ba4e-4badba9e06ad", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8ca33cfe-c12a-53c3-b0c6-9d70e3ce308d", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.558441558441558, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d34fe6f1-6159-5966-a9f5-0807ebfd6eed", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.659420289855072, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d30082cc-9cdb-529e-9044-a1755499a029", + "metric_name": "work_mean_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.785185185185185, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3f5fa26a-9b06-58d4-9124-ae4a5d7a5538", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.5, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "128be204-a584-50b6-ac46-6ed4bfb98fc4", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "824fe426-84fb-57c2-8487-c609bca14f65", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bec9d2e1-f3aa-5175-910a-7c0fdbb37423", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ba115eda-0680-58c1-9f04-70605ad36d88", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b696ca42-3c32-523b-a55f-6cd11de534ad", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "42188b26-b701-57db-8eea-195c7dac47b4", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3adbaa7c-d7fc-5d97-b29d-3f236d7dc75a", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ecfaf90b-6cb3-572c-97d3-64afcf2fdcbf", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9f4139de-7c1e-55af-a759-15bb27578f49", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 2, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "10c24426-4b1f-5b39-b029-1581dab4b24c", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 3, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eb4f63a2-e98e-5e8b-b097-c6f7aea31558", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 2, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6f3faf60-4313-5f71-aff3-0ab25fb6cbae", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e44bb7e3-6ef1-562e-8267-28dae21d877c", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c9a24105-12fc-540b-a53f-cf7268b3bf9c", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ead539b3-8b1e-5d0d-8681-cb0a6cd07dcf", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 3.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ad6c5ca5-bf59-5ff1-9cdf-371b0092fdec", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "76578263-6ff6-5f81-a80f-7ff3e2fe7d98", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a420466f-30e2-56b3-9852-bfcfbf648d2e", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d2c9f3f4-fefe-591a-9afd-f949bf7613a2", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 3, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "552529c7-229d-573a-92a5-7a974f49310d", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 2.0, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7145b996-894c-5874-b257-74216082d804", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 2, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a646a3ad-6fca-5237-ae5b-481c302679c2", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 6, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8c53acbf-8324-57fa-a639-38bc6c20871a", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "df5c0fa6-c086-5208-9013-620e4a0953df", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ce6da340-ef65-568b-a69d-2ecd46edc0dd", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "de29901b-f07b-5dec-80e9-b08d3fe5371a", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "48c811cd-58de-54b6-a3e4-c325248bdf3a", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f90fe7de-09a4-5263-867d-0a3619c659ab", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 7.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c6003598-0546-59d8-8df5-bce055b132c3", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cf1b5157-2f36-5085-9e8d-f625809a4c03", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 6, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e291e7a9-57ec-5913-974e-af66402a0400", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4a2fb4ac-4142-528c-9780-dab9010f9e76", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 6, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0a8d07e0-3709-5c34-ad00-b0684b3ab453", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 6, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a66b9a82-1afe-585e-a6b2-258c4b0039bf", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "89d78b33-623f-5bb5-bf12-e1715a7fb95b", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6c86281d-4e7d-59bf-a9c0-e34d9adba6e4", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6d37f307-be54-5d76-b5cb-52bc8bcd050e", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "68b84a4a-9e45-598f-8423-744800dc2839", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9741df1-dae7-50a1-aad7-399334e20c18", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 7.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "709b3789-aee4-5049-960b-1358626f6ec6", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6cf09e63-c034-5b6b-811c-5e5ebb5acbf5", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 6, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "58d69d1b-d00e-567e-950b-c650e95919e3", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 6.0, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "199f161b-9937-5fa9-9611-04da9ab2583e", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 6, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d3ec8753-1acc-5bf6-bc0c-fef5b035153b", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e58cfaef-616e-5244-aa57-a82e12e8ae72", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cfa5c554-cb7b-5505-b775-d0fee906239e", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "db703020-b18c-57c0-8d86-208d8b71bfce", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0832692d-1122-5c7d-a79b-acd8d951d89a", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b9dba6bd-468c-5a0d-9861-04fe80197eef", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bda5168d-e69d-5176-8444-7110525ef3a2", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "523d821d-345f-57c6-b84e-5a0f13866511", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "518c399c-1654-5218-b998-3371ef984390", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0fc7f57f-3a52-57fe-89fe-8b3904d4fb92", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "63eda542-8ba8-59b6-b641-9661b4f96416", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1ededade-022c-5ccb-9f44-3bbdadeca696", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "797bd40a-febc-5a7f-bd61-3450f4846032", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e77f1c3d-4bb7-528b-acff-07ee7ce674b5", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "37418aee-4525-5032-8a35-54a366e153ff", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7b390ec1-1c61-5b2e-b4ef-b7ee4818f2ce", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "93a92025-efb5-5ab2-ad75-9326239bde2f", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "882aeee4-dc17-5b35-8048-5e4af66f3510", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a2825bd2-bac9-5776-b3fc-509b48357d6d", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "24257686-1688-5b96-9680-a91137bd5cb0", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8bebaa66-e024-5687-bc51-0816c184b0c8", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9e1fabc9-2ab8-5d0e-8c46-fdcbe25990e6", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "66f130f5-250e-5176-8065-7dca8876736a", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2d11c603-c5a8-52ba-b57f-ea3668d4bb77", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9402de6f-da16-5b00-80ea-ea7b1db526e6", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4e8e6208-1662-5790-8568-cd07f56d3b43", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2524664e-80ca-525e-b0b5-ce08ab60eef2", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "45b04e0f-c0f9-543c-b84a-d60dd5f090c6", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f3649420-c3de-53ac-9219-5661a37cc82f", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6d515b4a-f26d-599e-a623-c4c08cec0b7a", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d128ac84-06fd-5a98-83c4-9d999567c904", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f07d2dd8-c18f-500f-8d44-3b7c2da83ca8", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1e52e794-562c-5151-b591-c17150ea93a5", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d591c15d-beda-505d-9441-ed0172878e6a", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9c0d8f7e-e5c6-5251-ac87-95242a328c1b", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "de3a2aae-0ad5-5e3d-8152-39b3138d25a0", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ed5fb22f-7b19-5460-adfd-627ab42c8483", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b8a7e992-c5e8-5bfa-a3fa-4164f408d271", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "22a79dd4-d5a0-5ebb-81e6-561387fbb7c0", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d5e634ac-72c5-5ef2-af59-2475b7f9895d", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items_per_transition", + "value": 4.5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "dcf97fdc-3649-51bc-be9e-4d30f51fcdb9", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3e9d2d0a-2e09-5737-946c-ebb2923c8cfa", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f72d12ff-9764-5aa9-8436-d45d8c3a9cfe", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items_per_transition", + "value": 5.0, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8b77d6a6-d3aa-5ef0-8538-37fe80ee4f86", + "metric_name": "work_median_per_transition", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items_per_transition", + "value": 5, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": 1694, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "eb47b873-3d57-57bd-984e-a4a0cffce503", + "metric_name": "work_saving_c_vs_b", + "numerator": 1283, + "scope": "common_prefix_B_C", + "stratum": "all", + "trial_id": null, + "unit": "saving", + "value": 0.24262101534828806, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "90b5404c-2ea0-5a71-9930-8dbffbeadc67", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 700, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "89073f80-5cc1-5359-8a6d-e772ec5f9047", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "164ab440-735f-5497-b45b-12788b20cec8", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 84, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "af1e0608-bb60-55d3-a3a8-6b107547647b", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "916dba5a-26b3-5b37-8507-f5ca853974ec", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 84, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c3492020-c1d0-558a-8a68-a84091c95921", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ad9c57cf-674b-569e-ba44-ca3b5ad6d173", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9716400b-0c47-55ea-a550-f2afa247dcf5", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2ddb784d-b849-51e9-83b6-19f530bbffdf", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 252, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39da1716-c9ba-51a9-8d90-63c400d71e4b", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 351, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "017311dc-516c-5106-a701-8df142c8393d", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 349, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "38e327d2-6293-5900-b366-0e6c9158f9d3", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 679, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2cab5fb6-3a18-5af6-bb04-2c0238bdbe07", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3f3ae705-e8ce-5a08-9732-83eec90229de", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 84, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0c37c320-3120-536e-aa96-95f45f471a0a", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "166c579b-1cec-5c67-851d-c9ba3483720e", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 84, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "9129bbe1-93d3-519a-9fc5-3fa7d0b4e2b2", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "50e06be5-8819-5c5c-b023-d4a895dc7466", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "7905efd9-ed94-5f07-834f-5df76403af83", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 56, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "97bb0e08-4e1a-5ded-a020-bfcc6c52866e", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 231, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "55a71ace-1f96-5890-b5e5-cdab62a56a19", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 342, + "wilson95": null + }, + { + "condition_id": "A", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c68c4f34-28df-5e30-a318-09ddc7975d20", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 337, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c487c512-191e-5b4e-9e54-03c594636225", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 1694, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "07ccfb59-2364-534a-b4ee-9c6350ee02bf", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 173, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b1f83b4b-6663-539b-ad3f-2e54d4d03826", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 168, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8ffeb762-6fc4-582c-89af-c2fcef0b095a", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 178, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "81a46cd1-0219-5c5f-8660-5c3e1c5c2fe3", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 176, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e145e409-201c-529e-925b-50552fefd7bc", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 172, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4b8ec487-860c-5ef2-90b3-88da00de69c1", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 191, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1d33daea-5708-5a5c-b949-07cc7e6ecef7", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 174, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5073d405-6875-5113-a9fb-ff4f1f5ed4d4", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 462, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6635c789-fab7-5659-87dd-10ec297858f1", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 856, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0758e1e7-fac9-5e9b-ac92-ed784b5ea1f0", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 838, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "5196f4ff-fd61-5cb6-8888-0b801a19a920", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 1694, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d68321a4-3fce-58d7-a628-423b9d5d8164", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 173, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "22607a48-d582-5fa6-ae6d-b5691c7792fa", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 168, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "07da3ad7-c64a-5b2f-ba47-d045588c6060", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 178, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "45d4b97d-4656-5610-acf2-9257f5eaee0d", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 176, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1ebf92cf-3958-541d-a1b0-9868f252873a", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 172, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0be4d27f-d912-5978-9ff5-b490bedd7fb7", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 191, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3f40b00f-efcf-5272-9c19-ba2968989dcd", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 174, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "fc536924-48ad-5d3c-b056-932984547772", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 462, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3c62da4f-80ef-5fdc-9dc1-fa4b15d9f932", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 856, + "wilson95": null + }, + { + "condition_id": "B", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1050b982-e81d-537a-89ac-d820331d3b5e", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 838, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d1cc6fed-c186-5510-aa3c-c89ff15961a2", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 1283, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "03b4c989-ce4d-59fb-b403-e60f11325be2", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 122, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "6fdf8718-3d81-5220-8988-d4b29050d999", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 130, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "13afeeff-3833-507c-88fb-a8ab520abc96", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 138, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "39fa5829-cc96-556c-b68c-6bbc69207c8e", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 149, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "219384ce-9728-5569-9aae-c4fa0ed9c4df", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 126, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "aad4a874-42cc-573a-9016-f084a79caf29", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 127, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c591071a-29fb-5e6c-94a6-848d0024c376", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 140, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f53ed9d3-4cc9-591a-a8f4-c4a724312a3f", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 351, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4efe629d-d483-576a-9fb2-df420bb4163b", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 640, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "59ab11eb-5059-5cc4-8426-ef433b6f430b", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 643, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bd88d197-3114-5f1b-84b3-647e6e64a44c", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 1283, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "4822fceb-afcd-5720-a198-486817c4da62", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 122, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "03d6fe96-59df-5513-8603-d7f4e1bad9d5", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 130, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "f7da16cc-18ef-5262-9439-e53424ae68ad", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 138, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e4b1d4c4-f26f-5724-ae00-6721caeec560", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 149, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "3f934e7b-ef05-5778-aea0-295a69b264c9", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 126, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "8f15b487-88d6-56bb-a334-20a4bca765f5", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 127, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "85261f34-d0c3-5760-b15a-46ee1ab0f3b8", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 140, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "bd78dfbe-27ad-590f-bc0c-5cf28a23b9ef", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 351, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "cd4e44db-dc96-5f5f-8d7b-fa5ae6ec8d96", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 640, + "wilson95": null + }, + { + "condition_id": "C", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ab47ad65-ea0e-5e8a-a3cb-0d851c760781", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 643, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2b93ed28-fc47-5971-827a-edca8a67c0aa", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 1289, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "c89b993e-fcf7-5fad-a398-4b3a52e11c3b", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 122, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ca574341-43c2-5d29-9633-e718b6555c1a", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 130, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "16d7b920-0f86-5f09-806b-6378577e7686", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 138, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "0b1f1a4d-e2d6-5c79-9e32-546d8bcfb372", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 149, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "38d43ad2-14e0-5322-9e83-1acda3ca2c3a", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 132, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "942fecf9-9847-58cd-a847-2710185f7f5c", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 127, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "2591d8ba-135c-504e-a6c4-7f71ba074e9d", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 140, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "ef975cc0-49f3-539c-8225-f47eebd1e37a", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 351, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "33680b16-92ce-5f28-bc8d-783a32351237", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 643, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "aaf2f822-6875-55f5-9a55-fa47529aa786", + "metric_name": "work_total", + "numerator": null, + "scope": "all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 646, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "a22649d1-1db0-5159-8515-7ad0207952a5", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "all", + "trial_id": null, + "unit": "items", + "value": 1289, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "40cc8afd-6596-5593-8c95-dbca5e6a5ab8", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F1", + "trial_id": null, + "unit": "items", + "value": 122, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "d3e9f448-c83e-5766-adfa-877cf03d76f3", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F2", + "trial_id": null, + "unit": "items", + "value": 130, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1e1db6a0-918f-5dbc-9706-75f3bf42d903", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F3", + "trial_id": null, + "unit": "items", + "value": 138, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "b63320bd-4e58-5ec9-b172-b8fa33a0e280", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F4", + "trial_id": null, + "unit": "items", + "value": 149, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "db8174db-55ab-54b5-9c8d-ff87180e9468", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F5", + "trial_id": null, + "unit": "items", + "value": 132, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e842b75f-af35-58bc-af93-9a96a0d3e8b8", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F6", + "trial_id": null, + "unit": "items", + "value": 127, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e95901a0-8a06-57ce-afe0-37d8e933388f", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F7", + "trial_id": null, + "unit": "items", + "value": 140, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "1262bf97-97c7-57ee-9b40-f7455c3ada2d", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "family=F8", + "trial_id": null, + "unit": "items", + "value": 351, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "e820825d-6c52-5b3a-a0fe-6610ad1a7b21", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=controlled", + "trial_id": null, + "unit": "items", + "value": 643, + "wilson95": null + }, + { + "condition_id": "C_PRIME", + "denominator": null, + "experiment_id": "c4c20b99-0324-58e0-937e-dba26ad5cf0b", + "extractor_version": "metrics-draft-1", + "metric_id": "961f2ede-0b5d-5089-9494-346d0180a540", + "metric_name": "work_total", + "numerator": null, + "scope": "common_prefix_all", + "stratum": "tier=real_world", + "trial_id": null, + "unit": "items", + "value": 646, + "wilson95": null + } + ], + "trial_count": 896 +} diff --git a/docs/P8_DEV_PILOT.md b/docs/P8_DEV_PILOT.md new file mode 100644 index 0000000..ce1f6be --- /dev/null +++ b/docs/P8_DEV_PILOT.md @@ -0,0 +1,406 @@ +# P8 dev pilot report + +> This report supersedes the one generated from run 2 (`p8a-dev-harness-check-2`, ec2b4fa8...). +> Run 2 was affected by a DAIL defect found by the external review of PR #31: under registry v1.1 every +> baseline item of a created security group or rule was UNSUPPORTED `invariant.unknown` and never +> evaluated (`core/verification/baseline.py` looked the definition up at version 1). The fix is +> commit a34e25a (C-216); this report is generated from run 3 (`p8b-dev-run-3`), run on that commit. +> Run 2 is kept unchanged under the experiment root. + +Generated by `python -m experiments.report` from the stored trial records; +do not edit by hand. +Descriptive only (C-214): counts, proportions with numerator and denominator, Wilson 95% +intervals. No hypothesis test. These are dev-split numbers of a draft protocol, +not findings. + +- Experiment: `p8b-dev-run-3` version 1, id `c4c20b99-0324-58e0-937e-dba26ad5cf0b` +- Extractor: `metrics-draft-1` +- Trials: 896 (included 896, excluded 0) +- Common prefix of all conditions: 273 transitions; of B and C: 273 + +## 1. Headline metrics per condition (all evaluated transitions) + +| Metric | A | B | C | C′ | +|---|---|---|---|---| +| Unsafe promoted / promoted | 105 / 266 = 0.395 [0.338, 0.455] | 0 / 159 = 0.000 [0.000, 0.024] | 0 / 159 = 0.000 [0.000, 0.024] | 0 / 157 = 0.000 [0.000, 0.024] | +| Unsafe promoted / evaluated | 105 / 280 = 0.375 [0.320, 0.433] | 0 / 273 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | +| Regressions / evaluated | 98 / 280 = 0.350 [0.297, 0.408] | 98 / 273 = 0.359 [0.304, 0.417] | 98 / 273 = 0.359 [0.304, 0.417] | 98 / 273 = 0.359 [0.304, 0.417] | +| Missed regressions / regressions | 98 / 98 = 1.000 [0.962, 1.000] | 0 / 98 = 0.000 [0.000, 0.038] | 0 / 98 = 0.000 [0.000, 0.038] | 0 / 98 = 0.000 [0.000, 0.038] | +| False rejections / safe | 0 / 161 = 0.000 [0.000, 0.023] | 2 / 161 = 0.012 [0.003, 0.044] | 2 / 161 = 0.012 [0.003, 0.044] | 4 / 161 = 0.025 [0.010, 0.062] | +| Repair success (repair) | 42 / 84 = 0.500 [0.395, 0.605] | 42 / 84 = 0.500 [0.395, 0.605] | 42 / 84 = 0.500 [0.395, 0.605] | 42 / 84 = 0.500 [0.395, 0.605] | +| Repair success (change request) | 70 / 140 = 0.500 [0.418, 0.582] | 68 / 140 = 0.486 [0.404, 0.568] | 68 / 140 = 0.486 [0.404, 0.568] | 66 / 140 = 0.471 [0.391, 0.554] | +| W total | 700 | 1694 | 1283 | 1289 | +| W mean per transition | 2.500 | 6.205 | 4.700 | 4.722 | +| W median per transition | 2.500 | 6 | 5 | 5 | + +TRUNCATED transitions per condition: + +| Condition | TRUNCATED / all transitions | +|---|---| +| A | 0 / 280 = 0.000 [0.000, 0.014] | +| B | 7 / 280 = 0.025 [0.012, 0.051] | +| C | 7 / 280 = 0.025 [0.012, 0.051] | +| C′ | 7 / 280 = 0.025 [0.012, 0.051] | + +Save(C vs B) on the common prefix of B and C: 1 − 1283 / 1694 = 0.243 + +## 2. The same on the common prefix of all conditions + +| Metric | A | B | C | C′ | +|---|---|---|---|---| +| Unsafe promoted / promoted | 98 / 259 = 0.378 [0.321, 0.439] | 0 / 159 = 0.000 [0.000, 0.024] | 0 / 159 = 0.000 [0.000, 0.024] | 0 / 157 = 0.000 [0.000, 0.024] | +| Unsafe promoted / evaluated | 98 / 273 = 0.359 [0.304, 0.417] | 0 / 273 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | +| Regressions / evaluated | 98 / 273 = 0.359 [0.304, 0.417] | 98 / 273 = 0.359 [0.304, 0.417] | 98 / 273 = 0.359 [0.304, 0.417] | 98 / 273 = 0.359 [0.304, 0.417] | +| Missed regressions / regressions | 98 / 98 = 1.000 [0.962, 1.000] | 0 / 98 = 0.000 [0.000, 0.038] | 0 / 98 = 0.000 [0.000, 0.038] | 0 / 98 = 0.000 [0.000, 0.038] | +| False rejections / safe | 0 / 161 = 0.000 [0.000, 0.023] | 2 / 161 = 0.012 [0.003, 0.044] | 2 / 161 = 0.012 [0.003, 0.044] | 4 / 161 = 0.025 [0.010, 0.062] | +| W total | 679 | 1694 | 1283 | 1289 | +| W mean per transition | 2.487 | 6.205 | 4.700 | 4.722 | +| W median per transition | 2 | 6 | 5 | 5 | + +## 3. Per family + +### Unsafe promoted / promoted + +| Family | A | B | C | C′ | +|---|---|---|---|---| +| F1 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F2 | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F3 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F4 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F5 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 13 = 0.000 [0.000, 0.228] | 0 / 13 = 0.000 [0.000, 0.228] | 0 / 11 = 0.000 [0.000, 0.259] | +| F6 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 13 = 0.000 [0.000, 0.228] | 0 / 13 = 0.000 [0.000, 0.228] | 0 / 13 = 0.000 [0.000, 0.228] | +| F7 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F8 | 21 / 84 = 0.250 [0.170, 0.352] | 0 / 63 = 0.000 [0.000, 0.057] | 0 / 63 = 0.000 [0.000, 0.057] | 0 / 63 = 0.000 [0.000, 0.057] | + +### Unsafe promoted / evaluated + +| Family | A | B | C | C′ | +|---|---|---|---|---| +| F1 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | +| F2 | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | +| F3 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | +| F4 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | +| F5 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | +| F6 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | +| F7 | 14 / 28 = 0.500 [0.326, 0.674] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | +| F8 | 21 / 84 = 0.250 [0.170, 0.352] | 0 / 77 = 0.000 [0.000, 0.048] | 0 / 77 = 0.000 [0.000, 0.048] | 0 / 77 = 0.000 [0.000, 0.048] | + +### Regressions / evaluated + +| Family | A | B | C | C′ | +|---|---|---|---|---| +| F1 | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | +| F2 | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | 0 / 28 = 0.000 [0.000, 0.121] | +| F3 | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | +| F4 | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | +| F5 | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | +| F6 | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | +| F7 | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | 14 / 28 = 0.500 [0.326, 0.674] | +| F8 | 14 / 84 = 0.167 [0.102, 0.261] | 14 / 77 = 0.182 [0.112, 0.282] | 14 / 77 = 0.182 [0.112, 0.282] | 14 / 77 = 0.182 [0.112, 0.282] | + +### Missed regressions / regressions + +| Family | A | B | C | C′ | +|---|---|---|---|---| +| F1 | 14 / 14 = 1.000 [0.785, 1.000] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F2 | 0 / 0 = n/a | 0 / 0 = n/a | 0 / 0 = n/a | 0 / 0 = n/a | +| F3 | 14 / 14 = 1.000 [0.785, 1.000] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F4 | 14 / 14 = 1.000 [0.785, 1.000] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F5 | 14 / 14 = 1.000 [0.785, 1.000] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F6 | 14 / 14 = 1.000 [0.785, 1.000] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F7 | 14 / 14 = 1.000 [0.785, 1.000] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F8 | 14 / 14 = 1.000 [0.785, 1.000] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | + +### False rejections / safe + +| Family | A | B | C | C′ | +|---|---|---|---|---| +| F1 | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F2 | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F3 | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F4 | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F5 | 0 / 14 = 0.000 [0.000, 0.215] | 1 / 14 = 0.071 [0.013, 0.315] | 1 / 14 = 0.071 [0.013, 0.315] | 3 / 14 = 0.214 [0.076, 0.476] | +| F6 | 0 / 14 = 0.000 [0.000, 0.215] | 1 / 14 = 0.071 [0.013, 0.315] | 1 / 14 = 0.071 [0.013, 0.315] | 1 / 14 = 0.071 [0.013, 0.315] | +| F7 | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | 0 / 14 = 0.000 [0.000, 0.215] | +| F8 | 0 / 63 = 0.000 [0.000, 0.057] | 0 / 63 = 0.000 [0.000, 0.057] | 0 / 63 = 0.000 [0.000, 0.057] | 0 / 63 = 0.000 [0.000, 0.057] | + +## 4. Per tier + +### Unsafe promoted / promoted + +| Tier | A | B | C | C′ | +|---|---|---|---|---| +| controlled | 53 / 133 = 0.398 [0.319, 0.483] | 0 / 80 = 0.000 [0.000, 0.046] | 0 / 80 = 0.000 [0.000, 0.046] | 0 / 79 = 0.000 [0.000, 0.046] | +| real_world | 52 / 133 = 0.391 [0.312, 0.476] | 0 / 79 = 0.000 [0.000, 0.046] | 0 / 79 = 0.000 [0.000, 0.046] | 0 / 78 = 0.000 [0.000, 0.047] | + +### Unsafe promoted / evaluated + +| Tier | A | B | C | C′ | +|---|---|---|---|---| +| controlled | 53 / 141 = 0.376 [0.300, 0.458] | 0 / 138 = 0.000 [0.000, 0.027] | 0 / 138 = 0.000 [0.000, 0.027] | 0 / 138 = 0.000 [0.000, 0.027] | +| real_world | 52 / 139 = 0.374 [0.298, 0.457] | 0 / 135 = 0.000 [0.000, 0.028] | 0 / 135 = 0.000 [0.000, 0.028] | 0 / 135 = 0.000 [0.000, 0.028] | + +### Regressions / evaluated + +| Tier | A | B | C | C′ | +|---|---|---|---|---| +| controlled | 50 / 141 = 0.355 [0.280, 0.436] | 50 / 138 = 0.362 [0.287, 0.445] | 50 / 138 = 0.362 [0.287, 0.445] | 50 / 138 = 0.362 [0.287, 0.445] | +| real_world | 48 / 139 = 0.345 [0.271, 0.428] | 48 / 135 = 0.356 [0.280, 0.439] | 48 / 135 = 0.356 [0.280, 0.439] | 48 / 135 = 0.356 [0.280, 0.439] | + +### Missed regressions / regressions + +| Tier | A | B | C | C′ | +|---|---|---|---|---| +| controlled | 50 / 50 = 1.000 [0.929, 1.000] | 0 / 50 = 0.000 [0.000, 0.071] | 0 / 50 = 0.000 [0.000, 0.071] | 0 / 50 = 0.000 [0.000, 0.071] | +| real_world | 48 / 48 = 1.000 [0.926, 1.000] | 0 / 48 = 0.000 [0.000, 0.074] | 0 / 48 = 0.000 [0.000, 0.074] | 0 / 48 = 0.000 [0.000, 0.074] | + +### False rejections / safe + +| Tier | A | B | C | C′ | +|---|---|---|---|---| +| controlled | 0 / 80 = 0.000 [0.000, 0.046] | 0 / 80 = 0.000 [0.000, 0.046] | 0 / 80 = 0.000 [0.000, 0.046] | 1 / 80 = 0.013 [0.002, 0.067] | +| real_world | 0 / 81 = 0.000 [0.000, 0.045] | 2 / 81 = 0.025 [0.007, 0.086] | 2 / 81 = 0.025 [0.007, 0.086] | 3 / 81 = 0.037 [0.013, 0.103] | + +## 5. Outcome rates (Doc 13 §28) + +| Rate | A | B | C | C′ | +|---|---|---|---|---| +| PROMOTE / evaluated | 266 / 280 = 0.950 [0.918, 0.970] | 159 / 273 = 0.582 [0.523, 0.639] | 159 / 273 = 0.582 [0.523, 0.639] | 157 / 273 = 0.575 [0.516, 0.632] | +| REJECT / evaluated | 14 / 280 = 0.050 [0.030, 0.082] | 114 / 273 = 0.418 [0.361, 0.477] | 114 / 273 = 0.418 [0.361, 0.477] | 116 / 273 = 0.425 [0.368, 0.484] | +| RETRY / evaluated | 0 / 280 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | +| ESCALATE / evaluated | 0 / 280 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | 0 / 273 = 0.000 [0.000, 0.014] | +| UNKNOWN / required checks | 0 / 700 = 0.000 [0.000, 0.005] | 0 / 1704 = 0.000 [0.000, 0.002] | 0 / 1704 = 0.000 [0.000, 0.002] | 0 / 1716 = 0.000 [0.000, 0.002] | +| UNSUPPORTED / required checks | 0 / 700 = 0.000 [0.000, 0.005] | 3 / 1704 = 0.002 [0.001, 0.005] | 3 / 1704 = 0.002 [0.001, 0.005] | 3 / 1716 = 0.002 [0.001, 0.005] | + +Non-PASS required checks (fresh results), by condition, kind, verifier, result and reason: + +| Condition | Kind | Verifier | Result | Reason | Count | +|---|---|---|---|---|---| +| A | INVARIANT:INV-SEC-001 | dail.verify.public_ingress | FAIL | predicate.violated | 60 | +| A | INVARIANT:INV-SEC-002 | dail.verify.public_ingress | FAIL | predicate.violated | 5 | +| A | INVARIANT:INV-SEC-003 | dail.verify.rds_public_flag | FAIL | predicate.violated | 5 | +| B | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | FAIL | path.missing | 58 | +| B | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | FAIL | port.mismatch | 3 | +| B | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | UNSUPPORTED | route.unsupported | 1 | +| B | INVARIANT:INV-SEC-001 | dail.verify.public_ingress | FAIL | predicate.violated | 74 | +| B | INVARIANT:INV-SEC-001 | dail.verify.public_ingress | UNSUPPORTED | rule.unsupported | 1 | +| B | INVARIANT:INV-SEC-002 | dail.verify.public_ingress | FAIL | predicate.violated | 19 | +| B | INVARIANT:INV-SEC-002 | dail.verify.public_ingress | UNSUPPORTED | rule.unsupported | 1 | +| B | INVARIANT:INV-SEC-003 | dail.verify.rds_public_flag | FAIL | predicate.violated | 14 | +| B | RESOURCE:aws_db_instance | dail.verify.rds_public_flag | FAIL | predicate.violated | 5 | +| B | RESOURCE:aws_security_group | dail.verify.public_ingress | FAIL | predicate.violated | 9 | +| B | RESOURCE:aws_security_group_rule | dail.verify.public_ingress | FAIL | predicate.violated | 13 | +| C | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | FAIL | path.missing | 58 | +| C | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | FAIL | port.mismatch | 3 | +| C | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | UNSUPPORTED | route.unsupported | 1 | +| C | INVARIANT:INV-SEC-001 | dail.verify.public_ingress | FAIL | predicate.violated | 74 | +| C | INVARIANT:INV-SEC-001 | dail.verify.public_ingress | UNSUPPORTED | rule.unsupported | 1 | +| C | INVARIANT:INV-SEC-002 | dail.verify.public_ingress | FAIL | predicate.violated | 19 | +| C | INVARIANT:INV-SEC-002 | dail.verify.public_ingress | UNSUPPORTED | rule.unsupported | 1 | +| C | INVARIANT:INV-SEC-003 | dail.verify.rds_public_flag | FAIL | predicate.violated | 14 | +| C | RESOURCE:aws_db_instance | dail.verify.rds_public_flag | FAIL | predicate.violated | 5 | +| C | RESOURCE:aws_security_group | dail.verify.public_ingress | FAIL | predicate.violated | 9 | +| C | RESOURCE:aws_security_group_rule | dail.verify.public_ingress | FAIL | predicate.violated | 13 | +| C′ | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | FAIL | endpoint.absent | 2 | +| C′ | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | FAIL | path.missing | 58 | +| C′ | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | FAIL | port.mismatch | 3 | +| C′ | INVARIANT:INV-FUNC-001 | dail.verify.reachability_cfg | UNSUPPORTED | route.unsupported | 1 | +| C′ | INVARIANT:INV-SEC-001 | dail.verify.public_ingress | FAIL | predicate.violated | 74 | +| C′ | INVARIANT:INV-SEC-001 | dail.verify.public_ingress | UNSUPPORTED | rule.unsupported | 1 | +| C′ | INVARIANT:INV-SEC-002 | dail.verify.public_ingress | FAIL | predicate.violated | 19 | +| C′ | INVARIANT:INV-SEC-002 | dail.verify.public_ingress | UNSUPPORTED | rule.unsupported | 1 | +| C′ | INVARIANT:INV-SEC-003 | dail.verify.rds_public_flag | FAIL | predicate.violated | 14 | +| C′ | RESOURCE:aws_db_instance | dail.verify.rds_public_flag | FAIL | predicate.violated | 5 | +| C′ | RESOURCE:aws_security_group | dail.verify.public_ingress | FAIL | predicate.violated | 9 | +| C′ | RESOURCE:aws_security_group_rule | dail.verify.public_ingress | FAIL | predicate.violated | 13 | + +## 6. Work by kind (oracle excluded) + +| Kind | A | B | C | C′ | +|---|---|---|---|---| +| gate items | 560 | 546 | 546 | 546 | +| invariant items from impact | 88 | 627 | 627 | 627 | +| invariant items from the condition | 52 | 465 | 0 | 0 | +| U_t items (not PROTECTED in parent) | 0 | 0 | 54 | 54 | +| applicable baseline items | 0 | 56 | 56 | 62 | +| not-applicable baseline items (outside W) | 0 | 10 | 10 | 16 | +| W total | 700 | 1694 | 1283 | 1289 | + +## 7. DAIL–oracle agreement under B (C-155): S0 and every evaluated step + +### INV-FUNC-001 + +| DAIL | oracle truth | count | +|---|---|---| +| FAIL | FAIL | 61 | +| PASS | PASS | 434 | +| UNSUPPORTED | PASS | 2 | + +### INV-SEC-001 + +| DAIL | oracle truth | count | +|---|---|---| +| FAIL | FAIL | 119 | +| PASS | PASS | 376 | +| UNSUPPORTED | PASS | 2 | + +### INV-SEC-002 + +| DAIL | oracle truth | count | +|---|---|---| +| FAIL | FAIL | 37 | +| PASS | PASS | 458 | +| UNSUPPORTED | PASS | 2 | + +### INV-SEC-003 + +| DAIL | oracle truth | count | +|---|---|---| +| FAIL | FAIL | 35 | +| PASS | PASS | 462 | + +Disagreements: 6 (INV-FUNC-001: 2, INV-SEC-001: 2, INV-SEC-002: 2) + +| Scenario | Step | Invariant | DAIL | Oracle truth | +|---|---|---|---|---| +| tp-v1-F5-0319 | 0 | INV-FUNC-001 | UNSUPPORTED | PASS | +| tp-v1-F5-0319 | 1 | INV-FUNC-001 | UNSUPPORTED | PASS | +| tp-v1-F6-0350 | 0 | INV-SEC-001 | UNSUPPORTED | PASS | +| tp-v1-F6-0350 | 0 | INV-SEC-002 | UNSUPPORTED | PASS | +| tp-v1-F6-0350 | 1 | INV-SEC-001 | UNSUPPORTED | PASS | +| tp-v1-F6-0350 | 1 | INV-SEC-002 | UNSUPPORTED | PASS | + +## 8. Impact against the oracle's affected set (C and C′) + +### Precision + +| Stratum | C | C′ | +|---|---|---| +| all | 462 / 627 = 0.737 [0.701, 0.770] | 462 / 627 = 0.737 [0.701, 0.770] | +| family=F1 | 34 / 61 = 0.557 [0.433, 0.675] | 34 / 61 = 0.557 [0.433, 0.675] | +| family=F2 | 57 / 74 = 0.770 [0.663, 0.851] | 57 / 74 = 0.770 [0.663, 0.851] | +| family=F3 | 60 / 72 = 0.833 [0.731, 0.902] | 60 / 72 = 0.833 [0.731, 0.902] | +| family=F4 | 76 / 85 = 0.894 [0.811, 0.943] | 76 / 85 = 0.894 [0.811, 0.943] | +| family=F5 | 48 / 66 = 0.727 [0.610, 0.820] | 48 / 66 = 0.727 [0.610, 0.820] | +| family=F6 | 44 / 46 = 0.957 [0.855, 0.988] | 44 / 46 = 0.957 [0.855, 0.988] | +| family=F7 | 71 / 78 = 0.910 [0.826, 0.956] | 71 / 78 = 0.910 [0.826, 0.956] | +| family=F8 | 72 / 145 = 0.497 [0.416, 0.577] | 72 / 145 = 0.497 [0.416, 0.577] | + +### Recall + +| Stratum | C | C′ | +|---|---|---| +| all | 462 / 462 = 1.000 [0.992, 1.000] | 462 / 462 = 1.000 [0.992, 1.000] | +| family=F1 | 34 / 34 = 1.000 [0.898, 1.000] | 34 / 34 = 1.000 [0.898, 1.000] | +| family=F2 | 57 / 57 = 1.000 [0.937, 1.000] | 57 / 57 = 1.000 [0.937, 1.000] | +| family=F3 | 60 / 60 = 1.000 [0.940, 1.000] | 60 / 60 = 1.000 [0.940, 1.000] | +| family=F4 | 76 / 76 = 1.000 [0.952, 1.000] | 76 / 76 = 1.000 [0.952, 1.000] | +| family=F5 | 48 / 48 = 1.000 [0.926, 1.000] | 48 / 48 = 1.000 [0.926, 1.000] | +| family=F6 | 44 / 44 = 1.000 [0.920, 1.000] | 44 / 44 = 1.000 [0.920, 1.000] | +| family=F7 | 71 / 71 = 1.000 [0.949, 1.000] | 71 / 71 = 1.000 [0.949, 1.000] | +| family=F8 | 72 / 72 = 1.000 [0.949, 1.000] | 72 / 72 = 1.000 [0.949, 1.000] | + +### Truth-change coverage + +| Stratum | C | C′ | +|---|---|---| +| all | 161 / 161 = 1.000 [0.977, 1.000] | 161 / 161 = 1.000 [0.977, 1.000] | +| family=F1 | 14 / 14 = 1.000 [0.785, 1.000] | 14 / 14 = 1.000 [0.785, 1.000] | +| family=F2 | 14 / 14 = 1.000 [0.785, 1.000] | 14 / 14 = 1.000 [0.785, 1.000] | +| family=F3 | 14 / 14 = 1.000 [0.785, 1.000] | 14 / 14 = 1.000 [0.785, 1.000] | +| family=F4 | 42 / 42 = 1.000 [0.916, 1.000] | 42 / 42 = 1.000 [0.916, 1.000] | +| family=F5 | 14 / 14 = 1.000 [0.785, 1.000] | 14 / 14 = 1.000 [0.785, 1.000] | +| family=F6 | 14 / 14 = 1.000 [0.785, 1.000] | 14 / 14 = 1.000 [0.785, 1.000] | +| family=F7 | 14 / 14 = 1.000 [0.785, 1.000] | 14 / 14 = 1.000 [0.785, 1.000] | +| family=F8 | 35 / 35 = 1.000 [0.901, 1.000] | 35 / 35 = 1.000 [0.901, 1.000] | + +Truth changes outside the verified set: 0 + +## 9. Identity (H3: C and C′) + +| Metric | C | C′ | +|---|---|---| +| Identity errors / true pairs | 3 / 80 = 0.037 [0.013, 0.105] | 23 / 80 = 0.287 [0.200, 0.395] | +| UNCERTAIN / true pairs | 0 / 80 = 0.000 [0.000, 0.046] | 0 / 80 = 0.000 [0.000, 0.046] | +| Evidence-transfer errors | 3 | 13 | +| Excluded pairs | 40 | 40 | + +Identity errors: 26 + +| Condition | Scenario | Step | Operator | Effect | Old | New | DAIL pairs | +|---|---|---|---|---|---|---|---| +| C | tp-v1-F5-0107 | 1 | op_f5_force_replacement_safe | REPLACE | aws_instance.acme_app | aws_instance.acme_app | aws_instance.acme_app -> aws_instance.acme_app MODIFIED | +| C | tp-v1-F5-0114 | 1 | op_f5_force_replacement_safe | REPLACE | aws_instance.prod_app_server | aws_instance.prod_app_server | aws_instance.prod_app_server -> aws_instance.prod_app_server MODIFIED | +| C | tp-v1-F5-0315 | 1 | op_f5_force_replacement_safe | REPLACE | aws_instance.tp_app | aws_instance.tp_app | aws_instance.tp_app -> aws_instance.tp_app MODIFIED | +| C′ | tp-v1-F5-0106 | 1 | op_f5_restructure_moved_safe | MOVE | aws_subnet.app | aws_subnet.app_app | aws_subnet.app -> None DELETED; None -> aws_subnet.app_app CREATED | +| C′ | tp-v1-F5-0107 | 1 | op_f5_force_replacement_safe | REPLACE | aws_instance.acme_app | aws_instance.acme_app | aws_instance.acme_app -> aws_instance.acme_app MODIFIED | +| C′ | tp-v1-F5-0112 | 1 | op_f5_restructure_moved_safe | MOVE | aws_subnet.acme_app_subnet | aws_subnet.acme_app_subnet_app | aws_subnet.acme_app_subnet -> None DELETED; None -> aws_subnet.acme_app_subnet_app CREATED | +| C′ | tp-v1-F5-0113 | 1 | op_f5_db_rule_moved_safe | MOVE | aws_security_group_rule.storage_access_from_app_5432 | aws_security_group_rule.storage_access_from_app_5432_v2 | aws_security_group_rule.storage_access_from_app_5432 -> None DELETED; None -> aws_security_group_rule.storage_access_from_app_5432_v2 CREATED | +| C′ | tp-v1-F5-0114 | 1 | op_f5_force_replacement_safe | REPLACE | aws_instance.prod_app_server | aws_instance.prod_app_server | aws_instance.prod_app_server -> aws_instance.prod_app_server MODIFIED | +| C′ | tp-v1-F5-0118 | 1 | op_f5_db_sg_replacement_unsafe | REPLACE | aws_security_group.sg_db | aws_security_group.sg_db | aws_security_group.sg_db -> aws_security_group.sg_db MODIFIED | +| C′ | tp-v1-F5-0119 | 1 | op_f5_db_sg_replacement_unsafe | REPLACE | aws_security_group.prod_db_sg | aws_security_group.prod_db_sg | aws_security_group.prod_db_sg -> aws_security_group.prod_db_sg MODIFIED | +| C′ | tp-v1-F5-0120 | 1 | op_f5_db_sg_replacement_unsafe | REPLACE | aws_security_group.backend_sg | aws_security_group.backend_sg | aws_security_group.backend_sg -> aws_security_group.backend_sg MODIFIED | +| C′ | tp-v1-F5-0123 | 1 | op_f5_db_sg_replacement_unsafe | REPLACE | aws_security_group.storage_access | aws_security_group.storage_access | aws_security_group.storage_access -> aws_security_group.storage_access MODIFIED | +| C′ | tp-v1-F5-0125 | 1 | op_f5_db_rule_moved_unsafe | MOVE | aws_security_group_rule.pg_from_app_5432 | aws_security_group_rule.pg_from_app_5432_v2 | aws_security_group_rule.pg_from_app_5432 -> None DELETED; None -> aws_security_group_rule.pg_from_app_5432_v2 CREATED | +| C′ | tp-v1-F5-0126 | 1 | op_f5_db_sg_replacement_unsafe | REPLACE | aws_security_group.storage_access | aws_security_group.storage_access | aws_security_group.storage_access -> aws_security_group.storage_access MODIFIED | +| C′ | tp-v1-F5-0127 | 1 | op_f5_db_sg_replacement_unsafe | REPLACE | aws_security_group.sg_db | aws_security_group.sg_db | aws_security_group.sg_db -> aws_security_group.sg_db MODIFIED | +| C′ | tp-v1-F5-0128 | 1 | op_f5_db_rule_moved_unsafe | MOVE | aws_security_group_rule.data_secgroup_from_app_5432 | aws_security_group_rule.data_secgroup_from_app_5432_v2 | aws_security_group_rule.data_secgroup_from_app_5432 -> None DELETED; None -> aws_security_group_rule.data_secgroup_from_app_5432_v2 CREATED | +| C′ | tp-v1-F5-0313 | 1 | op_f5_restructure_moved_safe | MOVE | aws_subnet.tp_app_subnet | aws_subnet.tp_app_subnet_app | aws_subnet.tp_app_subnet -> None DELETED; None -> aws_subnet.tp_app_subnet_app CREATED | +| C′ | tp-v1-F5-0314 | 1 | op_f5_db_rule_moved_safe | MOVE | aws_security_group_rule.tp_db_ingress_postgres | aws_security_group_rule.tp_db_ingress_postgres_v2 | aws_security_group_rule.tp_db_ingress_postgres -> None DELETED; None -> aws_security_group_rule.tp_db_ingress_postgres_v2 CREATED | +| C′ | tp-v1-F5-0315 | 1 | op_f5_force_replacement_safe | REPLACE | aws_instance.tp_app | aws_instance.tp_app | aws_instance.tp_app -> aws_instance.tp_app MODIFIED | +| C′ | tp-v1-F5-0316 | 1 | op_f5_db_sg_replacement_safe | REPLACE | aws_security_group.tp_db_sg | aws_security_group.tp_db_sg | aws_security_group.tp_db_sg -> aws_security_group.tp_db_sg MODIFIED | +| C′ | tp-v1-F5-0319 | 1 | op_f5_restructure_moved_safe | MOVE | aws_subnet.tp_app_subnet | aws_subnet.tp_app_subnet_app | aws_subnet.tp_app_subnet -> None DELETED; None -> aws_subnet.tp_app_subnet_app CREATED | +| C′ | tp-v1-F5-0326 | 1 | op_f5_db_sg_replacement_unsafe | REPLACE | aws_security_group.tp_db_sg | aws_security_group.tp_db_sg | aws_security_group.tp_db_sg -> aws_security_group.tp_db_sg MODIFIED | +| C′ | tp-v1-F5-0328 | 1 | op_f5_db_rule_moved_unsafe | MOVE | aws_security_group_rule.tp_db_ingress_postgres | aws_security_group_rule.tp_db_ingress_postgres_v2 | aws_security_group_rule.tp_db_ingress_postgres -> None DELETED; None -> aws_security_group_rule.tp_db_ingress_postgres_v2 CREATED | +| C′ | tp-v1-F5-0329 | 1 | op_f5_db_sg_replacement_unsafe | REPLACE | aws_security_group.tp_db_sg | aws_security_group.tp_db_sg | aws_security_group.tp_db_sg -> aws_security_group.tp_db_sg MODIFIED | +| C′ | tp-v1-F5-0331 | 1 | op_f5_db_rule_moved_unsafe | MOVE | aws_security_group_rule.tp_db_ingress_postgres | aws_security_group_rule.tp_db_ingress_postgres_v2 | aws_security_group_rule.tp_db_ingress_postgres -> None DELETED; None -> aws_security_group_rule.tp_db_ingress_postgres_v2 CREATED | +| C′ | tp-v1-F5-0332 | 1 | op_f5_db_sg_replacement_unsafe | REPLACE | aws_security_group.tp_db_sg | aws_security_group.tp_db_sg | aws_security_group.tp_db_sg -> aws_security_group.tp_db_sg MODIFIED | + +Excluded identity pairs by operator: 80 + +| Condition | Operator | Pairs | +|---|---|---| +| C | op_f1_storage_expand_unsafe | 10 | +| C | op_f1_vpc_dns_toggle_unsafe | 10 | +| C | op_f4_repair_db_port_open_ssh_unsafe | 8 | +| C | op_f4_repair_rds_public_open_db_unsafe | 10 | +| C | op_f5_count_each_safe | 2 | +| C′ | op_f1_storage_expand_unsafe | 10 | +| C′ | op_f1_vpc_dns_toggle_unsafe | 10 | +| C′ | op_f4_repair_db_port_open_ssh_unsafe | 8 | +| C′ | op_f4_repair_rds_public_open_db_unsafe | 10 | +| C′ | op_f5_count_each_safe | 2 | + +Work difference C′ − C on F5 by moved resource type: + +| Moved type | Transitions | W under C | W under C′ | C′ − C | +|---|---|---|---|---| +| aws_security_group_rule | 6 | 34 | 40 | 6 | +| aws_subnet | 4 | 12 | 12 | 0 | + +## 10. Unsafe promotions under C (Doc 13 §37, §49) + +none + +## 11. Clusters (C-214) + +| Condition | Tier | Clusters | Evaluated | Promoted | Unsafe promoted | Safe | Safe not promoted | +|---|---|---|---|---|---|---|---| +| A | controlled | 23 | 141 | 133 | 53 | 80 | 0 | +| A | real_world | 109 | 139 | 133 | 52 | 81 | 0 | +| B | controlled | 23 | 138 | 80 | 0 | 80 | 0 | +| B | real_world | 109 | 135 | 79 | 0 | 81 | 2 | +| C | controlled | 23 | 138 | 80 | 0 | 80 | 0 | +| C | real_world | 109 | 135 | 79 | 0 | 81 | 2 | +| C′ | controlled | 23 | 138 | 79 | 0 | 80 | 1 | +| C′ | real_world | 109 | 135 | 78 | 0 | 81 | 3 | + +## 12. Failed and excluded trials + +none: every trial is COMPLETED and included + +## 13. Reproducibility (Doc 13 §30) + +- extractor: metrics-draft-1; a second run of `extract` must write byte-identical records (it fails otherwise) +- rerun_20_trials: p8b-repro-check-3 (id 6764aa91-5745-548a-80a8-b46ed52b030a): 20 scenarios (one per family x class by lowest scenario id, plus 4 F8 with most steps) x 4 conditions = 80 trials, 136 transitions in common with this experiment; decision or status differences 0; invariant result differences 0 (python -m experiments.report compare) +- rerun_20_trials_scenarios: tp-v1-F1-0001, tp-v1-F1-0014, tp-v1-F2-0027, tp-v1-F2-0040, tp-v1-F3-0053, tp-v1-F3-0066, tp-v1-F4-0079, tp-v1-F4-0092, tp-v1-F5-0105, tp-v1-F5-0118, tp-v1-F6-0131, tp-v1-F6-0144, tp-v1-F7-0157, tp-v1-F7-0170, tp-v1-F8-0183, tp-v1-F8-0185, tp-v1-F8-0188, tp-v1-F8-0191, tp-v1-F8-0196, tp-v1-F8-0204 diff --git a/docs/PHASE_GATES.md b/docs/PHASE_GATES.md index 12a471d..44bfcd7 100644 --- a/docs/PHASE_GATES.md +++ b/docs/PHASE_GATES.md @@ -462,7 +462,35 @@ M4 "No model output can bypass promotion" ..... test_llm_candidate_e2e.TestSecur No item is marked passed here; the external review decides. ``` -## P8a — The experiment harness (pending external review) +## P8a — The experiment harness — passed with limits (verdict on PR #30) + +```text +P8a: passed with limits (verdict on PR #30). Verdict: APPROVED WITH LIMITS (external review, +12 Nov 2026 at d75b55e and 13 Nov 2026 at 68772e9; merged as 2db255b, whose message is the C-28 +approval record). The P8 gate and milestone M5 wait for P8b. +Limits carried forward (not blocking), from the verdict, with the package that closes each: +1. C-200 to C-204 and C-207 are Proposed. The authors must decide them. [Closed: Decided by the + authors on 13 Nov 2026, "accepted as built after external review", recorded in P8b.] +2. parent_refs defaults to empty. A caller that does not pass it gets impact's work set only. That + fails closed (INSUFFICIENT_EVIDENCE, never a wrong PROMOTE), and the orchestrator passes it (the + author's mutant U5 is caught). Open (stated limit). +3. Condition A is narrow by design (C-205, C-206). A's lineage tracks only the repair target, and a + change-request scenario under A starts from a baseline with no invariant. The paper must define + A exactly this way. Open (paper). +4. Automatic promotion is enabled only for the harness (C-207 (3)), through the reviewed file + experiments/config/promotion.toml, whose hash every definition pins. It must never be used + outside experiments/. Open (standing rule). +5. U_t items count as DAIL work (C-210). The paper's W must say so. Open (paper). +6. The dev-run counts are harness counts, not results. For example, C' rejects 4 more F5 + transitions than C. That gets explained only by P8b's scoring. Open (P8b). +7. For P8b, all from immutable records and never by hand: score every transition against the + labels; report DAIL-oracle disagreements at S0 and per step (C-155); impact precision and recall + against oracle_affected; H3 identity errors against mutation_record; the common-prefix analysis + for the 7 truncations per condition; the Doc 13 §49 check "no unsafe promotion" under C. On dev, + a failure is a defect to investigate, not to tune around. Open (P8b, this package). +``` + +The P8a author's input to the review, as written before the verdict: ```text P8a mini-gate: NOT DECIDED. Status written by the P8a author as an input to the external review. @@ -527,7 +555,87 @@ Limits: the open-target rule blocked C and C' on F8 (C-209, closed by C-210 in P Carry-forward: C-200 to C-204 and C-207 Proposed; C-209 closed by C-210; P8b scores the stored records. ``` -## P8b–P10 — NOT STARTED +## P8b — Scoring, metrics and the dev report (pending external review) + +```text +P8b mini-gate: NOT DECIDED. Status written by the P8b author as an input to the external review. +Fix round P8b-2 (13 Nov 2026, after the external review of PR #31 at 86a1e5a): baseline items under + registry v1.1 were never evaluated (core/verification/baseline.py looked the definition up at + version 1; C-216, fixed in a34e25a). Run 2 is superseded by run 3, p8b-dev-run-3 + (c4c20b99...), run on a34e25a: 896 trials, all COMPLETED, 339 s. docs/P8_DEV_PILOT.md is now + generated from run 3 and says so at the top. Against run 2: REJECT -> PROMOTE for 7 transitions + under B, 7 under C, 9 under C' (no other decision changed); every security baseline item is + evaluated (B and C: groups 23 PASS, 9 FAIL; rules 1 PASS, 13 FAIL; C': rules 7 PASS, 13 FAIL; + none invariant.unknown). Run 3 headline: unsafe promoted / promoted A 105/266, B 0/159, + C 0/159, C' 0/157; missed regressions / regressions B, C, C' 0/98; false rejections / safe + A 0/161, B 2/161, C 2/161, C' 4/161; W A 700, B 1694, C 1283, C' 1289; TRUNCATED A 0/280, + B, C, C' 7/280. Report guard: a table of every non-PASS required check, and a test that fails if + a baseline item is invariant.unknown while the definition is held (it flags 144 items in run 2, + none in run 3). Reproducibility against run 3: p8b-repro-check-3 (6764aa91...), 80 trials, + 136 common transitions, 0 differences. An extractor reading found in the fix round (C-217 (1)): + unproven required checks now count in the UNKNOWN/UNSUPPORTED rates. + The numbers below this paragraph are those of the first P8b report (run 2), kept as written. +What exists: + experiments/scoring (the only label reader: labels with the manifest and scenario hash checks, + per-transition truth, the identity truth table identity-truth-1 written from D4_OPERATORS.md, + the reproducibility selection); experiments/metrics (the extractor metrics-draft-1: MetricRecords + of Appendix B with scope, stratum and Wilson 95%; TrialOutcome of §31); experiments/report (the + ExperimentReport of §33 and its Markdown); python -m experiments.report extract|select|compare. + Blinding contract: (a) no module outside experiments/scoring names a label field; (b) no execution + module imports experiments.scoring, .metrics or .report (both shown failing on a mutant). +Dev report (13 Nov 2026): docs/P8_DEV_PILOT.md, generated from p8a-dev-harness-check-2 + (ec2b4fa8...), 896 trials, 1120 transitions, 1835 metric records; metric_records.json sha256 + 9f4d3d65..., report.json sha256 6fd62e2d...; a second extract wrote byte-identical files. + Headline (all evaluated transitions; numerator / denominator): unsafe promoted / promoted + A 105/266, B 0/152, C 0/152, C' 0/148; missed regressions / regressions A 98/98, B 0/98, + C 0/98, C' 0/98; false rejections / safe A 0/161, B 9/161, C 9/161, C' 13/161; W total A 700, + B 1694, C 1283, C' 1289; Save(C vs B) on the common prefix (273) = 1 - 1283/1694; TRUNCATED + A 0/280, B, C, C' 7/280. Unsafe promotions under C: none. Truth changes outside the verified + set: 0. DAIL-oracle disagreements under B: 6 (all DAIL UNSUPPORTED, oracle PASS; 2 scenarios). + Identity errors / true pairs: C 3/80, C' 23/80 (40 pairs excluded per condition). + These are dev numbers of a draft protocol, not findings (C-214). +Reproducibility: p8b-repro-check (308471ba...), 20 scenarios x 4 conditions = 80 trials on the + same code; 136 transitions in common with run 2, 0 decision and 0 invariant-result differences. +Tests: tests.unit.test_p8b_metrics (hand-computed record set), test_p8b_scoring_report, + tests.contract.test_experiment_blinding. Mutation checks: 12 of 12 metric mutants caught (3 only + after tests were added for the cases the record set lacked), 2 of 2 blinding mutants. +Limits: metrics-draft-1 is a draft (P10 freezes the definitions before the test split); the + identity table covers only what the catalogue states (5 operators excluded); evidence-transfer + error is the author's reading (C-215 (1)); descriptive statistics only. +Carry-forward: C-212 to C-215 Proposed. +``` + +## P8 — Experiment Harness (pending external review) + +```text +P8 gate (Doc 15 §15.3) and milestone M5 (Doc 15 §25): NOT DECIDED. Each item and what bears on it: +§15.3 "Trials are reproducible" ........ tests.unit.test_experiment_harness.TestReproducibility + (records identical across runs, concurrency, corrupted labels); the 20-trial re-run + (P8_DEV_PILOT.md section 13); the reviewer's independent re-run of both dev runs (PR #30). +§15.3 "All compared conditions share compatible fixtures" .. the one fixture_set of every + definition; test_experiment_harness.TestConditions.test_baselines_agree_across_conditions; + the common prefixes of the report (sections 1-2). +§15.3 "Metrics have frozen definitions" .. metrics-draft-1 is versioned and every record names it + (C-212; tests.unit.test_p8b_metrics.TestRecords); it is a draft for the dev pilot, and P10 + freezes the final definitions before the test split is opened. +§15.3 "Failed/excluded trials remain visible" .. experiments.report.build failures and + excluded_count; P8_DEV_PILOT.md section 12; test_p8b_scoring_report.TestReport; + test_experiment_harness.TestFailures. +§15.3 "Reports derive from persisted evidence" .. python -m experiments.report extract reads only + the stored records; test_p8b_scoring_report.TestDevPilotDocument; the byte-identical second + extract. +M5 "Experiments run reproducibly" ...... as §15.3 "Trials are reproducible". +M5 "Baselines work" .................... conditions A and B (tests.unit.test_p8a_conditions, + test_experiment_harness.TestConditions; P8_DEV_PILOT.md section 1). +M5 "Metrics generated" ................. metric_records.json (1835 records) of run 2. +M5 "Trial artifacts linked" ............ correlation ids and artifact_refs + (test_experiment_harness.TestConditions.test_a_transition_records_what_section_4_3_lists, + report artifact_refs). +M5 "Comparison report generated" ....... docs/P8_DEV_PILOT.md. +No item is marked passed here; the external review decides. +``` + +## P10 — NOT STARTED P9 (AWS integration) and milestone M6 (AWS Demonstration) are removed (C-71, Decided by the advisor, 11 Nov 2026): AWS stays fully offline, with no deployment, no AWS account and no calibration against diff --git a/experiments/metrics/__init__.py b/experiments/metrics/__init__.py new file mode 100644 index 0000000..ba5ec44 --- /dev/null +++ b/experiments/metrics/__init__.py @@ -0,0 +1,7 @@ +"""Metric extraction (P8b; Doc 13 §25-§35, §32, Appendix B; metrics-draft-1, C-212). + +"trial evidence -> metric extractor (versioned) -> metric record -> experiment aggregate" (§32): +``model`` reads the stored trial records into transitions; ``extractor`` turns transitions and the +truth that ``experiments.scoring`` derives into ``MetricRecord``s and the lists of Section 4. +Nothing here reads a label field (``tests/contract/test_experiment_blinding.py``, check (a)). +""" diff --git a/experiments/metrics/extractor.py b/experiments/metrics/extractor.py new file mode 100644 index 0000000..fb04153 --- /dev/null +++ b/experiments/metrics/extractor.py @@ -0,0 +1,691 @@ +"""The metric extractor, version ``metrics-draft-1`` (P8b work package Section 4; Doc 13 §25-§35, +Appendix B; C-212 to C-214). + +Input: the transitions of the stored trial records (``experiments.metrics.model``), the scenarios' +truth from ``experiments.scoring``, DAIL's S0 results under B, and the identity truth table. Output: +``MetricRecord``s (Appendix B, plus ``scope``, ``stratum`` and a Wilson 95% interval for +proportions) and the lists Section 4 asks for. Every proportion carries its numerator and +denominator; a denominator of 0 gives ``value = None``. Descriptive only (C-214): no test, no +p-value. + +Populations. ``scope = all``: every evaluated transition of the condition (TRUNCATED ones are +counted apart). ``common_prefix_all``: the (scenario, step) pairs evaluated under every condition +present. ``common_prefix_B_C``: those evaluated under both B and C (for ``work_saving_c_vs_b``). +Strata: ``all``, ``family=``, ``tier=``, and for ``repair_success`` +``kind=``. ``repair_success`` is per scenario and is reported on ``all`` +only. + +Definitions (Section 4): unsafe promotions per promotion and per evaluated transition; regressions +per evaluated transition and missed regressions per regression; false rejections per safe +transition; repair success per scenario; work by kind (gate; invariant items from impact, from the +condition's scope policy, and U_t items; applicable and not-applicable baseline items) with +W = gate + invariant + applicable baseline, its total, per-transition mean and median, and +Save(C vs B) = 1 - sum W_C / sum W_B on the common prefix of B and C; decision rates per evaluated +transition; UNKNOWN and UNSUPPORTED per required check (a carried-forward check has no result and is +neither; a check that is not proven has source None in DAIL's records and its fresh result +counts); TRUNCATED transitions per transition; the oracle agreement of B (confusion per invariant +at S0 and every evaluated step, every disagreement listed); impact precision, recall and +truth-change coverage for C and C' (pooled and per family; a truth change outside the verified set +listed); identity error rate, UNCERTAIN share and evidence-transfer errors for C and C' over the +pairs the identity table gives (excluded pairs counted), and the work difference C' - C on F5 by +moved resource type; every promoted unsafe transition under C listed with its trace. + +An evidence-transfer error (a reading for the authors, C-215): a true replacement that DAIL treats +as the same object (not REPLACED, not UNCERTAIN) in a transition where some required invariant was +CARRIED_FORWARD. +""" + +from __future__ import annotations + +import json +import math +import statistics +import uuid +from collections import Counter, defaultdict +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Final + +from experiments.metrics.model import Transition, invariant_of +from experiments.scoring.identity import IdentityTable, judge, true_pairs +from experiments.scoring.labels import ScenarioLabels +from experiments.scoring.truth import TransitionTruth, transition_truth + +EXTRACTOR_VERSION: Final = "metrics-draft-1" +NAMESPACE: Final = uuid.uuid5(uuid.NAMESPACE_URL, "dail/experiments/metrics") +Z95: Final = 1.959963984540054 +ORDER: Final = ("A", "B", "C", "C_PRIME") +NOT_PROTECTED: Final = "NOT_PROTECTED_IN_PARENT" + + +def wilson(k: int, n: int) -> tuple[float | None, float | None]: + """The Wilson score 95% interval of k successes in n.""" + if n == 0: + return None, None + p = k / n + z2 = Z95 * Z95 + centre = (p + z2 / (2 * n)) / (1 + z2 / n) + half = Z95 * math.sqrt(p * (1 - p) / n + z2 / (4 * n * n)) / (1 + z2 / n) + return max(0.0, centre - half), min(1.0, centre + half) # rounding never leaves [0, 1] + + +@dataclass +class Extraction: + experiment_id: str + metrics: list[dict[str, Any]] = field(default_factory=list) + lists: dict[str, Any] = field(default_factory=dict) + confusion: dict[str, dict[tuple[str, str], int]] = field(default_factory=dict) + outcomes: list[dict[str, Any]] = field(default_factory=list) + + +def find( + extraction: Extraction, name: str, condition: str, *, scope: str = "all", stratum: str = "all" +) -> dict[str, Any]: + for record in extraction.metrics: + if ( + record["metric_name"] == name + and record["condition_id"] == condition + and record["scope"] == scope + and record["stratum"] == stratum + ): + return record + raise KeyError((name, condition, scope, stratum)) + + +class _Out: + def __init__(self, experiment_id: str) -> None: + self.experiment_id = experiment_id + self.records: list[dict[str, Any]] = [] + + def add( + self, + name: str, + condition: str, + *, + scope: str, + stratum: str, + numerator: float | None, + denominator: float | None, + value: float | None, + unit: str, + ) -> None: + interval = None + if unit == "proportion" and denominator is not None: + interval = list(wilson(int(numerator or 0), int(denominator))) + self.records.append( + { + "metric_id": str( + uuid.uuid5( + NAMESPACE, f"{self.experiment_id}/{name}/{condition}/{scope}/{stratum}" + ) + ), + "experiment_id": self.experiment_id, + "trial_id": None, + "condition_id": condition, + "metric_name": name, + "scope": scope, + "stratum": stratum, + "numerator": numerator, + "denominator": denominator, + "value": value, + "unit": unit, + "wilson95": interval, + "extractor_version": EXTRACTOR_VERSION, + } + ) + + def ratio( + self, name: str, condition: str, num: int, den: int, *, scope: str, stratum: str + ) -> None: + self.add( + name, + condition, + scope=scope, + stratum=stratum, + numerator=num, + denominator=den, + value=None if den == 0 else num / den, + unit="proportion", + ) + + def amount( + self, name: str, condition: str, value: float, *, scope: str, stratum: str, unit: str + ) -> None: + self.add( + name, + condition, + scope=scope, + stratum=stratum, + numerator=None, + denominator=None, + value=value, + unit=unit, + ) + + +@dataclass(frozen=True) +class _Scored: + t: Transition + truth: TransitionTruth + scenario: ScenarioLabels + + +def _work(t: Transition) -> dict[str, int]: + counts = Counter( + { + "gate": 0, + "invariant_impact": 0, + "invariant_condition": 0, + "invariant_not_protected": 0, + "baseline_applicable": 0, + "baseline_not_applicable": 0, + } + ) + for w in t.work: + if w.origin == "gate": + counts["gate"] += 1 + elif w.target_type == "INVARIANT": + if w.selected_reason == NOT_PROTECTED: + counts["invariant_not_protected"] += 1 + elif w.origin == "impact": + counts["invariant_impact"] += 1 + else: + counts["invariant_condition"] += 1 + elif w.applicable: + counts["baseline_applicable"] += 1 + else: + counts["baseline_not_applicable"] += 1 + return dict(counts) + + +def work_w(t: Transition) -> int: + c = _work(t) + return ( + c["gate"] + + c["invariant_impact"] + + c["invariant_condition"] + + c["invariant_not_protected"] + + c["baseline_applicable"] + ) + + +def _strata(s: _Scored) -> list[str]: + return ["all", f"family={s.scenario.family}", f"tier={s.scenario.tier}"] + + +def _core(out: _Out, condition: str, items: Sequence[_Scored], scope: str) -> None: + """Section 4 items 1-3, 5 and 6 for one condition and population.""" + groups: dict[str, list[_Scored]] = defaultdict(list) + for s in items: + for stratum in _strata(s): + groups[stratum].append(s) + groups.setdefault("all", []) + for stratum, group in sorted(groups.items()): + promoted = [s for s in group if s.t.decision == "PROMOTE"] + unsafe_promoted = [s for s in promoted if s.truth.unsafe] + out.ratio( + "unsafe_promotions_per_promotion", + condition, + len(unsafe_promoted), + len(promoted), + scope=scope, + stratum=stratum, + ) + out.ratio( + "unsafe_promotions_per_evaluated", + condition, + len(unsafe_promoted), + len(group), + scope=scope, + stratum=stratum, + ) + regressions = [s for s in group if s.truth.regression] + missed = [s for s in regressions if s.t.decision == "PROMOTE"] + out.ratio( + "regressions", condition, len(regressions), len(group), scope=scope, stratum=stratum + ) + out.ratio( + "missed_regressions_per_regression", + condition, + len(missed), + len(regressions), + scope=scope, + stratum=stratum, + ) + safe = [s for s in group if not s.truth.unsafe] + rejected = [s for s in safe if s.t.decision != "PROMOTE"] + out.ratio( + "false_rejection_rate", + condition, + len(rejected), + len(safe), + scope=scope, + stratum=stratum, + ) + for decision in ("PROMOTE", "REJECT", "RETRY", "ESCALATE"): + count = sum(1 for s in group if s.t.decision == decision) + out.ratio( + f"{decision.lower()}_rate", + condition, + count, + len(group), + scope=scope, + stratum=stratum, + ) + checks, unknown, unsupported = 0, 0, 0 + for s in group: + results = {w.target_id: w.result for w in s.t.work} + for target_id, source in s.t.required: + checks += 1 + # a check that is not proven has source None in DAIL's records; only a carried + # one has no fresh result + result = results.get(target_id) if source != "CARRIED_FORWARD" else None + unknown += result == "UNKNOWN" + unsupported += result == "UNSUPPORTED" + out.ratio("unknown_rate", condition, unknown, checks, scope=scope, stratum=stratum) + out.ratio("unsupported_rate", condition, unsupported, checks, scope=scope, stratum=stratum) + kinds: Counter[str] = Counter() + totals = [] + for s in group: + kinds.update(_work(s.t)) + totals.append(work_w(s.t)) + for kind in ( + "gate", + "invariant_impact", + "invariant_condition", + "invariant_not_protected", + "baseline_applicable", + "baseline_not_applicable", + ): + out.amount( + f"work_{kind}", condition, kinds[kind], scope=scope, stratum=stratum, unit="items" + ) + out.amount("work_total", condition, sum(totals), scope=scope, stratum=stratum, unit="items") + out.amount( + "work_mean_per_transition", + condition, + sum(totals) / len(totals) if totals else 0, + scope=scope, + stratum=stratum, + unit="items_per_transition", + ) + out.amount( + "work_median_per_transition", + condition, + statistics.median(totals) if totals else 0, + scope=scope, + stratum=stratum, + unit="items_per_transition", + ) + + +def _repair_success( + out: _Out, condition: str, all_items: Sequence[Transition], scored: Mapping[str, ScenarioLabels] +) -> None: + by_scenario: dict[str, list[Transition]] = defaultdict(list) + for t in all_items: + by_scenario[t.scenario_id].append(t) + for kind in ("repair", "change_request"): + ids = sorted(sid for sid in by_scenario if scored[sid].kind == kind) + ok = 0 + for sid in ids: + steps = by_scenario[sid] + last = scored[sid].truth[-1] + if all(t.evaluated and t.decision == "PROMOTE" for t in steps) and all( + v == "PASS" for v in last.values() + ): + ok += 1 + out.ratio("repair_success", condition, ok, len(ids), scope="all", stratum=f"kind={kind}") + + +def extract( + *, + experiment_id: str, + transitions: Iterable[Transition], + scored: Mapping[str, ScenarioLabels], + baselines: Mapping[tuple[str, str], Mapping[str, str]], + identity: IdentityTable, +) -> Extraction: + """Every metric record and list of Section 4 (see the module docstring).""" + out = _Out(experiment_id) + x = Extraction(experiment_id) + everything = sorted(transitions, key=lambda t: (t.condition, t.scenario_id, t.step)) + conditions = [c for c in ORDER if any(t.condition == c for t in everything)] + by_condition = {c: [t for t in everything if t.condition == c] for c in conditions} + scored_items = { + c: [ + _Scored( + t, + transition_truth(scored[t.scenario_id], t.step, t.open_targets), + scored[t.scenario_id], + ) + for t in ts + if t.evaluated + ] + for c, ts in by_condition.items() + } + keys = {c: {(s.t.scenario_id, s.t.step) for s in items} for c, items in scored_items.items()} + prefix_all = set.intersection(*keys.values()) if keys else set() + for c in conditions: + items = scored_items[c] + _core(out, c, items, "all") + _core( + out, + c, + [s for s in items if (s.t.scenario_id, s.t.step) in prefix_all], + "common_prefix_all", + ) + _repair_success(out, c, by_condition[c], scored) + truncated = sum(1 for t in by_condition[c] if not t.evaluated) + out.ratio( + "truncated_transitions", c, truncated, len(by_condition[c]), scope="all", stratum="all" + ) + if "B" in keys and "C" in keys: + prefix = keys["B"] & keys["C"] + sums = { + c: sum(work_w(s.t) for s in scored_items[c] if (s.t.scenario_id, s.t.step) in prefix) + for c in ("B", "C") + } + out.add( + "work_saving_c_vs_b", + "C", + scope="common_prefix_B_C", + stratum="all", + numerator=sums["C"], + denominator=sums["B"], + value=None if sums["B"] == 0 else 1 - sums["C"] / sums["B"], + unit="saving", + ) + _agreement(x, scored_items.get("B", []), baselines, scored) + for c in ("C", "C_PRIME"): + if c in scored_items: + _impact(out, x, c, scored_items[c]) + _identity(out, x, c, scored_items[c], identity) + _work_difference_f5(x, scored_items, identity) + x.lists["unsafe_promotions_c"] = [ + { + "scenario_id": s.t.scenario_id, + "step": s.t.step, + "failing": sorted(s.truth.failing), + "impact": sorted(s.t.impact_affected), + "work": [ + { + "target_id": w.target_id, + "result": w.result, + "origin": w.origin, + "selected_reason": w.selected_reason, + } + for w in s.t.work + ], + "required": [{"target_id": r, "source": src} for r, src in s.t.required], + } + for s in scored_items.get("C", []) + if s.t.decision == "PROMOTE" and s.truth.unsafe + ] + x.lists["clusters"] = _clusters(scored_items) + x.lists["non_pass_checks"] = _non_pass(scored_items) + x.lists["prefix_sizes"] = { + "common_prefix_all": len(prefix_all), + "common_prefix_B_C": len(keys.get("B", set()) & keys.get("C", set())), + } + x.metrics = sorted( + out.records, + key=lambda r: (r["metric_name"], ORDER.index(r["condition_id"]), r["scope"], r["stratum"]), + ) + return x + + +def _agreement( + x: Extraction, + items: Sequence[_Scored], + baselines: Mapping[tuple[str, str], Mapping[str, str]], + scored: Mapping[str, ScenarioLabels], +) -> None: + table: dict[str, Counter[tuple[str, str]]] = defaultdict(Counter) + disagreements = [] + + def note(sid: str, step: int, invariant: str, dail: str) -> None: + truth = scored[sid].truth[step].get(invariant) + if truth is None: + return + table[invariant][(dail, truth)] += 1 + if dail != truth: + disagreements.append( + { + "scenario_id": sid, + "step": step, + "invariant_id": invariant, + "dail": dail, + "oracle_truth": truth, + } + ) + + for (sid, condition), results in sorted(baselines.items()): + if condition == "B" and sid in scored: + for invariant, dail in sorted(results.items()): + note(sid, 0, invariant, dail) + for s in items: + for w in s.t.work: + named = invariant_of(w.target_id) + if named is not None: + note(s.t.scenario_id, s.t.step, named, w.result) + x.confusion = {k: dict(v) for k, v in sorted(table.items())} + x.lists["disagreements"] = sorted( + disagreements, key=lambda d: (d["scenario_id"], d["step"], d["invariant_id"]) + ) + + +def _impact(out: _Out, x: Extraction, condition: str, items: Sequence[_Scored]) -> None: + pooled: dict[str, Counter[str]] = defaultdict(Counter) + outside = x.lists.setdefault("truth_changes_outside_v", []) + for s in items: + found, truth = s.t.impact_affected, s.truth.affected + verified = { + name + for w in s.t.work + if w.target_type == "INVARIANT" and (name := invariant_of(w.target_id)) is not None + } + for stratum in ("all", f"family={s.scenario.family}"): + c = pooled[stratum] + c["hit"] += len(found & truth) + c["found"] += len(found) + c["true"] += len(truth) + c["changed"] += len(s.truth.changed) + c["covered"] += len(s.truth.changed & verified) + for invariant in sorted(s.truth.changed - verified): + outside.append( + { + "condition": condition, + "scenario_id": s.t.scenario_id, + "step": s.t.step, + "invariant_id": invariant, + } + ) + for stratum, c in sorted(pooled.items()): + out.ratio("impact_precision", condition, c["hit"], c["found"], scope="all", stratum=stratum) + out.ratio("impact_recall", condition, c["hit"], c["true"], scope="all", stratum=stratum) + out.ratio( + "truth_change_coverage", + condition, + c["covered"], + c["changed"], + scope="all", + stratum=stratum, + ) + if "all" not in pooled: + for name in ("impact_precision", "impact_recall", "truth_change_coverage"): + out.ratio(name, condition, 0, 0, scope="all", stratum="all") + + +def _identity( + out: _Out, x: Extraction, condition: str, items: Sequence[_Scored], table: IdentityTable +) -> None: + pairs = wrong = uncertain = transfer = excluded = 0 + errors = x.lists.setdefault("identity_errors", []) + excluded_list = x.lists.setdefault("identity_excluded", []) + for s in items: + found, skipped = true_pairs(table, s.scenario, s.t.step) + excluded += len(skipped) + excluded_list += [ + {"condition": condition, "scenario_id": s.t.scenario_id, "step": s.t.step, **e} + for e in skipped + ] + carried = any(src == "CARRIED_FORWARD" for _, src in s.t.required) + for pair in found: + verdict = judge(pair, s.t.identity_pairs) + pairs += 1 + if verdict == "uncertain": + uncertain += 1 + if verdict == "wrong": + wrong += 1 + errors.append( + { + "condition": condition, + "scenario_id": s.t.scenario_id, + "step": s.t.step, + "operator": pair.operator, + "effect": pair.effect, + "old": pair.old, + "new": pair.new, + "dail_pairs": [ + list(p) + for p in s.t.identity_pairs + if pair.old in p[:2] or pair.new in p[:2] + ], + } + ) + if pair.effect == "REPLACE" and carried: + transfer += 1 + out.ratio("identity_error_rate", condition, wrong, pairs, scope="all", stratum="all") + out.ratio("identity_uncertain_share", condition, uncertain, pairs, scope="all", stratum="all") + out.amount( + "evidence_transfer_errors", condition, transfer, scope="all", stratum="all", unit="count" + ) + out.amount( + "identity_excluded_pairs", condition, excluded, scope="all", stratum="all", unit="count" + ) + + +def _work_difference_f5( + x: Extraction, scored_items: Mapping[str, Sequence[_Scored]], table: IdentityTable +) -> None: + result: dict[str, dict[str, int]] = {} + if "C" in scored_items and "C_PRIME" in scored_items: + prime = {(s.t.scenario_id, s.t.step): s for s in scored_items["C_PRIME"]} + for s in scored_items["C"]: + other = prime.get((s.t.scenario_id, s.t.step)) + if other is None or s.scenario.family != "F5": + continue + moved = sorted( + { + p.resource_type + for p in true_pairs(table, s.scenario, s.t.step)[0] + if p.effect == "MOVE" + } + ) + for kind in moved: + row = result.setdefault( + kind, {"transitions": 0, "work_c": 0, "work_c_prime": 0, "difference": 0} + ) + row["transitions"] += 1 + row["work_c"] += work_w(s.t) + row["work_c_prime"] += work_w(other.t) + row["difference"] = row["work_c_prime"] - row["work_c"] + x.lists["work_difference_f5"] = dict(sorted(result.items())) + + +def _clusters(scored_items: Mapping[str, Sequence[_Scored]]) -> list[dict[str, Any]]: + rows: dict[tuple[str, str, str], Counter[str]] = defaultdict(Counter) + for condition, items in scored_items.items(): + for s in items: + c = rows[(condition, s.scenario.tier, s.scenario.cluster)] + c["evaluated"] += 1 + c["promoted"] += s.t.decision == "PROMOTE" + c["unsafe_promoted"] += s.t.decision == "PROMOTE" and s.truth.unsafe + c["safe"] += not s.truth.unsafe + c["safe_not_promoted"] += (not s.truth.unsafe) and s.t.decision != "PROMOTE" + return [ + {"condition": c, "tier": tier, "cluster": cluster, **dict(sorted(v.items()))} + for (c, tier, cluster), v in sorted( + rows.items(), key=lambda kv: (ORDER.index(kv[0][0]), kv[0][1], kv[0][2]) + ) + ] + + +def _kind(target_id: str) -> str: + """INVARIANT:, RESOURCE:, or the gate's name.""" + if target_id.startswith("INVARIANT:"): + return "INVARIANT:" + target_id.split(":")[1] + if target_id.startswith("RESOURCE:"): + return "RESOURCE:" + target_id.removeprefix("RESOURCE:").split(".", 1)[0] + return target_id + + +def _non_pass(scored_items: Mapping[str, Sequence[_Scored]]) -> list[dict[str, Any]]: + """Every required check with a fresh non-PASS result, counted by condition, kind, verifier, + result and reason (P8b-2): a systemic error shows here instead of inside a rate.""" + counts: Counter[tuple[str, str, str, str, str]] = Counter() + for condition, items in scored_items.items(): + for s in items: + rows = {w.target_id: w for w in s.t.work} + for target_id, source in s.t.required: + w = rows.get(target_id) + if source != "CARRIED_FORWARD" and w is not None and w.result != "PASS": + key = ( + condition, + _kind(target_id), + w.verifier_id or "", + w.result, + w.reason or "", + ) + counts[key] += 1 + return [ + {"condition": c, "kind": k, "verifier_id": v, "result": r, "reason": why, "count": n} + for (c, k, v, r, why), n in sorted( + counts.items(), key=lambda kv: (ORDER.index(kv[0][0]), kv[0][1:]) + ) + ] + + +BASELINE_INVARIANT: Final = { + "aws_security_group": "INV-SEC-001", + "aws_security_group_rule": "INV-SEC-001", + "aws_db_instance": "INV-SEC-003", +} + + +def baseline_guard(report: Mapping[str, Any]) -> list[dict[str, Any]]: + """Baseline items with reason ``invariant.unknown`` although the run's definitions hold the + invariant they belong to (P8b-2): each is a systemic error, never a measured outcome.""" + held = set(report.get("definitions_held") or ()) + found = [] + for row in report["lists"].get("non_pass_checks", []): + kind = row["kind"] + if not kind.startswith("RESOURCE:") or row["reason"] != "invariant.unknown": + continue + if BASELINE_INVARIANT.get(kind.removeprefix("RESOURCE:")) in held: + found.append(dict(row)) + return found + + +def to_json(x: Extraction) -> dict[str, Any]: + return { + "extractor_version": EXTRACTOR_VERSION, + "experiment_id": x.experiment_id, + "metrics": x.metrics, + "lists": x.lists, + "confusion": { + inv: [{"dail": d, "oracle_truth": o, "count": n} for (d, o), n in sorted(cells.items())] + for inv, cells in x.confusion.items() + }, + "trial_outcomes": x.outcomes, + } + + +def dumps_metrics(x: Extraction) -> bytes: + return (json.dumps(to_json(x), sort_keys=True, indent=1, ensure_ascii=False) + "\n").encode( + "utf-8" + ) + + +Selector = Callable[[Transition], bool] diff --git a/experiments/metrics/model.py b/experiments/metrics/model.py new file mode 100644 index 0000000..ca0a6f6 --- /dev/null +++ b/experiments/metrics/model.py @@ -0,0 +1,130 @@ +"""The stored trial records as transitions (P8b; Doc 13 §31-§32). + +A ``Transition`` is one (scenario, condition, step >= 1) of a trial result (``result.json``, the +P8a record format): its status (EVALUATED or TRUNCATED), DAIL's decision type, the open targets, the +required checks with their proof source, every work item run (target, type, result, ``applicable``, +origin and the item's reason), the invariants impact listed, DAIL's identity pairs and the step's +duration. ``trial_outcome`` gives Doc 13 §31's TrialOutcome of one transition. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class WorkRow: + target_id: str + target_type: str + result: str + applicable: bool + origin: str + selected_reason: str | None + verifier_id: str | None = None + reason: str | None = None # the result's reason (for example invariant.unknown) + + +@dataclass(frozen=True) +class Transition: + trial_id: str + scenario_id: str + condition: str + step: int + status: str + decision: str | None + open_targets: tuple[str, ...] + required: tuple[tuple[str, str | None], ...] + work: tuple[WorkRow, ...] + impact_affected: frozenset[str] + identity_pairs: tuple[tuple[str | None, str | None, str, bool], ...] + latency: float | None + + @property + def evaluated(self) -> bool: + return self.status == "EVALUATED" + + +def invariant_of(target_id: str) -> str | None: + """``INVARIANT::`` to ````; None for a gate or a resource item.""" + if not target_id.startswith("INVARIANT:"): + return None + return target_id.split(":")[1] + + +def transitions_from_result( + result: Mapping[str, Any], durations: Mapping[str, float] | None = None +) -> list[Transition]: + found = [] + for t in result["transitions"]: + decision = t.get("decision") + found.append( + Transition( + trial_id=result["trial_id"], + scenario_id=result["scenario_id"], + condition=result["condition"], + step=int(t["step_index"]), + status=t["status"], + decision=None if decision is None else decision["type"], + open_targets=tuple(t.get("open_targets") or ()), + required=tuple((r["target_id"], r["source"]) for r in t.get("required") or ()), + work=tuple( + WorkRow( + w["target_id"], + w["target_type"], + w["result"], + bool(w["applicable"]), + w["origin"], + w.get("selected_reason"), + w.get("verifier_id"), + w.get("reason"), + ) + for w in t.get("work") or () + ), + impact_affected=frozenset( + a["invariant_id"] for a in ((t.get("impact") or {}).get("affected") or ()) + ), + identity_pairs=tuple( + (p["trusted_ref"], p["candidate_ref"], p["change_type"], bool(p["moved"])) + for p in ((t.get("identity") or {}).get("pairs") or ()) + ), + latency=None if durations is None else durations.get(f"step-{t['step_index']}"), + ) + ) + return found + + +def trial_outcome(transition: Transition) -> dict[str, Any]: + """Doc 13 §31 TrialOutcome for one transition of a fixed-patch trial.""" + if not transition.evaluated: + return { + "generation_status": "NOT_RUN", + "patch_status": "NOT_RUN", + "identity_status": "NOT_RUN", + "impact_status": "NOT_RUN", + "verification_status": "NOT_RUN", + "promotion_decision": None, + "invariant_results": [], + "latency": None, + "token_usage": None, + "failure_class": None, + } + results = [ + {"invariant_id": invariant_of(w.target_id), "result": w.result} + for w in transition.work + if w.target_type == "INVARIANT" + ] + worst = {w.result for w in transition.work} - {"PASS"} + return { + "generation_status": "FIXED_PATCH", # no generation in P8 (fixed patches, C-200) + "patch_status": "ADMITTED", # content hash checked (C-201) + "identity_status": "OK" if transition.identity_pairs else "NO_PAIRS", + "impact_status": "OK", + "verification_status": "ALL_PASS" if not worst else "+".join(sorted(worst)), + "promotion_decision": transition.decision, + "invariant_results": results, + "latency": transition.latency, + "token_usage": None, + "failure_class": None, + } diff --git a/experiments/report/__init__.py b/experiments/report/__init__.py new file mode 100644 index 0000000..64c878d --- /dev/null +++ b/experiments/report/__init__.py @@ -0,0 +1,5 @@ +"""The experiment report (P8b; Doc 13 §33, §35): ``build`` assembles an ExperimentReport from the +extractor's output, ``render`` turns it into Markdown, ``__main__`` is the command line +(``python -m experiments.report``). Nothing here names a label field; trial execution never imports +this package (``tests/contract/test_experiment_blinding.py``). +""" diff --git a/experiments/report/__main__.py b/experiments/report/__main__.py new file mode 100644 index 0000000..c69ef65 --- /dev/null +++ b/experiments/report/__main__.py @@ -0,0 +1,217 @@ +"""The report command line (P8b Steps 4-5). + + python -m experiments.report extract --name NAME [--version V] [--doc docs/P8_DEV_PILOT] + python -m experiments.report select --name NAME [--version V] + python -m experiments.report compare --name NAME --other OTHER [--version V] + +``extract`` reads the stored records of the experiment under ``$DAIL_EXPERIMENT_ROOT`` and the dev +labels under ``$TERRAPRESERVE_ROOT`` (through ``experiments.scoring``), runs the extractor and +writes ``report//{metric_records.json, report.json, report.md}`` once: run again, +it checks that what it would write is byte-identical to what is stored (Doc 13 §30) and fails if +not. With ``--doc`` it also writes ``.json`` and ``.md``. ``select`` prints the scenario +ids of the reproducibility re-run. ``compare`` compares the decisions and invariant results of two +experiments on their common (scenario, condition, step) and prints the counts. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +from experiments.artifacts import ArtifactError, dumps, read_record, write_or_same +from experiments.definition import experiment_id_for +from experiments.metrics.extractor import EXTRACTOR_VERSION, dumps_metrics, extract +from experiments.metrics.model import Transition, transitions_from_result, trial_outcome +from experiments.report.build import build_report +from experiments.report.render import render_markdown +from experiments.runner import Layout, read_events, read_result +from experiments.scoring.identity import load_identity_table +from experiments.scoring.labels import load_labels +from experiments.scoring.selection import reproducibility_selection + + +def _env(name: str) -> Path: + value = os.environ.get(name) + if not value: + raise SystemExit(f"set {name}") + return Path(value) + + +def collect( + layout: Layout, +) -> tuple[ + dict[str, Any], + dict[str, str], + list[Transition], + dict[tuple[str, str], dict[str, str]], + list[str], +]: + """The definition, the trial statuses, the transitions, DAIL's S0 results and the record + paths.""" + definition = read_record(layout.definition_path) + statuses: dict[str, str] = {} + transitions: list[Transition] = [] + baselines: dict[tuple[str, str], dict[str, str]] = {} + refs = [layout.definition_path.relative_to(layout.experiment_dir).as_posix()] + for trial in definition["trials"]: + tid = trial["trial_id"] + events = read_events(layout, tid) + statuses[tid] = events[-1]["status"] if events else "NOT_STARTED" + try: + result = read_result(layout, tid) + except Exception: + continue + attempt = layout.attempt_dir(tid, result["attempt"]) + refs.append((attempt / "result.json").relative_to(layout.experiment_dir).as_posix()) + timing = read_record(attempt / "timing.json") + transitions += transitions_from_result(result, timing.get("durations_seconds")) + baseline_file = attempt / "baseline" / "baseline.json" + if baseline_file.exists(): + stored = read_record(baseline_file) + results = (stored.get("verification") or {}).get("results") or [] + baselines[(result["scenario_id"], result["condition"])] = { + r["target_id"].split(":")[1]: r["result"] + for r in results + if r["target_id"].startswith("INVARIANT:") + } + return definition, statuses, transitions, baselines, refs + + +def cmd_extract(args: argparse.Namespace) -> int: + root, tp = _env("DAIL_EXPERIMENT_ROOT"), _env("TERRAPRESERVE_ROOT") + layout = Layout(root, experiment_id_for(args.name, args.version)) + definition, statuses, transitions, baselines, refs = collect(layout) + scored = load_labels(tp, {t.scenario_id for t in transitions}) + extraction = extract( + experiment_id=layout.experiment_id, + transitions=transitions, + scored=scored, + baselines=baselines, + identity=load_identity_table(), + ) + extraction.outcomes = [ + { + "trial_id": t.trial_id, + "condition": t.condition, + "scenario_id": t.scenario_id, + "step": t.step, + **trial_outcome(t), + } + for t in sorted(transitions, key=lambda t: (t.condition, t.scenario_id, t.step)) + ] + reproducibility = { + "extractor": ( + f"{EXTRACTOR_VERSION}; a second run of `extract` must write byte-identical records " + "(it fails otherwise)" + ), + } + if args.repro: + reproducibility.update(json.loads(Path(args.repro).read_text(encoding="utf-8"))) + report = build_report( + definition=definition, + statuses=statuses, + extraction=extraction, + reproducibility=reproducibility, + artifact_refs=refs[:1] + [f"{len(refs) - 1} trial result records under trials/"], + preface=( + Path(args.preface).read_text(encoding="utf-8").splitlines() if args.preface else None + ), + ) + out = layout.experiment_dir / "report" / EXTRACTOR_VERSION + out.mkdir(parents=True, exist_ok=True) + metrics_path = out / "metric_records.json" + if metrics_path.exists() and metrics_path.read_bytes() != dumps_metrics(extraction): + raise SystemExit(f"{metrics_path}: the extractor's output differs from the stored records") + if not metrics_path.exists(): + metrics_path.write_bytes(dumps_metrics(extraction)) + try: + write_or_same(out / "report.json", report) + except ArtifactError as changed: + raise SystemExit(str(changed)) from None + (out / "report.md").write_text(render_markdown(report), encoding="utf-8", newline="\n") + if args.doc: + Path(f"{args.doc}.json").write_bytes(dumps(report)) + Path(f"{args.doc}.md").write_text(render_markdown(report), encoding="utf-8", newline="\n") + print( + json.dumps( + { + "experiment_id": layout.experiment_id, + "metrics": len(extraction.metrics), + "report": str(out), + } + ) + ) + return 0 + + +def cmd_select(args: argparse.Namespace) -> int: + root, tp = _env("DAIL_EXPERIMENT_ROOT"), _env("TERRAPRESERVE_ROOT") + layout = Layout(root, experiment_id_for(args.name, args.version)) + definition = read_record(layout.definition_path) + scored = load_labels(tp, definition["fixture_set"]) + print(",".join(reproducibility_selection(scored))) + return 0 + + +def _decisions(layout: Layout) -> dict[tuple[str, str, int], tuple[Any, ...]]: + definition = read_record(layout.definition_path) + found: dict[tuple[str, str, int], tuple[Any, ...]] = {} + for trial in definition["trials"]: + result = read_result(layout, trial["trial_id"]) + for t in result["transitions"]: + results = tuple(sorted((w["target_id"], w["result"]) for w in t["work"])) + decision = None if t["decision"] is None else t["decision"]["type"] + found[(result["scenario_id"], result["condition"], t["step_index"])] = ( + t["status"], + decision, + results, + ) + return found + + +def cmd_compare(args: argparse.Namespace) -> int: + root = _env("DAIL_EXPERIMENT_ROOT") + one = _decisions(Layout(root, experiment_id_for(args.name, args.version))) + two = _decisions(Layout(root, experiment_id_for(args.other, args.version))) + common = sorted(set(one) & set(two)) + decision_diff = [k for k in common if one[k][:2] != two[k][:2]] + result_diff = [k for k in common if one[k][2] != two[k][2]] + print( + json.dumps( + { + "common_transitions": len(common), + "trials": len({(k[0], k[1]) for k in common}), + "decision_or_status_differences": len(decision_diff), + "invariant_result_differences": len(result_diff), + "examples": [list(k) for k in (decision_diff + result_diff)[:10]], + }, + indent=1, + ) + ) + return 0 if not decision_diff and not result_diff else 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m experiments.report") + commands = parser.add_subparsers(dest="command", required=True) + for name in ("extract", "select", "compare"): + command = commands.add_parser(name) + command.add_argument("--name", required=True) + command.add_argument("--version", default="1") + if name == "extract": + command.add_argument("--doc", default="") + command.add_argument("--repro", default="", help="a JSON file of reproducibility facts") + command.add_argument("--preface", default="", help="a text file shown at the top") + if name == "compare": + command.add_argument("--other", required=True) + args = parser.parse_args(argv) + handler = {"extract": cmd_extract, "select": cmd_select, "compare": cmd_compare}[args.command] + return handler(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/report/build.py b/experiments/report/build.py new file mode 100644 index 0000000..0cd6c5c --- /dev/null +++ b/experiments/report/build.py @@ -0,0 +1,93 @@ +"""The ExperimentReport (Doc 13 §33) built from the extractor's output only. + +"ExperimentReport { experiment_id, experiment_version, trial_count, included_count, excluded_count, +condition_summaries[], primary_metrics[], secondary_metrics[], failures[], reproducibility_summary, +artifact_refs[] }." Added: the extractor version, every metric record, the lists and the confusion +tables, so that the Markdown rendering needs nothing but this document. A trial is included when it +is COMPLETED; every other one is listed under ``failures`` and counted as excluded (none is dropped, +Doc 13 §34). +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Final + +from experiments.metrics.extractor import EXTRACTOR_VERSION, Extraction, to_json + +REPORT_KEYS: Final = ( + "experiment_id", + "experiment_version", + "trial_count", + "included_count", + "excluded_count", + "condition_summaries", + "primary_metrics", + "secondary_metrics", + "failures", + "reproducibility_summary", + "artifact_refs", +) +PRIMARY: Final = ( + "unsafe_promotions_per_promotion", + "unsafe_promotions_per_evaluated", + "regressions", + "missed_regressions_per_regression", + "false_rejection_rate", + "repair_success", +) + + +def build_report( + *, + definition: Mapping[str, Any], + statuses: Mapping[str, str], + extraction: Extraction, + reproducibility: Mapping[str, Any] | None = None, + preface: list[str] | None = None, + artifact_refs: list[str] | None = None, +) -> dict[str, Any]: + data = to_json(extraction) + metrics = data["metrics"] + failures = [ + {"trial_id": t, "status": s} for t, s in sorted(statuses.items()) if s != "COMPLETED" + ] + conditions = list(definition.get("condition_set", [])) + summaries = [] + for c in conditions: + mine = [ + m + for m in metrics + if m["condition_id"] == c and m["scope"] == "all" and m["stratum"] == "all" + ] + get = {m["metric_name"]: m for m in mine} + summaries.append( + { + "condition": c, + "evaluated_transitions": (get.get("promote_rate") or {}).get("denominator"), + "truncated_transitions": (get.get("truncated_transitions") or {}).get("numerator"), + "all_transitions": (get.get("truncated_transitions") or {}).get("denominator"), + } + ) + return { + "experiment_id": definition.get("experiment_id"), + "experiment_name": definition.get("name"), + "experiment_version": definition.get("version"), + "extractor_version": EXTRACTOR_VERSION, + "trial_count": len(statuses), + "included_count": len(statuses) - len(failures), + "excluded_count": len(failures), + "condition_summaries": summaries, + "primary_metrics": [m for m in metrics if m["metric_name"] in PRIMARY], + "secondary_metrics": [m for m in metrics if m["metric_name"] not in PRIMARY], + "failures": failures, + "reproducibility_summary": dict(reproducibility or {}), + "preface": list(preface or []), + "artifact_refs": list(artifact_refs or []), + "conditions": conditions, + "definitions_held": sorted( + ((definition.get("pins") or {}).get("verifiers") or {}).get("invariants") or {} + ), + "lists": data["lists"], + "confusion": data["confusion"], + } diff --git a/experiments/report/render.py b/experiments/report/render.py new file mode 100644 index 0000000..f32a76e --- /dev/null +++ b/experiments/report/render.py @@ -0,0 +1,386 @@ +"""The Markdown rendering of an ExperimentReport (P8b Step 5; Doc 13 §35). + +Every number comes from the report document. A proportion is shown as ``k / n = p [low, high]`` +(Wilson 95%); a count or an amount as it is. The rendering is deterministic: the same report gives +the same text, and ``tests/unit/test_p8b_scoring_report.py`` checks that ``docs/P8_DEV_PILOT.md`` is +the rendering of ``docs/P8_DEV_PILOT.json``. Descriptive only (C-214). +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +NAMES = {"A": "A", "B": "B", "C": "C", "C_PRIME": "C′"} + + +class _Index: + def __init__(self, report: Mapping[str, Any]) -> None: + self.by_key: dict[tuple[str, str, str, str], Mapping[str, Any]] = {} + for m in [*report["primary_metrics"], *report["secondary_metrics"]]: + self.by_key[(m["metric_name"], m["condition_id"], m["scope"], m["stratum"])] = m + + def get( + self, name: str, condition: str, scope: str = "all", stratum: str = "all" + ) -> Mapping[str, Any] | None: + return self.by_key.get((name, condition, scope, stratum)) + + def strata(self, prefix: str) -> list[str]: + return sorted({k[3] for k in self.by_key if k[3].startswith(prefix)}) + + +def _num(value: Any) -> str: + if value is None: + return "n/a" + if isinstance(value, float): + if value.is_integer(): + return str(int(value)) + return f"{value:.3f}" + return str(value) + + +def cell(m: Mapping[str, Any] | None) -> str: + if m is None: + return "—" + if m["unit"] == "proportion": + low, high = m["wilson95"] or (None, None) + interval = "" if low is None else f" [{low:.3f}, {high:.3f}]" + value = "n/a" if m["value"] is None else f"{m['value']:.3f}" + return f"{m['numerator']} / {m['denominator']} = {value}{interval}" + if m["unit"] == "saving": + value = "n/a" if m["value"] is None else f"{m['value']:.3f}" + return f"1 − {m['numerator']} / {m['denominator']} = {value}" + return _num(m["value"]) + + +def _table(head: Sequence[str], rows: Iterable[Sequence[str]]) -> list[str]: + out = ["| " + " | ".join(head) + " |", "|" + "---|" * len(head)] + out += ["| " + " | ".join(r) + " |" for r in rows] + return out + + +HEADLINE = ( + ("Unsafe promoted / promoted", "unsafe_promotions_per_promotion", "all"), + ("Unsafe promoted / evaluated", "unsafe_promotions_per_evaluated", "all"), + ("Regressions / evaluated", "regressions", "all"), + ("Missed regressions / regressions", "missed_regressions_per_regression", "all"), + ("False rejections / safe", "false_rejection_rate", "all"), + ("Repair success (repair)", "repair_success", "kind=repair"), + ("Repair success (change request)", "repair_success", "kind=change_request"), + ("W total", "work_total", "all"), + ("W mean per transition", "work_mean_per_transition", "all"), + ("W median per transition", "work_median_per_transition", "all"), +) +OUTCOMES = ( + ("PROMOTE / evaluated", "promote_rate"), + ("REJECT / evaluated", "reject_rate"), + ("RETRY / evaluated", "retry_rate"), + ("ESCALATE / evaluated", "escalate_rate"), + ("UNKNOWN / required checks", "unknown_rate"), + ("UNSUPPORTED / required checks", "unsupported_rate"), +) +WORK = ( + ("gate items", "work_gate"), + ("invariant items from impact", "work_invariant_impact"), + ("invariant items from the condition", "work_invariant_condition"), + ("U_t items (not PROTECTED in parent)", "work_invariant_not_protected"), + ("applicable baseline items", "work_baseline_applicable"), + ("not-applicable baseline items (outside W)", "work_baseline_not_applicable"), + ("W total", "work_total"), +) + + +def _condition_table( + ix: _Index, conds: Sequence[str], rows: Sequence[tuple[str, str, str]], scope: str +) -> list[str]: + return _table( + ["Metric", *[NAMES[c] for c in conds]], + ( + [label, *[cell(ix.get(name, c, scope, stratum)) for c in conds]] + for label, name, stratum in rows + ), + ) + + +def render_markdown(report: Mapping[str, Any]) -> str: + ix = _Index(report) + conds = list(report["conditions"]) + lists = report["lists"] + prefix = lists["prefix_sizes"] + saving = cell(ix.get("work_saving_c_vs_b", "C", "common_prefix_B_C")) + lines: list[str] = [ + "# P8 dev pilot report", + "", + *[f"> {line}" for line in report.get("preface") or []], + *([""] if report.get("preface") else []), + "Generated by `python -m experiments.report` from the stored trial records;", + "do not edit by hand.", + "Descriptive only (C-214): counts, proportions with numerator and denominator, Wilson 95%", + "intervals. No hypothesis test. These are dev-split numbers of a draft protocol,", + "not findings.", + "", + f"- Experiment: `{report['experiment_name']}` version {report['experiment_version']}, " + f"id `{report['experiment_id']}`", + f"- Extractor: `{report['extractor_version']}`", + f"- Trials: {report['trial_count']} (included {report['included_count']}, " + f"excluded {report['excluded_count']})", + f"- Common prefix of all conditions: {prefix['common_prefix_all']} transitions; " + f"of B and C: {prefix['common_prefix_B_C']}", + "", + "## 1. Headline metrics per condition (all evaluated transitions)", + "", + *_condition_table(ix, conds, HEADLINE, "all"), + "", + "TRUNCATED transitions per condition:", + "", + *_table( + ["Condition", "TRUNCATED / all transitions"], + ([NAMES[c], cell(ix.get("truncated_transitions", c))] for c in conds), + ), + "", + f"Save(C vs B) on the common prefix of B and C: {saving}", + "", + "## 2. The same on the common prefix of all conditions", + "", + *_condition_table( + ix, conds, [r for r in HEADLINE if r[1] != "repair_success"], "common_prefix_all" + ), + "", + "## 3. Per family", + "", + ] + for label, name, _ in HEADLINE[:5]: + lines += [f"### {label}", ""] + lines += _table( + ["Family", *[NAMES[c] for c in conds]], + ( + [s.removeprefix("family="), *[cell(ix.get(name, c, "all", s)) for c in conds]] + for s in ix.strata("family=") + ), + ) + lines.append("") + lines += ["## 4. Per tier", ""] + for label, name, _ in HEADLINE[:5]: + lines += [f"### {label}", ""] + lines += _table( + ["Tier", *[NAMES[c] for c in conds]], + ( + [s.removeprefix("tier="), *[cell(ix.get(name, c, "all", s)) for c in conds]] + for s in ix.strata("tier=") + ), + ) + lines.append("") + lines += ["## 5. Outcome rates (Doc 13 §28)", ""] + lines += _table( + ["Rate", *[NAMES[c] for c in conds]], + ([lab, *[cell(ix.get(n, c)) for c in conds]] for lab, n in OUTCOMES), + ) + lines += [ + "", + "Non-PASS required checks (fresh results), by condition, kind, verifier, result and " + "reason:", + "", + ] + non_pass = lists.get("non_pass_checks", []) + if non_pass: + lines += _table( + ["Condition", "Kind", "Verifier", "Result", "Reason", "Count"], + ( + [ + NAMES[r["condition"]], + r["kind"], + r["verifier_id"], + r["result"], + r["reason"] or "—", + str(r["count"]), + ] + for r in non_pass + ), + ) + else: + lines.append("none") + lines += ["", "## 6. Work by kind (oracle excluded)", ""] + lines += _table( + ["Kind", *[NAMES[c] for c in conds]], + ([lab, *[cell(ix.get(n, c)) for c in conds]] for lab, n in WORK), + ) + lines += ["", "## 7. DAIL–oracle agreement under B (C-155): S0 and every evaluated step", ""] + for invariant, cells in sorted(report["confusion"].items()): + lines += [f"### {invariant}", ""] + lines += _table( + ["DAIL", "oracle truth", "count"], + ([c["dail"], c["oracle_truth"], str(c["count"])] for c in cells), + ) + lines.append("") + disagreements = lists.get("disagreements", []) + by_inv = Counter(d["invariant_id"] for d in disagreements) + lines += [ + f"Disagreements: {len(disagreements)}" + + ( + "" + if not by_inv + else " (" + ", ".join(f"{k}: {v}" for k, v in sorted(by_inv.items())) + ")" + ), + "", + ] + if disagreements: + lines += _table( + ["Scenario", "Step", "Invariant", "DAIL", "Oracle truth"], + ( + [d["scenario_id"], str(d["step"]), d["invariant_id"], d["dail"], d["oracle_truth"]] + for d in disagreements + ), + ) + lines.append("") + lines += ["## 8. Impact against the oracle's affected set (C and C′)", ""] + impact_conds = [c for c in conds if c in ("C", "C_PRIME")] + strata = ["all", *ix.strata("family=")] + for label, name in ( + ("Precision", "impact_precision"), + ("Recall", "impact_recall"), + ("Truth-change coverage", "truth_change_coverage"), + ): + lines += [f"### {label}", ""] + lines += _table( + ["Stratum", *[NAMES[c] for c in impact_conds]], + ([s, *[cell(ix.get(name, c, "all", s)) for c in impact_conds]] for s in strata), + ) + lines.append("") + outside = lists.get("truth_changes_outside_v", []) + lines += [f"Truth changes outside the verified set: {len(outside)}", ""] + if outside: + lines += _table( + ["Condition", "Scenario", "Step", "Invariant"], + ( + [NAMES[o["condition"]], o["scenario_id"], str(o["step"]), o["invariant_id"]] + for o in outside + ), + ) + lines.append("") + lines += ["## 9. Identity (H3: C and C′)", ""] + lines += _table( + ["Metric", *[NAMES[c] for c in impact_conds]], + ( + [lab, *[cell(ix.get(n, c)) for c in impact_conds]] + for lab, n in ( + ("Identity errors / true pairs", "identity_error_rate"), + ("UNCERTAIN / true pairs", "identity_uncertain_share"), + ("Evidence-transfer errors", "evidence_transfer_errors"), + ("Excluded pairs", "identity_excluded_pairs"), + ) + ), + ) + errors = lists.get("identity_errors", []) + lines += ["", f"Identity errors: {len(errors)}", ""] + if errors: + lines += _table( + ["Condition", "Scenario", "Step", "Operator", "Effect", "Old", "New", "DAIL pairs"], + ( + [ + NAMES[e["condition"]], + e["scenario_id"], + str(e["step"]), + e["operator"], + e["effect"], + str(e["old"]), + str(e["new"]), + "; ".join(f"{p[0]} -> {p[1]} {p[2]}" for p in e["dail_pairs"]) or "none", + ] + for e in errors + ), + ) + lines.append("") + excluded = Counter((e["condition"], e["operator"]) for e in lists.get("identity_excluded", [])) + lines += [f"Excluded identity pairs by operator: {sum(excluded.values())}", ""] + if excluded: + lines += _table( + ["Condition", "Operator", "Pairs"], + ([NAMES[c], op, str(n)] for (c, op), n in sorted(excluded.items())), + ) + lines.append("") + diff = lists.get("work_difference_f5", {}) + lines += ["Work difference C′ − C on F5 by moved resource type:", ""] + if diff: + lines += _table( + ["Moved type", "Transitions", "W under C", "W under C′", "C′ − C"], + ( + [ + k, + str(v["transitions"]), + str(v["work_c"]), + str(v["work_c_prime"]), + str(v["difference"]), + ] + for k, v in diff.items() + ), + ) + else: + lines.append("none") + lines += ["", "## 10. Unsafe promotions under C (Doc 13 §37, §49)", ""] + unsafe = lists.get("unsafe_promotions_c", []) + if not unsafe: + lines.append("none") + for u in unsafe: + work = ", ".join(w["target_id"] + "=" + w["result"] for w in u["work"]) + lines += [ + f"- `{u['scenario_id']}` step {u['step']}: failing {', '.join(u['failing'])}; " + f"impact {{{', '.join(u['impact'])}}}; work {work}", + ] + lines += ["", "## 11. Clusters (C-214)", ""] + lines += _table( + [ + "Condition", + "Tier", + "Clusters", + "Evaluated", + "Promoted", + "Unsafe promoted", + "Safe", + "Safe not promoted", + ], + _cluster_rows(lists.get("clusters", []), conds), + ) + lines += ["", "## 12. Failed and excluded trials", ""] + if not report["failures"]: + lines.append("none: every trial is COMPLETED and included") + for f in report["failures"]: + lines.append(f"- `{f['trial_id']}`: {f['status']}") + repro = report.get("reproducibility_summary") or {} + lines += ["", "## 13. Reproducibility (Doc 13 §30)", ""] + if not repro: + lines.append("not recorded in this report") + for key, value in sorted(repro.items()): + lines.append(f"- {key}: {value}") + return "\n".join(lines) + "\n" + + +def _cluster_rows(rows: Sequence[Mapping[str, Any]], conds: Sequence[str]) -> list[list[str]]: + totals: dict[tuple[str, str], Counter[str]] = {} + for r in rows: + key = (r["condition"], r["tier"]) + c = totals.setdefault(key, Counter()) + c["clusters"] += 1 + for k in ("evaluated", "promoted", "unsafe_promoted", "safe", "safe_not_promoted"): + c[k] += r[k] + return [ + [ + NAMES[c], + tier, + *( + str(v[k]) + for k in ( + "clusters", + "evaluated", + "promoted", + "unsafe_promoted", + "safe", + "safe_not_promoted", + ) + ), + ] + for (c, tier), v in sorted( + totals.items(), + key=lambda kv: (conds.index(kv[0][0]) if kv[0][0] in conds else 9, kv[0][1]), + ) + ] diff --git a/experiments/scoring/__init__.py b/experiments/scoring/__init__.py new file mode 100644 index 0000000..0dc679d --- /dev/null +++ b/experiments/scoring/__init__.py @@ -0,0 +1,9 @@ +"""Scoring (P8b): the only part of the repository that reads TerraPreserve label fields. + +``labels`` reads the dev labels (oracle truth per step, the oracle's affected invariants, the +mutation records, the scenario class) from the frozen TerraPreserve files, with the same hash checks +as ``experiments.loader``; ``truth`` turns them into the truth of one transition (metrics-draft-1); +``identity`` holds the identity truth table of C-213. Trial execution never imports this package +(``tests/contract/test_experiment_blinding.py``, check (b)), and no module outside it names a label +field (check (a)). The test split is refused. +""" diff --git a/experiments/scoring/identity.py b/experiments/scoring/identity.py new file mode 100644 index 0000000..3b1e488 --- /dev/null +++ b/experiments/scoring/identity.py @@ -0,0 +1,159 @@ +"""Identity truth for H3 (P8b-2, C-213; metrics-draft-1 item 10). + +``identity_truth.json`` gives, per TerraPreserve operator, the identity effect the catalogue +(D4_OPERATORS.md) states. ``true_pairs`` applies it to a step. Appearing addresses are those the +step's configuration declares and the previous step's does not; vanishing ones the reverse +(``ScenarioLabels.declared``). A MOVE pairs the one vanishing address of the type with the one +appearing address of the type; a REPLACE names an address the operator's mutation rows change on the +forcing attribute (both ``before`` and ``after`` set); a CREATE (DELETE) names every appearing +(vanishing) address of the type that the operator's mutation rows name. A MOVE whose addresses +cannot be recovered that way, and every address of an excluded operator, is an excluded pair, +counted. + +``judge`` compares one true pair with DAIL's identity pairs of the transition (trusted ref, +candidate ref, change type), on addresses without instance keys: ``correct`` (MOVE: DAIL pairs the +old with the new address; REPLACE: REPLACED; CREATE: CREATED; DELETE: DELETED), ``uncertain`` (DAIL +says UNCERTAIN for it), or ``wrong``. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from experiments.scoring.labels import ScenarioLabels + +TABLE_PATH = Path(__file__).with_name("identity_truth.json") + + +@dataclass(frozen=True) +class IdentityTable: + version: str + rows: Mapping[str, Mapping[str, Any]] + excluded: Mapping[str, str] + + +@dataclass(frozen=True) +class TruePair: + operator: str + effect: str + resource_type: str + old: str | None + new: str | None + + +def load_identity_table(path: Path = TABLE_PATH) -> IdentityTable: + data = json.loads(path.read_text(encoding="utf-8")) + return IdentityTable(data["version"], data["rows"], data["excluded"]) + + +def _type(address: str) -> str: + return address.split(".", 1)[0] + + +def base(address: str | None) -> str | None: + return None if address is None else address.split("[", 1)[0] + + +def true_pairs( + table: IdentityTable, scenario: ScenarioLabels, step: int +) -> tuple[list[TruePair], list[dict[str, str]]]: + """The true identity pairs of ``step`` and the excluded ones (operator, address, reason).""" + by_op: dict[str, dict[str, list[Any]]] = defaultdict(lambda: defaultdict(list)) + for row in scenario.mutations: + if row.step == step: + by_op[row.operator][row.target_address].append(row) + pairs: list[TruePair] = [] + excluded: list[dict[str, str]] = [] + for operator in sorted(by_op): + addresses = by_op[operator] + if operator in table.excluded: + excluded += [ + {"operator": operator, "address": a, "reason": table.excluded[operator]} + for a in sorted(addresses) + ] + continue + spec = table.rows.get(operator) + if spec is None: + continue # no identity effect: an attribute edit + before, after = scenario.declared[step - 1], scenario.declared[step] + appearing, vanishing = set(after - before), set(before - after) + named = set(addresses) + used: set[str] = set() + for effect in spec["effects"]: + kind, wanted = effect["effect"], effect["resource_type"] + + def fits(address: str, wanted: str = wanted) -> bool: + return wanted == "*" or _type(address) == wanted + + if kind == "MOVE": + olds = sorted(a for a in vanishing - used if fits(a)) + news = sorted(a for a in appearing - used if fits(a)) + if not (named & set(olds + news)): + olds, news = [], [] + if len(olds) == 1 and len(news) == 1: + pairs.append(TruePair(operator, kind, wanted, olds[0], news[0])) + used |= {olds[0], news[0]} + else: + excluded.append( + { + "operator": operator, + "address": ",".join(olds + news) or "-", + "reason": "the renamed addresses cannot be recovered from the rows", + } + ) + elif kind == "REPLACE": + for address in sorted(addresses): + rows = addresses[address] + if fits(address) and any( + r.attribute == effect["attribute"] + and r.before is not None + and r.after is not None + for r in rows + ): + pairs.append(TruePair(operator, kind, wanted, address, address)) + used.add(address) + else: + pool = appearing if kind == "CREATE" else vanishing + for address in sorted(a for a in (pool & named) - used if fits(a)): + old, new = (None, address) if kind == "CREATE" else (address, None) + pairs.append(TruePair(operator, kind, wanted, old, new)) + used.add(address) + return pairs, excluded + + +_EXPECTED = {"REPLACE": "REPLACED", "CREATE": "CREATED", "DELETE": "DELETED"} + + +def judge(pair: TruePair, dail: Sequence[tuple[str | None, str | None, str, bool]]) -> str: + """``correct``, ``uncertain`` or ``wrong`` (see the module docstring).""" + found = [(base(t), base(c), kind) for t, c, kind, _ in dail] + touching = [ + kind + for t, c, kind in found + if (pair.old is not None and pair.old in (t, c)) + or (pair.new is not None and pair.new in (t, c)) + ] + if pair.effect == "MOVE": + if any(t == pair.old and c == pair.new for t, c, _ in found): + return "correct" + else: + own = [ + kind + for t, c, kind in found + if (t == pair.old if pair.old is not None else t is None) + and (c == pair.new if pair.new is not None else c is None) + ] + if _EXPECTED[pair.effect] in own: + return "correct" + return "uncertain" if "UNCERTAIN" in touching else "wrong" + + +def judge_all( + pairs: Iterable[TruePair], dail: Sequence[tuple[str | None, str | None, str, bool]] +) -> list[tuple[TruePair, str]]: + return [(p, judge(p, dail)) for p in pairs] diff --git a/experiments/scoring/identity_truth.json b/experiments/scoring/identity_truth.json new file mode 100644 index 0000000..254ff9c --- /dev/null +++ b/experiments/scoring/identity_truth.json @@ -0,0 +1,30 @@ +{ + "_comment": "P8b-2 (C-213): identity truth per TerraPreserve operator, written by hand from docs/D4_OPERATORS.md at terrapreserve-v1.0, never from DAIL's output. Effects: MOVE (the old and the new address are one object: the old address's mutation rows have no 'after', the new one's no 'before', same resource type), REPLACE (the address is replaced: a row on the named attribute with both 'before' and 'after'), CREATE (an address whose rows all have no 'before'), DELETE (an address whose rows all have no 'after'). resource_type '*' matches any type. 'sentence' is quoted from the catalogue. 'excluded' lists operators that may create or move a resource but whose identity truth the catalogue does not state; their pairs are excluded and counted.", + "version": "identity-truth-1", + "rows": { + "op_f3_db_ingress_peer_safe": {"effects": [{"effect": "CREATE", "resource_type": "*"}], "sentence": "adds a new empty client group and an extra 5432 ingress rule from it; the app rule is kept."}, + "op_f3_db_ingress_peer_unsafe": {"effects": [{"effect": "CREATE", "resource_type": "*"}], "sentence": "adds the same client group and re-points the existing 5432 ingress source to it (B4)."}, + "op_f5_separate_to_inline_rules_safe": {"effects": [{"effect": "DELETE", "resource_type": "aws_security_group_rule"}], "sentence": "moves every separate rule of the DB group into inline blocks with identical ports, protocols and sources, and deletes the rule resources."}, + "op_f5_separate_to_inline_rules_unsafe": {"effects": [{"effect": "DELETE", "resource_type": "aws_security_group_rule"}], "sentence": "the same restructuring, but the 5432 ingress rule is not carried into the inline form (B3)."}, + "op_f5_restructure_moved_safe": {"effects": [{"effect": "MOVE", "resource_type": "aws_subnet"}], "sentence": "renames the application subnet with a `moved` block and updates its references."}, + "op_f5_db_rule_moved_safe": {"effects": [{"effect": "MOVE", "resource_type": "aws_security_group_rule"}], "sentence": "renames the DB group's 5432 ingress rule resource with a `moved` block; every attribute is identical."}, + "op_f5_db_rule_moved_unsafe": {"effects": [{"effect": "MOVE", "resource_type": "aws_security_group_rule"}, {"effect": "CREATE", "resource_type": "aws_security_group"}], "sentence": "the same rename, but the recreated rule points to a newly added group outside `SG(S)` (B4)."}, + "op_f5_force_replacement_safe": {"effects": [{"effect": "REPLACE", "resource_type": "aws_instance", "attribute": "ami"}], "sentence": "changes `aws_instance.ami`, which forces replacement."}, + "op_f5_db_sg_replacement_safe": {"effects": [{"effect": "REPLACE", "resource_type": "aws_security_group", "attribute": "name"}], "sentence": "changes the DB group `name`, which forces replacement of the group; its rules are unchanged."}, + "op_f5_db_sg_replacement_unsafe": {"effects": [{"effect": "REPLACE", "resource_type": "aws_security_group", "attribute": "name"}], "sentence": "the same replacement, and the recreated 5432 ingress carries port 5433 (B3)."}, + "op_f6_add_monitoring_sg_safe": {"effects": [{"effect": "CREATE", "resource_type": "*"}], "sentence": "adds a monitoring security group with no ingress and attaches it to `S`."}, + "op_f6_add_monitoring_sg_unsafe": {"effects": [{"effect": "CREATE", "resource_type": "*"}], "sentence": "adds the group with an ingress from `0.0.0.0/0` to 5432."}, + "op_f6_add_bastion_host_safe": {"effects": [{"effect": "CREATE", "resource_type": "*"}], "sentence": "adds a bastion `aws_instance` with no security group changes."}, + "op_f6_add_bastion_host_unsafe": {"effects": [{"effect": "CREATE", "resource_type": "*"}], "sentence": "adds a bastion instance with a group that allows SSH from `0.0.0.0/0`."}, + "op_f6_add_read_replica_safe": {"effects": [{"effect": "CREATE", "resource_type": "*"}], "sentence": "adds a private read-replica `aws_db_instance` (`publicly_accessible = false`)."}, + "op_f6_add_read_replica_unsafe": {"effects": [{"effect": "CREATE", "resource_type": "*"}], "sentence": "adds a read replica with `publicly_accessible = true`."}, + "op_f7_db_ingress_source_unsafe": {"effects": [{"effect": "CREATE", "resource_type": "aws_security_group"}], "sentence": "re-points the DB group's 5432 ingress source to a newly added shared group outside `SG(S)` (B4)."} + }, + "excluded": { + "op_f5_count_each_safe": "adds or removes `count = 1` on `aws_instance`: the catalogue does not state whether the instance keeps its identity (an implicit count move) or is replaced.", + "op_f1_storage_expand_unsafe": "\"plus a new ingress from `0.0.0.0/0` to 5432 on the DB group\": the catalogue does not say whether the ingress is a new resource.", + "op_f1_vpc_dns_toggle_unsafe": "\"plus a new ingress from `0.0.0.0/0` to 22 on the app group\": the catalogue does not say whether the ingress is a new resource.", + "op_f4_repair_db_port_open_ssh_unsafe": "\"then opens port 22 to `0.0.0.0/0` on the app group\": the catalogue does not say whether a new resource is created.", + "op_f4_repair_rds_public_open_db_unsafe": "\"then opens 5432 to `0.0.0.0/0` on the DB group\": the catalogue does not say whether a new resource is created." + } +} diff --git a/experiments/scoring/labels.py b/experiments/scoring/labels.py new file mode 100644 index 0000000..1bba454 --- /dev/null +++ b/experiments/scoring/labels.py @@ -0,0 +1,139 @@ +"""The TerraPreserve dev labels (P8b; D5-11; TerraPreserve read-only, ADR-010, ADR-013). + +Read for scoring only. The files are the ones ``experiments.loader`` verified for execution: the +manifest's SHA-256 must be the pin and every scenario.json must have the manifest's SHA-256, or +``LoaderError`` is raised. The test split is refused. + +Per scenario: ``truth[k]`` is ``labels[k].invariant_truth`` (the oracle's result per invariant at +step k), ``oracle_affected[k]`` is ``labels[k].oracle_affected``, ``mutations`` are the +``mutation_record`` rows, ``gt_class`` the scenario class of D5-11; the tier and the cluster come +from ``source.tier`` (``real_world``, else ``controlled``) and from ``template_id`` (controlled) or +``source.repo_lineage_id`` (real-world), as C-214 defines the clusters. ``declared[k]`` is the set +of resource addresses (``type.name``) that the ``.tf`` files of step k declare with a top-level +``resource`` block: the identity truth uses it to tell a created or deleted resource from one that +only gains or loses an attribute. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Any + +from experiments.loader import DEV_SPLIT, MANIFEST_SHA256, REFUSED_SPLIT, LoaderError, sha256_hex + + +@dataclass(frozen=True) +class MutationRow: + step: int + operator: str + target_address: str + attribute: str + before: str | None + after: str | None + + +@dataclass(frozen=True) +class ScenarioLabels: + scenario_id: str + family: str + kind: str + target: str | None + tier: str + cluster: str + gt_class: str | None + truth: tuple[Mapping[str, str], ...] + oracle_affected: tuple[frozenset[str], ...] + mutations: tuple[MutationRow, ...] + declared: tuple[frozenset[str], ...] = () + + +_RESOURCE = re.compile(r'^resource\s+"([^"]+)"\s+"([^"]+)"', re.MULTILINE) + + +def declared_resources(directory: Path) -> frozenset[str]: + """The ``type.name`` of every top-level ``resource`` block in the step's ``.tf`` files.""" + found: set[str] = set() + for file in sorted(directory.rglob("*.tf")): + for kind, name in _RESOURCE.findall(file.read_text(encoding="utf-8")): + found.add(f"{kind}.{name}") + return frozenset(found) + + +def _scenario(sid: str, doc: Mapping[str, Any], base: Path | None = None) -> ScenarioLabels: + rows = sorted(doc["labels"], key=lambda r: r["step"]) + if [r["step"] for r in rows] != list(range(len(doc["steps"]))): + raise LoaderError(f"{sid}: labels do not cover steps 0..n") + source = doc.get("source") or {} + real = source.get("tier") == "real_world" + cluster = source.get("repo_lineage_id") if real else doc.get("template_id") + if not isinstance(cluster, str): + cluster = doc.get("template_id") or source.get("repo_lineage_id") or sid + target = doc["target"] + return ScenarioLabels( + scenario_id=sid, + family=doc["family"], + kind=target["kind"], + target=target.get("invariant_id"), + tier="real_world" if real else "controlled", + cluster=str(cluster), + gt_class=doc.get("gt_class"), + truth=tuple(MappingProxyType(dict(r["invariant_truth"])) for r in rows), + oracle_affected=tuple(frozenset(r.get("oracle_affected") or ()) for r in rows), + mutations=tuple( + MutationRow( + int(m["step"]), + m["operator"], + m["target_address"], + m["attribute"], + m.get("before"), + m.get("after"), + ) + for m in doc.get("mutation_record") or () + ), + declared=( + () + if base is None + else tuple( + declared_resources(base / step["terraform_dir"]) + for step in sorted(doc["steps"], key=lambda st: st["step"]) + ) + ), + ) + + +def load_labels( + root: Path, + scenario_ids: Iterable[str] | None = None, + *, + split: str = DEV_SPLIT, + manifest_sha256: str = MANIFEST_SHA256, + roles: Iterable[str] = ("target",), +) -> dict[str, ScenarioLabels]: + """The labels of the wanted dev scenarios (all targets by default), hash-checked.""" + if split == REFUSED_SPLIT or split != DEV_SPLIT: + raise LoaderError(f"split {split!r} is refused: scoring reads the dev split only") + path = Path(root) / "scenarios" / "V1_MANIFEST.json" + if not path.is_file(): + raise LoaderError(f"no TerraPreserve manifest at {path}") + raw = path.read_bytes() + if sha256_hex(raw) != manifest_sha256: + raise LoaderError("the manifest sha256 is not the pin") + wanted = None if scenario_ids is None else set(scenario_ids) + found: dict[str, ScenarioLabels] = {} + for entry in json.loads(raw)[DEV_SPLIT]: + sid = entry["scenario_id"] + if entry["v1_role"] not in set(roles) or (wanted is not None and sid not in wanted): + continue + file = Path(root) / "scenarios" / DEV_SPLIT / sid / "scenario.json" + data = file.read_bytes() + if sha256_hex(data) != entry["scenario_json_sha256"]: + raise LoaderError(f"{sid}: scenario.json sha256 is not the manifest's") + found[sid] = _scenario(sid, json.loads(data), file.parent) + if wanted is not None and set(found) != wanted: + raise LoaderError(f"no labels for {sorted(wanted - set(found))}") + return dict(sorted(found.items())) diff --git a/experiments/scoring/selection.py b/experiments/scoring/selection.py new file mode 100644 index 0000000..2d9c984 --- /dev/null +++ b/experiments/scoring/selection.py @@ -0,0 +1,28 @@ +"""The trials re-run for the reproducibility check (P8b Step 4). + +"re-run 20 trials (one per family x class, chosen by lowest scenario id, plus the 4 F8 with most +steps)". The class is the scenario's ``gt_class``, so the choice lives in scoring. The 4 F8 with +most steps are taken in order of steps (most first), then scenario id, among the F8 scenarios not +chosen already. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +from experiments.scoring.labels import ScenarioLabels + + +def reproducibility_selection( + scored: Mapping[str, ScenarioLabels], *, extra_f8: int = 4 +) -> list[str]: + chosen: dict[tuple[str, str], str] = {} + for sid in sorted(scored): + s = scored[sid] + chosen.setdefault((s.family, str(s.gt_class)), sid) + picked = set(chosen.values()) + f8 = sorted( + (sid for sid, s in scored.items() if s.family == "F8" and sid not in picked), + key=lambda sid: (-len(scored[sid].truth), sid), + ) + return sorted(picked | set(f8[:extra_f8])) diff --git a/experiments/scoring/truth.py b/experiments/scoring/truth.py new file mode 100644 index 0000000..a2d498a --- /dev/null +++ b/experiments/scoring/truth.py @@ -0,0 +1,48 @@ +"""The truth of one transition (P8b, metrics-draft-1, Section 4 "Truth per transition"; D5-11). + +For a transition to step i (i >= 1) with the open targets the harness gave it (C-196): + +* regression: an invariant PASS in T_{i-1} and FAIL in T_i; +* unsafe: some invariant FAIL in T_i other than an open target; on the last step there is no open + target, so the caller's open targets are ignored there; +* safe: not unsafe; +* changed (Delta): the invariants whose truth differs between T_{i-1} and T_i. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass + +from experiments.scoring.labels import ScenarioLabels + +PASS, FAIL = "PASS", "FAIL" + + +@dataclass(frozen=True) +class TransitionTruth: + regression: frozenset[str] + failing: frozenset[str] + unsafe: bool + changed: frozenset[str] + affected: frozenset[str] + last: bool + all_pass: bool + + +def transition_truth( + scenario: ScenarioLabels, step: int, open_targets: Iterable[str] +) -> TransitionTruth: + before, after = scenario.truth[step - 1], scenario.truth[step] + last = step == len(scenario.truth) - 1 + opened = frozenset() if last else frozenset(open_targets) + failing = frozenset(i for i, v in after.items() if v == FAIL) - opened + return TransitionTruth( + regression=frozenset(i for i, v in after.items() if v == FAIL and before.get(i) == PASS), + failing=failing, + unsafe=bool(failing), + changed=frozenset(i for i in set(before) | set(after) if before.get(i) != after.get(i)), + affected=scenario.oracle_affected[step], + last=last, + all_pass=all(v == PASS for v in after.values()), + ) diff --git a/tests/contract/test_experiment_blinding.py b/tests/contract/test_experiment_blinding.py index b558531..b46632f 100644 --- a/tests/contract/test_experiment_blinding.py +++ b/tests/contract/test_experiment_blinding.py @@ -7,11 +7,15 @@ mutation_record, intent, variable_values_alt or anything else. A contract test fails if any module used by trial execution reads one of those fields (for example by parsing the sources with ast)." -Every module of ``experiments/`` is used by trial execution in P8a (scoring is P8b, and a scoring -module would be listed here by name as the one exception). The check parses each source with ``ast`` -and reports any use of a forbidden field name: a string constant that is not a docstring (a key, a -``get`` or ``getattr`` argument, a set or list member), an attribute, a name or a keyword argument. -A self-test feeds sources that read each field, so a checker that can no longer fail is caught. +P8b (work package Step 2): ``experiments/scoring/`` is the only package that reads label fields. +(a) No other module of ``experiments/`` names one: neither the execution modules (the top-level +modules of ``experiments/``) nor ``experiments/metrics/`` and ``experiments/report/``, which consume +scoring's output. (b) No execution module imports ``experiments.scoring``, ``experiments.metrics`` +or ``experiments.report``, so trial execution cannot reach a label even indirectly. The check parses +each source with ``ast`` and reports any use of a forbidden field name: a string constant that is +not a docstring (a key, a ``get`` or ``getattr`` argument, a set or list member), an attribute, a +name or a keyword argument. A self-test feeds sources that read each field, so a checker that can no +longer fail is caught. """ from __future__ import annotations @@ -25,8 +29,9 @@ FORBIDDEN = frozenset( {"labels", "gt_class", "oracle", "mutation_record", "intent", "variable_values_alt"} ) -# P8b's scoring module(s) will be the only exception; none exists in P8a. -SCORING_MODULES: frozenset[str] = frozenset() +# P8b: the only package that reads label fields. +SCORING_PACKAGE = EXPERIMENTS / "scoring" +ANALYSIS_PACKAGES = ("experiments.scoring", "experiments.metrics", "experiments.report") def _docstrings(tree: ast.AST) -> set[int]: @@ -62,25 +67,66 @@ def label_reads(source: str) -> list[str]: return found -def execution_modules() -> list[Path]: +def non_scoring_modules() -> list[Path]: + """Every module of experiments/ outside experiments/scoring/ (check (a)).""" return sorted( p for p in EXPERIMENTS.rglob("*.py") - if "__pycache__" not in p.parts and p.stem not in SCORING_MODULES + if "__pycache__" not in p.parts and SCORING_PACKAGE not in p.parents ) +def execution_modules() -> list[Path]: + """The modules trial execution runs: the top-level modules of experiments/ (check (b)).""" + return sorted(EXPERIMENTS.glob("*.py")) + + +def analysis_imports(source: str) -> list[str]: + """Every import of experiments.scoring, experiments.metrics or experiments.report.""" + found: list[str] = [] + for node in ast.walk(ast.parse(source)): + names: list[str] = [] + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom): + if node.level: + names = [f"experiments.{node.module or ''}"] + [ + f"experiments.{alias.name}" for alias in node.names + ] + else: + base = node.module or "" + names = [base] + [f"{base}.{alias.name}" for alias in node.names] + for name in names: + if any(name == p or name.startswith(p + ".") for p in ANALYSIS_PACKAGES): + found.append(f"{node.lineno}: {name}") + return found + + class TestBlinding(unittest.TestCase): - def test_no_execution_module_reads_a_label_field(self) -> None: - modules = execution_modules() + def test_no_module_outside_scoring_reads_a_label_field(self) -> None: + modules = non_scoring_modules() self.assertTrue(modules, "experiments/ has no module to check") self.assertIn(EXPERIMENTS / "loader.py", modules) + self.assertIn(EXPERIMENTS / "metrics" / "extractor.py", modules) + self.assertIn(EXPERIMENTS / "report" / "build.py", modules) for path in modules: with self.subTest(module=path.relative_to(REPO_ROOT).as_posix()): self.assertEqual(label_reads(path.read_text(encoding="utf-8")), []) - def test_no_scoring_module_exists_yet(self) -> None: - self.assertEqual(SCORING_MODULES, frozenset()) + def test_the_scoring_package_is_the_one_exception(self) -> None: + # P8b replaces P8a's "no scoring module exists yet": the scoring package exists and reads + # labels; it is the only part of experiments/ the check (a) skips + self.assertTrue((SCORING_PACKAGE / "labels.py").is_file()) + reads = label_reads((SCORING_PACKAGE / "labels.py").read_text(encoding="utf-8")) + self.assertTrue(reads, "the scoring package should be where labels are read") + + def test_execution_never_imports_scoring_metrics_or_report(self) -> None: + modules = execution_modules() + self.assertIn(EXPERIMENTS / "trial.py", modules) + self.assertIn(EXPERIMENTS / "__main__.py", modules) + for path in modules: + with self.subTest(module=path.relative_to(REPO_ROOT).as_posix()): + self.assertEqual(analysis_imports(path.read_text(encoding="utf-8")), []) class TestTheCheckerCanFail(unittest.TestCase): @@ -99,6 +145,18 @@ def test_each_kind_of_read_is_found(self) -> None: with self.subTest(source=source.strip()): self.assertNotEqual(label_reads(source), []) + def test_each_kind_of_analysis_import_is_found(self) -> None: + for source in ( + "import experiments.scoring\n", + "from experiments.metrics import extractor\n", + "from experiments.report.build import build_report\n", + "from .scoring import labels\n", + "from . import metrics\n", + ): + with self.subTest(source=source.strip()): + self.assertNotEqual(analysis_imports(source), []) + self.assertEqual(analysis_imports("from experiments.loader import load\n"), []) + def test_docstrings_and_allowed_fields_are_not_findings(self) -> None: source = ( '"""This module never reads labels, gt_class or the oracle."""\n' diff --git a/tests/metrics_builders.py b/tests/metrics_builders.py new file mode 100644 index 0000000..e276c39 --- /dev/null +++ b/tests/metrics_builders.py @@ -0,0 +1,328 @@ +"""A hand-built record set for the P8b metric tests. Not a test module. + +Two invariants, S = INV-SEC-001 and F = INV-FUNC-001. Four scenarios: + +* X F8 repair (target S), controlled, cluster tpl-1, steps 0..2: + T0 {S FAIL, F PASS} T1 {S FAIL, F PASS} T2 {S PASS, F PASS}; affected O1 {F}, O2 {S}. +* Y F5 repair (target S), controlled, cluster tpl-2, steps 0..1: + T0 {S FAIL, F PASS} T1 {S PASS, F FAIL}; affected O1 {S, F}. Step 1 renames a rule with a moved + block and replaces a security group (mutation rows below). +* Z F1 change_request, real-world, cluster lin-1, steps 0..1: + T0 {S PASS, F PASS} T1 {S PASS, F PASS}; affected O1 {}. Step 1 adds a monitoring group. +* W F8 repair (target S), controlled, cluster tpl-1, steps 0..2: + T0 {S FAIL, F PASS} T1 {S FAIL, F FAIL} T2 {S PASS, F FAIL}; affected O1 {F}, O2 {S}. + Step 1 uses op_f5_count_each_safe, which the identity table excludes. + +DAIL's records, per condition (decision; work rows as invariant:origin:reason:result; gates are two +PASS rows in every evaluated transition): + +C X1 PROMOTE open[S] S:condition:NOT_PROTECTED_IN_PARENT:FAIL F:impact:PASS impact {F} + X2 PROMOTE S:impact:PASS (F carried) impact {S} + Y1 PROMOTE S:impact:PASS (F carried) impact {S} + Z1 RETRY F:impact:UNKNOWN (S carried) impact {F} + W1 REJECT open[S] S:condition:NOT_PROTECTED_IN_PARENT:FAIL F:impact:FAIL impact {F} + W2 TRUNCATED +B X1 PROMOTE S:condition:CONDITION_SCOPE:FAIL F:impact:PASS + X2 PROMOTE S:impact:PASS F:condition:CONDITION_SCOPE:PASS + Y1 REJECT S:impact:PASS F:condition:CONDITION_SCOPE:FAIL + Z1 PROMOTE S:condition:CONDITION_SCOPE:PASS F:impact:PASS + W1 REJECT S:condition:CONDITION_SCOPE:FAIL F:impact:FAIL + W2 TRUNCATED +A X1 X2 Y1 Z1 W1 W2 all PROMOTE; S:condition (X1, W1) or S:impact (X2, Y1, W2); Z1 gates only +C' as C, except Y1 also has F:impact:PASS (work 4), and its identity pairs differ. + +Identity pairs: Y1 under C: rule r5432 -> r5432_v2 paired (MODIFIED, moved), group db -> db +MODIFIED; under C': r5432 DELETED and r5432_v2 CREATED, group db REPLACED. Z1 under C: mon +CREATED; under C': mon UNCERTAIN. B's S0 results: as T0, except Z's F is UNKNOWN. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from types import MappingProxyType + +from experiments.metrics.model import Transition, WorkRow +from experiments.scoring.labels import MutationRow, ScenarioLabels + +S, F = "INV-SEC-001", "INV-FUNC-001" +P, X_ = "PASS", "FAIL" + + +def _t(**truth: str) -> Mapping[str, str]: + return MappingProxyType({S: truth["s"], F: truth["f"]}) + + +def _d(*addresses: str) -> frozenset[str]: + return frozenset(addresses) + + +BASE = ("aws_instance.app", "aws_security_group.db") + + +def scenarios() -> dict[str, ScenarioLabels]: + rename = ( + MutationRow( + 1, + "op_f5_db_rule_moved_safe", + "aws_security_group_rule.r5432", + "from_port", + "5432", + None, + ), + MutationRow( + 1, + "op_f5_db_rule_moved_safe", + "aws_security_group_rule.r5432_v2", + "from_port", + None, + "5432", + ), + MutationRow( + 1, "op_f5_db_sg_replacement_safe", "aws_security_group.db", "name", '"db"', '"db-v2"' + ), + ) + # Z: the monitoring group is new; the existing group db only gains an attribute (no CREATE) + z_rows = ( + MutationRow( + 1, "op_f6_add_monitoring_sg_safe", "aws_security_group.mon", "name", None, '"mon"' + ), + MutationRow( + 1, "op_f6_add_monitoring_sg_safe", "aws_security_group.db", "description", None, '"x"' + ), + ) + return { + "X": ScenarioLabels( + "X", + "F8", + "repair", + S, + "controlled", + "tpl-1", + "safe", + (_t(s=X_, f=P), _t(s=X_, f=P), _t(s=P, f=P)), + (frozenset(), frozenset({F}), frozenset({S})), + (), + (_d(*BASE), _d(*BASE), _d(*BASE)), + ), + "Y": ScenarioLabels( + "Y", + "F5", + "repair", + S, + "controlled", + "tpl-2", + "unsafe", + (_t(s=X_, f=P), _t(s=P, f=X_)), + (frozenset(), frozenset({S, F})), + rename, + ( + _d(*BASE, "aws_security_group_rule.r5432"), + _d(*BASE, "aws_security_group_rule.r5432_v2"), + ), + ), + "Z": ScenarioLabels( + "Z", + "F1", + "change_request", + None, + "real_world", + "lin-1", + "safe", + (_t(s=P, f=P), _t(s=P, f=P)), + (frozenset(), frozenset()), + z_rows, + (_d(*BASE), _d(*BASE, "aws_security_group.mon")), + ), + "W": ScenarioLabels( + "W", + "F8", + "repair", + S, + "controlled", + "tpl-1", + "unsafe", + (_t(s=X_, f=P), _t(s=X_, f=X_), _t(s=P, f=X_)), + (frozenset(), frozenset({F}), frozenset({S})), + (MutationRow(1, "op_f5_count_each_safe", "aws_instance.app", "count", None, "1"),), + (_d(*BASE), _d(*BASE), _d(*BASE)), + ), + } + + +GATES = ( + WorkRow("STRUCTURAL", "STRUCTURAL", "PASS", True, "gate", None), + WorkRow("CONSISTENCY", "STRUCTURAL", "PASS", True, "gate", None), +) + + +def inv(name: str, origin: str, result: str, reason: str | None = None) -> WorkRow: + version = 2 if name == S else 1 + chosen = reason or ("CONDITION_SCOPE" if origin == "condition" else "SCOPE_RESOURCE_CHANGED") + return WorkRow(f"INVARIANT:{name}:{version}", "INVARIANT", result, True, origin, chosen) + + +def target(name: str) -> str: + return f"INVARIANT:{name}:{2 if name == S else 1}" + + +def tr( + condition: str, + scenario: str, + step: int, + decision: str | None, + work: Iterable[WorkRow] = (), + *, + carried: Iterable[str] = (), + open_targets: Iterable[str] = (), + affected: Iterable[str] = (), + pairs: Iterable[tuple[str | None, str | None, str, bool]] = (), + required: Iterable[str] | None = None, +) -> Transition: + open_targets = tuple(open_targets) + rows = tuple(work) + if decision is None: + return Transition( + f"{condition}-{scenario}", + scenario, + condition, + step, + "TRUNCATED", + None, + (), + (), + (), + frozenset(), + (), + None, + ) + run = [w for w in rows if w.target_type == "INVARIANT"] + wanted = [target(n) for n in required] if required is not None else [w.target_id for w in run] + req = [("CONSISTENCY", "VERIFIED"), ("STRUCTURAL", "VERIFIED")] + # as DAIL records it: VERIFIED only when the result proves the check (a PASS, or a FAIL on an + # open target, C-196); an unproven check has no source + results = {w.target_id: w.result for w in rows} + opened = {target(n) for n in open_targets} + req += [ + ( + t, + ( + "VERIFIED" + if results.get(t) == "PASS" or (results.get(t) == "FAIL" and t in opened) + else None + ), + ) + for t in wanted + ] + req += [(target(n), "CARRIED_FORWARD") for n in carried] + return Transition( + f"{condition}-{scenario}", + scenario, + condition, + step, + "EVALUATED", + decision, + tuple(open_targets), + tuple(sorted(req)), + GATES + rows, + frozenset(affected), + tuple(pairs), + 0.5, + ) + + +NP = "NOT_PROTECTED_IN_PARENT" +Y_PAIRS_C = ( + ("aws_security_group_rule.r5432", "aws_security_group_rule.r5432_v2", "MODIFIED", True), + ("aws_security_group.db", "aws_security_group.db", "MODIFIED", False), +) +Y_PAIRS_CP = ( + ("aws_security_group_rule.r5432", None, "DELETED", False), + (None, "aws_security_group_rule.r5432_v2", "CREATED", False), + ("aws_security_group.db", "aws_security_group.db", "REPLACED", False), +) + + +def transitions() -> list[Transition]: + c_like = [] + for cond in ("C", "C_PRIME"): + y_work = [inv(S, "impact", P)] + ([inv(F, "impact", P)] if cond == "C_PRIME" else []) + c_like += [ + tr( + cond, + "X", + 1, + "PROMOTE", + [inv(S, "condition", X_, NP), inv(F, "impact", P)], + open_targets=[S], + affected=[F], + ), + tr(cond, "X", 2, "PROMOTE", [inv(S, "impact", P)], carried=[F], affected=[S]), + tr( + cond, + "Y", + 1, + "PROMOTE", + y_work, + carried=[] if cond == "C_PRIME" else [F], + affected=[S], + pairs=Y_PAIRS_CP if cond == "C_PRIME" else Y_PAIRS_C, + ), + tr( + cond, + "Z", + 1, + "RETRY", + [inv(F, "impact", "UNKNOWN")], + carried=[S], + affected=[F], + pairs=[ + ( + None, + "aws_security_group.mon", + "UNCERTAIN" if cond == "C_PRIME" else "CREATED", + False, + ) + ], + ), + tr( + cond, + "W", + 1, + "REJECT", + [inv(S, "condition", X_, NP), inv(F, "impact", X_)], + open_targets=[S], + affected=[F], + ), + tr(cond, "W", 2, None), + ] + b = [ + tr( + "B", "X", 1, "PROMOTE", [inv(S, "condition", X_), inv(F, "impact", P)], open_targets=[S] + ), + tr("B", "X", 2, "PROMOTE", [inv(S, "impact", P), inv(F, "condition", P)]), + tr("B", "Y", 1, "REJECT", [inv(S, "impact", P), inv(F, "condition", X_)]), + tr("B", "Z", 1, "PROMOTE", [inv(S, "condition", P), inv(F, "impact", P)]), + tr( + "B", "W", 1, "REJECT", [inv(S, "condition", X_), inv(F, "impact", X_)], open_targets=[S] + ), + tr("B", "W", 2, None), + ] + a = [ + tr("A", "X", 1, "PROMOTE", [inv(S, "condition", X_)], open_targets=[S]), + tr("A", "X", 2, "PROMOTE", [inv(S, "impact", P)]), + tr("A", "Y", 1, "PROMOTE", [inv(S, "impact", P)]), + tr("A", "Z", 1, "PROMOTE", []), + tr("A", "W", 1, "PROMOTE", [inv(S, "condition", X_)], open_targets=[S]), + tr("A", "W", 2, "PROMOTE", [inv(S, "impact", P)]), + ] + return a + b + c_like + + +def baselines() -> dict[tuple[str, str], Mapping[str, str]]: + """S0 results of DAIL under B, by (scenario, condition).""" + return { + ("X", "B"): {S: X_, F: P}, + ("Y", "B"): {S: X_, F: P}, + ("Z", "B"): {S: P, F: "UNKNOWN"}, + ("W", "B"): {S: X_, F: P}, + } diff --git a/tests/unit/test_p8b2_baseline_v1_1.py b/tests/unit/test_p8b2_baseline_v1_1.py new file mode 100644 index 0000000..dc5dcad --- /dev/null +++ b/tests/unit/test_p8b2_baseline_v1_1.py @@ -0,0 +1,200 @@ +"""Baseline items under registry v1.1 (P8b-2; C-216; Doc 08 §16; C-148; C-150, C-178). + +Expected values come from the decided text (C-216): "Baseline items (C-148) use the definition that +the run's definitions hold for the invariant they belong to (INV-SEC-001 for security groups and +rules, INV-SEC-003 for database instances). If more than one version is present, the highest is +used. The item's evidence records that id and version, and the §3.3 clause it applies is that +definition's registry (version 1 -> registry-v1.0, version 2 -> registry-v1.1) ... If no definition +is present, the item stays UNSUPPORTED invariant.unknown, as now." + +The v1.1 §3.3 clause (C-178): an indeterminate rule whose resolved IPv4 sources cover 0.0.0.0/0 is +``possible`` -> UNKNOWN; registry v1.0 has no such alternative -> PASS. +""" + +from __future__ import annotations + +import dataclasses +import functools +import shutil +import tempfile +import unittest +from pathlib import Path + +from core.application.orchestrator import ResultKind +from core.domain.enums import VerificationResult +from core.domain.registry_v1 import registry_v1_invariants +from core.domain.registry_v1_1 import REGISTRY_VERSION_V1_1, registry_v1_1_invariants +from core.domain.verification import VerificationReason +from core.promotion.policy import dail_policy +from core.verification.baseline import verify_baseline +from tests.identity_builders import NOW +from tests.pipeline_builders import inputs_for +from tests.promotion_orchestrator_builders import ( + DESTINATION, + PATCH_SAFE_DIR, + SOURCE, + OrchestratorCase, + bindings, + context_for, +) +from tests.security_builders import VARS, inline, sg + +P, F, U, S = ( + VerificationResult.PASS, + VerificationResult.FAIL, + VerificationResult.UNKNOWN, + VerificationResult.UNSUPPORTED, +) +R = VerificationReason +OLD = sg("db") +NEW = "aws_security_group.new" +RULE = "aws_security_group_rule.r" +V1_1 = registry_v1_1_invariants(NOW, source=SOURCE, destination=DESTINATION) +V1_0 = registry_v1_invariants(NOW, source=SOURCE, destination=DESTINATION) + +CLOSED = OLD + sg("new", inline("ingress", "tcp", 443, 443, ["10.0.0.0/8"])) +OPEN_22 = OLD + sg("new", inline("ingress", "tcp", 22, 22, ["0.0.0.0/0"])) +UNRESOLVED_PORT_RULE = ( + VARS + + OLD + + sg("new") + + 'resource "aws_security_group_rule" "r" {\n type = "ingress"\n' + + ' security_group_id = aws_security_group.new.id\n protocol = "tcp"\n' + + ' from_port = var.u\n to_port = 22\n cidr_blocks = ["0.0.0.0/0"]\n}\n' +) + + +def item(address: str, text: str, definitions): # type: ignore[no-untyped-def] + inputs = dataclasses.replace(inputs_for(text, OLD), invariants=tuple(definitions)) + return verify_baseline(address, inputs) + + +class TestUnderRegistryV1_1(unittest.TestCase): + def test_a_closed_created_group_passes_and_names_version_2(self) -> None: + outcome = item(NEW, CLOSED, V1_1) + self.assertEqual((outcome.result, outcome.reason), (P, None)) + self.assertEqual(outcome.observations["invariant_id"], "INV-SEC-001") + self.assertEqual(outcome.observations["invariant_version"], 2) + self.assertEqual(outcome.observations["registry"], "registry-v1.1") + self.assertEqual( + (outcome.verifier_id, outcome.verifier_version), ("dail.verify.public_ingress", "1") + ) + + def test_a_created_group_open_on_ssh_fails(self) -> None: + outcome = item(NEW, OPEN_22, V1_1) + self.assertEqual((outcome.result, outcome.reason), (F, R.PREDICATE_VIOLATED)) + + def test_a_created_rule_with_an_unresolved_port_and_a_public_source_is_unknown(self) -> None: + outcome = item(RULE, UNRESOLVED_PORT_RULE, V1_1) + self.assertEqual((outcome.result, outcome.reason), (U, R.RULE_INDETERMINATE)) + + def test_the_highest_version_is_used_when_both_are_present(self) -> None: + both = [*V1_0, *[d for d in V1_1 if d.version == 2]] + outcome = item(RULE, UNRESOLVED_PORT_RULE, both) + self.assertEqual(outcome.observations["invariant_version"], 2) + self.assertEqual(outcome.result, U) + + def test_a_created_database_names_inv_sec_003(self) -> None: + text = ( + OLD + + 'resource "aws_db_instance" "extra" {\n identifier = "e"\n' + + " publicly_accessible = false\n}\n" + ) + outcome = item("aws_db_instance.extra", text, V1_1) + self.assertEqual(outcome.result, P) + self.assertEqual( + (outcome.observations["invariant_id"], outcome.observations["invariant_version"]), + ("INV-SEC-003", 1), + ) + + def test_without_the_definition_it_stays_unsupported(self) -> None: + others = [d for d in V1_1 if d.invariant_id != "INV-SEC-001"] + outcome = item(NEW, CLOSED, others) + self.assertEqual((outcome.result, outcome.reason), (S, R.INVARIANT_UNKNOWN)) + + +class TestUnderRegistryV1_0(unittest.TestCase): + def test_the_same_three_give_the_v1_0_results(self) -> None: + self.assertEqual(item(NEW, CLOSED, V1_0).result, P) + self.assertEqual(item(NEW, OPEN_22, V1_0).result, F) + outcome = item(RULE, UNRESOLVED_PORT_RULE, V1_0) + self.assertEqual(outcome.result, P) # registry v1.0 has no 'possible' for this rule + self.assertEqual(outcome.observations["invariant_version"], 1) + self.assertEqual(outcome.observations["registry"], "registry-v1.0") + + +EXTRA_CLOSED = ( + 'resource "aws_security_group" "extra" {\n name = "extra"\n description = "extra group"\n' + " vpc_id = aws_vpc.main.id\n ingress = []\n}\n" +) +EXTRA_OPEN = ( + 'resource "aws_security_group" "extra" {\n name = "extra"\n description = "extra group"\n' + " vpc_id = aws_vpc.main.id\n ingress {\n from_port = 22\n to_port = 22\n" + ' protocol = "tcp"\n cidr_blocks = ["0.0.0.0/0"]\n }\n}\n' +) + + +class TestEndToEndUnderV1_1(OrchestratorCase): + """C under registry v1.1 through the orchestrator: v0 = patch_safe (every invariant + PROTECTED).""" + + def setUp(self) -> None: + super().setUp() + self._dirs = tempfile.TemporaryDirectory(prefix="dail-p8b2-") + self.addCleanup(self._dirs.cleanup) + with self.uow() as u: # version 1 before version 2 (Doc 06 §4.2, C-207 (2)) + for d in V1_0: + if d.invariant_id in ("INV-SEC-001", "INV-SEC-002"): + u.invariants.register_definition(d) + self.v0 = self.baseline( + PATCH_SAFE_DIR, definitions=V1_1, registry_version=REGISTRY_VERSION_V1_1 + ).state + + def candidate(self, name: str, extra: str) -> Path: + target = Path(self._dirs.name) / name + shutil.copytree(PATCH_SAFE_DIR, target, ignore=shutil.ignore_patterns("expected.json")) + (target / "extra.tf").write_text(extra, encoding="utf-8") + return target + + def evaluate_v1_1(self, directory: Path): # type: ignore[no-untyped-def] + request = dataclasses.replace( + self.request(self.v0, directory), + definitions=V1_1, + bindings=bindings(), + context=context_for(1), + ) + return self.evaluate( + request, + policy_factory=functools.partial( + dail_policy, invariant_registry_version=REGISTRY_VERSION_V1_1 + ), + ) + + def test_a_closed_new_group_promotes(self) -> None: + result = self.evaluate_v1_1(self.candidate("closed", EXTRA_CLOSED)) + self.assertIs(result.kind, ResultKind.PROMOTED) + assert result.verification is not None + row = next( + r + for r in result.verification.results + if r.target_id == "RESOURCE:aws_security_group.extra" + ) + self.assertIs(row.result, P) + + def test_a_new_group_open_on_ssh_rejects_on_the_resource_item(self) -> None: + result = self.evaluate_v1_1(self.candidate("open", EXTRA_OPEN)) + self.assertIs(result.kind, ResultKind.REJECTED) + assert result.decision is not None + on_item = { + c.code.value + for c in result.decision.blocking_conditions + if c.target == "RESOURCE:aws_security_group.extra" + } + self.assertEqual(on_item, {"REQUIRED_CHECK_FAILED"}) + self.assertNotIn( + "REQUIRED_UNSUPPORTED", {c.code.value for c in result.decision.blocking_conditions} + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_p8b2_report_guard.py b/tests/unit/test_p8b2_report_guard.py new file mode 100644 index 0000000..69e4302 --- /dev/null +++ b/tests/unit/test_p8b2_report_guard.py @@ -0,0 +1,111 @@ +"""The report guard (P8b-2, work package Change 2; C-216). + +"The dev report gets a table of every non-PASS required check by verifier, result and reason, per +condition (from the records). A test fails if any baseline item in a report has reason +invariant.unknown while the run's definitions hold the invariant. This makes a systemic error +visible instead of hiding it in a rate." +""" + +from __future__ import annotations + +import json +import unittest + +from experiments.metrics.extractor import baseline_guard, extract +from experiments.metrics.model import Transition, WorkRow +from experiments.report.build import build_report +from experiments.report.render import render_markdown +from experiments.scoring.identity import load_identity_table +from tests.fixture_catalog import REPO_ROOT +from tests.metrics_builders import GATES, scenarios + +UNKNOWN_DEF = "invariant.unknown" + + +def transition( + rows: tuple[WorkRow, ...], required: tuple[tuple[str, str | None], ...] +) -> Transition: + return Transition( + "t", "Z", "C", 1, "EVALUATED", "REJECT", (), required, GATES + rows, frozenset(), (), None + ) + + +def baseline_row(address: str, result: str, reason: str | None) -> WorkRow: + return WorkRow( + f"RESOURCE:{address}", "RESOURCE", result, True, "impact", "CREATED_BASELINE", + "dail.verify.public_ingress", reason, + ) # fmt: skip + + +def report_for(rows: tuple[WorkRow, ...], held: list[str]) -> dict: # type: ignore[type-arg] + # as DAIL records it: a check that is not proven (a FAIL or UNSUPPORTED) has no source + required = (("CONSISTENCY", "VERIFIED"), ("STRUCTURAL", "VERIFIED")) + tuple( + (r.target_id, "VERIFIED" if r.result == "PASS" else None) for r in rows + ) + x = extract( + experiment_id="e", + transitions=[transition(rows, required)], + scored=scenarios(), + baselines={}, + identity=load_identity_table(), + ) + definition = { + "experiment_id": "e", + "name": "n", + "version": "1", + "condition_set": ["C"], + "pins": {"verifiers": {"invariants": {i: {} for i in held}}}, + } + return build_report(definition=definition, statuses={"t": "COMPLETED"}, extraction=x) + + +class TestNonPassTable(unittest.TestCase): + def test_every_non_pass_required_check_is_tabled(self) -> None: + rows = ( + baseline_row("aws_security_group.a", "UNSUPPORTED", UNKNOWN_DEF), + baseline_row("aws_security_group.b", "UNSUPPORTED", UNKNOWN_DEF), + baseline_row("aws_security_group_rule.r", "FAIL", "predicate.violated"), + ) + report = report_for(rows, ["INV-SEC-001"]) + table = report["lists"]["non_pass_checks"] + self.assertEqual( + table, + [ + { + "condition": "C", + "kind": "RESOURCE:aws_security_group", + "verifier_id": "dail.verify.public_ingress", + "result": "UNSUPPORTED", + "reason": UNKNOWN_DEF, + "count": 2, + }, + { + "condition": "C", + "kind": "RESOURCE:aws_security_group_rule", + "verifier_id": "dail.verify.public_ingress", + "result": "FAIL", + "reason": "predicate.violated", + "count": 1, + }, + ], + ) + self.assertIn("Non-PASS required checks", render_markdown(report)) + + +class TestGuard(unittest.TestCase): + def test_an_unknown_definition_while_it_is_held_is_flagged(self) -> None: + rows = (baseline_row("aws_security_group.a", "UNSUPPORTED", UNKNOWN_DEF),) + self.assertEqual(len(baseline_guard(report_for(rows, ["INV-SEC-001"]))), 1) + + def test_it_is_not_flagged_when_the_definition_is_absent(self) -> None: + rows = (baseline_row("aws_security_group.a", "UNSUPPORTED", UNKNOWN_DEF),) + self.assertEqual(baseline_guard(report_for(rows, ["INV-SEC-003"])), []) + + def test_the_committed_dev_report_has_no_such_item(self) -> None: + report = json.loads((REPO_ROOT / "docs" / "P8_DEV_PILOT.json").read_text(encoding="utf-8")) + self.assertIn("non_pass_checks", report["lists"]) + self.assertEqual(baseline_guard(report), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_p8b_metrics.py b/tests/unit/test_p8b_metrics.py new file mode 100644 index 0000000..5ffb097 --- /dev/null +++ b/tests/unit/test_p8b_metrics.py @@ -0,0 +1,318 @@ +"""The P8b metrics on a hand-built record set (P8b work package Section 4, metrics-draft-1; C-212 to +C-214). The record set is described in tests/metrics_builders.py; every expected value below was +computed by hand from the Section 4 definitions, never from the extractor's output. + +Truth per transition (Section 4 "Truth per transition"; open target S on X1 and W1): + X1 safe, no regression | X2 safe | Y1 unsafe (F FAIL), regression F | Z1 safe | + W1 unsafe (F FAIL; S is the open target), regression F | W2 unsafe (F FAIL), no regression. +""" + +from __future__ import annotations + +import math +import unittest +from typing import Any + +from experiments.metrics.extractor import EXTRACTOR_VERSION, extract, find, wilson +from experiments.scoring.identity import load_identity_table +from experiments.scoring.truth import transition_truth +from tests.metrics_builders import F, S, baselines, scenarios, transitions + + +def run() -> Any: + return extract( + experiment_id="00000000-0000-4000-8000-0000000000e8", + transitions=transitions(), + scored=scenarios(), + baselines=baselines(), + identity=load_identity_table(), + ) + + +class Case(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.x = run() + + def ratio(self, name: str, condition: str, num: int, den: int, **where: str) -> None: + record = find(self.x, name, condition, **where) + self.assertEqual((record["numerator"], record["denominator"]), (num, den), name) + self.assertAlmostEqual(record["value"], num / den if den else 0.0) + + +class TestTruth(unittest.TestCase): + def test_per_transition_truth(self) -> None: + labels = scenarios() + x1 = transition_truth(labels["X"], 1, (S,)) + self.assertFalse(x1.unsafe) + self.assertEqual(x1.regression, frozenset()) + y1 = transition_truth(labels["Y"], 1, ()) + self.assertTrue(y1.unsafe) + self.assertEqual(y1.regression, frozenset({F})) + self.assertEqual(y1.changed, frozenset({S, F})) + w1 = transition_truth(labels["W"], 1, (S,)) + self.assertTrue(w1.unsafe) + self.assertEqual(w1.failing, frozenset({F})) # S is the open target + w2 = transition_truth(labels["W"], 2, ()) + self.assertTrue(w2.unsafe) + self.assertEqual(w2.regression, frozenset()) + + def test_on_the_last_step_there_is_no_open_target(self) -> None: + # a FAIL of the target on the last step is unsafe even if the caller names it open + labels = scenarios() + self.assertFalse(transition_truth(labels["X"], 1, (S,)).unsafe) + self.assertTrue(transition_truth(labels["W"], 1, ()).unsafe) + + +class TestSafety(Case): + def test_unsafe_promotions(self) -> None: + self.ratio("unsafe_promotions_per_promotion", "C", 1, 3) + self.ratio("unsafe_promotions_per_evaluated", "C", 1, 5) + self.ratio("unsafe_promotions_per_promotion", "B", 0, 3) + self.ratio("unsafe_promotions_per_promotion", "A", 3, 6) + self.ratio("unsafe_promotions_per_evaluated", "A", 3, 6) + + def test_missed_regressions(self) -> None: + self.ratio("regressions", "C", 2, 5) + self.ratio("missed_regressions_per_regression", "C", 1, 2) + self.ratio("missed_regressions_per_regression", "B", 0, 2) + self.ratio("missed_regressions_per_regression", "A", 2, 2) + + def test_false_rejection_rate(self) -> None: + self.ratio("false_rejection_rate", "C", 1, 3) + self.ratio("false_rejection_rate", "B", 0, 3) + self.ratio("false_rejection_rate", "A", 0, 3) + + def test_repair_success_by_stratum(self) -> None: + self.ratio("repair_success", "C", 1, 3, stratum="kind=repair") + self.ratio("repair_success", "C", 0, 1, stratum="kind=change_request") + self.ratio("repair_success", "B", 1, 3, stratum="kind=repair") + self.ratio("repair_success", "B", 1, 1, stratum="kind=change_request") + self.ratio("repair_success", "A", 1, 3, stratum="kind=repair") + + def test_unsafe_promotions_under_c_are_listed_with_their_trace(self) -> None: + listed = self.x.lists["unsafe_promotions_c"] + self.assertEqual([(u["scenario_id"], u["step"]) for u in listed], [("Y", 1)]) + self.assertEqual(listed[0]["failing"], [F]) + self.assertEqual(listed[0]["impact"], [S]) + self.assertIn("INVARIANT:INV-SEC-001:2", [w["target_id"] for w in listed[0]["work"]]) + + +class TestWork(Case): + def total(self, name: str, condition: str, **where: str) -> float: + return find(self.x, name, condition, **where)["value"] + + def test_work_by_kind_under_c(self) -> None: + self.assertEqual(self.total("work_gate", "C"), 10) + self.assertEqual(self.total("work_invariant_impact", "C"), 5) + self.assertEqual(self.total("work_invariant_condition", "C"), 0) + self.assertEqual(self.total("work_invariant_not_protected", "C"), 2) + self.assertEqual(self.total("work_baseline_applicable", "C"), 0) + self.assertEqual(self.total("work_total", "C"), 17) + self.assertAlmostEqual(self.total("work_mean_per_transition", "C"), 3.4) + self.assertEqual(self.total("work_median_per_transition", "C"), 3) + + def test_work_under_b(self) -> None: + self.assertEqual(self.total("work_total", "B"), 20) + self.assertEqual(self.total("work_invariant_condition", "B"), 5) + self.assertEqual(self.total("work_median_per_transition", "B"), 4) + + def test_saving_of_c_against_b_on_the_common_prefix(self) -> None: + record = find(self.x, "work_saving_c_vs_b", "C", scope="common_prefix_B_C") + self.assertEqual((record["numerator"], record["denominator"]), (17, 20)) + self.assertAlmostEqual(record["value"], 1 - 17 / 20) + + +class TestOutcomes(Case): + def test_decision_rates(self) -> None: + self.ratio("promote_rate", "C", 3, 5) + self.ratio("reject_rate", "C", 1, 5) + self.ratio("retry_rate", "C", 1, 5) + self.ratio("escalate_rate", "C", 0, 5) + self.ratio("reject_rate", "B", 2, 5) + + def test_unknown_and_unsupported_per_required_check(self) -> None: + self.ratio("unknown_rate", "C", 1, 20) + self.ratio("unsupported_rate", "C", 0, 20) + self.ratio("unknown_rate", "A", 0, 17) + + +class TestEarlyTermination(Case): + def test_truncated_counts(self) -> None: + for condition, count in (("A", 0), ("B", 1), ("C", 1), ("C_PRIME", 1)): + with self.subTest(condition=condition): + record = find(self.x, "truncated_transitions", condition) + self.assertEqual(record["numerator"], count) + self.assertEqual(record["denominator"], 6) + + def test_metrics_on_the_common_prefix_of_all_four(self) -> None: + self.ratio("unsafe_promotions_per_promotion", "A", 2, 5, scope="common_prefix_all") + self.assertEqual(find(self.x, "work_total", "A", scope="common_prefix_all")["value"], 14) + self.ratio("unsafe_promotions_per_promotion", "C", 1, 3, scope="common_prefix_all") + + +class TestAgreement(Case): + def test_confusion_tables_of_b(self) -> None: + table = self.x.confusion + self.assertEqual(table[S], {("FAIL", "FAIL"): 5, ("PASS", "PASS"): 4}) + self.assertEqual( + table[F], {("PASS", "PASS"): 6, ("FAIL", "FAIL"): 2, ("UNKNOWN", "PASS"): 1} + ) + + def test_every_disagreement_is_listed(self) -> None: + self.assertEqual( + self.x.lists["disagreements"], + [ + { + "scenario_id": "Z", + "step": 0, + "invariant_id": F, + "dail": "UNKNOWN", + "oracle_truth": "PASS", + } + ], + ) + + +class TestImpact(Case): + def test_precision_recall_and_coverage(self) -> None: + for condition in ("C", "C_PRIME"): + with self.subTest(condition=condition): + self.ratio("impact_precision", condition, 4, 5) + self.ratio("impact_recall", condition, 4, 5) + self.ratio("truth_change_coverage", "C", 3, 4) + self.ratio("truth_change_coverage", "C_PRIME", 4, 4) + self.ratio("impact_precision", "C", 3, 3, stratum="family=F8") + + def test_truth_changes_outside_v_are_listed(self) -> None: + self.assertEqual( + self.x.lists["truth_changes_outside_v"], + [{"condition": "C", "scenario_id": "Y", "step": 1, "invariant_id": F}], + ) + + +class TestIdentity(Case): + def test_error_rate_uncertain_share_and_evidence_transfer(self) -> None: + self.ratio("identity_error_rate", "C", 1, 3) + self.ratio("identity_uncertain_share", "C", 0, 3) + self.ratio("identity_error_rate", "C_PRIME", 1, 3) + self.ratio("identity_uncertain_share", "C_PRIME", 1, 3) + self.assertEqual(find(self.x, "evidence_transfer_errors", "C")["value"], 1) + self.assertEqual(find(self.x, "evidence_transfer_errors", "C_PRIME")["value"], 0) + self.assertEqual(find(self.x, "identity_excluded_pairs", "C")["value"], 1) + + def test_identity_errors_are_listed(self) -> None: + listed = {(e["condition"], e["effect"]) for e in self.x.lists["identity_errors"]} + self.assertEqual(listed, {("C", "REPLACE"), ("C_PRIME", "MOVE")}) + + def test_work_difference_on_f5_by_moved_type(self) -> None: + self.assertEqual( + self.x.lists["work_difference_f5"], + { + "aws_security_group_rule": { + "transitions": 1, + "work_c": 3, + "work_c_prime": 4, + "difference": 1, + } + }, + ) + + +class TestRecords(Case): + def test_metric_records_have_the_fields_of_appendix_b(self) -> None: + for record in self.x.metrics: + for key in ( + "metric_id", + "experiment_id", + "trial_id", + "condition_id", + "metric_name", + "numerator", + "denominator", + "value", + "unit", + "extractor_version", + ): + self.assertIn(key, record) + self.assertEqual(record["extractor_version"], "metrics-draft-1") + self.assertEqual(EXTRACTOR_VERSION, "metrics-draft-1") + + def test_the_extractor_is_deterministic(self) -> None: + from experiments.metrics.extractor import dumps_metrics + + self.assertEqual(dumps_metrics(run()), dumps_metrics(run())) + + def test_wilson_interval(self) -> None: + z = 1.959963984540054 + n, k = 3, 1 + p = k / n + centre = (p + z * z / (2 * n)) / (1 + z * z / n) + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n) + low, high = wilson(k, n) + self.assertAlmostEqual(low, centre - half) + self.assertAlmostEqual(high, centre + half) + self.assertEqual(wilson(0, 0), (None, None)) + record = find(self.x, "unsafe_promotions_per_promotion", "C") + self.assertAlmostEqual(record["wilson95"][0], centre - half) + + +class TestEdgeRules(unittest.TestCase): + """Cases the main record set does not contain (added after mutations M2, M6 and M7 survived).""" + + def test_a_target_still_failing_on_the_last_step_is_unsafe_even_if_named_open(self) -> None: + from types import MappingProxyType + + from experiments.scoring.labels import ScenarioLabels + + one = ScenarioLabels( + "V", "F8", "repair", S, "controlled", "tpl-1", "unsafe", + (MappingProxyType({S: "FAIL"}), MappingProxyType({S: "FAIL"})), + (frozenset(), frozenset()), (), (frozenset(), frozenset()), + ) # fmt: skip + self.assertTrue(transition_truth(one, 1, (S,)).unsafe) + + def test_save_is_computed_on_the_common_prefix_only(self) -> None: + # without C's X2: C evaluates X1 Y1 Z1 W1 (W 4+3+3+4 = 14); B on those four: 4 x 4 = 16 + kept = [ + t + for t in transitions() + if not (t.condition == "C" and t.scenario_id == "X" and t.step == 2) + ] + x = extract( + experiment_id="00000000-0000-4000-8000-0000000000e8", + transitions=kept, + scored=scenarios(), + baselines=baselines(), + identity=load_identity_table(), + ) + record = find(x, "work_saving_c_vs_b", "C", scope="common_prefix_B_C") + self.assertEqual((record["numerator"], record["denominator"]), (14, 16)) + + def test_a_carried_check_has_no_result(self) -> None: + # F carried forward, and a stale UNKNOWN row for F in the work list: not an UNKNOWN check + from tests.metrics_builders import inv, tr + + odd = tr( + "C", + "X", + 1, + "PROMOTE", + [inv(F, "impact", "UNKNOWN")], + carried=[F], + required=[], + open_targets=[S], + ) + x = extract( + experiment_id="00000000-0000-4000-8000-0000000000e8", + transitions=[odd], + scored=scenarios(), + baselines={}, + identity=load_identity_table(), + ) + record = find(x, "unknown_rate", "C") + self.assertEqual((record["numerator"], record["denominator"]), (0, 3)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_p8b_scoring_report.py b/tests/unit/test_p8b_scoring_report.py new file mode 100644 index 0000000..c0c2f19 --- /dev/null +++ b/tests/unit/test_p8b_scoring_report.py @@ -0,0 +1,220 @@ +"""Scoring inputs, the identity truth table, records to transitions, and the report (P8b; Doc 13 +§31-§33, §35; C-212 to C-214). + +* labels: only experiments.scoring reads TerraPreserve label fields; it verifies the manifest pin + and every scenario.json hash as the loader does, and refuses the test split. +* identity truth (C-213): "one row per TerraPreserve operator that moves, replaces, deletes or + creates a resource, giving the expected change type at the operator's target address, with the + sentence of D4_OPERATORS.md it comes from. An operator whose identity truth the catalogue does not + state gets no row, and its pairs are excluded and counted." +* Doc 13 §33 "ExperimentReport { experiment_id, experiment_version, trial_count, included_count, + excluded_count, condition_summaries[], primary_metrics[], secondary_metrics[], failures[], + reproducibility_summary, artifact_refs[] }." +* Step 5: "Write docs/P8_DEV_PILOT.md from the generated report only (no hand-typed numbers; a test + checks the file's numbers equal the extractor's JSON)." +""" + +from __future__ import annotations + +import json +import os +import unittest +from datetime import UTC, datetime +from pathlib import Path + +from experiments.definition import Condition, ExperimentSpec +from experiments.loader import LoaderError, load +from experiments.metrics.model import transitions_from_result +from experiments.report.build import REPORT_KEYS, build_report +from experiments.report.render import render_markdown +from experiments.runner import Layout, plan, read_result, run_experiment +from experiments.scoring.identity import load_identity_table, true_pairs +from experiments.scoring.labels import load_labels +from tests.experiment_builders import F2, F8, TempDirs, mini_root +from tests.fixture_catalog import REPO_ROOT +from tests.unit.test_p8b_metrics import run as extraction + +ROOT = os.environ.get("TERRAPRESERVE_ROOT") +SEC1, FUNC1 = "INV-SEC-001", "INV-FUNC-001" + + +def real_labels(scenario: dict[str, object]) -> None: + """Give a mini scenario well-formed labels: every invariant PASS except INV-SEC-001 at S0.""" + steps = scenario["steps"] + assert isinstance(steps, list) + scenario["labels"] = [ + { + "step": k, + "invariant_truth": { + SEC1: "FAIL" if k == 0 else "PASS", + "INV-SEC-002": "PASS", + "INV-SEC-003": "PASS", + FUNC1: "PASS", + }, + "oracle_affected": [SEC1] if k else [], + "target_safe": None, + } + for k in range(len(steps)) + ] + scenario["gt_class"] = "safe" + scenario["mutation_record"] = [] + + +class TestLabels(unittest.TestCase): + def setUp(self) -> None: + self.dirs = TempDirs() + self.addCleanup(self.dirs.close) + self.mini = mini_root(self.dirs.new(), edit=real_labels) + + def test_labels_are_read_and_checked(self) -> None: + found = load_labels(self.mini.root, manifest_sha256=self.mini.manifest_sha256) + f8 = found[F8] + self.assertEqual(f8.family, "F8") + self.assertEqual((f8.kind, f8.target), ("repair", SEC1)) + self.assertEqual(len(f8.truth), 3) + self.assertEqual(f8.truth[0][SEC1], "FAIL") + self.assertEqual(f8.oracle_affected[1], frozenset({SEC1})) + self.assertEqual((f8.tier, f8.cluster), ("controlled", "tpl-mini")) + self.assertEqual(f8.gt_class, "safe") + self.assertIn("aws_security_group.app", f8.declared[0]) + self.assertEqual(len(f8.declared), 3) + + def test_the_test_split_and_a_wrong_pin_are_refused(self) -> None: + with self.assertRaises(LoaderError): + load_labels(self.mini.root, manifest_sha256=self.mini.manifest_sha256, split="test") + with self.assertRaises(LoaderError): + load_labels(self.mini.root, manifest_sha256="0" * 64) + + def test_a_changed_scenario_file_is_refused(self) -> None: + path = self.mini.root / "scenarios" / "dev" / F2 / "scenario.json" + path.write_bytes(path.read_bytes() + b" ") + with self.assertRaises(LoaderError): + load_labels(self.mini.root, manifest_sha256=self.mini.manifest_sha256) + + +class TestIdentityTable(unittest.TestCase): + def setUp(self) -> None: + self.table = load_identity_table() + + def test_every_row_quotes_the_catalogue(self) -> None: + self.assertTrue(self.table.rows) + for operator, row in self.table.rows.items(): + with self.subTest(operator=operator): + self.assertTrue(operator.startswith("op_")) + self.assertTrue(row["sentence"]) + self.assertTrue(row["effects"]) + for effect in row["effects"]: + self.assertIn(effect["effect"], ("MOVE", "REPLACE", "CREATE", "DELETE")) + + def test_the_count_operator_is_excluded(self) -> None: + # "adds or removes count = 1 on aws_instance": the catalogue states no identity truth + self.assertIn("op_f5_count_each_safe", self.table.excluded) + self.assertNotIn("op_f5_count_each_safe", self.table.rows) + + @unittest.skipUnless( + ROOT, "set TERRAPRESERVE_ROOT to check the sentences against the catalogue" + ) + def test_each_sentence_is_in_the_catalogue(self) -> None: + assert ROOT is not None + text = (Path(ROOT) / "docs" / "D4_OPERATORS.md").read_text(encoding="utf-8") + for operator, row in self.table.rows.items(): + with self.subTest(operator=operator): + self.assertIn(row["sentence"], text) + self.assertIn(f"`{operator}`", text) + for operator in self.table.excluded: + self.assertIn(f"`{operator}`", text) + + def test_true_pairs_from_mutation_rows(self) -> None: + from tests.metrics_builders import scenarios + + pairs, excluded = true_pairs(self.table, scenarios()["Y"], 1) + effects = sorted((p.effect, p.old, p.new) for p in pairs) + self.assertEqual( + effects, + [ + ("MOVE", "aws_security_group_rule.r5432", "aws_security_group_rule.r5432_v2"), + ("REPLACE", "aws_security_group.db", "aws_security_group.db"), + ], + ) + self.assertEqual(excluded, []) + # an existing resource that only gains an attribute (no 'before' rows) is not created + created, _ = true_pairs(self.table, scenarios()["Z"], 1) + self.assertEqual( + [(p.effect, p.new) for p in created], [("CREATE", "aws_security_group.mon")] + ) + _, excluded_w = true_pairs(self.table, scenarios()["W"], 1) + self.assertEqual([e["operator"] for e in excluded_w], ["op_f5_count_each_safe"]) + + +class TestRecordsToTransitions(unittest.TestCase): + def test_a_stored_trial_result_becomes_transitions(self) -> None: + dirs = TempDirs() + self.addCleanup(dirs.close) + mini = mini_root(dirs.new()) + loaded = load(mini.root, manifest_sha256=mini.manifest_sha256) + experiment = plan( + ExperimentSpec("p8b-parse", "1", "draft", "o", ("h",), (Condition.C,)), + loaded, + environment={}, + created_at=datetime(2026, 11, 13, tzinfo=UTC), + only=(F8,), + ) + out = dirs.new() + run_experiment(experiment, loaded, out) + result = read_result( + Layout(out, experiment.experiment_id), experiment.trial_id(F8, Condition.C) + ) + found = transitions_from_result(result) + self.assertEqual( + [(t.step, t.status, t.decision) for t in found], + [(1, "EVALUATED", "PROMOTE"), (2, "EVALUATED", "PROMOTE")], + ) + first = found[0] + self.assertEqual(first.open_targets, (SEC1,)) + self.assertEqual(first.condition, "C") + self.assertIn(SEC1, first.impact_affected) + self.assertTrue(any(w.origin == "gate" for w in first.work)) + self.assertTrue(all(len(r) == 2 for r in first.required)) + + +class TestReport(unittest.TestCase): + def test_the_report_has_the_fields_of_section_33(self) -> None: + report = build_report( + definition={"experiment_id": "e", "name": "n", "version": "1", "condition_set": ["A"]}, + statuses={"t1": "COMPLETED"}, + extraction=extraction(), + ) + for key in REPORT_KEYS: + self.assertIn(key, report) + self.assertEqual( + (report["trial_count"], report["included_count"], report["excluded_count"]), (1, 1, 0) + ) + + def test_markdown_shows_numerators_and_denominators(self) -> None: + report = build_report( + definition={ + "experiment_id": "e", + "name": "n", + "version": "1", + "condition_set": ["A", "B", "C", "C_PRIME"], + }, + statuses={"t1": "COMPLETED"}, + extraction=extraction(), + ) + text = render_markdown(report) + self.assertIn("1 / 3", text) # unsafe promotions per promotion under C + self.assertEqual(text, render_markdown(json.loads(json.dumps(report)))) + + +class TestDevPilotDocument(unittest.TestCase): + def test_the_document_is_the_rendering_of_the_extractors_json(self) -> None: + doc = REPO_ROOT / "docs" / "P8_DEV_PILOT.md" + data = REPO_ROOT / "docs" / "P8_DEV_PILOT.json" + report = json.loads(data.read_text(encoding="utf-8")) + rendered = render_markdown(report) + self.assertEqual(doc.read_text(encoding="utf-8").replace("\r\n", "\n"), rendered) + self.assertEqual(report["extractor_version"], "metrics-draft-1") + + +if __name__ == "__main__": + unittest.main()