Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 88 additions & 8 deletions .github/workflows/c6-microscopic-tail-numerics.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,95 @@ jobs:
blob = hashlib.sha1(b'blob %d\0' % len(data) + data).hexdigest()
if len(data) != entry['bytes'] or hashlib.sha256(data).hexdigest() != entry['sha256'] or blob != entry['git_blob']:
raise ValueError('consumed source identity mismatch on this tree: ' + entry['path'])
validator_source = r'''
class Result:
def __init__(self, returncode, stdout=b"", stderr=b""):
self.returncode = returncode
self.stdout = stdout
self.stderr = stderr

def validate_baseline(result):
expected = b'{"check": "ok", "mutant": null}\n'
if result.returncode != 0 or result.stdout != expected or result.stderr != b"":
raise ValueError("baseline output contract mismatch")

def validate_mutant(result, diagnostic):
expected = ("FAIL: " + diagnostic + "\n").encode("utf-8")
if result.returncode != 1 or result.stdout != expected or result.stderr != b"":
raise ValueError("mutant output contract mismatch: " + diagnostic)

def reject(fn, result, *args):
try:
fn(result, *args)
except ValueError:
return
raise AssertionError("invalid child result was accepted")

def synthetic_controls():
baseline = b'{"check": "ok", "mutant": null}\n'
validate_baseline(Result(0, baseline, b""))
for bad in (
Result(0, b"not-json\n", b""),
Result(0, baseline, b"unexpected stderr\n"),
Result(0, baseline + b"extra\n", b""),
Result(1, baseline, b""),
):
reject(validate_baseline, bad)

expected = {
"shape-integral": "I = 246528/35",
"gamma-power": "cusp integral replay J_cusp k=0.5",
"cusp-shift": "cusp integral replay J_cusp k=0.5",
}
for mutant, diagnostic in expected.items():
valid = ("FAIL: " + diagnostic + "\n").encode("utf-8")
validate_mutant(Result(1, valid, b""), diagnostic)
wrong_label = b'{"check": "ok", "mutant": "wrong-mutant"}\n'
for bad in (
Result(1, b"", b"Traceback (most recent call last): RuntimeError\n"),
Result(1, b"", b""),
Result(1, b"FAIL: unrelated failure\n", b""),
Result(1, valid, b"unexpected stderr\n"),
Result(0, valid, b""),
Result(1, baseline, b""),
Result(1, wrong_label, b""),
):
reject(validate_mutant, bad, diagnostic)
'''

validator_namespace = {}
exec(validator_source, validator_namespace)
validate_baseline = validator_namespace['validate_baseline']
validate_mutant = validator_namespace['validate_mutant']
synthetic_code = validator_source + "\nsynthetic_controls()\nprint('synthetic process-predicate controls passed')\n"
for flags in (['-B', '-S'], ['-B', '-O', '-S']):
probe = subprocess.run([sys.executable, *flags, '-c', synthetic_code], capture_output=True, timeout=30)
if probe.returncode != 0 or probe.stderr != b'' or probe.stdout != b'synthetic process-predicate controls passed\n':
raise RuntimeError('synthetic process-predicate controls failed under ' + ' '.join(flags) +
': stdout=' + repr(probe.stdout) + ' stderr=' + repr(probe.stderr))

script = str(root / 'tail_constants.py')
expected_mutants = {
'shape-integral': 'I = 246528/35',
'gamma-power': 'cusp integral replay J_cusp k=0.5',
'cusp-shift': 'cusp integral replay J_cusp k=0.5',
}
for flags in (['-B', '-S'], ['-B', '-O', '-S']):
out = subprocess.run([sys.executable, *flags, script, '--check'], cwd=root, capture_output=True, timeout=900)
if out.returncode != 0:
raise RuntimeError('check failed under ' + ' '.join(flags) + ': ' + out.stdout.decode()[-400:])
for m in ['shape-integral', 'gamma-power', 'cusp-shift']:
rc = subprocess.run([sys.executable, *flags, script, '--check', '--mutant', m], cwd=root, capture_output=True, timeout=900).returncode
if rc != 1:
raise RuntimeError('mutant ' + m + ' not rejected')
print(json.dumps({'scientific_effect': 'NONE', 'certified': False}))
baseline = subprocess.run([sys.executable, *flags, script, '--check'], cwd=root, capture_output=True, timeout=900)
try:
validate_baseline(baseline)
except ValueError as exc:
raise RuntimeError('baseline contract failed under ' + ' '.join(flags) +
': rc=' + str(baseline.returncode) + ' stdout=' + repr(baseline.stdout) +
' stderr=' + repr(baseline.stderr)) from exc
for mutant, diagnostic in expected_mutants.items():
result = subprocess.run([sys.executable, *flags, script, '--check', '--mutant', mutant],
cwd=root, capture_output=True, timeout=900)
try:
validate_mutant(result, diagnostic)
except ValueError as exc:
raise RuntimeError('mutant ' + mutant + ' contract failed under ' + ' '.join(flags) +
': rc=' + str(result.returncode) + ' stdout=' + repr(result.stdout) +
' stderr=' + repr(result.stderr)) from exc
PY
git diff --exit-code
Loading