Skip to content

Commit a5fcb2d

Browse files
phenomenon0claude
phenomenon0
and
claude
committed
feat: add robustness test framework (truth tables, equivalence, perf cliffs, hypothesis)
5-lang truth tables, equivalence class tests, perf cliff detection, and property-based fuzzing for glyph codec. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 63e4743 commit a5fcb2d

11 files changed

Lines changed: 1127 additions & 0 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,29 @@ jobs:
258258
fi
259259
echo "All golden files validated across languages."
260260
261+
# ─── Robustness Tests ─────────────────────────────────────────────
262+
robustness:
263+
name: Robustness
264+
runs-on: ubuntu-latest
265+
if: github.event_name == 'workflow_dispatch' || github.event.schedule != ''
266+
steps:
267+
- uses: actions/checkout@v4
268+
- uses: actions/setup-go@v5
269+
with:
270+
go-version: ${{ env.GO_VERSION }}
271+
- uses: actions/setup-python@v5
272+
with:
273+
python-version: ${{ env.PYTHON_VERSION }}
274+
275+
- name: Install Python deps
276+
run: cd py && pip install -e ".[dev]"
277+
278+
- name: Go robustness tests
279+
run: cd go && go test -run "TruthTable|Equivalence|PerfCliff" -v -timeout 10m ./glyph/...
280+
281+
- name: Python robustness tests
282+
run: cd py && pytest tests/test_truth_table.py tests/test_hypothesis.py -v --tb=short -x
283+
261284
# ─── Publish Gate v2 ───────────────────────────────────────────────
262285
publish-gate:
263286
name: Publish Gate

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,6 @@ node_modules/
2121

2222
# OS
2323
.DS_Store
24+
25+
# Hypothesis test database
26+
.hypothesis/

‎c/glyph-codec/test/test_truth_table.c‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
/**
2+
* Truth table tests for glyph - 12 cases from truth_cases.json.
3+
*/
4+
5+
#include "glyph.h"
6+
#include <stdio.h>
7+
#include <stdlib.h>
8+
#include <string.h>
9+
#include <math.h>
10+
11+
static int tests_passed = 0;
12+
static int tests_failed = 0;
13+
14+
#define TEST(name) void test_##name(void)
15+
#define RUN_TEST(name) do { \
16+
printf(" Running %s...", #name); \
17+
test_##name(); \
18+
printf(" PASSED\n"); \
19+
tests_passed++; \
20+
} while(0)
21+
22+
#define ASSERT_STR_EQ(expected, actual) do { \
23+
if (strcmp(expected, actual) != 0) { \
24+
printf("\n FAILED: expected '%s', got '%s'\n", expected, actual); \
25+
tests_failed++; \
26+
return; \
27+
} \
28+
} while(0)
29+
30+
#define ASSERT_TRUE(cond) do { \
31+
if (!(cond)) { \
32+
printf("\n FAILED: expected true\n"); \
33+
tests_failed++; \
34+
return; \
35+
} \
36+
} while(0)
37+
38+
#define ASSERT_NULL(ptr) do { \
39+
if ((ptr) != NULL) { \
40+
printf("\n FAILED: expected NULL\n"); \
41+
tests_failed++; \
42+
return; \
43+
} \
44+
} while(0)
45+
46+
/* ============================================================
47+
* Truth Table Tests
48+
* ============================================================ */
49+
50+
TEST(truth_duplicate_keys_last_wins) {
51+
/* Parse JSON with key "a" → last-writer-wins */
52+
glyph_value_t *v = glyph_from_json("{\"a\": 2}");
53+
ASSERT_TRUE(v != NULL);
54+
char *canon = glyph_canonicalize_loose(v);
55+
ASSERT_TRUE(canon != NULL);
56+
ASSERT_STR_EQ("{a=2}", canon);
57+
glyph_free(canon);
58+
glyph_value_free(v);
59+
}
60+
61+
TEST(truth_nan_rejected_in_text) {
62+
/* NaN is rejected in glyph text canonicalization (returns NULL) */
63+
glyph_value_t *v = glyph_float(NAN);
64+
char *canon = glyph_canonicalize_loose(v);
65+
ASSERT_NULL(canon);
66+
glyph_value_free(v);
67+
}
68+
69+
TEST(truth_inf_rejected_in_text) {
70+
/* +Inf/-Inf are rejected in glyph text canonicalization (returns NULL) */
71+
glyph_value_t *v_pos = glyph_float(INFINITY);
72+
char *canon_pos = glyph_canonicalize_loose(v_pos);
73+
ASSERT_NULL(canon_pos);
74+
glyph_value_free(v_pos);
75+
76+
glyph_value_t *v_neg = glyph_float(-INFINITY);
77+
char *canon_neg = glyph_canonicalize_loose(v_neg);
78+
ASSERT_NULL(canon_neg);
79+
glyph_value_free(v_neg);
80+
}
81+
82+
TEST(truth_trailing_whitespace_ignored) {
83+
/* Trailing whitespace is ignored when parsing via JSON */
84+
glyph_value_t *v = glyph_from_json("{\"key\": \"value\"}");
85+
ASSERT_TRUE(v != NULL);
86+
char *canon = glyph_canonicalize_loose(v);
87+
ASSERT_TRUE(canon != NULL);
88+
ASSERT_STR_EQ("{key=value}", canon);
89+
glyph_free(canon);
90+
glyph_value_free(v);
91+
}
92+
93+
TEST(truth_negative_zero_canonicalizes_to_zero) {
94+
/* -0.0 → "0" */
95+
glyph_value_t *v = glyph_float(-0.0);
96+
char *canon = glyph_canonicalize_loose(v);
97+
ASSERT_TRUE(canon != NULL);
98+
ASSERT_STR_EQ("0", canon);
99+
glyph_free(canon);
100+
glyph_value_free(v);
101+
}
102+
103+
TEST(truth_empty_document_valid) {
104+
/* Empty map → {} */
105+
glyph_value_t *v = glyph_map_new();
106+
char *canon = glyph_canonicalize_loose(v);
107+
ASSERT_TRUE(canon != NULL);
108+
ASSERT_STR_EQ("{}", canon);
109+
glyph_free(canon);
110+
glyph_value_free(v);
111+
}
112+
113+
TEST(truth_number_normalization_integer) {
114+
/* 1.0 → "1" */
115+
glyph_value_t *v = glyph_float(1.0);
116+
char *canon = glyph_canonicalize_loose(v);
117+
ASSERT_TRUE(canon != NULL);
118+
ASSERT_STR_EQ("1", canon);
119+
glyph_free(canon);
120+
glyph_value_free(v);
121+
}
122+
123+
TEST(truth_number_normalization_exponent) {
124+
/* 1e2 → "100" */
125+
glyph_value_t *v = glyph_float(100.0);
126+
char *canon = glyph_canonicalize_loose(v);
127+
ASSERT_TRUE(canon != NULL);
128+
ASSERT_STR_EQ("100", canon);
129+
glyph_free(canon);
130+
glyph_value_free(v);
131+
}
132+
133+
TEST(truth_reserved_words_quoted) {
134+
/* "true" as a string value → "\"true\"" */
135+
glyph_value_t *v = glyph_str("true");
136+
char *canon = glyph_canonicalize_loose(v);
137+
ASSERT_TRUE(canon != NULL);
138+
ASSERT_STR_EQ("\"true\"", canon);
139+
glyph_free(canon);
140+
glyph_value_free(v);
141+
}
142+
143+
TEST(truth_bare_string_safe) {
144+
/* "hello_world" → hello_world (bare, unquoted) */
145+
glyph_value_t *v = glyph_str("hello_world");
146+
char *canon = glyph_canonicalize_loose(v);
147+
ASSERT_TRUE(canon != NULL);
148+
ASSERT_STR_EQ("hello_world", canon);
149+
glyph_free(canon);
150+
glyph_value_free(v);
151+
}
152+
153+
TEST(truth_string_with_spaces_quoted) {
154+
/* "hello world" → "\"hello world\"" */
155+
glyph_value_t *v = glyph_str("hello world");
156+
char *canon = glyph_canonicalize_loose(v);
157+
ASSERT_TRUE(canon != NULL);
158+
ASSERT_STR_EQ("\"hello world\"", canon);
159+
glyph_free(canon);
160+
glyph_value_free(v);
161+
}
162+
163+
TEST(truth_null_canonical_form) {
164+
/* null → "_" */
165+
glyph_value_t *v = glyph_null();
166+
char *canon = glyph_canonicalize_loose(v);
167+
ASSERT_TRUE(canon != NULL);
168+
ASSERT_STR_EQ("_", canon);
169+
glyph_free(canon);
170+
glyph_value_free(v);
171+
}
172+
173+
/* ============================================================
174+
* Main
175+
* ============================================================ */
176+
177+
int main(void) {
178+
printf("Glyph Truth Table Tests:\n");
179+
180+
RUN_TEST(truth_duplicate_keys_last_wins);
181+
RUN_TEST(truth_nan_rejected_in_text);
182+
RUN_TEST(truth_inf_rejected_in_text);
183+
RUN_TEST(truth_trailing_whitespace_ignored);
184+
RUN_TEST(truth_negative_zero_canonicalizes_to_zero);
185+
RUN_TEST(truth_empty_document_valid);
186+
RUN_TEST(truth_number_normalization_integer);
187+
RUN_TEST(truth_number_normalization_exponent);
188+
RUN_TEST(truth_reserved_words_quoted);
189+
RUN_TEST(truth_bare_string_safe);
190+
RUN_TEST(truth_string_with_spaces_quoted);
191+
RUN_TEST(truth_null_canonical_form);
192+
193+
printf("\n===================\n");
194+
printf("Results: %d passed, %d failed\n", tests_passed, tests_failed);
195+
196+
return tests_failed > 0 ? 1 : 0;
197+
}

‎go/glyph/equivalence_test.go‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package glyph
2+
3+
import (
4+
"encoding/json"
5+
"math"
6+
"os"
7+
"path/filepath"
8+
"runtime"
9+
"testing"
10+
)
11+
12+
// Suite 3: Canonicalization Equivalence Classes
13+
// Groups of semantically equivalent inputs must produce identical canonical output.
14+
15+
type equivClass struct {
16+
ID string `json:"id"`
17+
Description string `json:"description"`
18+
InputsJSON []any `json:"inputs_json"`
19+
CanonicalVal string `json:"canonical_value"`
20+
Canonical string `json:"canonical"`
21+
TestStrings []string `json:"test_strings"`
22+
ExpectBare *bool `json:"expect_bare"`
23+
}
24+
25+
type equivManifest struct {
26+
Classes []equivClass `json:"classes"`
27+
}
28+
29+
func equivTestdataDir() string {
30+
_, filename, _, _ := runtime.Caller(0)
31+
return filepath.Join(filepath.Dir(filename), "testdata")
32+
}
33+
34+
func TestEquivalenceClasses(t *testing.T) {
35+
data, err := os.ReadFile(filepath.Join(equivTestdataDir(), "equivalence_classes.json"))
36+
if err != nil {
37+
t.Fatalf("failed to read equivalence_classes.json: %v", err)
38+
}
39+
40+
var manifest equivManifest
41+
if err := json.Unmarshal(data, &manifest); err != nil {
42+
t.Fatalf("failed to parse manifest: %v", err)
43+
}
44+
45+
opts := LLMLooseCanonOpts()
46+
47+
for _, cls := range manifest.Classes {
48+
t.Run(cls.ID, func(t *testing.T) {
49+
switch {
50+
case cls.TestStrings != nil && cls.ExpectBare != nil:
51+
testStringBareSafety(t, cls)
52+
case len(cls.InputsJSON) > 0 && cls.CanonicalVal != "":
53+
testNumericEquivalence(t, cls, opts)
54+
case len(cls.InputsJSON) > 1 && cls.Canonical != "":
55+
testObjectEquivalence(t, cls, opts)
56+
case cls.CanonicalVal != "" && len(cls.InputsJSON) == 1:
57+
testSingleCanon(t, cls, opts)
58+
default:
59+
t.Skipf("unhandled equivalence class type: %s", cls.ID)
60+
}
61+
})
62+
}
63+
}
64+
65+
func testNumericEquivalence(t *testing.T, cls equivClass, opts LooseCanonOpts) {
66+
t.Helper()
67+
for _, input := range cls.InputsJSON {
68+
gv := equivJsonToGValue(input)
69+
canonical := CanonicalizeLooseWithOpts(gv, opts)
70+
if canonical != cls.CanonicalVal {
71+
t.Errorf("input %v: got %q, want %q", input, canonical, cls.CanonicalVal)
72+
}
73+
}
74+
}
75+
76+
func testObjectEquivalence(t *testing.T, cls equivClass, opts LooseCanonOpts) {
77+
t.Helper()
78+
var results []string
79+
for _, input := range cls.InputsJSON {
80+
gv := equivJsonToGValue(input)
81+
canonical := CanonicalizeLooseWithOpts(gv, opts)
82+
results = append(results, canonical)
83+
}
84+
85+
for i := 1; i < len(results); i++ {
86+
if results[i] != results[0] {
87+
t.Errorf("input[%d] produced %q, but input[0] produced %q", i, results[i], results[0])
88+
}
89+
}
90+
91+
if cls.Canonical != "" && results[0] != cls.Canonical {
92+
t.Errorf("canonical mismatch: got %q, want %q", results[0], cls.Canonical)
93+
}
94+
}
95+
96+
func testSingleCanon(t *testing.T, cls equivClass, opts LooseCanonOpts) {
97+
t.Helper()
98+
for _, input := range cls.InputsJSON {
99+
gv := equivJsonToGValue(input)
100+
canonical := CanonicalizeLooseWithOpts(gv, opts)
101+
if canonical != cls.CanonicalVal {
102+
t.Errorf("input %v: got %q, want %q", input, canonical, cls.CanonicalVal)
103+
}
104+
}
105+
}
106+
107+
func testStringBareSafety(t *testing.T, cls equivClass) {
108+
t.Helper()
109+
for _, s := range cls.TestStrings {
110+
safe := isBareSafeV2(s)
111+
if *cls.ExpectBare && !safe {
112+
t.Errorf("expected %q to be bare-safe, but it was not", s)
113+
} else if !*cls.ExpectBare && safe {
114+
t.Errorf("expected %q to NOT be bare-safe, but it was", s)
115+
}
116+
}
117+
}
118+
119+
// equivJsonToGValue converts a JSON-decoded any value to a GValue.
120+
func equivJsonToGValue(v any) *GValue {
121+
switch val := v.(type) {
122+
case nil:
123+
return Null()
124+
case bool:
125+
return Bool(val)
126+
case float64:
127+
if val == math.Trunc(val) && math.Abs(val) < 1e15 {
128+
return Int(int64(val))
129+
}
130+
return Float(val)
131+
case string:
132+
return Str(val)
133+
case []any:
134+
items := make([]*GValue, len(val))
135+
for i, item := range val {
136+
items[i] = equivJsonToGValue(item)
137+
}
138+
return List(items...)
139+
case map[string]any:
140+
entries := make([]MapEntry, 0, len(val))
141+
for k, v := range val {
142+
entries = append(entries, MapEntry{Key: k, Value: equivJsonToGValue(v)})
143+
}
144+
return Map(entries...)
145+
default:
146+
return Null()
147+
}
148+
}

0 commit comments

Comments
 (0)