From daf608a0606144bdc7e17f33153d83893f650360 Mon Sep 17 00:00:00 2001 From: ahmdkaml Date: Tue, 4 Aug 2026 23:34:16 +0300 Subject: [PATCH 1/2] Add HTML report for compare results --- a.csv | 21 ++++++ b.csv | 21 ++++++ faircode/cli.py | 9 ++- faircode/report.py | 160 +++++++++++++++++++++++++++++++++++++++++++++ profile.html | 40 ++++++++++++ report.html | 1 + 6 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 a.csv create mode 100644 b.csv create mode 100644 profile.html create mode 100644 report.html diff --git a/a.csv b/a.csv new file mode 100644 index 0000000..48090d4 --- /dev/null +++ b/a.csv @@ -0,0 +1,21 @@ +race,sex +White,M +White,M +White,F +White,F +White,M +Black,M +Black,F +Black,M +Black,F +Black,F +Asian,M +Asian,F +Asian,M +Asian,F +Asian,M +North,M +North,F +North,M +South,F +South,M diff --git a/b.csv b/b.csv new file mode 100644 index 0000000..8d393aa --- /dev/null +++ b/b.csv @@ -0,0 +1,21 @@ +race,sex +White,M +White,M +White,M +White,M +White,M +White,M +White,F +White,M +White,M +White,F +Asian,M +Asian,M +Asian,F +Asian,M +Asian,F +West,M +West,F +West,M +West,F +West,M diff --git a/faircode/cli.py b/faircode/cli.py index 01699c6..792a708 100644 --- a/faircode/cli.py +++ b/faircode/cli.py @@ -9,6 +9,7 @@ faircode profile data.csv --html report.html faircode compare train.csv prod.csv faircode compare train.csv prod.csv --json + faircode compare train.csv prod.csv --html report.html faircode benchmark faircode benchmark --out results/ faircode benchmark COMPAS/audit.yaml "German Credit Lending/audit.yaml" @@ -32,7 +33,7 @@ from .loaders_extra import read_table from .profiler import parse_reference, profile from .proxy import proxy_hints -from .report import compare_to_terminal, to_html, to_json, to_terminal +from .report import compare_to_terminal, to_html, compare_to_html, to_json, to_terminal _MAP_CHOICES = VALID_KINDS + ("ignore",) @@ -111,6 +112,8 @@ def main(argv: list[str] | None = None) -> int: c.add_argument("csv_a", help="baseline dataset A (.csv, .tsv, .xlsx, .json, or .parquet)") c.add_argument("csv_b", help="current dataset B (.csv, .tsv, .xlsx, .json, or .parquet)") c.add_argument("--json", action="store_true", help="emit JSON to stdout") + c.add_argument("--html", metavar="PATH", + help="write a standalone HTML report to PATH") b = sub.add_parser("benchmark", help="run the cross-domain fairness benchmark harness over every audit.yaml") @@ -188,6 +191,10 @@ def main(argv: list[str] | None = None) -> int: profile(_read_or_exit(args.csv_b)), name_a=args.csv_a, name_b=args.csv_b, ) + if args.html: + with open(args.html, "w", encoding="utf-8") as fh: + fh.write(compare_to_html(result)) + print(f"HTML report written to {args.html}", file=sys.stderr) if args.json: print(to_json(result)) else: diff --git a/faircode/report.py b/faircode/report.py index 22d3c27..2e2f50c 100644 --- a/faircode/report.py +++ b/faircode/report.py @@ -230,3 +230,163 @@ def esc(s) -> str:

Generated by Fair Code - diagnostic only.

""" +def compare_to_html(cmp: dict) -> str: + """A self-contained HTML report displaying representation drift between datasets A and B.""" + + def esc(s) -> str: + return html.escape(str(s)) + + def signed(val: float | int, dp: int = 1) -> str: + prefix = "+" if val > 0 else "" + return f"{prefix}{val:.{dp}f}" + + # Summary Header + a, b = cmp["a"], cmp["b"] + score_delta = cmp["score_delta"] + delta_class = "up" if score_delta > 0 else ("down" if score_delta < 0 else "flat") + arrow = "=" if score_delta == 0 else "→" + + summary_html = ( + '
' + f'
{a["overall_score"]}{esc(a["name"])} ({a["n_rows"]:,} rows, Grade {a["grade"]})
' + f'' + f'
{b["overall_score"]}{esc(b["name"])} ({b["n_rows"]:,} rows, Grade {b["grade"]})
' + f'
score {signed(score_delta, 0)} pts
' + '
' + ) + + # Dimension Cards + cards_html = [] + if not cmp["dimensions"]: + cards_html.append( + '

No demographic dimension is present in both datasets to compare.

' + ) + else: + for cd in cmp["dimensions"]: + max_share = max( + (max(g["share_a"], g["share_b"]) for g in cd["groups"]), default=1.0 + ) + if max_share <= 0: + max_share = 1.0 + + rows = [] + for g in cd["groups"][:DISPLAY_GROUPS]: + cls = ( + " gone" + if g["status"] == "disappeared" + else (" new" if g["status"] == "appeared" else "") + ) + wa = (g["share_a"] / max_share) * 100 + wb = (g["share_b"] / max_share) * 100 + delta_pp = g["share_delta"] * 100 + d_cls = "up" if delta_pp > 0 else ("down" if delta_pp < 0 else "") + + tag = ( + f' {esc(g["status"])}' + if g["status"] in ("appeared", "disappeared") + else "" + ) + + rows.append( + f'' + f'{esc(g["label"])}{tag}' + f'{g["share_a"] * 100:.1f}% → {g["share_b"] * 100:.1f}%' + f'{signed(delta_pp)} pp' + f'' + f'
' + f'' + f'' + f'
' + f'' + f'' + ) + + more_html = "" + if len(cd["groups"]) > DISPLAY_GROUPS: + more_html = f'
… and {len(cd["groups"]) - DISPLAY_GROUPS} more groups
' + + cards_html.append( + '
' + '
' + f'

{esc(cd["name"])} {esc(cd["kind"])} ' + f'{esc(cd["drift_level"])} drift

' + f'
PSI {cd["psi"]:.3f} · TVD {cd["tvd"]:.3f} · score {cd["dimension_score_a"]}→{cd["dimension_score_b"]} ({signed(cd["dimension_score_delta"], 0)})
' + '
' + f'{"".join(rows)}
' + f'{more_html}' + '
' + ) + + # Flags Block + flags_html = "" + if cmp["flags"]: + items = "".join(f"
  • {esc(f)}
    • " for f in cmp["flags"]) + flags_html = f'

    Drift Flags

    ' + + # Dimensions missing/added info + only_html = "" + if cmp["added_dimensions"]: + only_html += f'
    Only in B ({esc(b["name"])}): {", ".join(map(esc, cmp["added_dimensions"]))}
    ' + if cmp["removed_dimensions"]: + only_html += f'
    Only in A ({esc(a["name"])}): {", ".join(map(esc, cmp["removed_dimensions"]))}
    ' + + style = ( + ":root { --bg:#f4f1e8; --surface:#ebe7d9; --border:#d9d3c0; --accent:#a63a22; " + "--accent3:#2f6b4f; --text:#36321f; --muted:#7d7459; --bar-a:#7d7459; --bar-b:#2f6b4f; } " + "body { font-family:'Helvetica Neue',sans-serif; background:var(--bg); color:var(--text); " + "max-width:820px; margin:0 auto; padding:48px 24px; } " + "h1 { font-family:Georgia,serif; margin-bottom:8px; } " + ".head { border-bottom:2px solid var(--accent); padding-bottom:12px; margin-bottom:20px; } " + ".drift-summary { display:flex; align-items:center; justify-content:space-between; background:var(--surface); border:1px solid var(--border); border-radius:8px; padding:16px; margin-bottom:20px; } " + ".drift-score { font-weight:bold; font-size:14px; display:flex; flex-direction:column; } " + ".drift-score .n { font-size:24px; color:var(--accent3); } " + ".drift-score .l { font-size:12px; color:var(--muted); font-weight:normal; } " + ".drift-arrow { font-size:20px; color:var(--muted); } " + ".drift-delta { font-weight:bold; padding:4px 8px; border-radius:4px; font-size:14px; } " + ".drift-delta.down { color:var(--accent); background:#fbeae3; } " + ".drift-delta.up { color:var(--accent3); background:#e2f0e8; } " + ".drift-delta.flat { color:var(--muted); } " + ".kind { color:var(--muted); font-size:.6em; text-transform:uppercase; letter-spacing:.08em; font-weight:normal; } " + ".drift-badge { font-size:11px; padding:2px 6px; border-radius:4px; text-transform:uppercase; font-weight:bold; background:var(--border); margin-left:8px; } " + ".drift-badge.significant { background:var(--accent); color:#fff; } " + ".drift-card { background:var(--surface); border:1px solid var(--border); border-radius:8px; padding:16px 20px; margin:16px 0; } " + ".drift-card-head { display:flex; justify-content:space-between; align-items:baseline; border-bottom:1px solid var(--border); padding-bottom:8px; margin-bottom:12px; } " + ".drift-card-head h2 { margin:0; font-size:18px; } " + ".drift-metrics { font-size:12px; color:var(--muted); } " + "table { width:100%; border-collapse:collapse; } " + "td { padding:6px 8px; font-size:14px; border-bottom:1px solid var(--border); } " + "td.num { text-align:right; font-variant-numeric:tabular-nums; white-space:nowrap; font-size:13px; } " + "td.label { width:25%; } " + "td.bar { width:40%; } " + ".bar-container { display:flex; flex-direction:column; gap:3px; } " + ".bar-a { display:block; height:6px; background:var(--bar-a); border-radius:2px; opacity:0.6; } " + ".bar-b { display:block; height:6px; background:var(--bar-b); border-radius:2px; } " + "tr.gone td.label { color:var(--accent); text-decoration:line-through; } " + "tr.new td.label { color:var(--accent3); font-weight:bold; } " + ".tag { font-size:10px; font-weight:bold; text-transform:uppercase; padding:1px 4px; border-radius:3px; border:1px solid currentColor; margin-left:4px; } " + ".up { color:var(--accent3); } " + ".down { color:var(--accent); } " + ".dim-more { font-size:12px; color:var(--muted); margin-top:8px; font-style:italic; } " + ".flags ul { list-style:none; padding:0; } " + ".flags li { background:#fbeae3; border-left:3px solid var(--accent); padding:8px 12px; margin:6px 0; border-radius:0 4px 4px 0; font-size:14px; } " + ".drift-only { font-size:13px; color:var(--muted); margin-top:8px; } " + ".print-btn { position:fixed; top:16px; right:16px; background:var(--accent); color:#fff; border:0; border-radius:6px; padding:8px 14px; font-size:13px; cursor:pointer; font-family:inherit; } " + "@media print { .print-btn { display:none; } body { padding:24px; max-width:none; } }" + ) + + return ( + '' + '' + f'Fair Code - Representation Drift' + '' + '

    Representation Drift (A → B)

    ' + f'{summary_html}' + f'{"".join(cards_html)}' + f'{flags_html}' + f'{only_html}' + '

    ' + 'Generated by Fair Code - diagnostic only.

    ' + '' + ) + + diff --git a/profile.html b/profile.html new file mode 100644 index 0000000..caeb961 --- /dev/null +++ b/profile.html @@ -0,0 +1,40 @@ + + + +Fair Code - Dataset Profile + + +

    Dataset Representation Profile

    +

    20 rows · 2 columns · Score +98/100 (Grade A)

    +

    race race 97/100

    Asian25.0%11.2–46.9%5
    Black25.0%11.2–46.9%5
    White25.0%11.2–46.9%5
    North15.0%5.2–36.0%3
    South10.0%2.8–30.1%2

    sex sex 99/100

    M55.0%34.2–74.2%11
    F45.0%25.8–65.8%9
    +

    Flags

    +

    +Generated by Fair Code - diagnostic only.

    + \ No newline at end of file diff --git a/report.html b/report.html new file mode 100644 index 0000000..8522a9b --- /dev/null +++ b/report.html @@ -0,0 +1 @@ +Fair Code - Representation Drift

    Representation Drift (A → B)

    98a.csv (20 rows, Grade A)
    92b.csv (20 rows, Grade A)
    score -6 pts

    race race significant drift

    PSI 5.870 · TVD 0.500 · score 97→95 (-2)
    Black disappeared25.0% → 0.0%-25.0 pp
    West appeared0.0% → 25.0%+25.0 pp
    White25.0% → 50.0%+25.0 pp
    North disappeared15.0% → 0.0%-15.0 pp
    South disappeared10.0% → 0.0%-10.0 pp
    Asian25.0% → 25.0%0.0 pp

    sex sex none drift

    PSI 0.097 · TVD 0.150 · score 99→88 (-11)
    F45.0% → 30.0%-15.0 pp
    M55.0% → 70.0%+15.0 pp

    Drift Flags

    Generated by Fair Code - diagnostic only.

    \ No newline at end of file From 16fb3a3f4a520fe06f54af5f15de5a8ae7f2ec58 Mon Sep 17 00:00:00 2001 From: ahmdkaml Date: Tue, 4 Aug 2026 23:34:49 +0300 Subject: [PATCH 2/2] Add HTML report for compare results --- a.csv | 21 --------------------- b.csv | 21 --------------------- profile.html | 40 ---------------------------------------- report.html | 1 - 4 files changed, 83 deletions(-) delete mode 100644 a.csv delete mode 100644 b.csv delete mode 100644 profile.html delete mode 100644 report.html diff --git a/a.csv b/a.csv deleted file mode 100644 index 48090d4..0000000 --- a/a.csv +++ /dev/null @@ -1,21 +0,0 @@ -race,sex -White,M -White,M -White,F -White,F -White,M -Black,M -Black,F -Black,M -Black,F -Black,F -Asian,M -Asian,F -Asian,M -Asian,F -Asian,M -North,M -North,F -North,M -South,F -South,M diff --git a/b.csv b/b.csv deleted file mode 100644 index 8d393aa..0000000 --- a/b.csv +++ /dev/null @@ -1,21 +0,0 @@ -race,sex -White,M -White,M -White,M -White,M -White,M -White,M -White,F -White,M -White,M -White,F -Asian,M -Asian,M -Asian,F -Asian,M -Asian,F -West,M -West,F -West,M -West,F -West,M diff --git a/profile.html b/profile.html deleted file mode 100644 index caeb961..0000000 --- a/profile.html +++ /dev/null @@ -1,40 +0,0 @@ - - - -Fair Code - Dataset Profile - - -

    Dataset Representation Profile

    -

    20 rows · 2 columns · Score -98/100 (Grade A)

    -

    race race 97/100

    Asian25.0%11.2–46.9%5
    Black25.0%11.2–46.9%5
    White25.0%11.2–46.9%5
    North15.0%5.2–36.0%3
    South10.0%2.8–30.1%2

    sex sex 99/100

    M55.0%34.2–74.2%11
    F45.0%25.8–65.8%9
    -

    Flags

    • race: 'Asian' has only 5 rows; fairness metrics may be unreliable
    • race: 'Black' has only 5 rows; fairness metrics may be unreliable
    • race: 'White' has only 5 rows; fairness metrics may be unreliable
    • race: 'North' has only 3 rows; fairness metrics may be unreliable
    • race: 'South' has only 2 rows; fairness metrics may be unreliable
    • sex: 'M' has only 11 rows; fairness metrics may be unreliable
    • sex: 'F' has only 9 rows; fairness metrics may be unreliable
    -

    -Generated by Fair Code - diagnostic only.

    - \ No newline at end of file diff --git a/report.html b/report.html deleted file mode 100644 index 8522a9b..0000000 --- a/report.html +++ /dev/null @@ -1 +0,0 @@ -Fair Code - Representation Drift

    Representation Drift (A → B)

    98a.csv (20 rows, Grade A)
    92b.csv (20 rows, Grade A)
    score -6 pts

    race race significant drift

    PSI 5.870 · TVD 0.500 · score 97→95 (-2)
    Black disappeared25.0% → 0.0%-25.0 pp
    West appeared0.0% → 25.0%+25.0 pp
    White25.0% → 50.0%+25.0 pp
    North disappeared15.0% → 0.0%-15.0 pp
    South disappeared10.0% → 0.0%-10.0 pp
    Asian25.0% → 25.0%0.0 pp

    sex sex none drift

    PSI 0.097 · TVD 0.150 · score 99→88 (-11)
    F45.0% → 30.0%-15.0 pp
    M55.0% → 70.0%+15.0 pp

    Drift Flags

    • overall representation score dropped 6 points (98 → 92)
    • race: significant representation drift (PSI 5.87)
    • race: 'Black' disappeared (25.0% → 0.0%)
    • race: 'West' appeared (0.0% → 25.0%)
    • race: 'North' disappeared (15.0% → 0.0%)
    • race: 'South' disappeared (10.0% → 0.0%)

    Generated by Fair Code - diagnostic only.

    \ No newline at end of file