diff --git a/faircode/cli.py b/faircode/cli.py index 01699c6..792a708 100644 --- a/faircode/cli.py +++ b/faircode/cli.py @@ -9,6 +9,7 @@ faircode profile data.csv --html report.html faircode compare train.csv prod.csv faircode compare train.csv prod.csv --json + faircode compare train.csv prod.csv --html report.html faircode benchmark faircode benchmark --out results/ faircode benchmark COMPAS/audit.yaml "German Credit Lending/audit.yaml" @@ -32,7 +33,7 @@ from .loaders_extra import read_table from .profiler import parse_reference, profile from .proxy import proxy_hints -from .report import compare_to_terminal, to_html, to_json, to_terminal +from .report import compare_to_terminal, to_html, compare_to_html, to_json, to_terminal _MAP_CHOICES = VALID_KINDS + ("ignore",) @@ -111,6 +112,8 @@ def main(argv: list[str] | None = None) -> int: c.add_argument("csv_a", help="baseline dataset A (.csv, .tsv, .xlsx, .json, or .parquet)") c.add_argument("csv_b", help="current dataset B (.csv, .tsv, .xlsx, .json, or .parquet)") c.add_argument("--json", action="store_true", help="emit JSON to stdout") + c.add_argument("--html", metavar="PATH", + help="write a standalone HTML report to PATH") b = sub.add_parser("benchmark", help="run the cross-domain fairness benchmark harness over every audit.yaml") @@ -188,6 +191,10 @@ def main(argv: list[str] | None = None) -> int: profile(_read_or_exit(args.csv_b)), name_a=args.csv_a, name_b=args.csv_b, ) + if args.html: + with open(args.html, "w", encoding="utf-8") as fh: + fh.write(compare_to_html(result)) + print(f"HTML report written to {args.html}", file=sys.stderr) if args.json: print(to_json(result)) else: diff --git a/faircode/report.py b/faircode/report.py index 22d3c27..2e2f50c 100644 --- a/faircode/report.py +++ b/faircode/report.py @@ -230,3 +230,163 @@ def esc(s) -> str:

Generated by Fair Code - diagnostic only.

""" +def compare_to_html(cmp: dict) -> str: + """A self-contained HTML report displaying representation drift between datasets A and B.""" + + def esc(s) -> str: + return html.escape(str(s)) + + def signed(val: float | int, dp: int = 1) -> str: + prefix = "+" if val > 0 else "" + return f"{prefix}{val:.{dp}f}" + + # Summary Header + a, b = cmp["a"], cmp["b"] + score_delta = cmp["score_delta"] + delta_class = "up" if score_delta > 0 else ("down" if score_delta < 0 else "flat") + arrow = "=" if score_delta == 0 else "→" + + summary_html = ( + '
' + f'
{a["overall_score"]}{esc(a["name"])} ({a["n_rows"]:,} rows, Grade {a["grade"]})
' + f'' + f'
{b["overall_score"]}{esc(b["name"])} ({b["n_rows"]:,} rows, Grade {b["grade"]})
' + f'
score {signed(score_delta, 0)} pts
' + '
' + ) + + # Dimension Cards + cards_html = [] + if not cmp["dimensions"]: + cards_html.append( + '

No demographic dimension is present in both datasets to compare.

' + ) + else: + for cd in cmp["dimensions"]: + max_share = max( + (max(g["share_a"], g["share_b"]) for g in cd["groups"]), default=1.0 + ) + if max_share <= 0: + max_share = 1.0 + + rows = [] + for g in cd["groups"][:DISPLAY_GROUPS]: + cls = ( + " gone" + if g["status"] == "disappeared" + else (" new" if g["status"] == "appeared" else "") + ) + wa = (g["share_a"] / max_share) * 100 + wb = (g["share_b"] / max_share) * 100 + delta_pp = g["share_delta"] * 100 + d_cls = "up" if delta_pp > 0 else ("down" if delta_pp < 0 else "") + + tag = ( + f' {esc(g["status"])}' + if g["status"] in ("appeared", "disappeared") + else "" + ) + + rows.append( + f'' + f'{esc(g["label"])}{tag}' + f'{g["share_a"] * 100:.1f}% → {g["share_b"] * 100:.1f}%' + f'{signed(delta_pp)} pp' + f'' + f'
' + f'' + f'' + f'
' + f'' + f'' + ) + + more_html = "" + if len(cd["groups"]) > DISPLAY_GROUPS: + more_html = f'
… and {len(cd["groups"]) - DISPLAY_GROUPS} more groups
' + + cards_html.append( + '
' + '
' + f'

{esc(cd["name"])} {esc(cd["kind"])} ' + f'{esc(cd["drift_level"])} drift

' + f'
PSI {cd["psi"]:.3f} · TVD {cd["tvd"]:.3f} · score {cd["dimension_score_a"]}→{cd["dimension_score_b"]} ({signed(cd["dimension_score_delta"], 0)})
' + '
' + f'{"".join(rows)}
' + f'{more_html}' + '
' + ) + + # Flags Block + flags_html = "" + if cmp["flags"]: + items = "".join(f"
  • {esc(f)}
    • " for f in cmp["flags"]) + flags_html = f'

    Drift Flags

    ' + + # Dimensions missing/added info + only_html = "" + if cmp["added_dimensions"]: + only_html += f'
    Only in B ({esc(b["name"])}): {", ".join(map(esc, cmp["added_dimensions"]))}
    ' + if cmp["removed_dimensions"]: + only_html += f'
    Only in A ({esc(a["name"])}): {", ".join(map(esc, cmp["removed_dimensions"]))}
    ' + + style = ( + ":root { --bg:#f4f1e8; --surface:#ebe7d9; --border:#d9d3c0; --accent:#a63a22; " + "--accent3:#2f6b4f; --text:#36321f; --muted:#7d7459; --bar-a:#7d7459; --bar-b:#2f6b4f; } " + "body { font-family:'Helvetica Neue',sans-serif; background:var(--bg); color:var(--text); " + "max-width:820px; margin:0 auto; padding:48px 24px; } " + "h1 { font-family:Georgia,serif; margin-bottom:8px; } " + ".head { border-bottom:2px solid var(--accent); padding-bottom:12px; margin-bottom:20px; } " + ".drift-summary { display:flex; align-items:center; justify-content:space-between; background:var(--surface); border:1px solid var(--border); border-radius:8px; padding:16px; margin-bottom:20px; } " + ".drift-score { font-weight:bold; font-size:14px; display:flex; flex-direction:column; } " + ".drift-score .n { font-size:24px; color:var(--accent3); } " + ".drift-score .l { font-size:12px; color:var(--muted); font-weight:normal; } " + ".drift-arrow { font-size:20px; color:var(--muted); } " + ".drift-delta { font-weight:bold; padding:4px 8px; border-radius:4px; font-size:14px; } " + ".drift-delta.down { color:var(--accent); background:#fbeae3; } " + ".drift-delta.up { color:var(--accent3); background:#e2f0e8; } " + ".drift-delta.flat { color:var(--muted); } " + ".kind { color:var(--muted); font-size:.6em; text-transform:uppercase; letter-spacing:.08em; font-weight:normal; } " + ".drift-badge { font-size:11px; padding:2px 6px; border-radius:4px; text-transform:uppercase; font-weight:bold; background:var(--border); margin-left:8px; } " + ".drift-badge.significant { background:var(--accent); color:#fff; } " + ".drift-card { background:var(--surface); border:1px solid var(--border); border-radius:8px; padding:16px 20px; margin:16px 0; } " + ".drift-card-head { display:flex; justify-content:space-between; align-items:baseline; border-bottom:1px solid var(--border); padding-bottom:8px; margin-bottom:12px; } " + ".drift-card-head h2 { margin:0; font-size:18px; } " + ".drift-metrics { font-size:12px; color:var(--muted); } " + "table { width:100%; border-collapse:collapse; } " + "td { padding:6px 8px; font-size:14px; border-bottom:1px solid var(--border); } " + "td.num { text-align:right; font-variant-numeric:tabular-nums; white-space:nowrap; font-size:13px; } " + "td.label { width:25%; } " + "td.bar { width:40%; } " + ".bar-container { display:flex; flex-direction:column; gap:3px; } " + ".bar-a { display:block; height:6px; background:var(--bar-a); border-radius:2px; opacity:0.6; } " + ".bar-b { display:block; height:6px; background:var(--bar-b); border-radius:2px; } " + "tr.gone td.label { color:var(--accent); text-decoration:line-through; } " + "tr.new td.label { color:var(--accent3); font-weight:bold; } " + ".tag { font-size:10px; font-weight:bold; text-transform:uppercase; padding:1px 4px; border-radius:3px; border:1px solid currentColor; margin-left:4px; } " + ".up { color:var(--accent3); } " + ".down { color:var(--accent); } " + ".dim-more { font-size:12px; color:var(--muted); margin-top:8px; font-style:italic; } " + ".flags ul { list-style:none; padding:0; } " + ".flags li { background:#fbeae3; border-left:3px solid var(--accent); padding:8px 12px; margin:6px 0; border-radius:0 4px 4px 0; font-size:14px; } " + ".drift-only { font-size:13px; color:var(--muted); margin-top:8px; } " + ".print-btn { position:fixed; top:16px; right:16px; background:var(--accent); color:#fff; border:0; border-radius:6px; padding:8px 14px; font-size:13px; cursor:pointer; font-family:inherit; } " + "@media print { .print-btn { display:none; } body { padding:24px; max-width:none; } }" + ) + + return ( + '' + '' + f'Fair Code - Representation Drift' + '' + '

    Representation Drift (A → B)

    ' + f'{summary_html}' + f'{"".join(cards_html)}' + f'{flags_html}' + f'{only_html}' + '

    ' + 'Generated by Fair Code - diagnostic only.

    ' + '' + ) + +