diff --git a/CHANGELOG.md b/CHANGELOG.md index b4e01e6f..fd85fed3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,13 @@ Changes: * Fix: `StructuralAttack` now respects the `report_individual` flag. Per-record `record_level_results` and `attack_metrics["individual"]` are only populated when the flag is set to `True`, matching the behaviour of `LIRAAttack` and `QMIAAttack`. +* Breaking: `LIRAAttack` per-record output field renamed from `"score"` to + `"member_prob"` to standardise with `QMIAAttack` and `WorstCaseAttack`. Affects + the `individual` dict in `attack_metrics`, the `report.json` payload, and the + externalised `.npz` sidecar key (`individual.score` becomes + `individual.member_prob`). Existing LiRA `report.json` files written before this + release will not be readable by `MetaAttack`'s `use_existing_only` mode and + should be regenerated. ## Version 1.4.3 (Jan 29, 2026) diff --git a/sacroml/attacks/likelihood_attack.py b/sacroml/attacks/likelihood_attack.py index 735db5e5..f2f8b4b6 100644 --- a/sacroml/attacks/likelihood_attack.py +++ b/sacroml/attacks/likelihood_attack.py @@ -78,7 +78,7 @@ def __init__( self.result: dict = {} # individual record results if self.report_individual: - self.result["score"] = [] + self.result["member_prob"] = [] self.result["label"] = [] self.result["target_logit"] = [] self.result["out_p_norm"] = [] @@ -353,7 +353,7 @@ def _save_attack_metrics( self.attack_metrics[-1]["n_normal"] = n_normal / (n_train_rows + n_shadow_rows) if self.report_individual: - self.result["score"] = [score[1] for score in mia_scores] + self.result["member_prob"] = [score[1] for score in mia_scores] self.result["member"] = mia_labels self.attack_metrics[-1]["individual"] = self.result diff --git a/sacroml/attacks/meta_attack.py b/sacroml/attacks/meta_attack.py index 400f1bd0..a6844853 100644 --- a/sacroml/attacks/meta_attack.py +++ b/sacroml/attacks/meta_attack.py @@ -67,7 +67,7 @@ } _MIA_SCORE_FIELDS: dict[str, str] = { - "lira": "score", + "lira": "member_prob", "qmia": "member_prob", } """Maps factory key → field name inside ``attack_metrics[N]["individual"]``. @@ -419,7 +419,7 @@ def _extract_scores_from_report( # noqa: C901 continue if key == "lira": - raw = individual.get("score") + raw = individual.get("member_prob") if raw is not None: try: collected.append([max(0.0, min(1.0, float(s))) for s in raw]) diff --git a/sacroml/attacks/report.py b/sacroml/attacks/report.py index 20c03d1b..8e5299f2 100644 --- a/sacroml/attacks/report.py +++ b/sacroml/attacks/report.py @@ -548,7 +548,7 @@ def _add_log_roc_to_page(log_roc: str = None, pdf_obj: FPDF = None) -> None: def _plot_lira_individuals(metrics: dict, dest: str) -> None: """Create a plot of the individual record LiRA scores.""" - scores = np.array(metrics["individual"]["score"]) + scores = np.array(metrics["individual"]["member_prob"]) member = np.array(metrics["individual"]["member"]) _, axes = plt.subplots(1, 2, figsize=(12.4, 4.8)) diff --git a/tests/attacks/test_lira_attack.py b/tests/attacks/test_lira_attack.py index f381b605..eb94dc23 100644 --- a/tests/attacks/test_lira_attack.py +++ b/tests/attacks/test_lira_attack.py @@ -176,7 +176,7 @@ def test_lira_arrays_externalised_and_json(lira_classifier_setup): assert "fpr" in arrays assert "tpr" in arrays assert "roc_thresh" in arrays - assert "individual.score" in arrays + assert "individual.member_prob" in arrays assert "individual.member" in arrays diff --git a/tests/attacks/test_meta_attack.py b/tests/attacks/test_meta_attack.py index 0c675433..68b8fb79 100644 --- a/tests/attacks/test_meta_attack.py +++ b/tests/attacks/test_meta_attack.py @@ -1059,7 +1059,7 @@ def test_extract_scores_lira_valid(bare_meta): "metadata": {"attack_name": "LiRA Attack"}, "attack_experiment_logger": { "attack_instance_logger": { - "instance_0": {"individual": {"score": [-0.5, 0.5, 2.0]}} + "instance_0": {"individual": {"member_prob": [-0.5, 0.5, 2.0]}} } }, } @@ -1073,7 +1073,7 @@ def test_extract_scores_lira_non_numeric(bare_meta): "metadata": {"attack_name": "LiRA Attack"}, "attack_experiment_logger": { "attack_instance_logger": { - "instance_0": {"individual": {"score": ["a", "b"]}} + "instance_0": {"individual": {"member_prob": ["a", "b"]}} } }, } @@ -1126,14 +1126,14 @@ def attack(self, _target): def test_extract_mia_scores_skips_missing_then_reads(): """The first metrics dict without scores is skipped; the next is read.""" obj = SimpleNamespace( - attack_metrics=[{}, {"individual": {"score": [-1.0, 0.3, 5.0]}}] + attack_metrics=[{}, {"individual": {"member_prob": [-1.0, 0.3, 5.0]}}] ) assert MetaAttack._extract_mia_scores(obj, "lira") == [0.0, 0.3, 1.0] def test_extract_mia_scores_non_numeric(): """Non-numeric individual MIA scores return None.""" - obj = SimpleNamespace(attack_metrics=[{"individual": {"score": ["a", "b"]}}]) + obj = SimpleNamespace(attack_metrics=[{"individual": {"member_prob": ["a", "b"]}}]) assert MetaAttack._extract_mia_scores(obj, "lira") is None diff --git a/tests/attacks/test_report.py b/tests/attacks/test_report.py index cff8c9de..aaa0c1b6 100644 --- a/tests/attacks/test_report.py +++ b/tests/attacks/test_report.py @@ -126,7 +126,7 @@ def test_write_json_excludes_large_arrays(): "fpr": np.linspace(0, 1, 100).tolist(), "tpr": np.linspace(0, 1, 100).tolist(), "roc_thresh": np.linspace(1, 0, 100).tolist(), - "individual": {"score": [0.1, 0.9], "member": [0, 1]}, + "individual": {"member_prob": [0.1, 0.9], "member": [0, 1]}, } output = { "log_id": "abcdef1234567890",