nonlinear-axiom/plot_comparison.py at main · watml/nonlinear-axiom · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
import os
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

path_fig = os.path.join(
    'figs_comparison',
    'dataset_id={}.pdf'
    )

labels = ['1.5-norm', '2.5-norm', '5-norm', '10-norm', r'$\infty$-norm', 'ELC',
          'Banzhaf', 'Beta Shapley']
keys = [1.5, 2.5, 5, 10, np.inf, 'ELC', 0.5]

colors = sns.color_palette("tab10")
del colors[-3]
del colors[-4]


def plot_curves(curves, dataset_id):
    path_components = path_fig.split(os.sep)
    os.makedirs(os.sep.join(path_components[:-1]), exist_ok=True)

    curves_beta = []
    for key in [(16,1), (8,1), (4,1), (2,1), (1,1), (1,2), (1,4), (1,8), (1,16), (1,32)]:
        curves_beta.append(curves[key][:, -1, :])
    curves_beta = np.stack(curves_beta)
    auc = curves_beta.sum(axis=2).mean(axis=1)
    loc = np.argmax(auc)
    best_beta = curves_beta[loc, :, :]
    x = np.arange(curves_beta.shape[-1])


    fig, ax = plt.subplots(figsize=(32, 24))
    for i, key in enumerate(keys):
        curve = curves[key]
        if curve[0,0,0] == np.inf:
            curve = curve[:, -1, :]
            ax.plot(x, curve.mean(axis=0), linewidth=10, c=colors[i])
        else:
            curve = curve[:, :-1, :].mean(axis=0)
            curve_mean = curve.mean(axis=0)
            curve_std = curve.std(axis=0)
            ax.plot(x, curve_mean, linewidth=10, c=colors[i])
            ax.fill_between(x, curve_mean - curve_std, curve_mean + curve_std, alpha=0.2, color=colors[i])

    ax.plot(x, best_beta.mean(axis=0), linewidth=10, c=colors[-1])


    ax.tick_params(axis='x', labelsize=80)
    ax.tick_params(axis='y', labelsize=80)
    plt.xlabel('#players added', fontsize=100)
    plt.ylabel(r'$U(S)$')

    plt.grid()
    plt.savefig(path_fig.format(dataset_id), bbox_inches='tight')
    plt.close(fig)


def export_legend(legend, fig_saved):
    fig  = legend.figure
    fig.canvas.draw()
    bbox  = legend.get_window_extent().transformed(fig.dpi_scale_trans.inverted())
    fig.savefig(fig_saved, dpi="figure", bbox_inches=bbox)


if __name__ == '__main__':
    from main import arg_dict
    from args import process_arg_dict
    from collections import defaultdict

    for i, label in enumerate(labels):
        plt.plot([], [], label=label, color=colors[i], linewidth=30)
    legend = plt.legend(ncol=8, fontsize=100, loc="upper left", bbox_to_anchor=(1, 1))
    export_legend(legend, 'legend_comparison.pdf')


    args = process_arg_dict(arg_dict)

    args_dataset = defaultdict(list)
    for arg in args:
        args_dataset[arg['dataset_id']].append(arg)

    for dataset_id, args_2nd in args_dataset.items():
        print(dataset_id)
        data = np.load(args_2nd[0]['path_data'])
        n_features = len(data['attribution'])

        curves = defaultdict(lambda : np.full((100, 6, n_features+1), np.inf, dtype=np.float64))

        for arg in args_2nd:
            data = np.load(arg['path_data'])
            auc = data['auc']

            if arg['method'] == 'semivalue':
                curves[arg['weight']][arg['instance_id'], -1] = auc
            else:
                if arg['random_seed_sampling'] == 'exact':
                    curves[arg['p']][arg['instance_id'], -1] = auc
                else:
                    curves[arg['p']][arg['instance_id'], arg['random_seed_sampling']] = auc

        plot_curves(curves, dataset_id)