pheme_dataset_preprocessing/preprocessing.py at main · BWAAEEEK/pheme_dataset_preprocessing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
import os
import json
import pickle
from pprint import PrettyPrinter
from tqdm import tqdm
import networkx as nx


events = [i for i in os.listdir(".") if "." not in i]
graph_dict = {event: {"rumours": [], "non-rumours": []} for event in events}

for event in events:
    # non-rumours
    non_rumour_list = [i for i in os.listdir(f"./{event}/non-rumours") if "." not in i]

    for root_id in tqdm(non_rumour_list,
                        desc=f"preprocessing {event} non-rumours:"):
        graph = nx.DiGraph()

        # root node attribution
        with open(f"./{event}/non-rumours/{root_id}/source-tweets/{root_id}.json", "r") as f:
            root_info: dict = json.load(f)

        graph.add_node(root_info["id"])

        for k, v in root_info.items():
            graph.nodes[root_info["id"]][k] = v

        reactions = [i for i in os.listdir(f"./{event}/non-rumours/{root_id}/reactions") if "json" in i and "._" not in i]

        for reaction in reactions:
            with open(f"./{event}/non-rumours/{root_id}/reactions/{reaction}", "r") as f:
                data: dict = json.load(f)

            # node attributions
            graph.add_node(data["id"])

            for k, v in data.items():
                graph.nodes[data["id"]][k] = v

            try:
                graph.add_edge(data["id"], data["in_reply_to_status_id"])
            except ValueError:
                graph.add_node(data["id"])

        graph_dict[event]["non-rumours"].append(graph)

    # rumours
    rumour_list = [i for i in os.listdir(f"./{event}/rumours") if "." not in i]

    for root_id in tqdm(rumour_list,
                        desc=f"preprocessing {event} rumours:"):
        graph = nx.DiGraph()

        # root node attribution
        with open(f"./{event}/rumours/{root_id}/source-tweets/{root_id}.json", "r") as f:
            root_info: dict = json.load(f)

        graph.add_node(root_info["id"])

        for k, v in root_info.items():
            graph.nodes[root_info["id"]][k] = v

        reactions = [i for i in os.listdir(f"./{event}/rumours/{root_id}/reactions") if "json" in i and "._" not in i]

        for reaction in reactions:
            with open(f"./{event}/rumours/{root_id}/reactions/{reaction}", "r") as f:
                data: dict = json.load(f)

            # node attributions
            graph.add_node(data["id"])

            for k, v in data.items():
                graph.nodes[data["id"]][k] = v

            try:
                graph.add_edge(data["id"], data["in_reply_to_status_id"])
            except ValueError:
                graph.add_node(data["id"])

        graph_dict[event]["rumours"].append(graph)


with open("./graph_dict.pkl", "wb") as f:
    pickle.dump(graph_dict, f)

PrettyPrinter().pprint(graph_dict[events[0]]["rumours"][0].nodes)