Sentiment-Analysis/main_textblob.py at main · Fabiaho/Sentiment-Analysis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
from textblob import TextBlob
import pandas as pd

from sklearn.utils import shuffle

import time

import json


import dataProcesser as dp

# target: the polarity of the tweet (0 = negative, 2 = neutral, 4 = positive)
df = pd.read_csv('data/twitter/kaggle_twitter.csv', encoding='iso-8859-1')
df_tt = df.rename(columns={'clean_text': 'text', 'category': 'target'})
df_tt.dropna()
df_tt = shuffle(df_tt, random_state=42)
df_tt = df_tt.iloc[20000:25000]


# train, test = train_test_split(df_tt, test_size=0.2, random_state=42)

# expects a dataframe with columns 'text' and 'target'
def evaluate_with_textblob_3(df_tb):
    correct_assumptions = 0
    accuracy = 0
    start_time = time.time()
    i = 0

    for index, row in df_tb.iterrows():
        i += 1
        testimonial = TextBlob(row['text'])
        sentiment = 0

        if testimonial.sentiment.polarity < -0.33:
            sentiment = -1
        elif testimonial.sentiment.polarity > 0.33:
            sentiment = 1
        else:
            sentiment = 0

        if sentiment == row['target']:
            accuracy += 1

        if i % 20000 == 0 or i % df_tb.shape[0] == 0:
            elapsed = time.time() - start_time
            print('| {:7d}/{:7d} batches | time: {:5.2f}s |'.format(i, df_tb.shape[0], elapsed))
            start_time = time.time()

    accuracy = accuracy / df_tb.shape[0] * 100
    print(str(accuracy) + '%')


# expects a dataframe with columns 'text' and 'target'
def evaluate_with_textblob_5(df_tb):
    correct_assumptions = 0
    accuracy = 0
    start_time = time.time()
    i = 0

    for index, row in df_tb.iterrows():
        i += 1
        testimonial = TextBlob(row['text'])
        sentiment = -1

        if 0.6 < testimonial.sentiment.polarity <= 1.0:
            sentiment = 4
        elif 0.2 < testimonial.sentiment.polarity <= 0.6:
            sentiment = 3
        elif -0.2 <= testimonial.sentiment.polarity <= 0.2:
            sentiment = 2
        elif -0.6 <= testimonial.sentiment.polarity < -0.2:
            sentiment = 1
        elif -1 <= testimonial.sentiment.polarity < -0.6:

            sentiment = 0

        if sentiment == row['target']:
            accuracy += 1

        if i % 20000 == 0 or i % df_tb.shape[0] == 0:
            elapsed = time.time() - start_time
            print('| {:7d}/{:7d} batches | time: {:5.2f}s |'.format(i, df_tb.shape[0], elapsed))
            start_time = time.time()

    accuracy = accuracy / df_tb.shape[0] * 100
    print(str(accuracy) + '%')


train, test, dev = dp.load_movie_reviews2()
train = train.rename(columns={"sentence": "text", "label": "target"})
test = test.rename(columns={"sentence": "text", "label": "target"})
dev = dev.rename(columns={"sentence": "text", "label": "target"})

evaluate_with_textblob_3(df_tt)
evaluate_with_textblob_5(train)
evaluate_with_textblob_5(test)
evaluate_with_textblob_5(dev)