AnswerScriptEvaluation/w2vtest.py at main · aniruddhakj/AnswerScriptEvaluation · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
from nltk.corpus import stopwords
import re
import nltk
from gensim.models import Word2Vec, KeyedVectors

nltk.download('punkt')
nltk.download('stopwords')

text = ""
f = open('cn1.txt')
for line in f:
    text += line

f1 = open('cn2.txt')
for line in f1:
    text += line

f2 = open('cn3.txt')
for line in f2:
    text += line

f3 = open('cn4.txt')
for line in f3:
    text += line

f4 = open('cn5.txt')
for line in f4:
    text += line

f5 = open('cn6.txt')
for line in f5:
    text += line


processed_article = text.lower()
processed_article = re.sub('[^a-zA-Z]', ' ', processed_article)
processed_article = re.sub(r'\s+', ' ', processed_article)


# Preparing the dataset
all_sentences = nltk.sent_tokenize(processed_article)

all_words = [nltk.word_tokenize(sent) for sent in all_sentences]

for i in range(len(all_words)):
    all_words[i] = [w for w in all_words[i]
                    if w not in stopwords.words('english')]


# modify parameters
word2vec = Word2Vec(all_words, min_count=1, vector_size=700, epochs=100)
vocabulary = word2vec.wv.key_to_index
print(len(vocabulary.keys()))

word2vec.save('models_large.kv')