Repository navigation
Expand file tree
/
Copy pathtrain_Word2vecModel.py
More file actions
69 lines (55 loc) · 1.99 KB
/
Copy pathtrain_Word2vecModel.py
File metadata and controls
69 lines (55 loc) · 1.99 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# import modules & set up logging
import gensim
import os
import json
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import numpy as np
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
##################### Training ########################
dotaCounter = 0
csCounter = 0
lolCounter = 0
othersCounter = 0
totalWordCounter = 0
class MySentences(object):
def __init__(self, dirname):
self.dirname = dirname
def __iter__(self):
for fname in os.listdir(self.dirname):
for line in open(os.path.join(self.dirname, fname)):
# if counter < 5000:
#parse json msg
try:
parsed = json.loads(line)
parsed_msg = parsed['msg']
parsed_game = parsed['game']
except ValueError:
print "Error in line"
continue
if parsed_game == "Dota%202":
global dotaCounter
dotaCounter += 1
elif parsed_game == "League%20of%20Legends":
global lolCounter
lolCounter += 1
elif parsed_game == "Counter-Strike%3A%20Global%20Offensive":
global csCounter
csCounter += 1
else:
global othersCounter
othersCounter += 1
# counter += 1
# print counter
parsed_msg = parsed_msg.split()
global totalWordCounter
totalWordCounter += len(parsed_msg)
yield parsed_msg
# else:
# break
# sentences = MySentences('txtfiles/messages')
sentences = MySentences('txtfiles/messages')
model = gensim.models.Word2Vec(sentences, workers=4)
model.save('models/02.01_default')
print dotaCounter, csCounter, lolCounter, othersCounter, totalWordCounter