source.py

# -*- coding: utf-8 -*-
"""Q4.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1J0CJ2Za4m_Leizl1pCl-CaoQmeBJIXz9
"""

from google.colab import drive
drive.mount('/content/gdrive')
import os
import csv
import math
import random
import operator

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import pandas as pd
import collections
from collections import Counter 


train_x = '/content/gdrive/My Drive/Q4/question-4-train-features.csv'
train_y = '/content/gdrive/My Drive/Q4/question-4-train-labels.csv'
test_x = '/content/gdrive/My Drive/Q4/question-4-test-features.csv'
test_y = '/content/gdrive/My Drive/Q4/question-4-test-labels.csv'

train_y  = open(train_y, "r")
train_y = csv.reader(train_y)
train_y = list(train_y)[:-1]


train_x = open(train_x, "r")
train_x = csv.reader(train_x)
train_x = list(train_x)[:-1]

test_y  = open(test_y, "r")
test_y = csv.reader(test_y)
test_y = list(test_y)[:-1]


test_x = open(test_x, "r")
test_x = csv.reader(test_x)
test_x = list(test_x)[:-1]

#train_df = pd.DataFrame(train_y)
with open('/content/gdrive/My Drive/Q4/question-4-vocab.txt', 'r') as f:
    words = (f.readlines())
words = [str(w.replace('\n', '')) for w in words]
#train_df = pd.DataFrame({'features':train_x, 'labels':train_y})
#words.insert(len(words),'label')
#train_df.columns=words
#print(train_df)

#0:medicine
#1:space
#2:cryp
#3:electronic
total_dct = {}
for i in range(0,len(train_x)):
  for j in range(0,len(words)):
    if words[j] in total_dct:
      total_dct[words[j]] += int(train_x[i][j])
    else:
      total_dct[words[j]] = 1  
med_dct = {}
space_dct = {}
cryp_dct = {}
elec_dct = {}
total_emails_train = len(train_x)
total_voc = len(words)
number_of_space_emails = 0
number_of_med_emails = 0
number_of_cryp_emails = 0
number_of_elec_emails = 0


for i in range(0,len(train_x)):
    if (train_y[i] == ['0']): #med
      number_of_med_emails +=1
      for j in range(0,len(words)):
        if(train_x[i][j] != 0):
          if words[j] in med_dct:
            med_dct[words[j]] += int(train_x[i][j])
          else:
            med_dct[words[j]] = 1  
    elif (train_y[i] == ['1']): #space
      number_of_space_emails +=1
      for j in range(0,len(words)):
        if(train_x[i][j] != 0):
          if words[j] in space_dct:
            space_dct[words[j]] += int(train_x[i][j])
          else:
            space_dct[words[j]] = 1
    elif (train_y[i] == ['2']): #cryp
      number_of_cryp_emails +=1
      for j in range(0,len(words)):
        if(train_x[i][j] != 0):
          if words[j] in cryp_dct:
            cryp_dct[words[j]] += int(train_x[i][j])
          else:
            cryp_dct[words[j]] = 1   
    elif (train_y[i] == ['3']): #elec
      number_of_elec_emails +=1
      for j in range(0,len(words)):
        if(train_x[i][j] != 0):
          if words[j] in elec_dct:
            elec_dct[words[j]] += int(train_x[i][j])
          else:
            elec_dct[words[j]] = 1   


#%% 

total_med_voc = sum(med_dct.values()) + len(words)
total_space_voc = sum(space_dct.values()) + len(words)
total_cryp_voc = sum(cryp_dct.values()) + len(words)
total_elec_voc = sum(elec_dct.values()) + len(words)
total_voc = sum(total_dct.values())
#print(total_med_voc)
#print(total_space_voc)
#print(total_cryp_voc)
#print(total_elec_voc)
#print(total_voc)

theta_med_dct = {}
theta_space_dct = {}
theta_cryp_dct = {}
theta_elec_dct = {}
for word in med_dct.keys():
  theta_med_dct[word] = float((med_dct[word]+1)/total_med_voc)
for word in space_dct.keys():
  theta_space_dct[word] = float((space_dct[word]+1)/total_space_voc)
for word in cryp_dct.keys():
  theta_cryp_dct[word] = float((cryp_dct[word]+1)/total_cryp_voc)
for word in elec_dct.keys():
  theta_elec_dct[word] = float((elec_dct[word]+1)/total_elec_voc)


pi_med = number_of_med_emails/total_emails_train
pi_space = number_of_space_emails/total_emails_train
pi_cryp = number_of_cryp_emails/total_emails_train
pi_elec = number_of_elec_emails/total_emails_train

true_med = 0
true_space = 0
true_cryp = 0
true_elec = 0

false_med = 0
false_space = 0
false_cryp = 0
false_elec = 0

highest_med = -99999999
highest_space = -99999999
highest_cryp = -99999999
highest_elec = -99999999
highest_med_i = -99999999
highest_space_i = -99999999
highest_cryp_i = -99999999
highest_elec_i = -99999999

lowest_med = 99999999
lowest_space = 99999999
lowest_cryp = 99999999
lowest_elec = 99999999
lowest_med_i = 99999999
lowest_space_i = 99999999
lowest_cryp_i = 99999999
lowest_elec_i = 99999999
for i in range(0,len(test_x)):
  prob_med = float(math.log2(pi_med))
  prob_space = float(math.log2(pi_space))
  prob_cryp = float(math.log2(pi_cryp))
  prob_elec = float(math.log2(pi_elec)) 

  for j in range(0,len(test_x[i])):
    if(test_x[i][j]!=0):
      if words[j] in theta_med_dct:
        prob_med += abs(float(test_x[i][j]) * float(math.log2(theta_med_dct[words[j]])))
      if  words[j] in theta_space_dct:
        prob_space += abs(float(test_x[i][j]) * float(math.log2(theta_space_dct[words[j]])))
      if words[j] in theta_cryp_dct:
        prob_cryp += abs(float(test_x[i][j]) * float(math.log2(theta_cryp_dct[words[j]])))
      if words[j] in theta_elec_dct:
        prob_elec += abs(float(test_x[i][j]) * float(math.log2(theta_elec_dct[words[j]])))
#0:medicine
#1:space
#2:cryp
#3:electronic
  
  if max(prob_med,prob_space,prob_cryp,prob_elec) == prob_med:
    if test_y[i] == ['0']:
      true_med +=1
    else:
      false_med +=1
  elif max(prob_med,prob_space,prob_cryp,prob_elec) == prob_space:
    if test_y[i] == ['1']:
      true_space +=1
    else:
      false_space +=1
  elif max(prob_med,prob_space,prob_cryp,prob_elec) == prob_cryp:
    if test_y[i] == ['2']:
      true_cryp +=1
    else:
      false_cryp +=1
  elif max(prob_med,prob_space,prob_cryp,prob_elec) == prob_elec:
    if test_y[i] == ['3']:
      true_elec +=1
    else:
      false_elec +=1

  if test_y[i] == ['0']:
    if lowest_med > prob_med:
      lowest_med = prob_med
      lowest_med_i = i
    if highest_med <= prob_med:
      highest_med = prob_med
      highest_med_i = i

  if test_y[i] == ['1']:
    if lowest_space > prob_space:
      lowest_space = prob_space
      lowest_space_i = i
    if highest_space <= prob_space:
      highest_space = prob_space
      highest_space_i = i

  if test_y[i] == ['2']:
    if lowest_cryp > prob_cryp:
      lowest_cryp = prob_cryp
      lowest_cryp_i = i
    if highest_cryp <= prob_cryp:
      highest_cryp = prob_cryp
      highest_cryp_i = i

  if test_y[i] == ['3']:
    if lowest_elec > prob_elec:
      lowest_elec = prob_elec
      lowest_elec_i = i
    if highest_elec <= prob_elec:
      highest_elec = prob_elec
      highest_elec_i = i

accuracy = (true_med+true_space+true_cryp+true_elec)/(true_med+true_space+true_cryp+true_elec+false_med+false_space+false_cryp+false_elec)
print("accuracy",accuracy)
print("true med {},true space {},true cryp {},true elec {}".format(true_med,true_space,true_cryp,true_elec))
print("false med {},false space {},false cryp {},false elec {}".format(false_med,false_space,false_cryp,false_elec))

#most occured words in categories
most_med_list = []
most_space_list = []
most_cryp_list = []
most_elec_list = []

cd = sorted(med_dct.items(),key=operator.itemgetter(1),reverse=True)
print("\nMEDICAL MOST USED 20 WORDS")
for x in list(cd)[0:20]:
    print (x)

cd = sorted(space_dct.items(),key=operator.itemgetter(1),reverse=True)
print("\nSPACE MOST USED 20 WORDS")
for x in list(cd)[0:20]:
    print (x)

cd = sorted(cryp_dct.items(),key=operator.itemgetter(1),reverse=True)
print("\nCRYPTOLOGY MOST USED 20 WORDS")
for x in list(cd)[0:20]:
    print (x)

cd = sorted(elec_dct.items(),key=operator.itemgetter(1),reverse=True)
print("\nELECTRONICS MOST USED 20 WORDS")
for x in list(cd)[0:20]:
    print (x)

print("For Medical: Highest probability index{},lowest probability index{}".format(highest_med_i,lowest_med_i))
print("For Space: Highest probability index{},lowest probability index{}".format(highest_space_i,lowest_space_i))
print("For Cryptology: Highest probability index{},lowest probability index{}".format(highest_cryp_i,lowest_cryp_i))
print("For Electronics: Highest probability index{},lowest probability index{}".format(highest_elec_i,lowest_elec_i))