recommender.py

# -*- coding: utf-8 -*-
"""recommender.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1FKGFQpPyw6dPxCaFSpnm00p1SnMqguy5

# Recommender System
"""

# Commented out IPython magic to ensure Python compatibility.
# importing required libraries
# %matplotlib inline
import tensorflow as tf
import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import random
import os
import seaborn as sns
from scipy import stats
from ast import literal_eval
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.metrics.pairwise import linear_kernel, cosine_similarity
from nltk.stem.snowball import SnowballStemmer
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import wordnet

plt.switch_backend('agg')

user = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/users.csv'
userData = pd.read_csv(user)
userData

notifications = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/notifications.csv'
notificationsData = pd.read_csv(notifications)
notificationsData

posts = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/posts.csv'
postsData = pd.read_csv(posts)
postsData

to_reads = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/to_reads.csv'
to_reads = pd.read_csv(to_reads)
to_reads

to_reads = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/to_reads.csv'
to_reads = pd.read_csv(to_reads)
to_reads

"""# Data Preprocessing"""

#reading in of csv
contact_settings = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/contact_settings.csv'
contact_settings = pd.read_csv(contact_settings)

extfeeds = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/extfeeds.csv'
extfeeds = pd.read_csv(extfeeds)

following = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/following.csv'
following = pd.read_csv(following)

maillists = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/maillists.csv'
maillists = pd.read_csv(maillists)

migrations = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/migrations.csv'
migrations = pd.read_csv(migrations)

thoughts = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/thoughts.csv'
thoughts = pd.read_csv(thoughts)

user_settings = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/user_settings.csv'
user_settings = pd.read_csv(user_settings)

thoughts.head()    #checking first 5 rows

thoughts.nunique()  #checking  number of unique in each columns

sns.countplot(x='user_id',data=thoughts) #this counts the amount of time a user_id appears

corr=thoughts.corr()
sns.heatmap(corr.corr(),cmap='coolwarm',annot=True)   #check if positively or negetively corrralated

plt.figure(figsize = (15,10))
sns.countplot(y='created_at',hue ='user_id',data=thoughts) #counts the amount of created at was found also spliting it categories under user_id through hue

plt.figure(figsize = (15,10))
sns.countplot(y='updated_at',hue ='user_id',data=thoughts)

contact_settings.head()

extfeeds.head()

extfeeds.shape

plt.figure(figsize = (10,7))
sns.countplot(y='title',data=extfeeds)

plt.figure(figsize = (10,7))
sns.countplot(y='site',data=extfeeds)

plt.figure(figsize = (10,7))
sns.countplot(y='date',data=extfeeds)

plt.figure(figsize = (10,7))
sns.countplot(y='user_id',data=extfeeds)

plt.figure(figsize = (15,10))
sns.countplot(y='id',data=extfeeds)

corr=extfeeds.corr()
sns.heatmap(corr.corr(),cmap='coolwarm',annot=True)

maillists.head()

plt.figure(figsize = (10,12))
sns.countplot(y='id',hue ='email',data=maillists)

notificationsData.head()

corr=notificationsData.corr()
sns.heatmap(corr.corr(),cmap='coolwarm',annot=True)

plt.figure(figsize = (10,10))
sns.countplot(y='user_id',hue ='action',data=notificationsData)

plt.figure(figsize = (10,10))
sns.countplot(y='user_id',hue ='type',data=notificationsData)

plt.figure(figsize = (10,10))
sns.countplot(y='post_id',hue ='action',data=notificationsData)

plt.figure(figsize = (10,10))
sns.countplot(y='post_id',hue ='type',data=notificationsData)

plt.figure(figsize = (5,50))
sns.countplot(y='comment',hue ='post_id',data=notificationsData)

plt.figure(figsize = (5,50))
sns.countplot(y='comment',hue ='user_id',data=notificationsData)

thoughts.head()

corr=thoughts.corr()
sns.heatmap(corr.corr(),cmap='coolwarm',annot=True)

plt.figure(figsize = (5,50))
sns.countplot(y='content',hue ='user_id',data=thoughts)

ratings = 'https://raw.githubusercontent.com/Nedu/RecommenderSystemTeamJava/master/data/ratings.csv'
ratings = pd.read_csv(ratings)
ratings

# creating a new column with indexes to ease the process of creating our training data
ratings = ratings.reset_index(drop=True)
ratings['List Index'] = ratings.index
readers_group = ratings.groupby("user_id")
ratings

total = []

for userID, curReader in readers_group:
        temp = np.zeros(len(ratings))

        for num, notif in curReader.iterrows():
            
            temp[notif['List Index']] = notif['eventStrength'] / 5.0

        total.append(temp)
total

# Dividing the total data into train and validation sets
random.shuffle(total)
print("total size of the data is: {0}".format(len(total)))

train = total[:4]
valid = total[4:]
print("size of the training data is: {0}".format(len(train)))
print("size of the validation data is: {0}".format(len(valid)))

print("Setting the models Parameters")
hiddenUnits = 64
visibleUnits = len(ratings)

vb = tf.placeholder(tf.float32, [visibleUnits])
# Number of features were going to learn
hb = tf.placeholder(tf.float32, [hiddenUnits])
W = tf.placeholder(tf.float32, [visibleUnits, hiddenUnits])  # Weight Matrix


print("Phase 1: Input Processing")
v0 = tf.placeholder("float", [None, visibleUnits])
_h0 = tf.nn.sigmoid(tf.matmul(v0, W) + hb)  # Visible layer activation
# Gibb's Sampling
h0 = tf.nn.relu(tf.sign(_h0 - tf.random_uniform(tf.shape(_h0))))

print("Phase 2: Reconstruction")
_v1 = tf.nn.sigmoid(tf.matmul(h0, tf.transpose(W)) +
                    vb)  # Hidden layer activation
v1 = tf.nn.relu(tf.sign(_v1 - tf.random_uniform(tf.shape(_v1))))
h1 = tf.nn.sigmoid(tf.matmul(v1, W) + hb)

print("Setting RBM Training Parameters")
# Learning rate
alpha = 0.6

print("Creating the gradients")
w_pos_grad = tf.matmul(tf.transpose(v0), h0)
w_neg_grad = tf.matmul(tf.transpose(v1), h1)

# Calculate the Contrastive Divergence to maximize
CD = (w_pos_grad - w_neg_grad) / tf.to_float(tf.shape(v0)[0])

# Create methods to update the weights and biases
update_w = W + alpha * CD
update_vb = vb + alpha * tf.reduce_mean(v0 - v1, 0)
update_hb = hb + alpha * tf.reduce_mean(h0 - h1, 0)

# Set the error function, here we use Mean Absolute Error Function
err = v0 - v1
err_sum = tf.reduce_mean(err * err)

""" Initialize our Variables with Zeroes using Numpy Library """
# Current weight
cur_w = np.random.normal(loc=0, scale=0.01, size=[visibleUnits, hiddenUnits])

# Current visible unit biases
cur_vb = np.zeros([visibleUnits], np.float32)

# Current hidden unit biases
cur_hb = np.zeros([hiddenUnits], np.float32)

# Previous weight
prv_w = np.zeros([visibleUnits, hiddenUnits], np.float32)

# Previous visible unit biases
prv_vb = np.zeros([visibleUnits], np.float32)

# Previous hidden unit biases
prv_hb = np.zeros([hiddenUnits], np.float32)

print("Running the session")
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())

def free_energy(v_sample, W, vb, hb):
    ''' Function to compute the free energy '''
    wx_b = np.dot(v_sample, W) + hb
    vbias_term = np.dot(v_sample, vb)
    hidden_term = np.sum(np.log(1 + np.exp(wx_b)), axis = 1)
    return -hidden_term - vbias_term

# Training RBM with 80 Epochs, with Each Epoch using batch size of 100.
# After training print out the error with epoch number.
print("Starting the training process")
epochs = 80
batchsize = 100
errors = []
energy_train = []
energy_valid = []

for i in range(epochs):
    for start, end in zip(range(0, len(train), batchsize), range(batchsize, len(train), batchsize)):
        batch = train[start:end]
        cur_w = sess.run(update_w, feed_dict={
                         v0: batch, W: prv_w, vb: prv_vb, hb: prv_hb})
        print(cur_w)
        cur_vb = sess.run(update_vb, feed_dict={
                          v0: batch, W: prv_w, vb: prv_vb, hb: prv_hb})
        cur_hb = sess.run(update_hb, feed_dict={
                          v0: batch, W: prv_w, vb: prv_vb, hb: prv_hb})
        prv_w = cur_w
        prv_vb = cur_vb
        prv_hb = cur_hb

    energy_train.append(np.mean(free_energy(train, cur_w, cur_vb, cur_hb)))
#     print("Epoch: {0}, free energy: {1}".format(i, energy_train[i]))
    energy_valid.append(np.mean(free_energy(valid, cur_w, cur_vb, cur_hb)))

    errors.append(sess.run(err_sum, feed_dict={
                  v0: train, W: cur_w, vb: cur_vb, hb: cur_hb}))
    if i % 10 == 0:
        print("Error in epoch {0} is: {1}".format(i, errors[i]))

# This is the input that we need to provide manually, that is the user number
user = 3
inputUser = [train[user]]

# Feeding in the User and Reconstructing the input
hh0 = tf.nn.sigmoid(tf.matmul(v0, W) + hb)
vv1 = tf.nn.sigmoid(tf.matmul(hh0, tf.transpose(W)) + vb)
feed = sess.run(hh0, feed_dict={v0: inputUser, W: prv_w, hb: prv_hb})
rec = sess.run(vv1, feed_dict={hh0: feed, W: prv_w, vb: prv_vb})

"""# Similar Post Content Based Recommender"""

# clean up content
postsData['content'] = postsData['content'].str.replace(r'<[^>]*>', '')
postsData['content'] = postsData['content'].str.replace(r'\s', ' ')
postsData['content'] = postsData['content'].str.replace(r'\\', ' ')
postsData['content'] = postsData['content'].str.replace(r'\~', ' ')
postsData['content'] = postsData['content'].str.replace(r'\[.*?\]', '')
postsData['content'] = postsData['content'].str.replace(r'\(.*?\)', '')
postsData['content'].head()

#Define a TF-IDF Vectorizer Object. Remove all english stop words such as 'the', 'a'
tfidf_post = TfidfVectorizer(stop_words='english')

#Replace NaN with an empty string
postsData['content'] = postsData['content'].fillna('')

#Construct the required TF-IDF matrix by fitting and transforming the data
tfidf_post_matrix = tfidf_post.fit_transform(postsData['content'])

#Output the shape of tfidf_matrix
tfidf_post_matrix.shape

# Compute the cosine similarity matrix
cosine_sim = linear_kernel(tfidf_post_matrix, tfidf_post_matrix)

#Construct a reverse map of indices and posts titles
indices_post = pd.Series(postsData.index, index=postsData['title']).drop_duplicates()

# Function that takes in post title as input and outputs most similar posts
def get_article_recommendations_for_user(title, cosine_sim=cosine_sim):
    # Get the index of the post that matches the title
    idx = indices_post[title]

    # Get the pairwsie similarity scores of all posts with that post
    sim_scores = list(enumerate(cosine_sim[idx]))

    # Sort the posts based on the similarity scores
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)

    # Get the scores of the 10 most similar posts
    sim_scores = sim_scores[1:11]

    # Get the post indices
    post_indices = [i[0] for i in sim_scores]

    # Return the top 10 most similar posts
    return postsData['title'].iloc[post_indices]

# currentPost = 'What i have learnt so far on HTML'
# recommendSimilarPosts = get_article_recommendations_for_user('What i have learnt so far on HTML')

"""# FOLLOWER RECOMMENDER"""

userData.columns

userData.drop(labels=['image', 'provider', 'provider', 'provider_id', 'password', 'remember_token', 'created_at', 'updated_at'], axis=1, inplace=True )
userData.head(10)

userData.shape

userData['short_bio'].head(5)

#Define a TF-IDF Vectorizer Object. Remove all english stop words such as 'the', 'a'
tfidf = TfidfVectorizer(stop_words='english')
#Replace NaN with an empty string
userData['short_bio'] = userData['short_bio'].fillna('')
#Construct the required TF-IDF matrix by fitting and transforming the data
tfidf_matrix = tfidf.fit_transform(userData['short_bio'])
#Output the shape of tfidf_matrix
# this produces the number of different words used by user in the short_bio
tfidf_matrix.shape

# Compute the cosine similarity matrix
consine_sim = linear_kernel(tfidf_matrix, tfidf_matrix, True)
#Construct a reverse map of indices and user name
indices = pd.Series(userData.index, index=userData['id'])

def create_new_db(x):
    """
    this function takes the result from get_followers and transforms to a DB with other info
    paramx: recommendation Series
    """
    y = x.copy()
    z = x.copy()
    y.update(userData['name'])
    z.update(userData['short_bio'])
    xdf=pd.DataFrame(x)
    ydf=pd.DataFrame(y)
    zdf=pd.DataFrame(z)
    ydf.rename(columns={'id':'Name'}, inplace=True)
    xdf.rename(columns={'id':'User_Id'}, inplace=True)
    zdf.rename(columns={'id':'short_bio'}, inplace=True)
    frames = [xdf,ydf,zdf]
    new_df = pd.concat(frames, axis=1)
    new_df.set_index('User_Id', inplace=True)
    return new_df

# Function that takes in user name as input and outputs most similar users
def get_followers(id, consine_sim=consine_sim):
    # Get the index of the users that matches the user
    idx = indices[id]
    # Get the pairwsie similarity scores of all users with that user
    sim_scores = list(enumerate(consine_sim[idx]))
    # Sort the users based on the similarity scores
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    # Get the scores of the 10 most similar users
    sim_scores = sim_scores[1:11]
     # Get the user indices
    follow_indices = [i[0] for i in sim_scores]
      # Return the top 10 most similar user
    follower_id = userData['id'].iloc[follow_indices]
    # Create a new data frame containing the user id and name 
    follower_rec = create_new_db(follower_id)
    return follower_rec

# Use this function to get details of the user with their user id
def check_user(id):
    name = userData.loc[userData['id'] == id, 'name']
    bio = userData.loc[userData['id'] == id, 'short_bio']
    return name.iloc[0], bio.iloc[0]

check_user(23)

# Run Get_followers with user id as input
get_followers(23)

userData.loc[userData['id'] == 23, 'short_bio'].iloc[0]

"""# Recommendation Engine"""

# Creating recommendation score for posts in our data
ratings["Recommendation Score"] = rec[0]
ratings

""" Recommend User what posts he has not interacted with yet """
# Find the mock user's user_id from the data
cur_user_id = ratings.iloc[user]['user_id']
print ('cur_user_id: ', cur_user_id)

# Find all posts the mock user has interacted with before
read_posts = ratings[ratings['user_id'] == cur_user_id]['post_id']

# converting the pandas series object into a list
read_posts_id = read_posts.tolist()

# getting the post title for the posts already interacted on by the user
read_posts_titles = []
for post in read_posts_id:
    read_posts_titles.append(
        postsData[postsData['id'] == post]['title'].tolist()[0])

# Find all posts the mock user has 'not' interacted before using the to_reads data
unread_posts = to_reads[to_reads['user_id'] == cur_user_id]['post_id']
unread_posts_id = unread_posts.tolist()

# extract the ratings of all the uninteracted posts from ratings dataframe
unread_with_score = ratings[ratings['post_id'].isin(unread_posts_id)]

# grouping the unread data on post id and taking the mean of the recommendation scores for each post_id
grouped_unread = unread_with_score.groupby('post_id', as_index=False)[
    'Recommendation Score'].mean()
grouped_unread

# getting the titles of the unread posts
unread_posts_titles = []
unread_posts_scores = []
for post in grouped_unread['post_id']:
    unread_books_names.append(
        postsData[postsData['post_id'] == post]['title'].tolist()[0])
    unread_books_scores.append(
        grouped_unread[grouped_unread['post_id'] == post]['Recommendation Score'].tolist()[0])

# creating a data frame for unread posts with their titles and recommendation scores
unread_posts_with_scores = pd.DataFrame({
    'post_name': unread_posts_titles,
    'score': unread_posts_scores
})
unread_posts_with_scores

# creating a data frame for read posts with the titles
read_posts_with_titles = pd.DataFrame({
    'post_name': read_posts_titles,
})

read_posts_with_titles

# sort the result in descending order of the recommendation score
sorted_result = unread_posts_with_scores.sort_values(
    by='score', ascending=False)

"""# Testing"""

samplePost = 'What i have learnt so far on HTML'
recommendSimilarPosts = get_article_recommendations_for_user(samplePost)

print('The current post read by the user is:', currentPost)

print('The posts recommended to the user based on similarity to the current post are:')
print(recommendSimilarPosts)

# Commented out IPython magic to ensure Python compatibility.
# %matplotlib inline
# Saving the plots
plt.plot(errors)
plt.xlabel("Epoch")
plt.ylabel("Error")
plt.savefig("error.png")
plt.show()

fig, ax = plt.subplots()
ax.plot(energy_train, label='train')
ax.plot(energy_valid, label='valid')
leg = ax.legend()
plt.xlabel("Epoch")
plt.ylabel("Free Energy")
plt.savefig("free_energy.png")
plt.show()