From 2e05a2d6562b460027a48e788773fbbdbecdc097 Mon Sep 17 00:00:00 2001 From: krushnapavan <35088826+krushnapavan9@users.noreply.github.com> Date: Tue, 24 Mar 2020 19:53:13 +0530 Subject: [PATCH 1/3] Updating ML file cleared all the errors and bugs --- .../TickectClassificationML.ipynb | 1317 +++++++++++++++++ 1 file changed, 1317 insertions(+) create mode 100644 Machine Learning/TickectClassificationML.ipynb diff --git a/Machine Learning/TickectClassificationML.ipynb b/Machine Learning/TickectClassificationML.ipynb new file mode 100644 index 0000000..9ebecaa --- /dev/null +++ b/Machine Learning/TickectClassificationML.ipynb @@ -0,0 +1,1317 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Copy of Copy of TickectClassificationML.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "cell_type": "code", + "metadata": { + "id": "s34hKA4TfQsg", + "colab_type": "code", + "outputId": "f6a3e78d-64b8-4c8a-81b3-c0ace1cbc342", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 85 + } + }, + "source": [ + "import nltk\n", + "nltk.download('punkt')\n", + "import re\n", + "from gensim import models, corpora\n", + "from nltk import word_tokenize\n", + "from nltk.corpus import stopwords\n", + "nltk.download('brown')\n", + "import numpy as np\n", + "import pandas as pd\n", + "from matplotlib import pyplot as plt\n", + "from bs4 import BeautifulSoup\n", + "import re\n", + "from sklearn import preprocessing" + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package punkt to /root/nltk_data...\n", + "[nltk_data] Unzipping tokenizers/punkt.zip.\n", + "[nltk_data] Downloading package brown to /root/nltk_data...\n", + "[nltk_data] Unzipping corpora/brown.zip.\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "9p2jq05Oi418", + "colab_type": "code", + "outputId": "b1f73cd4-9cbe-457c-a050-763d6ce45f59", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 204 + } + }, + "source": [ + "datasource=pd.read_csv('https://raw.githubusercontent.com/Killer2499/Support-Tickets-Topic-Classification/master/Deep%20Learning/latest_ticket_data.csv')\n", + "datasource.head()" + ], + "execution_count": 2, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DescriptionCategory
0hi since recruiter lead permission approve req...Application
1re expire days hi ask help update passwords co...Application
2verification warning hi has got attached pleas...Application
3please dear looks blacklisted receiving mails ...Application
4dear modules report report cost thank much reg...Application
\n", + "
" + ], + "text/plain": [ + " Description Category\n", + "0 hi since recruiter lead permission approve req... Application\n", + "1 re expire days hi ask help update passwords co... Application\n", + "2 verification warning hi has got attached pleas... Application\n", + "3 please dear looks blacklisted receiving mails ... Application\n", + "4 dear modules report report cost thank much reg... Application" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 2 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "xYzX96pWjzA6", + "colab_type": "code", + "colab": {} + }, + "source": [ + "features=datasource['Description']\n", + "labels=datasource['Category']\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "rgBWLrvOlQS8", + "colab_type": "code", + "outputId": "d331e5b5-842c-494e-e5b2-049b067c182b", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "cList = {\n", + " \"ain't\": \"am not\",\n", + " \"aren't\": \"are not\",\n", + " \"can't\": \"cannot\",\n", + " \"can't've\": \"cannot have\",\n", + " \"'cause\": \"because\",\n", + " \"could've\": \"could have\",\n", + " \"couldn't\": \"could not\",\n", + " \"couldn't've\": \"could not have\",\n", + " \"didn't\": \"did not\",\n", + " \"doesn't\": \"does not\",\n", + " \"don't\": \"do not\",\n", + " \"hadn't\": \"had not\",\n", + " \"hadn't've\": \"had not have\",\n", + " \"hasn't\": \"has not\",\n", + " \"haven't\": \"have not\",\n", + " \"he'd\": \"he would\",\n", + " \"he'd've\": \"he would have\",\n", + " \"he'll\": \"he will\",\n", + " \"he'll've\": \"he will have\",\n", + " \"he's\": \"he is\",\n", + " \"how'd\": \"how did\",\n", + " \"how'd'y\": \"how do you\",\n", + " \"how'll\": \"how will\",\n", + " \"how's\": \"how is\",\n", + " \"I'd\": \"I would\",\n", + " \"I'd've\": \"I would have\",\n", + " \"I'll\": \"I will\",\n", + " \"I'll've\": \"I will have\",\n", + " \"I'm\": \"I am\",\n", + " \"I've\": \"I have\",\n", + " \"isn't\": \"is not\",\n", + " \"it'd\": \"it had\",\n", + " \"it'd've\": \"it would have\",\n", + " \"it'll\": \"it will\",\n", + " \"it'll've\": \"it will have\",\n", + " \"it's\": \"it is\",\n", + " \"let's\": \"let us\",\n", + " \"ma'am\": \"madam\",\n", + " \"mayn't\": \"may not\",\n", + " \"might've\": \"might have\",\n", + " \"mightn't\": \"might not\",\n", + " \"mightn't've\": \"might not have\",\n", + " \"must've\": \"must have\",\n", + " \"mustn't\": \"must not\",\n", + " \"mustn't've\": \"must not have\",\n", + " \"needn't\": \"need not\",\n", + " \"needn't've\": \"need not have\",\n", + " \"o'clock\": \"of the clock\",\n", + " \"oughtn't\": \"ought not\",\n", + " \"oughtn't've\": \"ought not have\",\n", + " \"shan't\": \"shall not\",\n", + " \"sha'n't\": \"shall not\",\n", + " \"shan't've\": \"shall not have\",\n", + " \"she'd\": \"she would\",\n", + " \"she'd've\": \"she would have\",\n", + " \"she'll\": \"she will\",\n", + " \"she'll've\": \"she will have\",\n", + " \"she's\": \"she is\",\n", + " \"should've\": \"should have\",\n", + " \"shouldn't\": \"should not\",\n", + " \"shouldn't've\": \"should not have\",\n", + " \"so've\": \"so have\",\n", + " \"so's\": \"so is\",\n", + " \"that'd\": \"that would\",\n", + " \"that'd've\": \"that would have\",\n", + " \"that's\": \"that is\",\n", + " \"there'd\": \"there had\",\n", + " \"there'd've\": \"there would have\",\n", + " \"there's\": \"there is\",\n", + " \"they'd\": \"they would\",\n", + " \"they'd've\": \"they would have\",\n", + " \"they'll\": \"they will\",\n", + " \"they'll've\": \"they will have\",\n", + " \"they're\": \"they are\",\n", + " \"they've\": \"they have\",\n", + " \"to've\": \"to have\",\n", + " \"wasn't\": \"was not\",\n", + " \"we'd\": \"we had\",\n", + " \"we'd've\": \"we would have\",\n", + " \"we'll\": \"we will\",\n", + " \"we'll've\": \"we will have\",\n", + " \"we're\": \"we are\",\n", + " \"we've\": \"we have\",\n", + " \"weren't\": \"were not\",\n", + " \"what'll\": \"what will\",\n", + " \"what'll've\": \"what will have\",\n", + " \"what're\": \"what are\",\n", + " \"what's\": \"what is\",\n", + " \"what've\": \"what have\",\n", + " \"when's\": \"when is\",\n", + " \"when've\": \"when have\",\n", + " \"where'd\": \"where did\",\n", + " \"where's\": \"where is\",\n", + " \"where've\": \"where have\",\n", + " \"who'll\": \"who will\",\n", + " \"who'll've\": \"who will have\",\n", + " \"who's\": \"who is\",\n", + " \"who've\": \"who have\",\n", + " \"why's\": \"why is\",\n", + " \"why've\": \"why have\",\n", + " \"will've\": \"will have\",\n", + " \"won't\": \"will not\",\n", + " \"won't've\": \"will not have\",\n", + " \"would've\": \"would have\",\n", + " \"wouldn't\": \"would not\",\n", + " \"wouldn't've\": \"would not have\",\n", + " \"y'all\": \"you all\",\n", + " \"y'alls\": \"you alls\",\n", + " \"y'all'd\": \"you all would\",\n", + " \"y'all'd've\": \"you all would have\",\n", + " \"y'all're\": \"you all are\",\n", + " \"y'all've\": \"you all have\",\n", + " \"you'd\": \"you had\",\n", + " \"you'd've\": \"you would have\",\n", + " \"you'll\": \"you you will\",\n", + " \"you'll've\": \"you you will have\",\n", + " \"you're\": \"you are\",\n", + " \"you've\": \"you have\"\n", + "}\n", + "\n", + "c_re = re.compile('(%s)' % '|'.join(cList.keys()))\n", + " \n", + "def expandContractions(text, c_re=c_re):\n", + " def replace(match):\n", + " return cList[match.group(0)]\n", + " return c_re.sub(replace, text)\n", + " \n", + "expandContractions(\"I ain't going anywhere\")" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "'I am not going anywhere'" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 4 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "IFQTgXh7lUXE", + "colab_type": "code", + "colab": {} + }, + "source": [ + "def clean_text(text):\n", + "\n", + " \n", + " #Removing comments fom body\n", + " cleaned_text = text.replace(r'From:.*$', '')\n", + "\n", + "\n", + " #Expanding the sentence\n", + " cleaned_text = (re.sub(\"([A-Z]{1}[a-z])\",\" \\\\1\",i) for i in cleaned_text)\n", + "\n", + " #To Lower case\n", + " #cleaned_text = cleaned_text.lower()\n", + "\n", + " #Removing email ID's\n", + " cleaned_text = (re.sub(\"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\\\.[a-zA-Z0-9.]+\",\"\",i) for i in cleaned_text)\n", + "\n", + " #Removing hyperlinks\n", + " cleaned_text = (re.sub(\"(http:\\/\\/www\\.|https:\\/\\/www\\.|http:\\/\\/|https:\\/\\/)?[a-z0-9]+([\\-\\.]{1}[a-zA-Z0-9]+)*\\.[a-z]{2,5}(:[0-9]{1,5})?(\\/.*)?\",\"\",i) for i in cleaned_text)\n", + "\n", + " cleaned_text = (re.sub(\"http\\S+\",\"\",i) for i in cleaned_text)\n", + "\n", + " #cleaned_text = cleaned_text.replace(r'url= .', '')\n", + "\n", + " #Replacing Contractions\n", + " cleaned_text = (re.sub(\"’\",\"'\",i) for i in cleaned_text)\n", + "\n", + " cleaned_text = (expandContractions(i) for i in cleaned_text)\n", + "\n", + " #Removing unnecessary punctuations\n", + " cleaned_text = (re.sub(\"[\\(\\)\\-\\[\\]\\{\\}\\;\\:\\\"\\\\\\<\\>\\/\\@\\#\\$\\%\\^\\&\\*\\_\\~\\–]+\",\" \",i) for i in cleaned_text)\n", + "\n", + " cleaned_text = (re.sub(\"(\\?)([a-zA-Z0-9]+)\",\"\\\\1 \\\\2\",i) for i in cleaned_text)\n", + "\n", + " cleaned_text = (re.sub(\"([A-Za-z]{1,})(\\.)([A-Za-z]{2,})\",\"\\\\1 \\\\2 \\\\3\",i) for i in cleaned_text)\n", + "\n", + " cleaned_text = (re.sub(\"[\\']+\",\"\",i) for i in cleaned_text)\n", + "\n", + " #cleaned_text = cleaned_text.replace(r'\\xa0', ' ')\n", + "\n", + " cleaned_text = (re.sub(\"(\\.)(\\,|\\?|\\'|\\!|\\s){1,}\",\"\\\\1 \",i) for i in cleaned_text)\n", + "\n", + " cleaned_text = (re.sub(\"(!){1}(\\s)*(\\=)\",\" not equals \",i) for i in cleaned_text)\n", + "\n", + " cleaned_text = (re.sub(\"(\\=)\",\" equals \",i) for i in cleaned_text)\n", + "\n", + "\n", + " #Replacing _ with \" \"\n", + " cleaned_text = (re.sub(\"_\",\" \",i) for i in cleaned_text)\n", + " \n", + " \n", + " #Replacing @–\n", + " #cleaned_text = cleaned_text.apply(lambda i: re.sub(r'[@–]','',i))\n", + "\n", + " #Removing ASCII\n", + " #cleaned_text = cleaned_text.apply(lambda i: re.sub(r'[^\\x00-\\x7F]+',' ', i))\n", + "\n", + " #Replacing multiple '.' with single '.'\n", + " cleaned_text = (re.sub(\"\\s\\.+\",\" .\",i) for i in cleaned_text)\n", + "\n", + " cleaned_text = (re.sub(\"\\\\.+\",\".\",i) for i in cleaned_text)\n", + "\n", + " #Replacing . follower by characters with spaces followed by . followed by characters\n", + " cleaned_text = (re.sub(\"([0-9]+)(\\.{1})([^0-9]+)\",\"\\\\1 \\\\2 \\\\3\",i) for i in cleaned_text)\n", + "\n", + "\n", + " # remove after regards\n", + " cleaned_text = (re.sub(r'regards.*$','',i) for i in cleaned_text)\n", + " cleaned_text = (re.sub(r'tel \\+.*$','',i) for i in cleaned_text) \n", + " cleaned_text = (re.sub(r'fax \\+.*$','',i) for i in cleaned_text)\n", + " \n", + " #print (cleaned_text)\n", + " \n", + " # Remove all the special characters\n", + " cleaned_text = re.sub(r'\\W', ' ', (\"\").join(cleaned_text))\n", + " #print (cleaned_text)\n", + " \n", + " \n", + " # Remove single characters from the start\n", + " cleaned_text = re.sub(r'\\^[a-zA-Z]\\s+', ' ', cleaned_text) \n", + "\n", + " # Substituting multiple spaces with single space\n", + " cleaned_text = re.sub(r'\\s+', ' ', cleaned_text, flags=re.I)\n", + "\n", + " \n", + " # remove all single characters\n", + " cleaned_text= re.sub(r'\\s+[a-zA-Z]\\s+', ' ', cleaned_text)\n", + " \n", + " \n", + " #Getting to root word \n", + " #cleaned_text = (lemmatizeCust(i) for i in cleaned_text)\n", + " \n", + " #Removing stopwords\n", + " #cleaned_text = (removeStopWord(i) for i in cleaned_text)\n", + "\n", + " #Replacing muiltple white spaces\n", + " #cleaned_text = (re.sub(\" +\",\" \",i) for i in cleaned_text)\n", + "\n", + "\n", + " \n", + "\n", + " return(cleaned_text)\n", + " \n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "i_exG7nxqP4k", + "colab_type": "code", + "colab": {} + }, + "source": [ + "stopWordlist =[\n", + " 'thank',\n", + " 'regards',\n", + " 'ocp',\n", + " 'thanks',\n", + " 'hi',\n", + " 'hello',\n", + " 'team',\n", + " 'team',\n", + " 'team',\n", + " 'team',\n", + " 'team',\n", + " 'team',\n", + " 'team',\n", + " 'team',\n", + " 'i', \n", + " 'me',\n", + " 'my',\n", + " 'myself',\n", + " 'we',\n", + " 'our',\n", + " 'ours',\n", + " 'ourselves',\n", + " 'you',\n", + " \"you're\",\n", + " \"you've\",\n", + " \"you'll\",\n", + " \"you'd\",\n", + " 'your',\n", + " 'yours',\n", + " 'yourself',\n", + " 'yourselves',\n", + " 'he',\n", + " 'him',\n", + " 'his',\n", + " 'himself',\n", + " 'she',\n", + " \"she's\",\n", + " 'her',\n", + " 'hers',\n", + " 'herself',\n", + " 'it',\n", + " \"it's\",\n", + " 'its',\n", + " 'itself',\n", + " 'they',\n", + " 'them',\n", + " 'their',\n", + " 'theirs',\n", + " 'themselves',\n", + " 'what',\n", + " 'which',\n", + " 'who',\n", + " 'whom',\n", + " 'this',\n", + " 'that',\n", + " \"that'll\",\n", + " 'these',\n", + " 'those',\n", + " 'am',\n", + " 'is',\n", + " 'are',\n", + " 'was',\n", + " 'were',\n", + " 'be',\n", + " 'been',\n", + " 'being',\n", + " 'have',\n", + " 'has',\n", + " 'had',\n", + " 'having',\n", + " 'do',\n", + " 'does',\n", + " 'did',\n", + " 'doing',\n", + " 'a',\n", + " 'an',\n", + " 'the',\n", + " 'and',\n", + " 'if',\n", + " 'or',\n", + " 'because',\n", + " 'as',\n", + " 'until',\n", + " 'while',\n", + " 'of',\n", + " 'at',\n", + " 'by',\n", + " 'for',\n", + " 'with',\n", + " 'about',\n", + " 'against',\n", + " 'between',\n", + " 'into',\n", + " 'through',\n", + " 'during',\n", + " 'before',\n", + " 'after',\n", + " 'above',\n", + " 'below',\n", + " 'to',\n", + " 'from',\n", + " 'up',\n", + " 'down',\n", + " 'in',\n", + " 'out',\n", + " 'on',\n", + " 'off',\n", + " 'over',\n", + " 'under',\n", + " 'again',\n", + " 'further',\n", + " 'then',\n", + " 'once',\n", + " 'here',\n", + " 'there',\n", + " 'when',\n", + " 'where',\n", + " 'why',\n", + " 'how',\n", + " 'all',\n", + " 'any',\n", + " 'both',\n", + " 'each',\n", + " 'few',\n", + " 'more',\n", + " 'most',\n", + " 'other',\n", + " 'some',\n", + " 'such',\n", + " 'no',\n", + " 'nor',\n", + " 'only',\n", + " 'own',\n", + " 'same',\n", + " 'so',\n", + " 'than',\n", + " 'too',\n", + " 'very',\n", + " 's',\n", + " 't',\n", + " 'can',\n", + " 'will',\n", + " 'just',\n", + " 'don',\n", + " \"don't\",\n", + " 'should',\n", + " \"should've\",\n", + " 'now',\n", + " 'd',\n", + " 'll',\n", + " 'm',\n", + " 'o',\n", + " 're',\n", + " 've',\n", + " 'y',\n", + " 'ain',\n", + " 'aren',\n", + " \"aren't\",\n", + " 'couldn',\n", + " \"couldn't\",\n", + " 'didn',\n", + " \"didn't\",\n", + " 'doesn',\n", + " \"doesn't\",\n", + " 'hadn',\n", + " \"hadn't\",\n", + " 'hasn',\n", + " \"hasn't\",\n", + " 'haven',\n", + " \"haven't\",\n", + " 'isn',\n", + " \"isn't\",\n", + " 'ma',\n", + " 'mightn',\n", + " \"mightn't\",\n", + " 'mustn',\n", + " \"mustn't\",\n", + " 'needn',\n", + " \"needn't\",\n", + " 'shan',\n", + " \"shan't\",\n", + " 'shouldn',\n", + " \"shouldn't\",\n", + " 'wasn',\n", + " \"wasn't\",\n", + " 'weren',\n", + " \"weren't\",\n", + " 'won',\n", + " \"won't\",\n", + " 'wouldn',\n", + " \"wouldn't\",\n", + " \"xa0\",\n", + " \"nbsp\"]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "rcGuiRfNqTbs", + "colab_type": "code", + "colab": {} + }, + "source": [ + "def removeStopWord(text):\n", + " string = \"\"\n", + " for word in text.split(\" \"):\n", + " if(word in stopWordlist):\n", + " string = string + \"\"\n", + " else:\n", + " string = string + \" \" + word\n", + " return string " + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "0xOq8-urlXYh", + "colab_type": "code", + "outputId": "7230e887-0578-4d55-89d7-85bf0718fc6d", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "from nltk.stem import PorterStemmer\n", + "from nltk.tokenize import sent_tokenize, word_tokenize\n", + "\n", + "porter=PorterStemmer()\n", + "def stemSentence(sentence):\n", + " token_words=word_tokenize(sentence)\n", + " token_words\n", + " stem_sentence=[]\n", + " for word in token_words:\n", + " stem_sentence.append(porter.stem(word))\n", + " stem_sentence.append(\" \")\n", + " return \"\".join(stem_sentence)\n", + "\n", + "x=stemSentence(\"I am loving it\")\n", + "print(x)\n" + ], + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "text": [ + "I am love it \n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "KAOd4FKLloy1", + "colab_type": "code", + "outputId": "924626ae-ef97-4d2c-f21c-a31911a046c6", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 68 + } + }, + "source": [ + "import nltk\n", + "nltk.download('punkt')\n" + ], + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package punkt to /root/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "True" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 9 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Wh71uuIbit-f", + "colab_type": "code", + "colab": {} + }, + "source": [ + "cleaned_data_train=[]\n", + "for sentence in range(0,len(features)): #Define Your features above\n", + " cleaned_sentence=expandContractions(str(features[sentence]))\n", + " \n", + " cleaned_sentence=clean_text(cleaned_sentence)\n", + " cleaned_sentence=stemSentence(cleaned_sentence)\n", + " cleaned_sentence=removeStopWord(cleaned_sentence)\n", + " \n", + " \n", + " cleaned_data_train.append(cleaned_sentence)\n", + " " + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Sa4ID7Vnru4f", + "colab_type": "code", + "colab": {} + }, + "source": [ + "\n", + "from sklearn.feature_extraction.text import CountVectorizer\n", + "from nltk.corpus import stopwords \n", + "\n", + "stop_words = ['in', 'of', 'at', 'a', 'the','an','or','i','and','has','he','will','was','with','is','its','if']\n", + "\n", + "cv_vect = CountVectorizer(binary=True,ngram_range=(1,2),stop_words=stop_words)\n", + "\n", + "cv_vect.fit(features)\n", + "cleaned_data_vector= cv_vect.transform(features)\n", + "labels = labels.factorize()[0]" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "oJzcspVGK1tc", + "colab_type": "code", + "outputId": "94097a6c-ef5c-466f-842f-fb8e7169a22b", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 102 + } + }, + "source": [ + "print(labels[:100])\n", + "#print(labels.factorize()[1])\n", + "print(labels.shape)\n", + "y = np.bincount(labels[:600])\n", + "ii = np.nonzero(y)[0]\n", + "for a,b in zip(ii,y[ii]):\n", + " print(a,b)" + ], + "execution_count": 47, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n", + "(3000,)\n", + "0 600\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "O3uZngo8pSNd", + "colab_type": "text" + }, + "source": [ + "**Importing Models**" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "xO7HxxRgn3RI", + "colab_type": "code", + "colab": {} + }, + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.svm import LinearSVC\n", + "from sklearn.svm import SVC\n", + "from sklearn.naive_bayes import GaussianNB \n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.model_selection import KFold\n", + "\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kkrax42fpWUt", + "colab_type": "text" + }, + "source": [ + "**Splitting dataset into training and testing set**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RVBqvO1fpIAT", + "colab_type": "text" + }, + "source": [ + "**\"Random Forest Classifier\"**" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "XV2YiFJJn8pR", + "colab_type": "code", + "colab": {} + }, + "source": [ + "text_classifier_RF= RandomForestClassifier(n_estimators=200, random_state=0) \n", + "rf_scores=[]\n", + "cv = KFold(n_splits=5, random_state=42,shuffle = True)\n", + "for train_index, test_index in cv.split(cleaned_data_vector):\n", + " #print(\"Train Index: \", train_index, \"\\n\")\n", + " #print(\"Test Index: \", test_index)\n", + "\n", + " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", + " text_classifier_RF.fit(X_train, y_train)\n", + " rf_scores.append(text_classifier_RF.score(X_test, y_test))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "kKjIq5SLsvQH", + "colab_type": "code", + "outputId": "2cac99a5-a879-49e3-c45e-4a24182d5565", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "print(rf_scores)" + ], + "execution_count": 65, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[0.72, 0.6983333333333334, 0.735, 0.725, 0.6883333333333334]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7DwRGDwJpFBp", + "colab_type": "text" + }, + "source": [ + "**\"Linear Support Vector Classifier\"**" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "YbJO51ETf1gs", + "colab_type": "code", + "outputId": "e753ddaa-2194-4a16-bd46-e0313c198a2c", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + } + }, + "source": [ + "text_classifier_SVC= LinearSVC(C=1)\n", + "svc_scores=[]\n", + "cv = KFold(n_splits=5, random_state=42, shuffle=True)\n", + "\n", + "for train_index, test_index in cv.split(cleaned_data_vector):\n", + " #print(\"Train Index: \", train_index, \"\\n\")\n", + " #print(\"Test Index: \", test_index)\n", + "\n", + " X_train, X_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index]\n", + " y_train, y_test =labels[train_index], labels[test_index]\n", + " text_classifier_SVC.fit(X_train, y_train)\n", + " svc_scores.append(text_classifier_SVC.score(X_test, y_test))\n", + "\n" + ], + "execution_count": 66, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.6/dist-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n", + " \"the number of iterations.\", ConvergenceWarning)\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "8tBuw1Wv6AlR", + "colab_type": "code", + "outputId": "b982e769-6c6b-4278-dfd2-1337262495a1", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + } + }, + "source": [ + "print(sum(svc_scores) / len(svc_scores))\n", + "print (svc_scores)\n" + ], + "execution_count": 67, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.7276666666666667\n", + "[0.7233333333333334, 0.7466666666666667, 0.73, 0.7083333333333334, 0.73]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TuPnp1yMo7ui", + "colab_type": "text" + }, + "source": [ + "**\"Support Vector Machine\"**" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WPFfKBunoSj9", + "colab_type": "code", + "colab": {} + }, + "source": [ + "text_classifier_SVM= SVC()\n", + "svm_scores=[]\n", + "cv = KFold(n_splits=5, random_state=42, shuffle=True)\n", + "\n", + "for train_index, test_index in cv.split(cleaned_data_vector):\n", + " # print(\"Train Index: \", train_index, \"\\n\")\n", + " # print(\"Test Index: \", test_index)\n", + "\n", + " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", + " text_classifier_SVM.fit(X_train, y_train)\n", + " svm_scores.append(text_classifier_SVM.score(X_test, y_test))\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "HUo7hbjovlod", + "colab_type": "code", + "outputId": "8e9c2cae-d171-426c-e6e8-80be179597c9", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + } + }, + "source": [ + "print(sum(svm_scores) / len(svm_scores))\n", + "print(svm_scores)" + ], + "execution_count": 76, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.6803333333333333\n", + "[0.6866666666666666, 0.6683333333333333, 0.6866666666666666, 0.675, 0.685]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gqApRDYEo5gL", + "colab_type": "text" + }, + "source": [ + "**\"Naive Bayes Algorithm\"**" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "HIPeIQKioe89", + "colab_type": "code", + "colab": {} + }, + "source": [ + "from sklearn.naive_bayes import GaussianNB \n", + "text_classifier_NB = GaussianNB() \n", + "nb_scores=[]\n", + "cv = KFold(n_splits=5, random_state=42, shuffle=True)\n", + "\n", + "for train_index, test_index in cv.split(cleaned_data_vector):\n", + " #print(\"Train Index: \", train_index, \"\\n\")\n", + " #print(\"Test Index: \", test_index)\n", + "\n", + " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", + " text_classifier_NB.fit(X_train.toarray(), y_train)\n", + " nb_scores.append(text_classifier_NB.score(X_test.toarray(), y_test))\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "ppGIRkjsvYlj", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + }, + "outputId": "c0133a98-fe2a-4605-c83a-8cfc647b5984" + }, + "source": [ + "print(sum(nb_scores) / len(nb_scores))\n", + "print(nb_scores)" + ], + "execution_count": 78, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.6976666666666667\n", + "[0.7133333333333334, 0.7133333333333334, 0.71, 0.6733333333333333, 0.6783333333333333]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "9dOhyFFr1Xqr", + "colab_type": "code", + "colab": {} + }, + "source": [ + "from sklearn.naive_bayes import MultinomialNB\n", + "text_classifier_MNB = MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True) \n", + "mnb_scores=[]\n", + "cv = KFold(n_splits=5, random_state=42, shuffle=True)\n", + "\n", + "for train_index, test_index in cv.split(cleaned_data_vector):\n", + " #print(\"Train Index: \", train_index, \"\\n\")\n", + " #print(\"Test Index: \", test_index)\n", + "\n", + " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", + " text_classifier_MNB.fit(X_train.toarray(), y_train)\n", + " mnb_scores.append(text_classifier_MNB.score(X_test.toarray(), y_test))\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "bRuFHslD1hCl", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + }, + "outputId": "6f9d7796-801f-40ca-ead4-4367bb205ae0" + }, + "source": [ + "print(sum(mnb_scores)/len(mnb_scores))\n", + "print(mnb_scores)" + ], + "execution_count": 80, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.7443333333333333\n", + "[0.7533333333333333, 0.7466666666666667, 0.7683333333333333, 0.7066666666666667, 0.7466666666666667]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RlfBUQKIoyKY", + "colab_type": "text" + }, + "source": [ + "**\"Logistic Regression Algorithm\"**" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "S7lkFdJPorrY", + "colab_type": "code", + "colab": {} + }, + "source": [ + "text_classifier_LR= LogisticRegression(solver='lbfgs', multi_class='multinomial',random_state=1)\n", + "lr_scores=[]\n", + "cv = KFold(n_splits=5, random_state=42, shuffle=True)\n", + "\n", + "for train_index, test_index in cv.split(cleaned_data_vector):\n", + " #print(\"Train Index: \", train_index, \"\\n\")\n", + " #print(\"Test Index: \", test_index)\n", + "\n", + " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", + " text_classifier_LR.fit(X_train, y_train)\n", + " lr_scores.append(text_classifier_LR.score(X_test, y_test))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "WkUcvE6mt4ZS", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + }, + "outputId": "cfc8ec65-404b-4b14-9329-5bc50054ad48" + }, + "source": [ + "print(sum(lr_scores)/len(lr_scores))\n", + "print(lr_scores)" + ], + "execution_count": 81, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0.7473333333333334\n", + "[0.7483333333333333, 0.755, 0.7616666666666667, 0.7183333333333334, 0.7533333333333333]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "wNC-83Viisvh", + "colab_type": "code", + "colab": {} + }, + "source": [ + "text=[ \"Sentence_1\",\"Sentence_2\" ]\n", + "scores=[]\n", + "\n", + "for sentence in range(0,len(text)):\n", + " \n", + " cleaned_input_sentence=expandContractions(str(text[sentence]))\n", + " cleaned_input_sentence=clean_text(cleaned_input_sentence)\n", + " cleaned_input_sentence=stemSentence(cleaned_input_sentence)\n", + " cleaned_input_sentence=removeStopWord(cleaned_input_sentence)\n", + " scores.append(cleaned_input_sentence)\n", + " \n", + " \n", + " \n", + "cleaned_input_vector= cv_vect.transform(scores)\n", + "\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "U2NghXuXris7", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "abf54acb-c31f-4f08-981c-32ebcf82756f" + }, + "source": [ + "text_classifier_LR.predict(cleaned_input_vector)" + ], + "execution_count": 63, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "array([0, 0])" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 63 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "pPNm8HWXrmQ8", + "colab_type": "code", + "colab": {} + }, + "source": [ + "text_classifier_SVC.predict(cleaned_input_vector)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "dP7nuzrNpPqG", + "colab_type": "code", + "colab": {} + }, + "source": [ + "text_classifier_RF.predict(cleaned_input_vector)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "3FLXJDdEyKWR", + "colab_type": "code", + "colab": {} + }, + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file From 84bb74b6a6abe9ed2b808b8e75a0f5c9a345283a Mon Sep 17 00:00:00 2001 From: krushnapavan <35088826+krushnapavan9@users.noreply.github.com> Date: Tue, 24 Mar 2020 19:54:02 +0530 Subject: [PATCH 2/3] deleting the old ML file --- .../SupportTickect_Classification_ML.ipynb | 1222 ----------------- 1 file changed, 1222 deletions(-) delete mode 100644 Machine Learning/SupportTickect_Classification_ML.ipynb diff --git a/Machine Learning/SupportTickect_Classification_ML.ipynb b/Machine Learning/SupportTickect_Classification_ML.ipynb deleted file mode 100644 index 8757b2f..0000000 --- a/Machine Learning/SupportTickect_Classification_ML.ipynb +++ /dev/null @@ -1,1222 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "Copy of TickectClassificationML.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "code", - "metadata": { - "id": "s34hKA4TfQsg", - "colab_type": "code", - "outputId": "0262b84d-1166-4c4e-8f6f-0e91158eb32c", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 52 - } - }, - "source": [ - "import nltk\n", - "import re\n", - "from gensim import models, corpora\n", - "from nltk import word_tokenize\n", - "from nltk.corpus import stopwords\n", - "nltk.download('brown')\n", - "import numpy as np\n", - "import pandas as pd\n", - "from matplotlib import pyplot as plt\n", - "from bs4 import BeautifulSoup\n", - "import re\n", - "from sklearn import preprocessing" - ], - "execution_count": 2, - "outputs": [ - { - "output_type": "stream", - "text": [ - "[nltk_data] Downloading package brown to /root/nltk_data...\n", - "[nltk_data] Package brown is already up-to-date!\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "9p2jq05Oi418", - "colab_type": "code", - "outputId": "469c1428-f0e7-4bb8-93f8-47910cc1138d", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 197 - } - }, - "source": [ - "datasource=pd.read_csv('upload_your_file')\n", - "datasource.head()\n" - ], - "execution_count": 3, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DescriptionCategory
0hi since recruiter lead permission approve req...Application
1re expire days hi ask help update passwords co...Application
2verification warning hi has got attached pleas...Application
3please dear looks blacklisted receiving mails ...Application
4dear modules report report cost thank much reg...Application
\n", - "
" - ], - "text/plain": [ - " Description Category\n", - "0 hi since recruiter lead permission approve req... Application\n", - "1 re expire days hi ask help update passwords co... Application\n", - "2 verification warning hi has got attached pleas... Application\n", - "3 please dear looks blacklisted receiving mails ... Application\n", - "4 dear modules report report cost thank much reg... Application" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 3 - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "xYzX96pWjzA6", - "colab_type": "code", - "colab": {} - }, - "source": [ - "features='Features Over Here'\n", - "labels='Labels Over Here'\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "rgBWLrvOlQS8", - "colab_type": "code", - "outputId": "6b21068d-cd68-4251-8908-00315858a754", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - } - }, - "source": [ - "cList = {\n", - " \"ain't\": \"am not\",\n", - " \"aren't\": \"are not\",\n", - " \"can't\": \"cannot\",\n", - " \"can't've\": \"cannot have\",\n", - " \"'cause\": \"because\",\n", - " \"could've\": \"could have\",\n", - " \"couldn't\": \"could not\",\n", - " \"couldn't've\": \"could not have\",\n", - " \"didn't\": \"did not\",\n", - " \"doesn't\": \"does not\",\n", - " \"don't\": \"do not\",\n", - " \"hadn't\": \"had not\",\n", - " \"hadn't've\": \"had not have\",\n", - " \"hasn't\": \"has not\",\n", - " \"haven't\": \"have not\",\n", - " \"he'd\": \"he would\",\n", - " \"he'd've\": \"he would have\",\n", - " \"he'll\": \"he will\",\n", - " \"he'll've\": \"he will have\",\n", - " \"he's\": \"he is\",\n", - " \"how'd\": \"how did\",\n", - " \"how'd'y\": \"how do you\",\n", - " \"how'll\": \"how will\",\n", - " \"how's\": \"how is\",\n", - " \"I'd\": \"I would\",\n", - " \"I'd've\": \"I would have\",\n", - " \"I'll\": \"I will\",\n", - " \"I'll've\": \"I will have\",\n", - " \"I'm\": \"I am\",\n", - " \"I've\": \"I have\",\n", - " \"isn't\": \"is not\",\n", - " \"it'd\": \"it had\",\n", - " \"it'd've\": \"it would have\",\n", - " \"it'll\": \"it will\",\n", - " \"it'll've\": \"it will have\",\n", - " \"it's\": \"it is\",\n", - " \"let's\": \"let us\",\n", - " \"ma'am\": \"madam\",\n", - " \"mayn't\": \"may not\",\n", - " \"might've\": \"might have\",\n", - " \"mightn't\": \"might not\",\n", - " \"mightn't've\": \"might not have\",\n", - " \"must've\": \"must have\",\n", - " \"mustn't\": \"must not\",\n", - " \"mustn't've\": \"must not have\",\n", - " \"needn't\": \"need not\",\n", - " \"needn't've\": \"need not have\",\n", - " \"o'clock\": \"of the clock\",\n", - " \"oughtn't\": \"ought not\",\n", - " \"oughtn't've\": \"ought not have\",\n", - " \"shan't\": \"shall not\",\n", - " \"sha'n't\": \"shall not\",\n", - " \"shan't've\": \"shall not have\",\n", - " \"she'd\": \"she would\",\n", - " \"she'd've\": \"she would have\",\n", - " \"she'll\": \"she will\",\n", - " \"she'll've\": \"she will have\",\n", - " \"she's\": \"she is\",\n", - " \"should've\": \"should have\",\n", - " \"shouldn't\": \"should not\",\n", - " \"shouldn't've\": \"should not have\",\n", - " \"so've\": \"so have\",\n", - " \"so's\": \"so is\",\n", - " \"that'd\": \"that would\",\n", - " \"that'd've\": \"that would have\",\n", - " \"that's\": \"that is\",\n", - " \"there'd\": \"there had\",\n", - " \"there'd've\": \"there would have\",\n", - " \"there's\": \"there is\",\n", - " \"they'd\": \"they would\",\n", - " \"they'd've\": \"they would have\",\n", - " \"they'll\": \"they will\",\n", - " \"they'll've\": \"they will have\",\n", - " \"they're\": \"they are\",\n", - " \"they've\": \"they have\",\n", - " \"to've\": \"to have\",\n", - " \"wasn't\": \"was not\",\n", - " \"we'd\": \"we had\",\n", - " \"we'd've\": \"we would have\",\n", - " \"we'll\": \"we will\",\n", - " \"we'll've\": \"we will have\",\n", - " \"we're\": \"we are\",\n", - " \"we've\": \"we have\",\n", - " \"weren't\": \"were not\",\n", - " \"what'll\": \"what will\",\n", - " \"what'll've\": \"what will have\",\n", - " \"what're\": \"what are\",\n", - " \"what's\": \"what is\",\n", - " \"what've\": \"what have\",\n", - " \"when's\": \"when is\",\n", - " \"when've\": \"when have\",\n", - " \"where'd\": \"where did\",\n", - " \"where's\": \"where is\",\n", - " \"where've\": \"where have\",\n", - " \"who'll\": \"who will\",\n", - " \"who'll've\": \"who will have\",\n", - " \"who's\": \"who is\",\n", - " \"who've\": \"who have\",\n", - " \"why's\": \"why is\",\n", - " \"why've\": \"why have\",\n", - " \"will've\": \"will have\",\n", - " \"won't\": \"will not\",\n", - " \"won't've\": \"will not have\",\n", - " \"would've\": \"would have\",\n", - " \"wouldn't\": \"would not\",\n", - " \"wouldn't've\": \"would not have\",\n", - " \"y'all\": \"you all\",\n", - " \"y'alls\": \"you alls\",\n", - " \"y'all'd\": \"you all would\",\n", - " \"y'all'd've\": \"you all would have\",\n", - " \"y'all're\": \"you all are\",\n", - " \"y'all've\": \"you all have\",\n", - " \"you'd\": \"you had\",\n", - " \"you'd've\": \"you would have\",\n", - " \"you'll\": \"you you will\",\n", - " \"you'll've\": \"you you will have\",\n", - " \"you're\": \"you are\",\n", - " \"you've\": \"you have\"\n", - "}\n", - "\n", - "c_re = re.compile('(%s)' % '|'.join(cList.keys()))\n", - " \n", - "def expandContractions(text, c_re=c_re):\n", - " def replace(match):\n", - " return cList[match.group(0)]\n", - " return c_re.sub(replace, text)\n", - " \n", - "expandContractions(\"I ain't going anywhere\")" - ], - "execution_count": 5, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "'I am not going anywhere'" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 5 - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "IFQTgXh7lUXE", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def clean_text(text):\n", - "\n", - " \n", - " #Removing comments fom body\n", - " cleaned_text = text.replace(r'From:.*$', '')\n", - "\n", - "\n", - " #Expanding the sentence\n", - " cleaned_text = (re.sub(\"([A-Z]{1}[a-z])\",\" \\\\1\",i) for i in cleaned_text)\n", - "\n", - " #To Lower case\n", - " #cleaned_text = cleaned_text.lower()\n", - "\n", - " #Removing email ID's\n", - " cleaned_text = (re.sub(\"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\\\.[a-zA-Z0-9.]+\",\"\",i) for i in cleaned_text)\n", - "\n", - " #Removing hyperlinks\n", - " cleaned_text = (re.sub(\"(http:\\/\\/www\\.|https:\\/\\/www\\.|http:\\/\\/|https:\\/\\/)?[a-z0-9]+([\\-\\.]{1}[a-zA-Z0-9]+)*\\.[a-z]{2,5}(:[0-9]{1,5})?(\\/.*)?\",\"\",i) for i in cleaned_text)\n", - "\n", - " cleaned_text = (re.sub(\"http\\S+\",\"\",i) for i in cleaned_text)\n", - "\n", - " #cleaned_text = cleaned_text.replace(r'url= .', '')\n", - "\n", - " #Replacing Contractions\n", - " cleaned_text = (re.sub(\"’\",\"'\",i) for i in cleaned_text)\n", - "\n", - " cleaned_text = (expandContractions(i) for i in cleaned_text)\n", - "\n", - " #Removing unnecessary punctuations\n", - " cleaned_text = (re.sub(\"[\\(\\)\\-\\[\\]\\{\\}\\;\\:\\\"\\\\\\<\\>\\/\\@\\#\\$\\%\\^\\&\\*\\_\\~\\–]+\",\" \",i) for i in cleaned_text)\n", - "\n", - " cleaned_text = (re.sub(\"(\\?)([a-zA-Z0-9]+)\",\"\\\\1 \\\\2\",i) for i in cleaned_text)\n", - "\n", - " cleaned_text = (re.sub(\"([A-Za-z]{1,})(\\.)([A-Za-z]{2,})\",\"\\\\1 \\\\2 \\\\3\",i) for i in cleaned_text)\n", - "\n", - " cleaned_text = (re.sub(\"[\\']+\",\"\",i) for i in cleaned_text)\n", - "\n", - " #cleaned_text = cleaned_text.replace(r'\\xa0', ' ')\n", - "\n", - " cleaned_text = (re.sub(\"(\\.)(\\,|\\?|\\'|\\!|\\s){1,}\",\"\\\\1 \",i) for i in cleaned_text)\n", - "\n", - " cleaned_text = (re.sub(\"(!){1}(\\s)*(\\=)\",\" not equals \",i) for i in cleaned_text)\n", - "\n", - " cleaned_text = (re.sub(\"(\\=)\",\" equals \",i) for i in cleaned_text)\n", - "\n", - "\n", - " #Replacing _ with \" \"\n", - " cleaned_text = (re.sub(\"_\",\" \",i) for i in cleaned_text)\n", - " \n", - " \n", - " #Replacing @–\n", - " #cleaned_text = cleaned_text.apply(lambda i: re.sub(r'[@–]','',i))\n", - "\n", - " #Removing ASCII\n", - " #cleaned_text = cleaned_text.apply(lambda i: re.sub(r'[^\\x00-\\x7F]+',' ', i))\n", - "\n", - " #Replacing multiple '.' with single '.'\n", - " cleaned_text = (re.sub(\"\\s\\.+\",\" .\",i) for i in cleaned_text)\n", - "\n", - " cleaned_text = (re.sub(\"\\\\.+\",\".\",i) for i in cleaned_text)\n", - "\n", - " #Replacing . follower by characters with spaces followed by . followed by characters\n", - " cleaned_text = (re.sub(\"([0-9]+)(\\.{1})([^0-9]+)\",\"\\\\1 \\\\2 \\\\3\",i) for i in cleaned_text)\n", - "\n", - "\n", - " # remove after regards\n", - " cleaned_text = (re.sub(r'regards.*$','',i) for i in cleaned_text)\n", - " cleaned_text = (re.sub(r'tel \\+.*$','',i) for i in cleaned_text) \n", - " cleaned_text = (re.sub(r'fax \\+.*$','',i) for i in cleaned_text)\n", - " \n", - " #print (cleaned_text)\n", - " \n", - " # Remove all the special characters\n", - " cleaned_text = re.sub(r'\\W', ' ', (\"\").join(cleaned_text))\n", - " #print (cleaned_text)\n", - " \n", - " \n", - " # Remove single characters from the start\n", - " cleaned_text = re.sub(r'\\^[a-zA-Z]\\s+', ' ', cleaned_text) \n", - "\n", - " # Substituting multiple spaces with single space\n", - " cleaned_text = re.sub(r'\\s+', ' ', cleaned_text, flags=re.I)\n", - "\n", - " \n", - " # remove all single characters\n", - " cleaned_text= re.sub(r'\\s+[a-zA-Z]\\s+', ' ', cleaned_text)\n", - " \n", - " \n", - " #Getting to root word \n", - " #cleaned_text = (lemmatizeCust(i) for i in cleaned_text)\n", - " \n", - " #Removing stopwords\n", - " #cleaned_text = (removeStopWord(i) for i in cleaned_text)\n", - "\n", - " #Replacing muiltple white spaces\n", - " #cleaned_text = (re.sub(\" +\",\" \",i) for i in cleaned_text)\n", - "\n", - "\n", - " \n", - "\n", - " return(cleaned_text)\n", - " \n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "i_exG7nxqP4k", - "colab_type": "code", - "colab": {} - }, - "source": [ - "stopWordlist =[\n", - " 'thank',\n", - " 'regards',\n", - " 'ocp',\n", - " 'thanks',\n", - " 'hi',\n", - " 'hello',\n", - " 'team',\n", - " 'team',\n", - " 'team',\n", - " 'team',\n", - " 'team',\n", - " 'team',\n", - " 'team',\n", - " 'team',\n", - " 'i', \n", - " 'me',\n", - " 'my',\n", - " 'myself',\n", - " 'we',\n", - " 'our',\n", - " 'ours',\n", - " 'ourselves',\n", - " 'you',\n", - " \"you're\",\n", - " \"you've\",\n", - " \"you'll\",\n", - " \"you'd\",\n", - " 'your',\n", - " 'yours',\n", - " 'yourself',\n", - " 'yourselves',\n", - " 'he',\n", - " 'him',\n", - " 'his',\n", - " 'himself',\n", - " 'she',\n", - " \"she's\",\n", - " 'her',\n", - " 'hers',\n", - " 'herself',\n", - " 'it',\n", - " \"it's\",\n", - " 'its',\n", - " 'itself',\n", - " 'they',\n", - " 'them',\n", - " 'their',\n", - " 'theirs',\n", - " 'themselves',\n", - " 'what',\n", - " 'which',\n", - " 'who',\n", - " 'whom',\n", - " 'this',\n", - " 'that',\n", - " \"that'll\",\n", - " 'these',\n", - " 'those',\n", - " 'am',\n", - " 'is',\n", - " 'are',\n", - " 'was',\n", - " 'were',\n", - " 'be',\n", - " 'been',\n", - " 'being',\n", - " 'have',\n", - " 'has',\n", - " 'had',\n", - " 'having',\n", - " 'do',\n", - " 'does',\n", - " 'did',\n", - " 'doing',\n", - " 'a',\n", - " 'an',\n", - " 'the',\n", - " 'and',\n", - " 'if',\n", - " 'or',\n", - " 'because',\n", - " 'as',\n", - " 'until',\n", - " 'while',\n", - " 'of',\n", - " 'at',\n", - " 'by',\n", - " 'for',\n", - " 'with',\n", - " 'about',\n", - " 'against',\n", - " 'between',\n", - " 'into',\n", - " 'through',\n", - " 'during',\n", - " 'before',\n", - " 'after',\n", - " 'above',\n", - " 'below',\n", - " 'to',\n", - " 'from',\n", - " 'up',\n", - " 'down',\n", - " 'in',\n", - " 'out',\n", - " 'on',\n", - " 'off',\n", - " 'over',\n", - " 'under',\n", - " 'again',\n", - " 'further',\n", - " 'then',\n", - " 'once',\n", - " 'here',\n", - " 'there',\n", - " 'when',\n", - " 'where',\n", - " 'why',\n", - " 'how',\n", - " 'all',\n", - " 'any',\n", - " 'both',\n", - " 'each',\n", - " 'few',\n", - " 'more',\n", - " 'most',\n", - " 'other',\n", - " 'some',\n", - " 'such',\n", - " 'no',\n", - " 'nor',\n", - " 'only',\n", - " 'own',\n", - " 'same',\n", - " 'so',\n", - " 'than',\n", - " 'too',\n", - " 'very',\n", - " 's',\n", - " 't',\n", - " 'can',\n", - " 'will',\n", - " 'just',\n", - " 'don',\n", - " \"don't\",\n", - " 'should',\n", - " \"should've\",\n", - " 'now',\n", - " 'd',\n", - " 'll',\n", - " 'm',\n", - " 'o',\n", - " 're',\n", - " 've',\n", - " 'y',\n", - " 'ain',\n", - " 'aren',\n", - " \"aren't\",\n", - " 'couldn',\n", - " \"couldn't\",\n", - " 'didn',\n", - " \"didn't\",\n", - " 'doesn',\n", - " \"doesn't\",\n", - " 'hadn',\n", - " \"hadn't\",\n", - " 'hasn',\n", - " \"hasn't\",\n", - " 'haven',\n", - " \"haven't\",\n", - " 'isn',\n", - " \"isn't\",\n", - " 'ma',\n", - " 'mightn',\n", - " \"mightn't\",\n", - " 'mustn',\n", - " \"mustn't\",\n", - " 'needn',\n", - " \"needn't\",\n", - " 'shan',\n", - " \"shan't\",\n", - " 'shouldn',\n", - " \"shouldn't\",\n", - " 'wasn',\n", - " \"wasn't\",\n", - " 'weren',\n", - " \"weren't\",\n", - " 'won',\n", - " \"won't\",\n", - " 'wouldn',\n", - " \"wouldn't\",\n", - " \"xa0\",\n", - " \"nbsp\"]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "rcGuiRfNqTbs", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def removeStopWord(text):\n", - " string = \"\"\n", - " for word in text.split(\" \"):\n", - " if(word in stopWordlist):\n", - " string = string + \"\"\n", - " else:\n", - " string = string + \" \" + word\n", - " return string " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "0xOq8-urlXYh", - "colab_type": "code", - "outputId": "ec3827d9-77be-4064-b795-78a93acb5984", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - } - }, - "source": [ - "from nltk.stem import PorterStemmer\n", - "from nltk.tokenize import sent_tokenize, word_tokenize\n", - "\n", - "porter=PorterStemmer()\n", - "def stemSentence(sentence):\n", - " token_words=word_tokenize(sentence)\n", - " token_words\n", - " stem_sentence=[]\n", - " for word in token_words:\n", - " stem_sentence.append(porter.stem(word))\n", - " stem_sentence.append(\" \")\n", - " return \"\".join(stem_sentence)\n", - "\n", - "x=stemSentence(\"I am loving it\")\n", - "print(x)\n" - ], - "execution_count": 19, - "outputs": [ - { - "output_type": "stream", - "text": [ - "I am love it \n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "KAOd4FKLloy1", - "colab_type": "code", - "outputId": "bb84d83a-dde7-44a8-842e-3412d02f6142", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 70 - } - }, - "source": [ - "import nltk\n", - "nltk.download('punkt')\n" - ], - "execution_count": 10, - "outputs": [ - { - "output_type": "stream", - "text": [ - "[nltk_data] Downloading package punkt to /root/nltk_data...\n", - "[nltk_data] Package punkt is already up-to-date!\n" - ], - "name": "stdout" - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "True" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 10 - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Wh71uuIbit-f", - "colab_type": "code", - "colab": {} - }, - "source": [ - "cleaned_data_train=[]\n", - "for sentence in range(0,len(features)): #Define Your features above\n", - " cleaned_sentence=expandContractions(str(features[sentence]))\n", - " \n", - " cleaned_sentence=clean_text(cleaned_sentence)\n", - " cleaned_sentence=stemSentence(cleaned_sentence)\n", - " cleaned_sentence=removeStopWord(cleaned_sentence)\n", - " \n", - " \n", - " cleaned_data_train.append(cleaned_sentence)\n", - " " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "Sa4ID7Vnru4f", - "colab_type": "code", - "colab": {} - }, - "source": [ - "\n", - "from sklearn.feature_extraction.text import CountVectorizer\n", - "from nltk.corpus import stopwords \n", - "\n", - "stop_words = ['in', 'of', 'at', 'a', 'the','an','or','i','and','has','he','will','was','with','is','its','if']\n", - "\n", - "cv_vect = CountVectorizer(binary=True,ngram_range=(1,2),stop_words=stop_words)\n", - "\n", - "cv_vect.fit(features)\n", - "cleaned_data_vector= cv_vect.transform(features)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "O3uZngo8pSNd", - "colab_type": "text" - }, - "source": [ - "**Importing Models**" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "xO7HxxRgn3RI", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.ensemble import RandomForestClassifier\n", - "from sklearn.svm import LinearSVC\n", - "from sklearn.svm import SVC\n", - "from sklearn.naive_bayes import GaussianNB \n", - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.model_selection import KFold\n", - "\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "kkrax42fpWUt", - "colab_type": "text" - }, - "source": [ - "**Splitting dataset into training and testing set**" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RVBqvO1fpIAT", - "colab_type": "text" - }, - "source": [ - "**\"Random Forest Classifier\"**" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "XV2YiFJJn8pR", - "colab_type": "code", - "colab": {} - }, - "source": [ - "text_classifier_RF= RandomForestClassifier(n_estimators=200, random_state=0) \n", - "rf_scores=[]\n", - "cv = KFold(n_splits=5, random_state=42, shuffle=False)\n", - "for train_index, test_index in cv.split(cleaned_data_vector):\n", - " #print(\"Train Index: \", train_index, \"\\n\")\n", - " #print(\"Test Index: \", test_index)\n", - "\n", - " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", - " text_classifier_RF.fit(X_train, y_train)\n", - " rf_scores.append(text_classifier_RF.score(X_test, y_test))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "kKjIq5SLsvQH", - "colab_type": "code", - "colab": {} - }, - "source": [ - "print(rf_scores)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7DwRGDwJpFBp", - "colab_type": "text" - }, - "source": [ - "**\"Linear Support Vector Classifier\"**" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "YbJO51ETf1gs", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 52 - }, - "outputId": "4d7ef75d-33e6-40ba-9b99-0c56d93d0378" - }, - "source": [ - "text_classifier_SVC= LinearSVC(C=1)\n", - "svc_scores=[]\n", - "cv = KFold(n_splits=5, random_state=42, shuffle=False)\n", - "\n", - "for train_index, test_index in cv.split(cleaned_data_vector):\n", - " #print(\"Train Index: \", train_index, \"\\n\")\n", - " #print(\"Test Index: \", test_index)\n", - "\n", - " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", - " text_classifier_SVC.fit(X_train, y_train)\n", - " svc_scores.append(text_classifier_SVC.score(X_test, y_test))\n", - "\n" - ], - "execution_count": 14, - "outputs": [ - { - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n", - " \"the number of iterations.\", ConvergenceWarning)\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "8tBuw1Wv6AlR", - "colab_type": "code", - "colab": {} - }, - "source": [ - "print(sum(svc_scores) / len(svc_scores))\n", - "print (svc_scores)\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TuPnp1yMo7ui", - "colab_type": "text" - }, - "source": [ - "**\"Support Vector Machine\"**" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "WPFfKBunoSj9", - "colab_type": "code", - "outputId": "35c561e1-b52a-44b4-87c3-23917a914d2e", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 214 - } - }, - "source": [ - "text_classifier_SVM= SVC()\n", - "svm_scores=[]\n", - "cv = KFold(n_splits=5, random_state=42, shuffle=False)\n", - "\n", - "for train_index, test_index in cv.split(cleaned_data_vector):\n", - " #print(\"Train Index: \", train_index, \"\\n\")\n", - " #print(\"Test Index: \", test_index)\n", - "\n", - " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", - " text_classifier_SVM.fit(X_train, y_train)\n", - " svm_scores.append(text_classifier_SVM.score(X_test, y_test))\n" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n", - " \"avoid this warning.\", FutureWarning)\n", - "/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n", - " \"avoid this warning.\", FutureWarning)\n", - "/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n", - " \"avoid this warning.\", FutureWarning)\n", - "/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n", - " \"avoid this warning.\", FutureWarning)\n", - "/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n", - " \"avoid this warning.\", FutureWarning)\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "HUo7hbjovlod", - "colab_type": "code", - "colab": {} - }, - "source": [ - "print(svm_scores)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gqApRDYEo5gL", - "colab_type": "text" - }, - "source": [ - "**\"Naive Bayes Algorithm\"**" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "HIPeIQKioe89", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from sklearn.naive_bayes import GaussianNB \n", - "text_classifier_NB = GaussianNB() \n", - "nb_scores=[]\n", - "cv = KFold(n_splits=5, random_state=42, shuffle=False)\n", - "\n", - "for train_index, test_index in cv.split(cleaned_data_vector):\n", - " #print(\"Train Index: \", train_index, \"\\n\")\n", - " #print(\"Test Index: \", test_index)\n", - "\n", - " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", - " text_classifier_NB.fit(X_train.toarray(), y_train)\n", - " nb_scores.append(text_classifier_NB.score(X_test.toarray(), y_test))\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "ppGIRkjsvYlj", - "colab_type": "code", - "colab": {} - }, - "source": [ - "print(nb_scores)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "9dOhyFFr1Xqr", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from sklearn.naive_bayes import MultinomialNB\n", - "text_classifier_MNB = MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True) \n", - "mnb_scores=[]\n", - "cv = KFold(n_splits=5, random_state=42, shuffle=False)\n", - "\n", - "for train_index, test_index in cv.split(cleaned_data_vector):\n", - " #print(\"Train Index: \", train_index, \"\\n\")\n", - " #print(\"Test Index: \", test_index)\n", - "\n", - " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", - " text_classifier_MNB.fit(X_train.toarray(), y_train)\n", - " mnb_scores.append(text_classifier_MNB.score(X_test.toarray(), y_test))\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "bRuFHslD1hCl", - "colab_type": "code", - "colab": {} - }, - "source": [ - "print(mnb_scores)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RlfBUQKIoyKY", - "colab_type": "text" - }, - "source": [ - "**\"Logistic Regression Algorithm\"**" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "S7lkFdJPorrY", - "colab_type": "code", - "outputId": "ab8a67c9-f70b-4520-d4b6-bdbe8a4cb039", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 194 - } - }, - "source": [ - "text_classifier_LR= LogisticRegression(solver='lbfgs', multi_class='multinomial',random_state=1)\n", - "lr_scores=[]\n", - "cv = KFold(n_splits=5, random_state=42, shuffle=False)\n", - "\n", - "for train_index, test_index in cv.split(cleaned_data_vector):\n", - " #print(\"Train Index: \", train_index, \"\\n\")\n", - " #print(\"Test Index: \", test_index)\n", - "\n", - " X_train, X_test, y_train, y_test = cleaned_data_vector[train_index], cleaned_data_vector[test_index], labels[train_index], labels[test_index]\n", - " text_classifier_LR.fit(X_train, y_train)\n", - " lr_scores.append(text_classifier_LR.score(X_test, y_test))" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n", - " \"of iterations.\", ConvergenceWarning)\n", - "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n", - " \"of iterations.\", ConvergenceWarning)\n", - "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n", - " \"of iterations.\", ConvergenceWarning)\n", - "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n", - " \"of iterations.\", ConvergenceWarning)\n", - "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n", - " \"of iterations.\", ConvergenceWarning)\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "WkUcvE6mt4ZS", - "colab_type": "code", - "colab": {} - }, - "source": [ - "print(lr_scores)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "wNC-83Viisvh", - "colab_type": "code", - "colab": {} - }, - "source": [ - "text=[ \"Sentence_1\",\"Sentence_2\" ]\n", - "scores=[]\n", - "\n", - "for sentence in range(0,len(text)):\n", - " \n", - " cleaned_input_sentence=expandContractions(str(text[sentence]))\n", - " cleaned_input_sentence=clean_text(cleaned_input_sentence)\n", - " cleaned_input_sentence=stemSentence(cleaned_input_sentence)\n", - " cleaned_input_sentence=removeStopWord(cleaned_input_sentence)\n", - " scores.append(cleaned_input_sentence)\n", - " \n", - " \n", - " \n", - "cleaned_input_vector= cv_vect.transform(scores)\n", - "\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "U2NghXuXris7", - "colab_type": "code", - "colab": {} - }, - "source": [ - "text_classifier_LR.predict(cleaned_input_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "pPNm8HWXrmQ8", - "colab_type": "code", - "colab": {} - }, - "source": [ - "text_classifier_SVC.predict(cleaned_input_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "dP7nuzrNpPqG", - "colab_type": "code", - "colab": {} - }, - "source": [ - "text_classifier_RF.predict(cleaned_input_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "3FLXJDdEyKWR", - "colab_type": "code", - "colab": {} - }, - "source": [ - "" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file From 2ee5ab6563d2ae85a31f5be575cd8a711a8bcc83 Mon Sep 17 00:00:00 2001 From: krushnapavan <35088826+krushnapavan9@users.noreply.github.com> Date: Tue, 24 Mar 2020 19:58:34 +0530 Subject: [PATCH 3/3] Rename Machine Learning/TickectClassificationML.ipynb to Machine Learning/Machine Learning/SupportTickect_Classification_ML.ipynb --- .../SupportTickect_Classification_ML.ipynb} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Machine Learning/{TickectClassificationML.ipynb => Machine Learning/SupportTickect_Classification_ML.ipynb} (99%) diff --git a/Machine Learning/TickectClassificationML.ipynb b/Machine Learning/Machine Learning/SupportTickect_Classification_ML.ipynb similarity index 99% rename from Machine Learning/TickectClassificationML.ipynb rename to Machine Learning/Machine Learning/SupportTickect_Classification_ML.ipynb index 9ebecaa..28d2202 100644 --- a/Machine Learning/TickectClassificationML.ipynb +++ b/Machine Learning/Machine Learning/SupportTickect_Classification_ML.ipynb @@ -1314,4 +1314,4 @@ "outputs": [] } ] -} \ No newline at end of file +}