diff --git a/your-code/challenge-1.ipynb b/your-code/[challenge-1] Bruno.ipynb similarity index 69% rename from your-code/challenge-1.ipynb rename to your-code/[challenge-1] Bruno.ipynb index 0808166..9742aa5 100644 --- a/your-code/challenge-1.ipynb +++ b/your-code/[challenge-1] Bruno.ipynb @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -82,6 +82,95 @@ " \"\"\"" ] }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import nltk \n", + "from nltk.stem import WordNetLemmatizer\n", + "from nltk.corpus import stopwords\n", + "from sklearn.feature_extraction.text import CountVectorizer\n", + "#from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from nltk.tokenize import word_tokenize \n", + "from nltk.stem import PorterStemmer, LancasterStemmer, SnowballStemmer" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package stopwords to\n", + "[nltk_data] C:\\Users\\btdjf\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package stopwords is already up-to-date!\n", + "[nltk_data] Downloading package wordnet to\n", + "[nltk_data] C:\\Users\\btdjf\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package punkt to\n", + "[nltk_data] C:\\Users\\btdjf\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to\n", + "[nltk_data] C:\\Users\\btdjf\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nltk.download('stopwords')\n", + "nltk.download('wordnet')\n", + "nltk.download('punkt') \n", + "nltk.download('omw-1.4')" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ironhack s q website is\n" + ] + } + ], + "source": [ + "import re\n", + "\n", + "text = \"@Ironhack's-#Q website 776-is http://ironhack.com [(2018)]\\\")\"\n", + "\n", + "def clean_up(text):\n", + " # removing URLs\n", + " text = re.sub(r\"http\\S+|www\\S+|https\\S+\", \"\", text)\n", + " # removing special characters\n", + " text = re.sub(r\"[^a-zA-Z ]+\", \" \", text)\n", + " # converting to lowercase\n", + " text = text.lower()\n", + " # Remove extra whitespaces\n", + " text = re.sub(r\"\\s+\", \" \", text)\n", + " return text.strip()\n", + "\n", + "print(clean_up(text))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -101,7 +190,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ @@ -117,6 +206,28 @@ " \"\"\"" ] }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['ironhack', 's', 'q', 'website', 'is']\n" + ] + } + ], + "source": [ + "text = \"ironhack s q website is\"\n", + "\n", + "def tokenize(text):\n", + " return word_tokenize(text)\n", + "\n", + "print(tokenize(text))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -145,7 +256,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 67, "metadata": {}, "outputs": [], "source": [ @@ -161,6 +272,37 @@ " \"\"\"" ] }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['ironhack', 's', 'q', 'websit', 'is']\n" + ] + } + ], + "source": [ + "words = ['ironhack', 's', 'q', 'website', 'is']\n", + "\n", + "def stem_and_lemmatize(words):\n", + " stemmer = PorterStemmer()\n", + " lemmatizer = WordNetLemmatizer()\n", + " stemmed_and_lemmatized = []\n", + "\n", + " for word in words:\n", + " stemmed_word = stemmer.stem(word)\n", + " lemmatized_word = lemmatizer.lemmatize(stemmed_word)\n", + " stemmed_and_lemmatized.append((lemmatized_word))\n", + "\n", + " return stemmed_and_lemmatized\n", + "\n", + "print(stem_and_lemmatize(words))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -176,7 +318,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -192,6 +334,30 @@ " \"\"\"" ] }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['ironhack', 'websit']\n" + ] + } + ], + "source": [ + "words = ['ironhack', 's', 'q', 'websit', 'is']\n", + "\n", + "def remove_stopwords(words):\n", + " stop_words = set(stopwords.words('english'))\n", + " filtered_words = [word for word in words if word not in stop_words and len(word) > 1]\n", + " return filtered_words\n", + "\n", + "print(remove_stopwords(words))\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -204,7 +370,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -218,7 +384,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/your-code/[challenge-2] Bruno.ipynb b/your-code/[challenge-2] Bruno.ipynb new file mode 100644 index 0000000..a0dc5c3 --- /dev/null +++ b/your-code/[challenge-2] Bruno.ipynb @@ -0,0 +1,5069 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 2: Sentiment Analysis\n", + "\n", + "In this challenge we will learn sentiment analysis and practice performing sentiment analysis on Twitter tweets. \n", + "\n", + "## Introduction\n", + "\n", + "Sentiment analysis is to *systematically identify, extract, quantify, and study affective states and subjective information* based on texts ([reference](https://en.wikipedia.org/wiki/Sentiment_analysis)). In simple words, it's to understand whether a person is happy or unhappy in producing the piece of text. Why we (or rather, companies) care about sentiment in texts? It's because by understanding the sentiments in texts, we will be able to know if our customers are happy or unhappy about our products and services. If they are unhappy, the subsequent action is to figure out what have caused the unhappiness and make improvements.\n", + "\n", + "Basic sentiment analysis only understands the *positive* or *negative* (sometimes *neutral* too) polarities of the sentiment. More advanced sentiment analysis will also consider dimensions such as agreement, subjectivity, confidence, irony, and so on. In this challenge we will conduct the basic positive vs negative sentiment analysis based on real Twitter tweets.\n", + "\n", + "NLTK comes with a [sentiment analysis package](https://www.nltk.org/api/nltk.sentiment.html). This package is great for dummies to perform sentiment analysis because it requires only the textual data to make predictions. For example:\n", + "\n", + "```python\n", + ">>> from nltk.sentiment.vader import SentimentIntensityAnalyzer\n", + ">>> txt = \"Ironhack is a Global Tech School ranked num 2 worldwide. 
", + "
", + "Our mission is to help people transform their careers and join a thriving community of tech professionals that love what they do.\"\n", + ">>> analyzer = SentimentIntensityAnalyzer()\n", + ">>> analyzer.polarity_scores(txt)\n", + "{'neg': 0.0, 'neu': 0.741, 'pos': 0.259, 'compound': 0.8442}\n", + "```\n", + "\n", + "In this challenge, however, you will not use NLTK's sentiment analysis package because in your Machine Learning training in the past 2 weeks you have learned how to make predictions more accurate than that. The [tweets data](https://www.kaggle.com/kazanova/sentiment140) we will be using today are already coded for the positive/negative sentiment. You will be able to use the Naïve Bayes classifier you learned in the lesson to predict the sentiment of tweets based on the labels." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conducting Sentiment Analysis\n", + "\n", + "### Loading and Exploring Data\n", + "\n", + "The dataset we'll be using today is located on Kaggle (https://www.kaggle.com/kazanova/sentiment140). Once you have downloaded and imported the dataset, it you will need to define the columns names: df.columns = ['target','id','date','flag','user','text']\n", + "\n", + "*Notes:* \n", + "\n", + "* The dataset is huuuuge (1.6m tweets). When you develop your data analysis codes, you can sample a subset of the data (e.g. 20k records) so that you will save a lot of time when you test your codes." + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
targetiddateflagusertext
001467810369Mon Apr 06 22:19:45 PDT 2009NO_QUERY_TheSpecialOne_@switchfoot http://twitpic.com/2y1zl - Awww, t...
101467810672Mon Apr 06 22:19:49 PDT 2009NO_QUERYscotthamiltonis upset that he can't update his Facebook by ...
201467810917Mon Apr 06 22:19:53 PDT 2009NO_QUERYmattycus@Kenichan I dived many times for the ball. Man...
301467811184Mon Apr 06 22:19:57 PDT 2009NO_QUERYElleCTFmy whole body feels itchy and like its on fire
401467811193Mon Apr 06 22:19:57 PDT 2009NO_QUERYKaroli@nationwideclass no, it's not behaving at all....
.....................
159999542193601966Tue Jun 16 08:40:49 PDT 2009NO_QUERYAmandaMarie1028Just woke up. Having no school is the best fee...
159999642193601969Tue Jun 16 08:40:49 PDT 2009NO_QUERYTheWDBoardsTheWDB.com - Very cool to hear old Walt interv...
159999742193601991Tue Jun 16 08:40:49 PDT 2009NO_QUERYbpbabeAre you ready for your MoJo Makeover? Ask me f...
159999842193602064Tue Jun 16 08:40:49 PDT 2009NO_QUERYtinydiamondzHappy 38th Birthday to my boo of alll time!!! ...
159999942193602129Tue Jun 16 08:40:50 PDT 2009NO_QUERYRyanTrevMorrishappy #charitytuesday @theNSPCC @SparksCharity...
\n", + "

1600000 rows × 6 columns

\n", + "
" + ], + "text/plain": [ + " target id date flag \\\n", + "0 0 1467810369 Mon Apr 06 22:19:45 PDT 2009 NO_QUERY \n", + "1 0 1467810672 Mon Apr 06 22:19:49 PDT 2009 NO_QUERY \n", + "2 0 1467810917 Mon Apr 06 22:19:53 PDT 2009 NO_QUERY \n", + "3 0 1467811184 Mon Apr 06 22:19:57 PDT 2009 NO_QUERY \n", + "4 0 1467811193 Mon Apr 06 22:19:57 PDT 2009 NO_QUERY \n", + "... ... ... ... ... \n", + "1599995 4 2193601966 Tue Jun 16 08:40:49 PDT 2009 NO_QUERY \n", + "1599996 4 2193601969 Tue Jun 16 08:40:49 PDT 2009 NO_QUERY \n", + "1599997 4 2193601991 Tue Jun 16 08:40:49 PDT 2009 NO_QUERY \n", + "1599998 4 2193602064 Tue Jun 16 08:40:49 PDT 2009 NO_QUERY \n", + "1599999 4 2193602129 Tue Jun 16 08:40:50 PDT 2009 NO_QUERY \n", + "\n", + " user text \n", + "0 _TheSpecialOne_ @switchfoot http://twitpic.com/2y1zl - Awww, t... \n", + "1 scotthamilton is upset that he can't update his Facebook by ... \n", + "2 mattycus @Kenichan I dived many times for the ball. Man... \n", + "3 ElleCTF my whole body feels itchy and like its on fire \n", + "4 Karoli @nationwideclass no, it's not behaving at all.... \n", + "... ... ... \n", + "1599995 AmandaMarie1028 Just woke up. Having no school is the best fee... \n", + "1599996 TheWDBoards TheWDB.com - Very cool to hear old Walt interv... \n", + "1599997 bpbabe Are you ready for your MoJo Makeover? Ask me f... \n", + "1599998 tinydiamondz Happy 38th Birthday to my boo of alll time!!! ... \n", + "1599999 RyanTrevMorris happy #charitytuesday @theNSPCC @SparksCharity... \n", + "\n", + "[1600000 rows x 6 columns]" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "column_names = ['target', 'id', 'date', 'flag', 'user', 'text']\n", + "\n", + "data = pd.read_csv(r'C:\\Users\\btdjf\\Desktop\\Ironhack 2\\Lab NLP\\lab-nlp\\your-code/training.1600000.processed.noemoticon.csv', names=column_names, encoding='latin-1')\n", + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
targetiddateflagusertext
149582142069999955Sun Jun 07 16:55:18 PDT 2009NO_QUERYyokomarie510On my way up to [his] job, he called talk`n bo...
238801468367488Tue Apr 07 01:14:20 PDT 2009NO_QUERYjoacimhagg@theresev solen var inge varm
122705441990937963Mon Jun 01 06:04:20 PDT 2009NO_QUERYKumiyo_In one of those sick moods yaaaaayyy
53490702197731667Tue Jun 16 15:04:56 PDT 2009NO_QUERYlinnanmGot home from Trondheim 2night! We had so much...
62658002230639116Thu Jun 18 17:23:49 PDT 2009NO_QUERYelchupahuesoGot a B in my skills class. Very disappointed...
.....................
131493542013971796Wed Jun 03 00:05:26 PDT 2009NO_QUERYazaxacavabanama@iruleatwork the hindi one i guess. was browsi...
36793202049351547Fri Jun 05 16:43:48 PDT 2009NO_QUERYMarkSheppard@ElaineGiles We don't buy our Macs, we just re...
7915801751707334Sat May 09 20:05:03 PDT 2009NO_QUERYJoannethecat@roccoman so sad please be safe, me luvs U ro...
155049342183795447Mon Jun 15 14:59:51 PDT 2009NO_QUERYa_l_wellsmock the week baybieeee
35532402043889856Fri Jun 05 08:33:50 PDT 2009NO_QUERYjfmanzoPLEASE go to http://krupskupofexcellence.com/ ...
\n", + "

20000 rows × 6 columns

\n", + "
" + ], + "text/plain": [ + " target id date flag \\\n", + "1495821 4 2069999955 Sun Jun 07 16:55:18 PDT 2009 NO_QUERY \n", + "2388 0 1468367488 Tue Apr 07 01:14:20 PDT 2009 NO_QUERY \n", + "1227054 4 1990937963 Mon Jun 01 06:04:20 PDT 2009 NO_QUERY \n", + "534907 0 2197731667 Tue Jun 16 15:04:56 PDT 2009 NO_QUERY \n", + "626580 0 2230639116 Thu Jun 18 17:23:49 PDT 2009 NO_QUERY \n", + "... ... ... ... ... \n", + "1314935 4 2013971796 Wed Jun 03 00:05:26 PDT 2009 NO_QUERY \n", + "367932 0 2049351547 Fri Jun 05 16:43:48 PDT 2009 NO_QUERY \n", + "79158 0 1751707334 Sat May 09 20:05:03 PDT 2009 NO_QUERY \n", + "1550493 4 2183795447 Mon Jun 15 14:59:51 PDT 2009 NO_QUERY \n", + "355324 0 2043889856 Fri Jun 05 08:33:50 PDT 2009 NO_QUERY \n", + "\n", + " user text \n", + "1495821 yokomarie510 On my way up to [his] job, he called talk`n bo... \n", + "2388 joacimhagg @theresev solen var inge varm \n", + "1227054 Kumiyo_ In one of those sick moods yaaaaayyy \n", + "534907 linnanm Got home from Trondheim 2night! We had so much... \n", + "626580 elchupahueso Got a B in my skills class. Very disappointed... \n", + "... ... ... \n", + "1314935 azaxacavabanama @iruleatwork the hindi one i guess. was browsi... \n", + "367932 MarkSheppard @ElaineGiles We don't buy our Macs, we just re... \n", + "79158 Joannethecat @roccoman so sad please be safe, me luvs U ro... \n", + "1550493 a_l_wells mock the week baybieeee \n", + "355324 jfmanzo PLEASE go to http://krupskupofexcellence.com/ ... \n", + "\n", + "[20000 rows x 6 columns]" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = data.sample(n=20000)\n", + "data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Prepare Textual Data for Sentiment Analysis\n", + "\n", + "Now, apply the functions you have written in Challenge 1 to your whole data set. These functions include:\n", + "\n", + "* `clean_up()`\n", + "\n", + "* `tokenize()`\n", + "\n", + "* `stem_and_lemmatize()`\n", + "\n", + "* `remove_stopwords()`\n", + "\n", + "Create a new column called `text_processed` in the dataframe to contain the processed data. At the end, your `text_processed` column should contain lists of word tokens that are cleaned up. Your data should look like below:\n", + "\n", + "![Processed Data](data-cleaning-results.png)" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "from nltk.tokenize import word_tokenize\n", + "from nltk.stem import PorterStemmer, WordNetLemmatizer\n", + "from nltk.corpus import stopwords" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [], + "source": [ + "def clean_up(text):\n", + " cleaned_text = re.sub('[^a-zA-Z]', ' ', text)\n", + " cleaned_text = re.sub(r'http\\S+', '', cleaned_text)\n", + " cleaned_text = cleaned_text.lower()\n", + " return cleaned_text" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [], + "source": [ + "def tokenize(text):\n", + " tokens = word_tokenize(text)\n", + " return tokens" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [], + "source": [ + "def stem_and_lemmatize(words):\n", + " stemmer = PorterStemmer()\n", + " lemmatizer = WordNetLemmatizer()\n", + " stemmed_and_lemmatized = []\n", + "\n", + " for word in words:\n", + " stemmed_word = stemmer.stem(word)\n", + " lemmatized_word = lemmatizer.lemmatize(stemmed_word)\n", + " stemmed_and_lemmatized.append((lemmatized_word))\n", + "\n", + " return stemmed_and_lemmatized" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [], + "source": [ + "def remove_stopwords(tokens):\n", + " stop_words = set(stopwords.words('english'))\n", + " filtered_tokens = [word for word in tokens if word not in stop_words and len(word) > 1]\n", + " return filtered_tokens" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [], + "source": [ + "data['text_processed'] = data['text'].apply(lambda x: remove_stopwords(stem_and_lemmatize(tokenize(clean_up(x)))))" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
targetiddateflagusertexttext_processed
149582142069999955Sun Jun 07 16:55:18 PDT 2009NO_QUERYyokomarie510On my way up to [his] job, he called talk`n bo...[way, hi, job, call, talk, bout, quot, momma, ...
238801468367488Tue Apr 07 01:14:20 PDT 2009NO_QUERYjoacimhagg@theresev solen var inge varm[theresev, solen, var, ing, varm]
122705441990937963Mon Jun 01 06:04:20 PDT 2009NO_QUERYKumiyo_In one of those sick moods yaaaaayyy[one, sick, mood, yaaaaayyy]
53490702197731667Tue Jun 16 15:04:56 PDT 2009NO_QUERYlinnanmGot home from Trondheim 2night! We had so much...[got, home, trondheim, night, much, fun, got, ...
62658002230639116Thu Jun 18 17:23:49 PDT 2009NO_QUERYelchupahuesoGot a B in my skills class. Very disappointed...[got, skill, class, veri, disappoint, mean, ne...
........................
131493542013971796Wed Jun 03 00:05:26 PDT 2009NO_QUERYazaxacavabanama@iruleatwork the hindi one i guess. was browsi...[iruleatwork, hindi, one, guess, wa, brow, thr...
36793202049351547Fri Jun 05 16:43:48 PDT 2009NO_QUERYMarkSheppard@ElaineGiles We don't buy our Macs, we just re...[elainegil, buy, mac, rent, even, get, use, mu...
7915801751707334Sat May 09 20:05:03 PDT 2009NO_QUERYJoannethecat@roccoman so sad please be safe, me luvs U ro...[roccoman, sad, plea, safe, luv, roccoman, hea...
155049342183795447Mon Jun 15 14:59:51 PDT 2009NO_QUERYa_l_wellsmock the week baybieeee[mock, week, baybieee]
35532402043889856Fri Jun 05 08:33:50 PDT 2009NO_QUERYjfmanzoPLEASE go to http://krupskupofexcellence.com/ ...[plea, go, http, krupskupofexcel, com, vote, k...
\n", + "

20000 rows × 7 columns

\n", + "
" + ], + "text/plain": [ + " target id date flag \\\n", + "1495821 4 2069999955 Sun Jun 07 16:55:18 PDT 2009 NO_QUERY \n", + "2388 0 1468367488 Tue Apr 07 01:14:20 PDT 2009 NO_QUERY \n", + "1227054 4 1990937963 Mon Jun 01 06:04:20 PDT 2009 NO_QUERY \n", + "534907 0 2197731667 Tue Jun 16 15:04:56 PDT 2009 NO_QUERY \n", + "626580 0 2230639116 Thu Jun 18 17:23:49 PDT 2009 NO_QUERY \n", + "... ... ... ... ... \n", + "1314935 4 2013971796 Wed Jun 03 00:05:26 PDT 2009 NO_QUERY \n", + "367932 0 2049351547 Fri Jun 05 16:43:48 PDT 2009 NO_QUERY \n", + "79158 0 1751707334 Sat May 09 20:05:03 PDT 2009 NO_QUERY \n", + "1550493 4 2183795447 Mon Jun 15 14:59:51 PDT 2009 NO_QUERY \n", + "355324 0 2043889856 Fri Jun 05 08:33:50 PDT 2009 NO_QUERY \n", + "\n", + " user text \\\n", + "1495821 yokomarie510 On my way up to [his] job, he called talk`n bo... \n", + "2388 joacimhagg @theresev solen var inge varm \n", + "1227054 Kumiyo_ In one of those sick moods yaaaaayyy \n", + "534907 linnanm Got home from Trondheim 2night! We had so much... \n", + "626580 elchupahueso Got a B in my skills class. Very disappointed... \n", + "... ... ... \n", + "1314935 azaxacavabanama @iruleatwork the hindi one i guess. was browsi... \n", + "367932 MarkSheppard @ElaineGiles We don't buy our Macs, we just re... \n", + "79158 Joannethecat @roccoman so sad please be safe, me luvs U ro... \n", + "1550493 a_l_wells mock the week baybieeee \n", + "355324 jfmanzo PLEASE go to http://krupskupofexcellence.com/ ... \n", + "\n", + " text_processed \n", + "1495821 [way, hi, job, call, talk, bout, quot, momma, ... \n", + "2388 [theresev, solen, var, ing, varm] \n", + "1227054 [one, sick, mood, yaaaaayyy] \n", + "534907 [got, home, trondheim, night, much, fun, got, ... \n", + "626580 [got, skill, class, veri, disappoint, mean, ne... \n", + "... ... \n", + "1314935 [iruleatwork, hindi, one, guess, wa, brow, thr... \n", + "367932 [elainegil, buy, mac, rent, even, get, use, mu... \n", + "79158 [roccoman, sad, plea, safe, luv, roccoman, hea... \n", + "1550493 [mock, week, baybieee] \n", + "355324 [plea, go, http, krupskupofexcel, com, vote, k... \n", + "\n", + "[20000 rows x 7 columns]" + ] + }, + "execution_count": 76, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating Bag of Words\n", + "\n", + "The purpose of this step is to create a [bag of words](https://en.wikipedia.org/wiki/Bag-of-words_model) from the processed data. The bag of words contains all the unique words in your whole text body (a.k.a. *corpus*) with the number of occurrence of each word. It will allow you to understand which words are the most important features across the whole corpus.\n", + "\n", + "Also, you can imagine you will have a massive set of words. The less important words (i.e. those of very low number of occurrence) do not contribute much to the sentiment. Therefore, you only need to use the most important words to build your feature set in the next step. In our case, we will use the top 5,000 words with the highest frequency to build the features.\n", + "\n", + "In the cell below, combine all the words in `text_processed` and calculate the frequency distribution of all words. A convenient library to calculate the term frequency distribution is NLTK's `FreqDist` class ([documentation](https://www.nltk.org/api/nltk.html#module-nltk.probability)). Then select the top 5,000 words from the frequency distribution." + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [], + "source": [ + "from nltk import FreqDist" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['way',\n", + " 'hi',\n", + " 'job',\n", + " 'call',\n", + " 'talk',\n", + " 'bout',\n", + " 'quot',\n", + " 'momma',\n", + " 'bore',\n", + " 'quot',\n", + " 'sooo',\n", + " 'go',\n", + " 'hi',\n", + " 'entertain',\n", + " 'ili',\n", + " 'daddi',\n", + " 'theresev',\n", + " 'solen',\n", + " 'var',\n", + " 'ing',\n", + " 'varm',\n", + " 'one',\n", + " 'sick',\n", + " 'mood',\n", + " 'yaaaaayyy',\n", + " 'got',\n", + " 'home',\n", + " 'trondheim',\n", + " 'night',\n", + " 'much',\n", + " 'fun',\n", + " 'got',\n", + " 'cold',\n", + " 'though',\n", + " 'got',\n", + " 'skill',\n", + " 'class',\n", + " 'veri',\n", + " 'disappoint',\n", + " 'mean',\n", + " 'need',\n", + " 'serious',\n", + " 'focu',\n", + " 'next',\n", + " 'quarter',\n", + " 'le',\n", + " 'zealotri',\n", + " 'knife',\n", + " 'skill',\n", + " 'wait',\n", + " 'till',\n", + " 'th',\n", + " 'june',\n", + " 'jonassss',\n", + " 'enjoy',\n", + " 'pre',\n", + " 'lucki',\n", + " 'peopl',\n", + " 'wait',\n", + " 'month',\n", + " 'befor',\n", + " 'sprint',\n", + " 'give',\n", + " 'enough',\n", + " 'discount',\n", + " 'upgrad',\n", + " 'prelaunch',\n", + " 'read',\n", + " 'quot',\n", + " 'untitl',\n", + " 'continu',\n", + " 'quot',\n", + " 'sleepi',\n", + " 'time',\n", + " 'stuf',\n", + " 'face',\n", + " 'pasta',\n", + " 'red',\n", + " 'velvet',\n", + " 'cake',\n", + " 'amp',\n", + " 'ice',\n", + " 'cream',\n", + " 'sick',\n", + " 'stomach',\n", + " 'headach',\n", + " 'wish',\n", + " 'someon',\n", + " 'could',\n", + " 'come',\n", + " 'rub',\n", + " 'templ',\n", + " 'rain',\n", + " 'drop',\n", + " 'keep',\n", + " 'fallin',\n", + " 'head',\n", + " 'omg',\n", + " 'censor',\n", + " 'omg',\n", + " 'sexiest',\n", + " 'men',\n", + " 'aliv',\n", + " 'censor',\n", + " 'sing',\n", + " 'part',\n", + " 'quot',\n", + " 'fuck',\n", + " 'quot',\n", + " 'sad',\n", + " 'never',\n", + " 'ever',\n", + " 'ha',\n", + " 'censor',\n", + " 'anyth',\n", + " 'look',\n", + " 'global',\n", + " 'like',\n", + " 'minunet',\n", + " 'longest',\n", + " 'day',\n", + " 'ever',\n", + " 'go',\n", + " 'bed',\n", + " 'chang',\n", + " 'chang',\n", + " 'met',\n", + " 'lot',\n", + " 'amaz',\n", + " 'new',\n", + " 'peopl',\n", + " 'thi',\n", + " 'year',\n", + " 'realli',\n", + " 'need',\n", + " 'let',\n", + " 'thi',\n", + " 'friendship',\n", + " 'go',\n", + " 'alreadi',\n", + " 'today',\n", + " 'wa',\n", + " 'sad',\n", + " 'day',\n", + " 'dajbelshaw',\n", + " 'hope',\n", + " 'week',\n", + " 'goe',\n", + " 'well',\n", + " 'workinnnn',\n", + " 'ugh',\n", + " 'oh',\n", + " 'well',\n", + " 'money',\n", + " 'may',\n", + " 'go',\n", + " 'adventur',\n", + " 'laterrrrr',\n", + " 'terribl',\n", + " 'night',\n", + " 'hate',\n", + " 'today',\n", + " 'alreadi',\n", + " 'mayb',\n", + " 'stay',\n", + " 'bed',\n", + " 'tomorrow',\n", + " 'melani',\n", + " 'lol',\n", + " 'didnt',\n", + " 'think',\n", + " 'howev',\n", + " 'mayb',\n", + " 'websit',\n", + " 'semi',\n", + " 'funki',\n", + " 'intuit',\n", + " 'take',\n", + " 'look',\n", + " 'later',\n", + " 'see',\n", + " 'think',\n", + " 'stop',\n", + " 'ignor',\n", + " 'foot',\n", + " 'pile',\n", + " 'cloth',\n", + " 'iron',\n", + " 'thee',\n", + " 'sun',\n", + " 'time',\n", + " 'low',\n", + " 'tuesday',\n", + " 'panera',\n", + " 'homi',\n", + " 'totita',\n", + " 'vote',\n", + " 'time',\n", + " 'wait',\n", + " 'minuto',\n", + " 'vote',\n", + " 'thank',\n", + " 'vote',\n", + " 'spread',\n", + " 'word',\n", + " 'haha',\n", + " 'xx',\n", + " 'back',\n", + " 'taip',\n", + " 'haze',\n", + " 'eye',\n", + " 'feel',\n", + " 'lil',\n", + " 'burn',\n", + " 'itchi',\n", + " 'sir',\n", + " 'benji',\n", + " 'marshal',\n", + " 'chat',\n", + " 'thi',\n", + " 'arvo',\n", + " 'talk',\n", + " 'nba',\n", + " 'kirk',\n", + " 'penney',\n", + " 'shizz',\n", + " 'get',\n", + " 'rid',\n", + " 'mothersday',\n", + " 'trixi',\n", + " 'nooo',\n", + " 'late',\n", + " 'oh',\n", + " 'well',\n", + " 'look',\n", + " 'like',\n", + " 'good',\n", + " 'time',\n", + " 'wa',\n", + " 'shut',\n", + " 'due',\n", + " 'douchbaggeri',\n", + " 'realli',\n", + " 'jim',\n", + " 'derogoti',\n", + " 'chicago',\n", + " 'tribun',\n", + " 'opinion',\n", + " 'stop',\n", + " 'jona',\n", + " 'hate',\n", + " 'catesong',\n", + " 'miss',\n", + " 'next',\n", + " 'tuesday',\n", + " 'kimkimz',\n", + " 'oh',\n", + " 'well',\n", + " 'think',\n", + " 'grandpar',\n", + " 'slaughter',\n", + " 'find',\n", + " 'uhmanduhpleas',\n", + " 'rorsley',\n", + " 'real',\n", + " 'thread',\n", + " 'made',\n", + " 'drammz',\n", + " 'soon',\n", + " 'snookca',\n", + " 'add',\n", + " 'nois',\n", + " 'via',\n", + " 'filter',\n", + " 'dialog',\n", + " 'pi',\n", + " 'add',\n", + " 'non',\n", + " 'destruct',\n", + " 'alway',\n", + " 'edit',\n", + " 'nois',\n", + " 'effect',\n", + " 'woke',\n", + " 'soccer',\n", + " 'practic',\n", + " 'todayyyi',\n", + " 'inluvwithjon',\n", + " 'miss',\n", + " 'wont',\n", + " 'comput',\n", + " 'long',\n", + " 'time',\n", + " 'wtf',\n", + " 'go',\n", + " 'glad',\n", + " 'got',\n", + " 'work',\n", + " 'earli',\n", + " 'im',\n", + " 'sure',\n", + " 'wa',\n", + " 'treat',\n", + " 'ashbymh',\n", + " 'must',\n", + " 'let',\n", + " 'know',\n", + " 'watch',\n", + " 'grey',\n", + " 'final',\n", + " 'total',\n", + " 'bawl',\n", + " 'eye',\n", + " 'im',\n", + " 'play',\n", + " 'hooki',\n", + " 'work',\n", + " 'feel',\n", + " 'good',\n", + " 'gon',\n", + " 'na',\n", + " 'go',\n", + " 'get',\n", + " 'hair',\n", + " 'wonder',\n", + " 'fitzsimmon',\n", + " 'honshi',\n", + " 'need',\n", + " 'flash',\n", + " 'work',\n", + " 'done',\n", + " 'send',\n", + " 'im',\n", + " 'get',\n", + " 'thi',\n", + " 'fever',\n", + " 'bore',\n", + " 'alreadi',\n", + " 'read',\n", + " 'weekli',\n", + " 'cover',\n", + " 'til',\n", + " 'end',\n", + " 'quot',\n", + " 'massag',\n", + " 'therapi',\n", + " 'quot',\n", + " 'ad',\n", + " 'start',\n", + " 'debasispradhan',\n", + " 'heheh',\n", + " 'agre',\n", + " 'ryohakkai',\n", + " 'alway',\n", + " 'afraid',\n", + " 'use',\n", + " 'countri',\n", + " 'name',\n", + " 'mention',\n", + " 'pair',\n", + " 'wuss',\n", + " 'oneblackdaria',\n", + " 'whi',\n", + " 'limit',\n", + " 'stori',\n", + " 'ch',\n", + " 'tell',\n", + " 'annoy',\n", + " 'www',\n", + " 'iamsoannoy',\n", + " 'com',\n", + " 'help',\n", + " 'reliev',\n", + " 'stress',\n", + " 'good',\n", + " 'morn',\n", + " 'got',\n", + " 'errand',\n", + " 'run',\n", + " 'soon',\n", + " 'http',\n", + " 'twitpic',\n", + " 'com',\n", + " 'rbhn',\n", + " 'stupid',\n", + " 'year',\n", + " 'project',\n", + " 'feel',\n", + " 'like',\n", + " 'conquer',\n", + " 'someth',\n", + " 'realis',\n", + " 'onli',\n", + " 'year',\n", + " 'dmcunningham',\n", + " 'get',\n", + " 'msg',\n", + " 'ask',\n", + " 'band',\n", + " 'wa',\n", + " 'listen',\n", + " 'last',\n", + " 'wk',\n", + " 'horrifi',\n", + " 'trae',\n", + " 'rememb',\n", + " 'richardarnatt',\n", + " 'good',\n", + " 'enough',\n", + " 'haha',\n", + " 'rubbish',\n", + " 'game',\n", + " 'like',\n", + " 'must',\n", + " 'fall',\n", + " 'waaaaaay',\n", + " 'late',\n", + " 'twitter',\n", + " 'night',\n", + " 'sinc',\n", + " 'lost',\n", + " 'ep',\n", + " 'still',\n", + " 'dl',\n", + " 'decid',\n", + " 'pop',\n", + " 'dvd',\n", + " 'classic',\n", + " 'cheech',\n", + " 'amp',\n", + " 'chong',\n", + " 'smoke',\n", + " 'cheech',\n", + " 'hurley',\n", + " 'day',\n", + " 'lost',\n", + " 'right',\n", + " 'taralynnfoxx',\n", + " 'full',\n", + " 'protein',\n", + " 'mirrormyey',\n", + " 'aw',\n", + " 'sorri',\n", + " 'good',\n", + " 'luck',\n", + " 'get',\n", + " 'accept',\n", + " 'swag',\n", + " 'slip',\n", + " 'thing',\n", + " 'eh',\n", + " 'whatev',\n", + " 'knew',\n", + " 'happen',\n", + " 'meggx',\n", + " 'paid',\n", + " 'new',\n", + " 'subscript',\n", + " 'back',\n", + " 'begin',\n", + " 'march',\n", + " 'gone',\n", + " 'sinc',\n", + " 'mid',\n", + " 'april',\n", + " 'sad',\n", + " 'hate',\n", + " 'back',\n", + " 'pain',\n", + " 'got',\n", + " 'ta',\n", + " 'get',\n", + " 'use',\n", + " 'phone',\n", + " 'givin',\n", + " 'eye',\n", + " 'like',\n", + " 'japanes',\n", + " 'sniper',\n", + " 'wish',\n", + " 'someon',\n", + " 'wa',\n", + " 'actual',\n", + " 'thi',\n", + " 'late',\n", + " 'lone',\n", + " 'lol',\n", + " 'modwif',\n", + " 'oh',\n", + " 'think',\n", + " 'plea',\n", + " 'found',\n", + " 'good',\n", + " 'home',\n", + " 'cat',\n", + " 'tho',\n", + " 'know',\n", + " 'gr',\n", + " 'adventur',\n", + " 'sethsimond',\n", + " 'mean',\n", + " 'follow',\n", + " 'goodnight',\n", + " 'seth',\n", + " 'gigglegirlnoel',\n", + " 'vote',\n", + " 'hope',\n", + " 'get',\n", + " 'ye',\n", + " 'drunken',\n", + " 'stori',\n", + " 'night',\n", + " 'done',\n", + " 'sober',\n", + " 'like',\n", + " 'mofo',\n", + " 'lol',\n", + " 'rsn',\n", + " 'miss',\n", + " 'ralph',\n", + " 'pogi',\n", + " 'haha',\n", + " 'aw',\n", + " 'still',\n", + " 'owe',\n", + " 'song',\n", + " 'song',\n", + " 'want',\n", + " 'sing',\n", + " 'four',\n", + " 'half',\n", + " 'hour',\n", + " 'till',\n", + " 'shot',\n", + " 'time',\n", + " 'cakest',\n", + " 'oh',\n", + " 'pm',\n", + " 'time',\n", + " 'like',\n", + " 'someth',\n", + " 'mayb',\n", + " 'total',\n", + " 'wrong',\n", + " 'lol',\n", + " 'jonasbroth',\n", + " 'wish',\n", + " 'could',\n", + " 'friend',\n", + " 'ddlovato',\n", + " 'nice',\n", + " 'live',\n", + " 'world',\n", + " 'one',\n", + " 'get',\n", + " 'critic',\n", + " 'wear',\n", + " 'date',\n", + " 'thank',\n", + " 'quot',\n", + " 'much',\n", + " 'fun',\n", + " 'club',\n", + " 'last',\n", + " 'night',\n", + " 'time',\n", + " 'sleep',\n", + " 'sadli',\n", + " 'onli',\n", + " 'hr',\n", + " 'sleep',\n", + " 'quot',\n", + " 'feel',\n", + " 'well',\n", + " 'get',\n", + " 'readi',\n", + " 'school',\n", + " 'tomorrow',\n", + " 'cant',\n", + " 'wait',\n", + " 'see',\n", + " 'bestfriend',\n", + " 'im',\n", + " 'go',\n", + " 'colleg',\n", + " 'next',\n", + " 'year',\n", + " 'yeehaa',\n", + " 'hate',\n", + " 'think',\n", + " 'kid',\n", + " 'might',\n", + " 'get',\n", + " 'morrow',\n", + " 'look',\n", + " 'kitten',\n", + " 'find',\n", + " 'last',\n", + " 'night',\n", + " 'petra',\n", + " 'ericleamen',\n", + " 'kind',\n", + " 'need',\n", + " 'one',\n", + " 'mac',\n", + " 'ashavan',\n", + " 'watch',\n", + " 'video',\n", + " 'today',\n", + " 'forget',\n", + " 'http',\n", + " 'bit',\n", + " 'ly',\n", + " 'whkdo',\n", + " 'iranelect',\n", + " 'kwalsham',\n", + " 'know',\n", + " 'seriou',\n", + " 'situat',\n", + " 'go',\n", + " 'ridicul',\n", + " 'extremist',\n", + " 'rout',\n", + " 'tv',\n", + " 'mtv',\n", + " 'movi',\n", + " 'award',\n", + " 'dinner',\n", + " 'rest',\n", + " 'good',\n", + " 'morn',\n", + " 'twitter',\n", + " 'hr',\n", + " 'sleep',\n", + " 'amp',\n", + " 'befor',\n", + " 'alarm',\n", + " 'goe',\n", + " 'excit',\n", + " 'like',\n", + " 'th',\n", + " 'grader',\n", + " 'first',\n", + " 'day',\n", + " 'school',\n", + " 'poke',\n", + " 'eye',\n", + " 'owchh',\n", + " 'bfflbwash',\n", + " 'made',\n", + " 'day',\n", + " 'better',\n", + " 'themunni',\n", + " 'libyan',\n", + " 'cool',\n", + " 'wa',\n", + " 'born',\n", + " 'rome',\n", + " 'ya',\n", + " 'like',\n", + " 'quot',\n", + " 'rom',\n", + " 'antic',\n", + " 'quot',\n", + " 'citi',\n", + " 'stevepavlina',\n", + " 'product',\n", + " 'come',\n", + " 'along',\n", + " 'say',\n", + " 'good',\n", + " 'morn',\n", + " 'day',\n", + " 'nle',\n", + " 'http',\n", + " 'plurk',\n", + " 'com',\n", + " 'ypv',\n", + " 'brentmayn',\n", + " 'big',\n", + " 'boy',\n", + " 'alright',\n", + " 'got',\n", + " 'good',\n", + " 'mamma',\n", + " 'home',\n", + " 'cook',\n", + " 'everi',\n", + " 'night',\n", + " 'sinc',\n", + " 'retir',\n", + " 'thelindsaylohan',\n", + " 'plan',\n", + " 'go',\n", + " 'dubai',\n", + " 'miss',\n", + " 'sam',\n", + " 'parti',\n", + " 'sanctuari',\n", + " 'wont',\n", + " 'miss',\n", + " 'thi',\n", + " 'time',\n", + " 'cookiemunstro',\n", + " 'ye',\n", + " 'come',\n", + " 'goa',\n", + " 'tri',\n", + " 'goan',\n", + " 'blog',\n", + " 'http',\n", + " 'goankrazi',\n", + " 'blogspot',\n", + " 'com',\n", + " 'http',\n", + " 'www',\n", + " 'goa',\n", + " 'meu',\n", + " 'amor',\n", + " 'blogspot',\n", + " 'com',\n", + " 'tomorrow',\n", + " 'colinbaylen',\n", + " 'turn',\n", + " 'yay',\n", + " 'happi',\n", + " 'birthday',\n", + " 'colin',\n", + " 'alway',\n", + " 'feel',\n", + " 'guilti',\n", + " 'whenev',\n", + " 'miss',\n", + " 'school',\n", + " 'even',\n", + " 'sick',\n", + " 'whi',\n", + " 'doe',\n", + " 'throat',\n", + " 'hurt',\n", + " 'whi',\n", + " 'ever',\n", + " 'go',\n", + " 'sleep',\n", + " 'befor',\n", + " 'catch',\n", + " 'flight',\n", + " 'hour',\n", + " 'surpris',\n", + " 'woke',\n", + " 'short',\n", + " 'nap',\n", + " 'bag',\n", + " 'pack',\n", + " 'thank',\n", + " 'mr',\n", + " 'ash',\n", + " 'ill',\n", + " 'miss',\n", + " 'go',\n", + " 'grandpa',\n", + " 'haha',\n", + " 'back',\n", + " 'soon',\n", + " 'lay',\n", + " 'bed',\n", + " 'sleepi',\n", + " 'yet',\n", + " 'unabl',\n", + " 'sleep',\n", + " 'yet',\n", + " 'beach',\n", + " 'egmontusa',\n", + " 'omg',\n", + " 'come',\n", + " 'see',\n", + " 'guy',\n", + " 'today',\n", + " 'candor',\n", + " 'didnt',\n", + " 'get',\n", + " 'chanc',\n", + " 'yesterday',\n", + " 'asfaq',\n", + " 'irohan',\n", + " 'vivekk',\n", + " 'ye',\n", + " 'ye',\n", + " 'need',\n", + " 'get',\n", + " 'number',\n", + " 'got',\n", + " 'joan',\n", + " 'whole',\n", + " 'break',\n", + " 'room',\n", + " 'cu',\n", + " 'dread',\n", + " 'stick',\n", + " 'stright',\n", + " 'say',\n", + " 'want',\n", + " 'mac',\n", + " 'chees',\n", + " 'plea',\n", + " 'http',\n", + " 'plurk',\n", + " 'com',\n", + " 'dfo',\n", + " 'miss',\n", + " 'mom',\n", + " 'broke',\n", + " 'televis',\n", + " 'turman',\n", + " 'realli',\n", + " 'break',\n", + " 'get',\n", + " 'back',\n", + " 'cabl',\n", + " 'oop',\n", + " 'nick',\n", + " 'nite',\n", + " 'go',\n", + " 'phoenix',\n", + " 'park',\n", + " 'get',\n", + " 'tan',\n", + " 'mistakepro',\n", + " 'http',\n", + " 'twitpic',\n", + " 'com',\n", + " 'kmt',\n", + " 'get',\n", + " 'dndgod',\n", + " 'lmao',\n", + " 'thank',\n", + " 'abcmsaj',\n", + " 'gon',\n", + " 'na',\n", + " 'back',\n", + " 'end',\n", + " 'juli',\n", + " 'easier',\n", + " 'nicer',\n", + " 'could',\n", + " 'ship',\n", + " 'go',\n", + " 'though',\n", + " 'snakob',\n", + " 'yup',\n", + " 'work',\n", + " 'weekend',\n", + " 'alway',\n", + " 'bummer',\n", + " 'gon',\n", + " 'na',\n", + " 'kick',\n", + " 'bowl',\n", + " 'tonit',\n", + " 'trishay',\n", + " 'rage',\n", + " 'pant',\n", + " 'rage',\n", + " 'glad',\n", + " 'see',\n", + " 'side',\n", + " 'would',\n", + " 'pheesh',\n", + " 'show',\n", + " 'woke',\n", + " 'shop',\n", + " 'today',\n", + " 'hope',\n", + " 'cuz',\n", + " 'desper',\n", + " 'need',\n", + " 'wardrob',\n", + " 'lookn',\n", + " 'kinda',\n", + " 'skimp',\n", + " 'got',\n", + " 'thing',\n", + " 'mind',\n", + " 'phase',\n", + " 'think',\n", + " 'adopt',\n", + " 'golden',\n", + " 'retriev',\n", + " 'miss',\n", + " 'gsd',\n", + " 'leilaboujnan',\n", + " 'ha',\n", + " 'beyond',\n", + " 'help',\n", + " 'serious',\n", + " 'bad',\n", + " 'thi',\n", + " 'guy',\n", + " 'http',\n", + " 'twitpic',\n", + " 'com',\n", + " 'nh',\n", + " 'jaredleto',\n", + " 'ha',\n", + " 'make',\n", + " 'better',\n", + " 'vampir',\n", + " 'xo',\n", + " 'fia',\n", + " 'mbase',\n", + " 'victorquest',\n", + " 'http',\n", + " 'twitpic',\n", + " 'com',\n", + " 'eep',\n", + " 'lizzi',\n", + " 'back',\n", + " 'littl',\n", + " 'late',\n", + " 'anyway',\n", + " 'yay',\n", + " 'love',\n", + " 'hair',\n", + " 'ticket',\n", + " 'lolla',\n", + " 'cost',\n", + " 'damn',\n", + " 'need',\n", + " 'way',\n", + " 'make',\n", + " 'money',\n", + " 'say',\n", + " 'helloooooo',\n", + " 'summervac',\n", + " 'littl',\n", + " 'miss',\n", + " 'sunshin',\n", + " 'giggl',\n", + " 'readyy',\n", + " 'great',\n", + " 'month',\n", + " 'aww',\n", + " 'gut',\n", + " 'didnt',\n", + " 'want',\n", + " 'either',\n", + " 'go',\n", + " 'last',\n", + " 'day',\n", + " 'juxt',\n", + " 'intern',\n", + " 'itsmeleighton',\n", + " 'wait',\n", + " 'hear',\n", + " 'nikkitabandita',\n", + " 'hey',\n", + " 'congrat',\n", + " 'graduat',\n", + " 'doe',\n", + " 'feel',\n", + " 'quot',\n", + " 'free',\n", + " 'last',\n", + " 'free',\n", + " 'last',\n", + " 'quot',\n", + " 'haha',\n", + " 'meredi',\n", + " 'hope',\n", + " 'treat',\n", + " 'hair',\n", + " 'way',\n", + " 'treat',\n", + " 'websit',\n", + " 'mileycyru',\n", + " 'http',\n", + " 'twitpic',\n", + " 'com',\n", + " 'dvj',\n", + " 'look',\n", + " 'fabul',\n", + " 'congrat',\n", + " 'award',\n", + " 'ili',\n", + " 'ulcer',\n", + " 'epicli',\n", + " 'lame',\n", + " 'onli',\n", + " 'imagin',\n", + " 'thi',\n", + " 'must',\n", + " 'feel',\n", + " 'like',\n", + " 'shot',\n", + " 'face',\n", + " 'close',\n", + " 'rang',\n", + " 'billyraycyru',\n", + " 'hi',\n", + " 'im',\n", + " ...]" + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "all_words = [word for tokens in data['text_processed'] for word in tokens]\n", + "all_words" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "FreqDist({'go': 1681, 'get': 1446, 'wa': 1330, 'day': 1312, 'thi': 1214, 'good': 1161, 'work': 1096, 'like': 1093, 'love': 1026, 'got': 964, ...})" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "freq_dist = FreqDist(all_words)\n", + "freq_dist" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('go', 1681),\n", + " ('get', 1446),\n", + " ('wa', 1330),\n", + " ('day', 1312),\n", + " ('thi', 1214),\n", + " ('good', 1161),\n", + " ('work', 1096),\n", + " ('like', 1093),\n", + " ('love', 1026),\n", + " ('got', 964),\n", + " ('quot', 905),\n", + " ('http', 877),\n", + " ('today', 844),\n", + " ('one', 796),\n", + " ('time', 793),\n", + " ('lol', 768),\n", + " ('thank', 765),\n", + " ('back', 726),\n", + " ('miss', 714),\n", + " ('want', 687),\n", + " ('know', 679),\n", + " ('think', 668),\n", + " ('realli', 655),\n", + " ('feel', 635),\n", + " ('com', 634),\n", + " ('im', 627),\n", + " ('amp', 623),\n", + " ('see', 611),\n", + " ('night', 582),\n", + " ('well', 582),\n", + " ('need', 578),\n", + " ('na', 549),\n", + " ('oh', 545),\n", + " ('still', 540),\n", + " ('new', 529),\n", + " ('come', 528),\n", + " ('watch', 525),\n", + " ('hope', 524),\n", + " ('home', 518),\n", + " ('look', 511),\n", + " ('make', 500),\n", + " ('ha', 452),\n", + " ('twitter', 448),\n", + " ('last', 446),\n", + " ('much', 435),\n", + " ('morn', 433),\n", + " ('tomorrow', 423),\n", + " ('great', 417),\n", + " ('wait', 411),\n", + " ('sleep', 406),\n", + " ('haha', 403),\n", + " ('fun', 393),\n", + " ('wish', 383),\n", + " ('whi', 371),\n", + " ('sad', 367),\n", + " ('week', 353),\n", + " ('follow', 352),\n", + " ('right', 345),\n", + " ('tri', 345),\n", + " ('thing', 343),\n", + " ('friend', 338),\n", + " ('would', 338),\n", + " ('gon', 333),\n", + " ('onli', 331),\n", + " ('happi', 330),\n", + " ('sorri', 327),\n", + " ('say', 327),\n", + " ('way', 326),\n", + " ('tonight', 325),\n", + " ('bit', 324),\n", + " ('bad', 317),\n", + " ('nice', 310),\n", + " ('tweet', 308),\n", + " ('though', 307),\n", + " ('hi', 301),\n", + " ('veri', 300),\n", + " ('even', 297),\n", + " ('take', 292),\n", + " ('start', 286),\n", + " ('better', 283),\n", + " ('peopl', 276),\n", + " ('bed', 275),\n", + " ('yeah', 275),\n", + " ('could', 273),\n", + " ('hour', 271),\n", + " ('guy', 270),\n", + " ('school', 262),\n", + " ('play', 256),\n", + " ('use', 252),\n", + " ('show', 252),\n", + " ('hey', 248),\n", + " ('twitpic', 247),\n", + " ('rain', 246),\n", + " ('cant', 246),\n", + " ('awesom', 246),\n", + " ('ye', 240),\n", + " ('next', 238),\n", + " ('hate', 237),\n", + " ('soon', 236),\n", + " ('weekend', 233),\n", + " ('find', 231),\n", + " ('lt', 228),\n", + " ('never', 227),\n", + " ('let', 226),\n", + " ('sick', 225),\n", + " ('year', 220),\n", + " ('final', 216),\n", + " ('plea', 215),\n", + " ('yay', 213),\n", + " ('littl', 211),\n", + " ('ani', 210),\n", + " ('long', 204),\n", + " ('head', 203),\n", + " ('first', 203),\n", + " ('life', 203),\n", + " ('keep', 201),\n", + " ('girl', 201),\n", + " ('alway', 200),\n", + " ('movi', 200),\n", + " ('ok', 200),\n", + " ('alreadi', 198),\n", + " ('dont', 197),\n", + " ('phone', 195),\n", + " ('best', 195),\n", + " ('talk', 194),\n", + " ('bore', 193),\n", + " ('eat', 193),\n", + " ('befor', 192),\n", + " ('someth', 192),\n", + " ('wan', 192),\n", + " ('read', 190),\n", + " ('ever', 190),\n", + " ('listen', 190),\n", + " ('everyon', 190),\n", + " ('leav', 190),\n", + " ('sure', 189),\n", + " ('done', 188),\n", + " ('made', 185),\n", + " ('ly', 185),\n", + " ('man', 185),\n", + " ('anoth', 184),\n", + " ('tire', 184),\n", + " ('call', 183),\n", + " ('cool', 182),\n", + " ('went', 182),\n", + " ('hurt', 180),\n", + " ('help', 179),\n", + " ('song', 178),\n", + " ('yet', 178),\n", + " ('suck', 178),\n", + " ('mayb', 177),\n", + " ('lot', 175),\n", + " ('sound', 174),\n", + " ('ur', 173),\n", + " ('live', 171),\n", + " ('away', 170),\n", + " ('hous', 164),\n", + " ('guess', 163),\n", + " ('excit', 162),\n", + " ('pretti', 162),\n", + " ('finish', 161),\n", + " ('enjoy', 159),\n", + " ('someon', 159),\n", + " ('summer', 159),\n", + " ('thought', 159),\n", + " ('left', 157),\n", + " ('old', 155),\n", + " ('omg', 154),\n", + " ('babi', 154),\n", + " ('tell', 151),\n", + " ('parti', 150),\n", + " ('amaz', 149),\n", + " ('big', 149),\n", + " ('two', 148),\n", + " ('pic', 148),\n", + " ('earli', 147),\n", + " ('hear', 146),\n", + " ('put', 146),\n", + " ('becaus', 146),\n", + " ('end', 144),\n", + " ('readi', 143),\n", + " ('th', 142),\n", + " ('lost', 142),\n", + " ('check', 142),\n", + " ('ya', 141),\n", + " ('doe', 141),\n", + " ('mean', 139),\n", + " ('wow', 139),\n", + " ('hot', 138),\n", + " ('kid', 137),\n", + " ('yesterday', 137),\n", + " ('god', 137),\n", + " ('sunday', 136),\n", + " ('updat', 136),\n", + " ('saw', 135),\n", + " ('glad', 133),\n", + " ('run', 133),\n", + " ('book', 133),\n", + " ('stuff', 132),\n", + " ('later', 131),\n", + " ('shop', 131),\n", + " ('game', 130),\n", + " ('car', 129),\n", + " ('music', 129),\n", + " ('ugh', 128),\n", + " ('sun', 128),\n", + " ('sinc', 128),\n", + " ('job', 127),\n", + " ('stop', 127),\n", + " ('ta', 127),\n", + " ('iphon', 127),\n", + " ('hard', 126),\n", + " ('happen', 125),\n", + " ('actual', 125),\n", + " ('weather', 125),\n", + " ('give', 123),\n", + " ('stay', 123),\n", + " ('must', 123),\n", + " ('wonder', 123),\n", + " ('www', 122),\n", + " ('world', 122),\n", + " ('mom', 122),\n", + " ('monday', 122),\n", + " ('also', 121),\n", + " ('exam', 121),\n", + " ('might', 120),\n", + " ('beauti', 120),\n", + " ('post', 120),\n", + " ('seem', 120),\n", + " ('video', 119),\n", + " ('noth', 118),\n", + " ('fuck', 117),\n", + " ('ask', 117),\n", + " ('damn', 117),\n", + " ('birthday', 116),\n", + " ('buy', 116),\n", + " ('late', 115),\n", + " ('found', 115),\n", + " ('free', 115),\n", + " ('around', 113),\n", + " ('friday', 113),\n", + " ('total', 112),\n", + " ('cute', 112),\n", + " ('meet', 112),\n", + " ('drink', 111),\n", + " ('dad', 109),\n", + " ('sweet', 109),\n", + " ('woke', 108),\n", + " ('mani', 108),\n", + " ('said', 108),\n", + " ('cri', 108),\n", + " ('poor', 108),\n", + " ('till', 107),\n", + " ('boy', 107),\n", + " ('aww', 107),\n", + " ('clean', 106),\n", + " ('pictur', 106),\n", + " ('plan', 105),\n", + " ('luck', 104),\n", + " ('dinner', 103),\n", + " ('welcom', 103),\n", + " ('anyth', 102),\n", + " ('famili', 102),\n", + " ('without', 101),\n", + " ('busi', 101),\n", + " ('name', 100),\n", + " ('studi', 100),\n", + " ('face', 99),\n", + " ('place', 99),\n", + " ('may', 98),\n", + " ('least', 98),\n", + " ('win', 98),\n", + " ('lunch', 98),\n", + " ('gt', 98),\n", + " ('enough', 97),\n", + " ('funni', 96),\n", + " ('okay', 96),\n", + " ('food', 95),\n", + " ('cold', 94),\n", + " ('gone', 94),\n", + " ('tv', 94),\n", + " ('walk', 94),\n", + " ('eye', 93),\n", + " ('tho', 93),\n", + " ('hahaha', 93),\n", + " ('die', 93),\n", + " ('caus', 92),\n", + " ('drive', 92),\n", + " ('shit', 92),\n", + " ('turn', 91),\n", + " ('believ', 91),\n", + " ('money', 90),\n", + " ('wrong', 90),\n", + " ('ill', 90),\n", + " ('saturday', 90),\n", + " ('heart', 90),\n", + " ('sit', 90),\n", + " ('class', 89),\n", + " ('didnt', 89),\n", + " ('xx', 89),\n", + " ('real', 89),\n", + " ('hair', 89),\n", + " ('aw', 89),\n", + " ('mine', 89),\n", + " ('dream', 89),\n", + " ('repli', 88),\n", + " ('move', 88),\n", + " ('chang', 87),\n", + " ('forward', 87),\n", + " ('rock', 87),\n", + " ('month', 86),\n", + " ('stupid', 86),\n", + " ('beach', 86),\n", + " ('open', 86),\n", + " ('write', 86),\n", + " ('kinda', 85),\n", + " ('idea', 85),\n", + " ('outsid', 84),\n", + " ('tinyurl', 84),\n", + " ('brother', 84),\n", + " ('almost', 83),\n", + " ('came', 83),\n", + " ('wake', 83),\n", + " ('seen', 82),\n", + " ('word', 81),\n", + " ('whole', 81),\n", + " ('dog', 81),\n", + " ('hang', 81),\n", + " ('send', 80),\n", + " ('plurk', 80),\n", + " ('everyth', 80),\n", + " ('person', 80),\n", + " ('list', 80),\n", + " ('part', 79),\n", + " ('everi', 79),\n", + " ('probabl', 79),\n", + " ('anyon', 78),\n", + " ('onc', 78),\n", + " ('comput', 77),\n", + " ('goodnight', 77),\n", + " ('quit', 77),\n", + " ('fan', 77),\n", + " ('super', 77),\n", + " ('full', 76),\n", + " ('minut', 76),\n", + " ('offic', 76),\n", + " ('danc', 76),\n", + " ('trip', 76),\n", + " ('awww', 76),\n", + " ('true', 76),\n", + " ('pick', 75),\n", + " ('far', 75),\n", + " ('pain', 74),\n", + " ('room', 74),\n", + " ('facebook', 74),\n", + " ('favorit', 74),\n", + " ('coffe', 74),\n", + " ('news', 74),\n", + " ('took', 74),\n", + " ('headach', 73),\n", + " ('blog', 73),\n", + " ('pas', 73),\n", + " ('care', 73),\n", + " ('site', 72),\n", + " ('fail', 71),\n", + " ('mother', 71),\n", + " ('hello', 71),\n", + " ('boo', 71),\n", + " ('forgot', 71),\n", + " ('dude', 70),\n", + " ('bring', 70),\n", + " ('el', 70),\n", + " ('vote', 69),\n", + " ('la', 69),\n", + " ('hit', 69),\n", + " ('hug', 69),\n", + " ('learn', 69),\n", + " ('heard', 69),\n", + " ('lucki', 68),\n", + " ('pm', 68),\n", + " ('crazi', 68),\n", + " ('anymor', 68),\n", + " ('fall', 67),\n", + " ('ticket', 67),\n", + " ('ago', 67),\n", + " ('concert', 67),\n", + " ('st', 67),\n", + " ('test', 67),\n", + " ('hehe', 67),\n", + " ('break', 66),\n", + " ('sister', 66),\n", + " ('abl', 66),\n", + " ('text', 66),\n", + " ('worri', 66),\n", + " ('sooo', 65),\n", + " ('serious', 65),\n", + " ('lil', 65),\n", + " ('anyway', 65),\n", + " ('suppos', 65),\n", + " ('asleep', 65),\n", + " ('alon', 65),\n", + " ('shower', 65),\n", + " ('spend', 65),\n", + " ('rest', 64),\n", + " ('either', 64),\n", + " ('pay', 64),\n", + " ('interest', 64),\n", + " ('awak', 64),\n", + " ('wear', 63),\n", + " ('cuz', 63),\n", + " ('photo', 63),\n", + " ('visit', 63),\n", + " ('smile', 63),\n", + " ('fine', 63),\n", + " ('mind', 62),\n", + " ('internet', 62),\n", + " ('album', 62),\n", + " ('instead', 62),\n", + " ('rememb', 61),\n", + " ('cat', 61),\n", + " ('catch', 61),\n", + " ('btw', 61),\n", + " ('email', 61),\n", + " ('ah', 61),\n", + " ('set', 61),\n", + " ('link', 61),\n", + " ('onlin', 61),\n", + " ('red', 60),\n", + " ('add', 60),\n", + " ('kind', 60),\n", + " ('mileycyru', 60),\n", + " ('train', 60),\n", + " ('hell', 60),\n", + " ('sign', 60),\n", + " ('agre', 59),\n", + " ('half', 59),\n", + " ('close', 59),\n", + " ('sunni', 59),\n", + " ('bout', 58),\n", + " ('june', 58),\n", + " ('ice', 58),\n", + " ('fix', 58),\n", + " ('top', 58),\n", + " ('ladi', 58),\n", + " ('min', 57),\n", + " ('season', 57),\n", + " ('wont', 56),\n", + " ('lmao', 56),\n", + " ('bought', 56),\n", + " ('hand', 56),\n", + " ('ride', 56),\n", + " ('dress', 56),\n", + " ('annoy', 55),\n", + " ('decid', 55),\n", + " ('short', 55),\n", + " ('blip', 55),\n", + " ('ipod', 55),\n", + " ('wed', 55),\n", + " ('kill', 55),\n", + " ('cut', 55),\n", + " ('nap', 54),\n", + " ('broke', 54),\n", + " ('co', 54),\n", + " ('second', 54),\n", + " ('scare', 54),\n", + " ('card', 54),\n", + " ('join', 54),\n", + " ('cook', 53),\n", + " ('fm', 53),\n", + " ('holiday', 53),\n", + " ('togeth', 53),\n", + " ('sometim', 53),\n", + " ('youtub', 53),\n", + " ('order', 53),\n", + " ('soo', 52),\n", + " ('offici', 52),\n", + " ('weird', 52),\n", + " ('band', 51),\n", + " ('nite', 51),\n", + " ('page', 51),\n", + " ('high', 51),\n", + " ('sore', 51),\n", + " ('problem', 51),\n", + " ('stori', 50),\n", + " ('tommcfli', 50),\n", + " ('save', 50),\n", + " ('reason', 50),\n", + " ('nd', 50),\n", + " ('laptop', 50),\n", + " ('side', 49),\n", + " ('graduat', 49),\n", + " ('yea', 49),\n", + " ('crap', 49),\n", + " ('told', 49),\n", + " ('question', 49),\n", + " ('goe', 48),\n", + " ('forget', 48),\n", + " ('throat', 48),\n", + " ('congrat', 48),\n", + " ('homework', 48),\n", + " ('relax', 48),\n", + " ('hmm', 48),\n", + " ('shirt', 48),\n", + " ('sim', 48),\n", + " ('bye', 48),\n", + " ('stuck', 48),\n", + " ('tuesday', 47),\n", + " ('til', 47),\n", + " ('pack', 47),\n", + " ('figur', 47),\n", + " ('gym', 47),\n", + " ('air', 47),\n", + " ('breakfast', 47),\n", + " ('broken', 47),\n", + " ('past', 47),\n", + " ('ive', 47),\n", + " ('worst', 47),\n", + " ('award', 46),\n", + " ('citi', 46),\n", + " ('park', 46),\n", + " ('dear', 46),\n", + " ('laugh', 46),\n", + " ('sigh', 46),\n", + " ('download', 46),\n", + " ('perfect', 46),\n", + " ('le', 45),\n", + " ('definit', 45),\n", + " ('afternoon', 45),\n", + " ('celebr', 45),\n", + " ('mad', 45),\n", + " ('david', 45),\n", + " ('sing', 44),\n", + " ('answer', 44),\n", + " ('slow', 44),\n", + " ('differ', 44),\n", + " ('fast', 44),\n", + " ('cours', 44),\n", + " ('hungri', 44),\n", + " ('church', 44),\n", + " ('lie', 44),\n", + " ('fb', 44),\n", + " ('luv', 44),\n", + " ('chocol', 44),\n", + " ('chanc', 43),\n", + " ('goin', 43),\n", + " ('record', 43),\n", + " ('comment', 43),\n", + " ('travel', 43),\n", + " ('dm', 43),\n", + " ('uk', 43),\n", + " ('moon', 43),\n", + " ('complet', 43),\n", + " ('sleepi', 42),\n", + " ('foot', 42),\n", + " ('ddlovato', 42),\n", + " ('mr', 42),\n", + " ('tea', 42),\n", + " ('manag', 42),\n", + " ('fli', 42),\n", + " ('easi', 42),\n", + " ('store', 42),\n", + " ('math', 42),\n", + " ('depress', 42),\n", + " ('jealou', 42),\n", + " ('line', 42),\n", + " ('cake', 41),\n", + " ('tour', 41),\n", + " ('freak', 41),\n", + " ('dead', 41),\n", + " ('understand', 41),\n", + " ('worth', 41),\n", + " ('team', 41),\n", + " ('plu', 41),\n", + " ('box', 41),\n", + " ('longer', 41),\n", + " ('account', 41),\n", + " ('ppl', 41),\n", + " ('water', 41),\n", + " ('moment', 41),\n", + " ('support', 41),\n", + " ('fell', 41),\n", + " ('mood', 40),\n", + " ('sadli', 40),\n", + " ('messag', 40),\n", + " ('rather', 40),\n", + " ('woman', 40),\n", + " ('black', 40),\n", + " ('upset', 40),\n", + " ('yep', 40),\n", + " ('version', 40),\n", + " ('three', 40),\n", + " ('episod', 40),\n", + " ('radio', 40),\n", + " ('unfortun', 40),\n", + " ('except', 40),\n", + " ('mum', 40),\n", + " ('cream', 39),\n", + " ('met', 39),\n", + " ('chat', 39),\n", + " ('begin', 39),\n", + " ('shot', 39),\n", + " ('mac', 39),\n", + " ('cousin', 39),\n", + " ('camp', 39),\n", + " ('share', 39),\n", + " ('app', 39),\n", + " ('load', 39),\n", + " ('appar', 39),\n", + " ('shame', 39),\n", + " ('leg', 39),\n", + " ('beer', 39),\n", + " ('jona', 38),\n", + " ('ad', 38),\n", + " ('colleg', 38),\n", + " ('kick', 38),\n", + " ('count', 38),\n", + " ('coupl', 38),\n", + " ('felt', 38),\n", + " ('point', 38),\n", + " ('nope', 38),\n", + " ('bum', 38),\n", + " ('ach', 38),\n", + " ('due', 37),\n", + " ('via', 37),\n", + " ('stress', 37),\n", + " ('hr', 37),\n", + " ('flight', 37),\n", + " ('father', 37),\n", + " ('babe', 37),\n", + " ('hubbi', 37),\n", + " ('design', 37),\n", + " ('ate', 37),\n", + " ('blue', 37),\n", + " ('bike', 37),\n", + " ('bitch', 37),\n", + " ('absolut', 37),\n", + " ('town', 37),\n", + " ('upload', 37),\n", + " ('bless', 37),\n", + " ('xd', 37),\n", + " ('burn', 36),\n", + " ('club', 36),\n", + " ('lay', 36),\n", + " ('special', 36),\n", + " ('flu', 36),\n", + " ('fantast', 36),\n", + " ('pool', 36),\n", + " ('warm', 36),\n", + " ('bet', 36),\n", + " ('son', 36),\n", + " ('lazi', 36),\n", + " ('soooo', 36),\n", + " ('ear', 36),\n", + " ('followfriday', 36),\n", + " ('power', 36),\n", + " ('forev', 36),\n", + " ('smell', 36),\n", + " ('lone', 35),\n", + " ('date', 35),\n", + " ('twit', 35),\n", + " ('vacat', 35),\n", + " ('wit', 35),\n", + " ('john', 35),\n", + " ('miley', 35),\n", + " ('swim', 35),\n", + " ('sort', 35),\n", + " ('cheer', 35),\n", + " ('wors', 35),\n", + " ('hotel', 35),\n", + " ('bodi', 35),\n", + " ('crash', 35),\n", + " ('usual', 35),\n", + " ('practic', 34),\n", + " ('wtf', 34),\n", + " ('knew', 34),\n", + " ('alright', 34),\n", + " ('type', 34),\n", + " ('yo', 34),\n", + " ('horribl', 34),\n", + " ('plane', 34),\n", + " ('road', 34),\n", + " ('green', 34),\n", + " ('meant', 34),\n", + " ('yummi', 34),\n", + " ('realiz', 34),\n", + " ('myspac', 34),\n", + " ('english', 34),\n", + " ('ju', 34),\n", + " ('shoot', 34),\n", + " ('connect', 34),\n", + " ('websit', 33),\n", + " ('cover', 33),\n", + " ('dvd', 33),\n", + " ('product', 33),\n", + " ('juli', 33),\n", + " ('safe', 33),\n", + " ('ahhh', 33),\n", + " ('note', 33),\n", + " ('thx', 33),\n", + " ('beat', 33),\n", + " ('pull', 33),\n", + " ('woo', 33),\n", + " ('parent', 33),\n", + " ('sent', 33),\n", + " ('present', 33),\n", + " ('art', 33),\n", + " ('prob', 33),\n", + " ('cd', 33),\n", + " ('appl', 33),\n", + " ('film', 33),\n", + " ('bbq', 33),\n", + " ('fact', 33),\n", + " ('stomach', 32),\n", + " ('drop', 32),\n", + " ('chees', 32),\n", + " ('yup', 32),\n", + " ('sunshin', 32),\n", + " ('child', 32),\n", + " ('tom', 32),\n", + " ('sat', 32),\n", + " ('remind', 32),\n", + " ('histori', 32),\n", + " ('sold', 32),\n", + " ('paper', 32),\n", + " ('bird', 32),\n", + " ('addict', 32),\n", + " ('tear', 32),\n", + " ('doctor', 32),\n", + " ('doesnt', 32),\n", + " ('ahh', 32),\n", + " ('step', 32),\n", + " ('inspir', 32),\n", + " ('star', 32),\n", + " ('fair', 32),\n", + " ('disappoint', 31),\n", + " ('terribl', 31),\n", + " ('mtv', 31),\n", + " ('da', 31),\n", + " ('spent', 31),\n", + " ('white', 31),\n", + " ('especi', 31),\n", + " ('pink', 31),\n", + " ('garden', 31),\n", + " ('lose', 31),\n", + " ('interview', 31),\n", + " ('laker', 31),\n", + " ('idk', 31),\n", + " ('shoe', 31),\n", + " ('allow', 31),\n", + " ('alot', 31),\n", + " ('tummi', 31),\n", + " ('appreci', 31),\n", + " ('block', 31),\n", + " ('goodby', 31),\n", + " ('watchin', 31),\n", + " ('dark', 31),\n", + " ('hahah', 31),\n", + " ('bar', 31),\n", + " ('door', 31),\n", + " ('bb', 31),\n", + " ('gd', 31),\n", + " ('london', 31),\n", + " ('cancel', 31),\n", + " ('project', 30),\n", + " ('surpris', 30),\n", + " ('promis', 30),\n", + " ('stand', 30),\n", + " ('yr', 30),\n", + " ('angel', 30),\n", + " ('cooki', 30),\n", + " ('chill', 30),\n", + " ('land', 30),\n", + " ('dure', 30),\n", + " ('event', 30),\n", + " ('havent', 30),\n", + " ('storm', 30),\n", + " ('background', 30),\n", + " ('airport', 30),\n", + " ('si', 30),\n", + " ('edit', 29),\n", + " ('mention', 29),\n", + " ('number', 29),\n", + " ('futur', 29),\n", + " ('becom', 29),\n", + " ('possibl', 29),\n", + " ('revis', 29),\n", + " ('ouch', 29),\n", + " ('extra', 29),\n", + " ('slept', 29),\n", + " ('light', 29),\n", + " ('window', 29),\n", + " ('age', 29),\n", + " ('boyfriend', 29),\n", + " ('act', 29),\n", + " ('small', 28),\n", + " ('bu', 28),\n", + " ('gave', 28),\n", + " ('demi', 28),\n", + " ('current', 28),\n", + " ('mommi', 28),\n", + " ('mess', 28),\n", + " ('uh', 28),\n", + " ('channel', 28),\n", + " ('search', 28),\n", + " ('sale', 28),\n", + " ('price', 28),\n", + " ('fri', 28),\n", + " ('touch', 28),\n", + " ('ball', 28),\n", + " ('chicken', 28),\n", + " ('finger', 28),\n", + " ('huh', 28),\n", + " ('fight', 28),\n", + " ('mail', 28),\n", + " ('front', 28),\n", + " ('web', 28),\n", + " ('voic', 28),\n", + " ('piss', 28),\n", + " ('daddi', 27),\n", + " ('eh', 27),\n", + " ('chip', 27),\n", + " ('recommend', 27),\n", + " ('wine', 27),\n", + " ('tr', 27),\n", + " ('puppi', 27),\n", + " ('street', 27),\n", + " ('scari', 27),\n", + " ('arm', 27),\n", + " ('hmmm', 27),\n", + " ('expect', 27),\n", + " ('twilight', 27),\n", + " ('kitti', 27),\n", + " ('round', 27),\n", + " ('bug', 27),\n", + " ('pizza', 27),\n", + " ('itun', 27),\n", + " ('king', 27),\n", + " ('huge', 27),\n", + " ('coz', 27),\n", + " ('color', 27),\n", + " ('kati', 27),\n", + " ('confus', 27),\n", + " ('speak', 27),\n", + " ('tip', 27),\n", + " ('xxx', 27),\n", + " ('jonasbroth', 26),\n", + " ('bag', 26),\n", + " ('nick', 26),\n", + " ('xo', 26),\n", + " ('ny', 26),\n", + " ('kate', 26),\n", + " ('caught', 26),\n", + " ('hospit', 26),\n", + " ('deserv', 26),\n", + " ('gay', 26),\n", + " ('xoxo', 26),\n", + " ('nobodi', 26),\n", + " ('bday', 26),\n", + " ('wast', 26),\n", + " ('servic', 26),\n", + " ('invit', 26),\n", + " ('id', 26),\n", + " ('hill', 26),\n", + " ('offer', 26),\n", + " ('low', 25),\n", + " ('along', 25),\n", + " ('thursday', 25),\n", + " ('kiss', 25),\n", + " ('jb', 25),\n", + " ('exhaust', 25),\n", + " ('drunk', 25),\n", + " ('impress', 25),\n", + " ('rip', 25),\n", + " ('code', 25),\n", + " ('paint', 25),\n", + " ('em', 25),\n", + " ('matter', 25),\n", + " ('dunno', 25),\n", + " ('jon', 25),\n", + " ('mile', 25),\n", + " ('wednesday', 25),\n", + " ('blood', 25),\n", + " ('ahead', 25),\n", + " ('googl', 25),\n", + " ('avail', 25),\n", + " ('england', 25),\n", + " ('case', 25),\n", + " ('exactli', 25),\n", + " ('hun', 25),\n", + " ('wash', 25),\n", + " ('magic', 25),\n", + " ('delet', 25),\n", + " ('tast', 25),\n", + " ('wet', 25),\n", + " ('notic', 25),\n", + " ('proud', 25),\n", + " ('jame', 25),\n", + " ('fire', 25),\n", + " ('continu', 24),\n", + " ('non', 24),\n", + " ('countri', 24),\n", + " ('vip', 24),\n", + " ('taylor', 24),\n", + " ('earlier', 24),\n", + " ('chri', 24),\n", + " ('kno', 24),\n", + " ('ruin', 24),\n", + " ('direct', 24),\n", + " ('arriv', 24),\n", + " ('heat', 24),\n", + " ('prepar', 24),\n", + " ('fit', 24),\n", + " ('pray', 24),\n", + " ('doubt', 24),\n", + " ('hold', 24),\n", + " ('race', 24),\n", + " ('everybodi', 24),\n", + " ('fav', 24),\n", + " ('daughter', 24),\n", + " ('track', 24),\n", + " ('honey', 24),\n", + " ('troubl', 24),\n", + " ('taken', 24),\n", + " ('brain', 24),\n", + " ('tune', 24),\n", + " ('consid', 24),\n", + " ('bro', 24),\n", + " ('net', 24),\n", + " ('joe', 24),\n", + " ('husband', 24),\n", + " ('sky', 24),\n", + " ('shut', 23),\n", + " ('tonit', 23),\n", + " ('normal', 23),\n", + " ('gorgeou', 23),\n", + " ('apart', 23),\n", + " ('ff', 23),\n", + " ('choic', 23),\n", + " ('etc', 23),\n", + " ('jordanknight', 23),\n", + " ('somewher', 23),\n", + " ('behind', 23),\n", + " ('lesson', 23),\n", + " ('fill', 23),\n", + " ('camera', 23),\n", + " ('nail', 23),\n", + " ('nah', 23),\n", + " ('cross', 23),\n", + " ('couldnt', 23),\n", + " ('ohh', 23),\n", + " ('delici', 23),\n", + " ('instal', 23),\n", + " ('secret', 23),\n", + " ('screen', 23),\n", + " ('marri', 23),\n", + " ('forc', 23),\n", + " ('hangov', 23),\n", + " ('issu', 23),\n", + " ('paul', 23),\n", + " ('darn', 23),\n", + " ('profil', 23),\n", + " ('group', 23),\n", + " ('suggest', 23),\n", + " ('aint', 23),\n", + " ('memori', 23),\n", + " ('tweep', 23),\n", + " ('spam', 23),\n", + " ('winter', 23),\n", + " ('smoke', 22),\n", + " ('sam', 22),\n", + " ('compani', 22),\n", + " ('ryan', 22),\n", + " ('self', 22),\n", + " ('hannah', 22),\n", + " ('bath', 22),\n", + " ('prefer', 22),\n", + " ('sweeti', 22),\n", + " ('death', 22),\n", + " ('blow', 22),\n", + " ('shall', 22),\n", + " ('screw', 22),\n", + " ('often', 22),\n", + " ('dat', 22),\n", + " ('system', 22),\n", + " ('disney', 22),\n", + " ('state', 22),\n", + " ('sa', 22),\n", + " ('result', 22),\n", + " ('peac', 22),\n", + " ('bunch', 22),\n", + " ('creat', 22),\n", + " ('american', 22),\n", + " ('sushi', 22),\n", + " ('adam', 22),\n", + " ('nyc', 22),\n", + " ('ooh', 22),\n", + " ('crappi', 22),\n", + " ('mo', 22),\n", + " ('experi', 22),\n", + " ('dang', 22),\n", + " ...]" + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "top_words = freq_dist.most_common(5000)\n", + "top_words" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['go',\n", + " 'get',\n", + " 'wa',\n", + " 'day',\n", + " 'thi',\n", + " 'good',\n", + " 'work',\n", + " 'like',\n", + " 'love',\n", + " 'got',\n", + " 'quot',\n", + " 'http',\n", + " 'today',\n", + " 'one',\n", + " 'time',\n", + " 'lol',\n", + " 'thank',\n", + " 'back',\n", + " 'miss',\n", + " 'want',\n", + " 'know',\n", + " 'think',\n", + " 'realli',\n", + " 'feel',\n", + " 'com',\n", + " 'im',\n", + " 'amp',\n", + " 'see',\n", + " 'night',\n", + " 'well',\n", + " 'need',\n", + " 'na',\n", + " 'oh',\n", + " 'still',\n", + " 'new',\n", + " 'come',\n", + " 'watch',\n", + " 'hope',\n", + " 'home',\n", + " 'look',\n", + " 'make',\n", + " 'ha',\n", + " 'twitter',\n", + " 'last',\n", + " 'much',\n", + " 'morn',\n", + " 'tomorrow',\n", + " 'great',\n", + " 'wait',\n", + " 'sleep',\n", + " 'haha',\n", + " 'fun',\n", + " 'wish',\n", + " 'whi',\n", + " 'sad',\n", + " 'week',\n", + " 'follow',\n", + " 'right',\n", + " 'tri',\n", + " 'thing',\n", + " 'friend',\n", + " 'would',\n", + " 'gon',\n", + " 'onli',\n", + " 'happi',\n", + " 'sorri',\n", + " 'say',\n", + " 'way',\n", + " 'tonight',\n", + " 'bit',\n", + " 'bad',\n", + " 'nice',\n", + " 'tweet',\n", + " 'though',\n", + " 'hi',\n", + " 'veri',\n", + " 'even',\n", + " 'take',\n", + " 'start',\n", + " 'better',\n", + " 'peopl',\n", + " 'bed',\n", + " 'yeah',\n", + " 'could',\n", + " 'hour',\n", + " 'guy',\n", + " 'school',\n", + " 'play',\n", + " 'use',\n", + " 'show',\n", + " 'hey',\n", + " 'twitpic',\n", + " 'rain',\n", + " 'cant',\n", + " 'awesom',\n", + " 'ye',\n", + " 'next',\n", + " 'hate',\n", + " 'soon',\n", + " 'weekend',\n", + " 'find',\n", + " 'lt',\n", + " 'never',\n", + " 'let',\n", + " 'sick',\n", + " 'year',\n", + " 'final',\n", + " 'plea',\n", + " 'yay',\n", + " 'littl',\n", + " 'ani',\n", + " 'long',\n", + " 'head',\n", + " 'first',\n", + " 'life',\n", + " 'keep',\n", + " 'girl',\n", + " 'alway',\n", + " 'movi',\n", + " 'ok',\n", + " 'alreadi',\n", + " 'dont',\n", + " 'phone',\n", + " 'best',\n", + " 'talk',\n", + " 'bore',\n", + " 'eat',\n", + " 'befor',\n", + " 'someth',\n", + " 'wan',\n", + " 'read',\n", + " 'ever',\n", + " 'listen',\n", + " 'everyon',\n", + " 'leav',\n", + " 'sure',\n", + " 'done',\n", + " 'made',\n", + " 'ly',\n", + " 'man',\n", + " 'anoth',\n", + " 'tire',\n", + " 'call',\n", + " 'cool',\n", + " 'went',\n", + " 'hurt',\n", + " 'help',\n", + " 'song',\n", + " 'yet',\n", + " 'suck',\n", + " 'mayb',\n", + " 'lot',\n", + " 'sound',\n", + " 'ur',\n", + " 'live',\n", + " 'away',\n", + " 'hous',\n", + " 'guess',\n", + " 'excit',\n", + " 'pretti',\n", + " 'finish',\n", + " 'enjoy',\n", + " 'someon',\n", + " 'summer',\n", + " 'thought',\n", + " 'left',\n", + " 'old',\n", + " 'omg',\n", + " 'babi',\n", + " 'tell',\n", + " 'parti',\n", + " 'amaz',\n", + " 'big',\n", + " 'two',\n", + " 'pic',\n", + " 'earli',\n", + " 'hear',\n", + " 'put',\n", + " 'becaus',\n", + " 'end',\n", + " 'readi',\n", + " 'th',\n", + " 'lost',\n", + " 'check',\n", + " 'ya',\n", + " 'doe',\n", + " 'mean',\n", + " 'wow',\n", + " 'hot',\n", + " 'kid',\n", + " 'yesterday',\n", + " 'god',\n", + " 'sunday',\n", + " 'updat',\n", + " 'saw',\n", + " 'glad',\n", + " 'run',\n", + " 'book',\n", + " 'stuff',\n", + " 'later',\n", + " 'shop',\n", + " 'game',\n", + " 'car',\n", + " 'music',\n", + " 'ugh',\n", + " 'sun',\n", + " 'sinc',\n", + " 'job',\n", + " 'stop',\n", + " 'ta',\n", + " 'iphon',\n", + " 'hard',\n", + " 'happen',\n", + " 'actual',\n", + " 'weather',\n", + " 'give',\n", + " 'stay',\n", + " 'must',\n", + " 'wonder',\n", + " 'www',\n", + " 'world',\n", + " 'mom',\n", + " 'monday',\n", + " 'also',\n", + " 'exam',\n", + " 'might',\n", + " 'beauti',\n", + " 'post',\n", + " 'seem',\n", + " 'video',\n", + " 'noth',\n", + " 'fuck',\n", + " 'ask',\n", + " 'damn',\n", + " 'birthday',\n", + " 'buy',\n", + " 'late',\n", + " 'found',\n", + " 'free',\n", + " 'around',\n", + " 'friday',\n", + " 'total',\n", + " 'cute',\n", + " 'meet',\n", + " 'drink',\n", + " 'dad',\n", + " 'sweet',\n", + " 'woke',\n", + " 'mani',\n", + " 'said',\n", + " 'cri',\n", + " 'poor',\n", + " 'till',\n", + " 'boy',\n", + " 'aww',\n", + " 'clean',\n", + " 'pictur',\n", + " 'plan',\n", + " 'luck',\n", + " 'dinner',\n", + " 'welcom',\n", + " 'anyth',\n", + " 'famili',\n", + " 'without',\n", + " 'busi',\n", + " 'name',\n", + " 'studi',\n", + " 'face',\n", + " 'place',\n", + " 'may',\n", + " 'least',\n", + " 'win',\n", + " 'lunch',\n", + " 'gt',\n", + " 'enough',\n", + " 'funni',\n", + " 'okay',\n", + " 'food',\n", + " 'cold',\n", + " 'gone',\n", + " 'tv',\n", + " 'walk',\n", + " 'eye',\n", + " 'tho',\n", + " 'hahaha',\n", + " 'die',\n", + " 'caus',\n", + " 'drive',\n", + " 'shit',\n", + " 'turn',\n", + " 'believ',\n", + " 'money',\n", + " 'wrong',\n", + " 'ill',\n", + " 'saturday',\n", + " 'heart',\n", + " 'sit',\n", + " 'class',\n", + " 'didnt',\n", + " 'xx',\n", + " 'real',\n", + " 'hair',\n", + " 'aw',\n", + " 'mine',\n", + " 'dream',\n", + " 'repli',\n", + " 'move',\n", + " 'chang',\n", + " 'forward',\n", + " 'rock',\n", + " 'month',\n", + " 'stupid',\n", + " 'beach',\n", + " 'open',\n", + " 'write',\n", + " 'kinda',\n", + " 'idea',\n", + " 'outsid',\n", + " 'tinyurl',\n", + " 'brother',\n", + " 'almost',\n", + " 'came',\n", + " 'wake',\n", + " 'seen',\n", + " 'word',\n", + " 'whole',\n", + " 'dog',\n", + " 'hang',\n", + " 'send',\n", + " 'plurk',\n", + " 'everyth',\n", + " 'person',\n", + " 'list',\n", + " 'part',\n", + " 'everi',\n", + " 'probabl',\n", + " 'anyon',\n", + " 'onc',\n", + " 'comput',\n", + " 'goodnight',\n", + " 'quit',\n", + " 'fan',\n", + " 'super',\n", + " 'full',\n", + " 'minut',\n", + " 'offic',\n", + " 'danc',\n", + " 'trip',\n", + " 'awww',\n", + " 'true',\n", + " 'pick',\n", + " 'far',\n", + " 'pain',\n", + " 'room',\n", + " 'facebook',\n", + " 'favorit',\n", + " 'coffe',\n", + " 'news',\n", + " 'took',\n", + " 'headach',\n", + " 'blog',\n", + " 'pas',\n", + " 'care',\n", + " 'site',\n", + " 'fail',\n", + " 'mother',\n", + " 'hello',\n", + " 'boo',\n", + " 'forgot',\n", + " 'dude',\n", + " 'bring',\n", + " 'el',\n", + " 'vote',\n", + " 'la',\n", + " 'hit',\n", + " 'hug',\n", + " 'learn',\n", + " 'heard',\n", + " 'lucki',\n", + " 'pm',\n", + " 'crazi',\n", + " 'anymor',\n", + " 'fall',\n", + " 'ticket',\n", + " 'ago',\n", + " 'concert',\n", + " 'st',\n", + " 'test',\n", + " 'hehe',\n", + " 'break',\n", + " 'sister',\n", + " 'abl',\n", + " 'text',\n", + " 'worri',\n", + " 'sooo',\n", + " 'serious',\n", + " 'lil',\n", + " 'anyway',\n", + " 'suppos',\n", + " 'asleep',\n", + " 'alon',\n", + " 'shower',\n", + " 'spend',\n", + " 'rest',\n", + " 'either',\n", + " 'pay',\n", + " 'interest',\n", + " 'awak',\n", + " 'wear',\n", + " 'cuz',\n", + " 'photo',\n", + " 'visit',\n", + " 'smile',\n", + " 'fine',\n", + " 'mind',\n", + " 'internet',\n", + " 'album',\n", + " 'instead',\n", + " 'rememb',\n", + " 'cat',\n", + " 'catch',\n", + " 'btw',\n", + " 'email',\n", + " 'ah',\n", + " 'set',\n", + " 'link',\n", + " 'onlin',\n", + " 'red',\n", + " 'add',\n", + " 'kind',\n", + " 'mileycyru',\n", + " 'train',\n", + " 'hell',\n", + " 'sign',\n", + " 'agre',\n", + " 'half',\n", + " 'close',\n", + " 'sunni',\n", + " 'bout',\n", + " 'june',\n", + " 'ice',\n", + " 'fix',\n", + " 'top',\n", + " 'ladi',\n", + " 'min',\n", + " 'season',\n", + " 'wont',\n", + " 'lmao',\n", + " 'bought',\n", + " 'hand',\n", + " 'ride',\n", + " 'dress',\n", + " 'annoy',\n", + " 'decid',\n", + " 'short',\n", + " 'blip',\n", + " 'ipod',\n", + " 'wed',\n", + " 'kill',\n", + " 'cut',\n", + " 'nap',\n", + " 'broke',\n", + " 'co',\n", + " 'second',\n", + " 'scare',\n", + " 'card',\n", + " 'join',\n", + " 'cook',\n", + " 'fm',\n", + " 'holiday',\n", + " 'togeth',\n", + " 'sometim',\n", + " 'youtub',\n", + " 'order',\n", + " 'soo',\n", + " 'offici',\n", + " 'weird',\n", + " 'band',\n", + " 'nite',\n", + " 'page',\n", + " 'high',\n", + " 'sore',\n", + " 'problem',\n", + " 'stori',\n", + " 'tommcfli',\n", + " 'save',\n", + " 'reason',\n", + " 'nd',\n", + " 'laptop',\n", + " 'side',\n", + " 'graduat',\n", + " 'yea',\n", + " 'crap',\n", + " 'told',\n", + " 'question',\n", + " 'goe',\n", + " 'forget',\n", + " 'throat',\n", + " 'congrat',\n", + " 'homework',\n", + " 'relax',\n", + " 'hmm',\n", + " 'shirt',\n", + " 'sim',\n", + " 'bye',\n", + " 'stuck',\n", + " 'tuesday',\n", + " 'til',\n", + " 'pack',\n", + " 'figur',\n", + " 'gym',\n", + " 'air',\n", + " 'breakfast',\n", + " 'broken',\n", + " 'past',\n", + " 'ive',\n", + " 'worst',\n", + " 'award',\n", + " 'citi',\n", + " 'park',\n", + " 'dear',\n", + " 'laugh',\n", + " 'sigh',\n", + " 'download',\n", + " 'perfect',\n", + " 'le',\n", + " 'definit',\n", + " 'afternoon',\n", + " 'celebr',\n", + " 'mad',\n", + " 'david',\n", + " 'sing',\n", + " 'answer',\n", + " 'slow',\n", + " 'differ',\n", + " 'fast',\n", + " 'cours',\n", + " 'hungri',\n", + " 'church',\n", + " 'lie',\n", + " 'fb',\n", + " 'luv',\n", + " 'chocol',\n", + " 'chanc',\n", + " 'goin',\n", + " 'record',\n", + " 'comment',\n", + " 'travel',\n", + " 'dm',\n", + " 'uk',\n", + " 'moon',\n", + " 'complet',\n", + " 'sleepi',\n", + " 'foot',\n", + " 'ddlovato',\n", + " 'mr',\n", + " 'tea',\n", + " 'manag',\n", + " 'fli',\n", + " 'easi',\n", + " 'store',\n", + " 'math',\n", + " 'depress',\n", + " 'jealou',\n", + " 'line',\n", + " 'cake',\n", + " 'tour',\n", + " 'freak',\n", + " 'dead',\n", + " 'understand',\n", + " 'worth',\n", + " 'team',\n", + " 'plu',\n", + " 'box',\n", + " 'longer',\n", + " 'account',\n", + " 'ppl',\n", + " 'water',\n", + " 'moment',\n", + " 'support',\n", + " 'fell',\n", + " 'mood',\n", + " 'sadli',\n", + " 'messag',\n", + " 'rather',\n", + " 'woman',\n", + " 'black',\n", + " 'upset',\n", + " 'yep',\n", + " 'version',\n", + " 'three',\n", + " 'episod',\n", + " 'radio',\n", + " 'unfortun',\n", + " 'except',\n", + " 'mum',\n", + " 'cream',\n", + " 'met',\n", + " 'chat',\n", + " 'begin',\n", + " 'shot',\n", + " 'mac',\n", + " 'cousin',\n", + " 'camp',\n", + " 'share',\n", + " 'app',\n", + " 'load',\n", + " 'appar',\n", + " 'shame',\n", + " 'leg',\n", + " 'beer',\n", + " 'jona',\n", + " 'ad',\n", + " 'colleg',\n", + " 'kick',\n", + " 'count',\n", + " 'coupl',\n", + " 'felt',\n", + " 'point',\n", + " 'nope',\n", + " 'bum',\n", + " 'ach',\n", + " 'due',\n", + " 'via',\n", + " 'stress',\n", + " 'hr',\n", + " 'flight',\n", + " 'father',\n", + " 'babe',\n", + " 'hubbi',\n", + " 'design',\n", + " 'ate',\n", + " 'blue',\n", + " 'bike',\n", + " 'bitch',\n", + " 'absolut',\n", + " 'town',\n", + " 'upload',\n", + " 'bless',\n", + " 'xd',\n", + " 'burn',\n", + " 'club',\n", + " 'lay',\n", + " 'special',\n", + " 'flu',\n", + " 'fantast',\n", + " 'pool',\n", + " 'warm',\n", + " 'bet',\n", + " 'son',\n", + " 'lazi',\n", + " 'soooo',\n", + " 'ear',\n", + " 'followfriday',\n", + " 'power',\n", + " 'forev',\n", + " 'smell',\n", + " 'lone',\n", + " 'date',\n", + " 'twit',\n", + " 'vacat',\n", + " 'wit',\n", + " 'john',\n", + " 'miley',\n", + " 'swim',\n", + " 'sort',\n", + " 'cheer',\n", + " 'wors',\n", + " 'hotel',\n", + " 'bodi',\n", + " 'crash',\n", + " 'usual',\n", + " 'practic',\n", + " 'wtf',\n", + " 'knew',\n", + " 'alright',\n", + " 'type',\n", + " 'yo',\n", + " 'horribl',\n", + " 'plane',\n", + " 'road',\n", + " 'green',\n", + " 'meant',\n", + " 'yummi',\n", + " 'realiz',\n", + " 'myspac',\n", + " 'english',\n", + " 'ju',\n", + " 'shoot',\n", + " 'connect',\n", + " 'websit',\n", + " 'cover',\n", + " 'dvd',\n", + " 'product',\n", + " 'juli',\n", + " 'safe',\n", + " 'ahhh',\n", + " 'note',\n", + " 'thx',\n", + " 'beat',\n", + " 'pull',\n", + " 'woo',\n", + " 'parent',\n", + " 'sent',\n", + " 'present',\n", + " 'art',\n", + " 'prob',\n", + " 'cd',\n", + " 'appl',\n", + " 'film',\n", + " 'bbq',\n", + " 'fact',\n", + " 'stomach',\n", + " 'drop',\n", + " 'chees',\n", + " 'yup',\n", + " 'sunshin',\n", + " 'child',\n", + " 'tom',\n", + " 'sat',\n", + " 'remind',\n", + " 'histori',\n", + " 'sold',\n", + " 'paper',\n", + " 'bird',\n", + " 'addict',\n", + " 'tear',\n", + " 'doctor',\n", + " 'doesnt',\n", + " 'ahh',\n", + " 'step',\n", + " 'inspir',\n", + " 'star',\n", + " 'fair',\n", + " 'disappoint',\n", + " 'terribl',\n", + " 'mtv',\n", + " 'da',\n", + " 'spent',\n", + " 'white',\n", + " 'especi',\n", + " 'pink',\n", + " 'garden',\n", + " 'lose',\n", + " 'interview',\n", + " 'laker',\n", + " 'idk',\n", + " 'shoe',\n", + " 'allow',\n", + " 'alot',\n", + " 'tummi',\n", + " 'appreci',\n", + " 'block',\n", + " 'goodby',\n", + " 'watchin',\n", + " 'dark',\n", + " 'hahah',\n", + " 'bar',\n", + " 'door',\n", + " 'bb',\n", + " 'gd',\n", + " 'london',\n", + " 'cancel',\n", + " 'project',\n", + " 'surpris',\n", + " 'promis',\n", + " 'stand',\n", + " 'yr',\n", + " 'angel',\n", + " 'cooki',\n", + " 'chill',\n", + " 'land',\n", + " 'dure',\n", + " 'event',\n", + " 'havent',\n", + " 'storm',\n", + " 'background',\n", + " 'airport',\n", + " 'si',\n", + " 'edit',\n", + " 'mention',\n", + " 'number',\n", + " 'futur',\n", + " 'becom',\n", + " 'possibl',\n", + " 'revis',\n", + " 'ouch',\n", + " 'extra',\n", + " 'slept',\n", + " 'light',\n", + " 'window',\n", + " 'age',\n", + " 'boyfriend',\n", + " 'act',\n", + " 'small',\n", + " 'bu',\n", + " 'gave',\n", + " 'demi',\n", + " 'current',\n", + " 'mommi',\n", + " 'mess',\n", + " 'uh',\n", + " 'channel',\n", + " 'search',\n", + " 'sale',\n", + " 'price',\n", + " 'fri',\n", + " 'touch',\n", + " 'ball',\n", + " 'chicken',\n", + " 'finger',\n", + " 'huh',\n", + " 'fight',\n", + " 'mail',\n", + " 'front',\n", + " 'web',\n", + " 'voic',\n", + " 'piss',\n", + " 'daddi',\n", + " 'eh',\n", + " 'chip',\n", + " 'recommend',\n", + " 'wine',\n", + " 'tr',\n", + " 'puppi',\n", + " 'street',\n", + " 'scari',\n", + " 'arm',\n", + " 'hmmm',\n", + " 'expect',\n", + " 'twilight',\n", + " 'kitti',\n", + " 'round',\n", + " 'bug',\n", + " 'pizza',\n", + " 'itun',\n", + " 'king',\n", + " 'huge',\n", + " 'coz',\n", + " 'color',\n", + " 'kati',\n", + " 'confus',\n", + " 'speak',\n", + " 'tip',\n", + " 'xxx',\n", + " 'jonasbroth',\n", + " 'bag',\n", + " 'nick',\n", + " 'xo',\n", + " 'ny',\n", + " 'kate',\n", + " 'caught',\n", + " 'hospit',\n", + " 'deserv',\n", + " 'gay',\n", + " 'xoxo',\n", + " 'nobodi',\n", + " 'bday',\n", + " 'wast',\n", + " 'servic',\n", + " 'invit',\n", + " 'id',\n", + " 'hill',\n", + " 'offer',\n", + " 'low',\n", + " 'along',\n", + " 'thursday',\n", + " 'kiss',\n", + " 'jb',\n", + " 'exhaust',\n", + " 'drunk',\n", + " 'impress',\n", + " 'rip',\n", + " 'code',\n", + " 'paint',\n", + " 'em',\n", + " 'matter',\n", + " 'dunno',\n", + " 'jon',\n", + " 'mile',\n", + " 'wednesday',\n", + " 'blood',\n", + " 'ahead',\n", + " 'googl',\n", + " 'avail',\n", + " 'england',\n", + " 'case',\n", + " 'exactli',\n", + " 'hun',\n", + " 'wash',\n", + " 'magic',\n", + " 'delet',\n", + " 'tast',\n", + " 'wet',\n", + " 'notic',\n", + " 'proud',\n", + " 'jame',\n", + " 'fire',\n", + " 'continu',\n", + " 'non',\n", + " 'countri',\n", + " 'vip',\n", + " 'taylor',\n", + " 'earlier',\n", + " 'chri',\n", + " 'kno',\n", + " 'ruin',\n", + " 'direct',\n", + " 'arriv',\n", + " 'heat',\n", + " 'prepar',\n", + " 'fit',\n", + " 'pray',\n", + " 'doubt',\n", + " 'hold',\n", + " 'race',\n", + " 'everybodi',\n", + " 'fav',\n", + " 'daughter',\n", + " 'track',\n", + " 'honey',\n", + " 'troubl',\n", + " 'taken',\n", + " 'brain',\n", + " 'tune',\n", + " 'consid',\n", + " 'bro',\n", + " 'net',\n", + " 'joe',\n", + " 'husband',\n", + " 'sky',\n", + " 'shut',\n", + " 'tonit',\n", + " 'normal',\n", + " 'gorgeou',\n", + " 'apart',\n", + " 'ff',\n", + " 'choic',\n", + " 'etc',\n", + " 'jordanknight',\n", + " 'somewher',\n", + " 'behind',\n", + " 'lesson',\n", + " 'fill',\n", + " 'camera',\n", + " 'nail',\n", + " 'nah',\n", + " 'cross',\n", + " 'couldnt',\n", + " 'ohh',\n", + " 'delici',\n", + " 'instal',\n", + " 'secret',\n", + " 'screen',\n", + " 'marri',\n", + " 'forc',\n", + " 'hangov',\n", + " 'issu',\n", + " 'paul',\n", + " 'darn',\n", + " 'profil',\n", + " 'group',\n", + " 'suggest',\n", + " 'aint',\n", + " 'memori',\n", + " 'tweep',\n", + " 'spam',\n", + " 'winter',\n", + " 'smoke',\n", + " 'sam',\n", + " 'compani',\n", + " 'ryan',\n", + " 'self',\n", + " 'hannah',\n", + " 'bath',\n", + " 'prefer',\n", + " 'sweeti',\n", + " 'death',\n", + " 'blow',\n", + " 'shall',\n", + " 'screw',\n", + " 'often',\n", + " 'dat',\n", + " 'system',\n", + " 'disney',\n", + " 'state',\n", + " 'sa',\n", + " 'result',\n", + " 'peac',\n", + " 'bunch',\n", + " 'creat',\n", + " 'american',\n", + " 'sushi',\n", + " 'adam',\n", + " 'nyc',\n", + " 'ooh',\n", + " 'crappi',\n", + " 'mo',\n", + " 'experi',\n", + " 'dang',\n", + " ...]" + ] + }, + "execution_count": 81, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "selected_words = [word for word, frequency in top_words]\n", + "selected_words" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Building Features\n", + "\n", + "Now let's build the features. Using the top 5,000 words, create a 2-dimensional matrix to record whether each of those words is contained in each document (tweet). Then you also have an output column to indicate whether the sentiment in each tweet is positive. For example, assuming your bag of words has 5 items (`['one', 'two', 'three', 'four', 'five']`) out of 4 documents (`['A', 'B', 'C', 'D']`), your feature set is essentially:\n", + "\n", + "| Doc | one | two | three | four | five | is_positive |\n", + "|---|---|---|---|---|---|---|\n", + "| A | True | False | False | True | False | True |\n", + "| B | False | False | False | True | True | False |\n", + "| C | False | True | False | False | False | True |\n", + "| D | True | False | False | False | True | False|\n", + "\n", + "However, because the `nltk.NaiveBayesClassifier.train` class we will use in the next step does not work with Pandas dataframe, the structure of your feature set should be converted to the Python list looking like below:\n", + "\n", + "```python\n", + "[\n", + "\t({\n", + "\t\t'one': True,\n", + "\t\t'two': False,\n", + "\t\t'three': False,\n", + "\t\t'four': True,\n", + "\t\t'five': False\n", + "\t}, True),\n", + "\t({\n", + "\t\t'one': False,\n", + "\t\t'two': False,\n", + "\t\t'three': False,\n", + "\t\t'four': True,\n", + "\t\t'five': True\n", + "\t}, False),\n", + "\t({\n", + "\t\t'one': False,\n", + "\t\t'two': True,\n", + "\t\t'three': False,\n", + "\t\t'four': False,\n", + "\t\t'five': False\n", + "\t}, True),\n", + "\t({\n", + "\t\t'one': True,\n", + "\t\t'two': False,\n", + "\t\t'three': False,\n", + "\t\t'four': False,\n", + "\t\t'five': True\n", + "\t}, False)\n", + "]\n", + "```\n", + "\n", + "To help you in this step, watch the [following video](https://www.youtube.com/watch?v=-vVskDsHcVc) to learn how to build the feature set with Python and NLTK. The source code in this video can be found [here](https://pythonprogramming.net/words-as-features-nltk-tutorial/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Building Features](building-features.jpg)](https://www.youtube.com/watch?v=-vVskDsHcVc)" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "({'go': True,\n", + " 'get': False,\n", + " 'wa': False,\n", + " 'day': False,\n", + " 'thi': False,\n", + " 'good': False,\n", + " 'work': False,\n", + " 'like': False,\n", + " 'love': False,\n", + " 'got': False,\n", + " 'quot': False,\n", + " 'http': True,\n", + " 'today': False,\n", + " 'one': False,\n", + " 'time': False,\n", + " 'lol': False,\n", + " 'thank': False,\n", + " 'back': False,\n", + " 'miss': False,\n", + " 'want': False,\n", + " 'know': False,\n", + " 'think': False,\n", + " 'realli': False,\n", + " 'feel': False,\n", + " 'com': True,\n", + " 'im': False,\n", + " 'amp': True,\n", + " 'see': False,\n", + " 'night': False,\n", + " 'well': False,\n", + " 'need': False,\n", + " 'na': False,\n", + " 'oh': False,\n", + " 'still': False,\n", + " 'new': False,\n", + " 'come': False,\n", + " 'watch': False,\n", + " 'hope': False,\n", + " 'home': False,\n", + " 'look': False,\n", + " 'make': False,\n", + " 'ha': False,\n", + " 'twitter': False,\n", + " 'last': False,\n", + " 'much': False,\n", + " 'morn': False,\n", + " 'tomorrow': False,\n", + " 'great': False,\n", + " 'wait': False,\n", + " 'sleep': False,\n", + " 'haha': False,\n", + " 'fun': False,\n", + " 'wish': False,\n", + " 'whi': False,\n", + " 'sad': False,\n", + " 'week': False,\n", + " 'follow': False,\n", + " 'right': False,\n", + " 'tri': False,\n", + " 'thing': False,\n", + " 'friend': False,\n", + " 'would': False,\n", + " 'gon': False,\n", + " 'onli': False,\n", + " 'happi': False,\n", + " 'sorri': False,\n", + " 'say': False,\n", + " 'way': False,\n", + " 'tonight': False,\n", + " 'bit': False,\n", + " 'bad': False,\n", + " 'nice': False,\n", + " 'tweet': False,\n", + " 'though': False,\n", + " 'hi': False,\n", + " 'veri': False,\n", + " 'even': False,\n", + " 'take': False,\n", + " 'start': False,\n", + " 'better': False,\n", + " 'peopl': False,\n", + " 'bed': False,\n", + " 'yeah': False,\n", + " 'could': False,\n", + " 'hour': False,\n", + " 'guy': False,\n", + " 'school': False,\n", + " 'play': False,\n", + " 'use': False,\n", + " 'show': False,\n", + " 'hey': False,\n", + " 'twitpic': False,\n", + " 'rain': False,\n", + " 'cant': False,\n", + " 'awesom': False,\n", + " 'ye': False,\n", + " 'next': False,\n", + " 'hate': False,\n", + " 'soon': False,\n", + " 'weekend': False,\n", + " 'find': False,\n", + " 'lt': False,\n", + " 'never': False,\n", + " 'let': False,\n", + " 'sick': False,\n", + " 'year': False,\n", + " 'final': False,\n", + " 'plea': True,\n", + " 'yay': False,\n", + " 'littl': False,\n", + " 'ani': False,\n", + " 'long': False,\n", + " 'head': False,\n", + " 'first': False,\n", + " 'life': False,\n", + " 'keep': False,\n", + " 'girl': False,\n", + " 'alway': False,\n", + " 'movi': False,\n", + " 'ok': False,\n", + " 'alreadi': False,\n", + " 'dont': False,\n", + " 'phone': False,\n", + " 'best': False,\n", + " 'talk': False,\n", + " 'bore': False,\n", + " 'eat': False,\n", + " 'befor': False,\n", + " 'someth': False,\n", + " 'wan': False,\n", + " 'read': False,\n", + " 'ever': False,\n", + " 'listen': False,\n", + " 'everyon': False,\n", + " 'leav': False,\n", + " 'sure': False,\n", + " 'done': False,\n", + " 'made': False,\n", + " 'ly': False,\n", + " 'man': False,\n", + " 'anoth': False,\n", + " 'tire': False,\n", + " 'call': False,\n", + " 'cool': False,\n", + " 'went': False,\n", + " 'hurt': False,\n", + " 'help': False,\n", + " 'song': False,\n", + " 'yet': False,\n", + " 'suck': False,\n", + " 'mayb': False,\n", + " 'lot': False,\n", + " 'sound': False,\n", + " 'ur': False,\n", + " 'live': False,\n", + " 'away': False,\n", + " 'hous': False,\n", + " 'guess': False,\n", + " 'excit': False,\n", + " 'pretti': False,\n", + " 'finish': False,\n", + " 'enjoy': False,\n", + " 'someon': False,\n", + " 'summer': False,\n", + " 'thought': False,\n", + " 'left': False,\n", + " 'old': False,\n", + " 'omg': False,\n", + " 'babi': False,\n", + " 'tell': False,\n", + " 'parti': False,\n", + " 'amaz': False,\n", + " 'big': False,\n", + " 'two': False,\n", + " 'pic': False,\n", + " 'earli': False,\n", + " 'hear': False,\n", + " 'put': False,\n", + " 'becaus': False,\n", + " 'end': False,\n", + " 'readi': False,\n", + " 'th': False,\n", + " 'lost': False,\n", + " 'check': False,\n", + " 'ya': False,\n", + " 'doe': False,\n", + " 'mean': False,\n", + " 'wow': False,\n", + " 'hot': False,\n", + " 'kid': False,\n", + " 'yesterday': False,\n", + " 'god': False,\n", + " 'sunday': False,\n", + " 'updat': False,\n", + " 'saw': False,\n", + " 'glad': False,\n", + " 'run': False,\n", + " 'book': False,\n", + " 'stuff': False,\n", + " 'later': False,\n", + " 'shop': False,\n", + " 'game': False,\n", + " 'car': False,\n", + " 'music': False,\n", + " 'ugh': False,\n", + " 'sun': False,\n", + " 'sinc': False,\n", + " 'job': False,\n", + " 'stop': False,\n", + " 'ta': False,\n", + " 'iphon': False,\n", + " 'hard': False,\n", + " 'happen': False,\n", + " 'actual': False,\n", + " 'weather': False,\n", + " 'give': False,\n", + " 'stay': False,\n", + " 'must': False,\n", + " 'wonder': False,\n", + " 'www': False,\n", + " 'world': False,\n", + " 'mom': False,\n", + " 'monday': False,\n", + " 'also': False,\n", + " 'exam': False,\n", + " 'might': False,\n", + " 'beauti': False,\n", + " 'post': False,\n", + " 'seem': False,\n", + " 'video': False,\n", + " 'noth': False,\n", + " 'fuck': False,\n", + " 'ask': False,\n", + " 'damn': False,\n", + " 'birthday': False,\n", + " 'buy': False,\n", + " 'late': False,\n", + " 'found': False,\n", + " 'free': False,\n", + " 'around': False,\n", + " 'friday': False,\n", + " 'total': False,\n", + " 'cute': False,\n", + " 'meet': False,\n", + " 'drink': False,\n", + " 'dad': False,\n", + " 'sweet': False,\n", + " 'woke': False,\n", + " 'mani': False,\n", + " 'said': False,\n", + " 'cri': False,\n", + " 'poor': False,\n", + " 'till': False,\n", + " 'boy': False,\n", + " 'aww': False,\n", + " 'clean': False,\n", + " 'pictur': False,\n", + " 'plan': False,\n", + " 'luck': False,\n", + " 'dinner': False,\n", + " 'welcom': False,\n", + " 'anyth': False,\n", + " 'famili': False,\n", + " 'without': False,\n", + " 'busi': False,\n", + " 'name': False,\n", + " 'studi': False,\n", + " 'face': False,\n", + " 'place': False,\n", + " 'may': False,\n", + " 'least': False,\n", + " 'win': True,\n", + " 'lunch': False,\n", + " 'gt': False,\n", + " 'enough': False,\n", + " 'funni': False,\n", + " 'okay': False,\n", + " 'food': False,\n", + " 'cold': False,\n", + " 'gone': False,\n", + " 'tv': False,\n", + " 'walk': False,\n", + " 'eye': False,\n", + " 'tho': False,\n", + " 'hahaha': False,\n", + " 'die': False,\n", + " 'caus': False,\n", + " 'drive': False,\n", + " 'shit': False,\n", + " 'turn': False,\n", + " 'believ': False,\n", + " 'money': False,\n", + " 'wrong': False,\n", + " 'ill': False,\n", + " 'saturday': False,\n", + " 'heart': False,\n", + " 'sit': False,\n", + " 'class': False,\n", + " 'didnt': False,\n", + " 'xx': False,\n", + " 'real': False,\n", + " 'hair': False,\n", + " 'aw': False,\n", + " 'mine': False,\n", + " 'dream': False,\n", + " 'repli': False,\n", + " 'move': False,\n", + " 'chang': False,\n", + " 'forward': False,\n", + " 'rock': False,\n", + " 'month': False,\n", + " 'stupid': False,\n", + " 'beach': False,\n", + " 'open': False,\n", + " 'write': False,\n", + " 'kinda': False,\n", + " 'idea': False,\n", + " 'outsid': False,\n", + " 'tinyurl': False,\n", + " 'brother': False,\n", + " 'almost': False,\n", + " 'came': False,\n", + " 'wake': False,\n", + " 'seen': False,\n", + " 'word': False,\n", + " 'whole': False,\n", + " 'dog': False,\n", + " 'hang': False,\n", + " 'send': False,\n", + " 'plurk': False,\n", + " 'everyth': False,\n", + " 'person': False,\n", + " 'list': False,\n", + " 'part': False,\n", + " 'everi': False,\n", + " 'probabl': False,\n", + " 'anyon': False,\n", + " 'onc': False,\n", + " 'comput': False,\n", + " 'goodnight': False,\n", + " 'quit': False,\n", + " 'fan': False,\n", + " 'super': False,\n", + " 'full': False,\n", + " 'minut': False,\n", + " 'offic': False,\n", + " 'danc': False,\n", + " 'trip': False,\n", + " 'awww': False,\n", + " 'true': False,\n", + " 'pick': False,\n", + " 'far': False,\n", + " 'pain': False,\n", + " 'room': False,\n", + " 'facebook': False,\n", + " 'favorit': False,\n", + " 'coffe': False,\n", + " 'news': False,\n", + " 'took': False,\n", + " 'headach': False,\n", + " 'blog': False,\n", + " 'pas': False,\n", + " 'care': False,\n", + " 'site': False,\n", + " 'fail': False,\n", + " 'mother': False,\n", + " 'hello': False,\n", + " 'boo': False,\n", + " 'forgot': False,\n", + " 'dude': False,\n", + " 'bring': False,\n", + " 'el': False,\n", + " 'vote': True,\n", + " 'la': False,\n", + " 'hit': False,\n", + " 'hug': False,\n", + " 'learn': False,\n", + " 'heard': False,\n", + " 'lucki': False,\n", + " 'pm': False,\n", + " 'crazi': False,\n", + " 'anymor': False,\n", + " 'fall': False,\n", + " 'ticket': False,\n", + " 'ago': False,\n", + " 'concert': False,\n", + " 'st': False,\n", + " 'test': False,\n", + " 'hehe': False,\n", + " 'break': False,\n", + " 'sister': False,\n", + " 'abl': False,\n", + " 'text': False,\n", + " 'worri': False,\n", + " 'sooo': False,\n", + " 'serious': False,\n", + " 'lil': False,\n", + " 'anyway': False,\n", + " 'suppos': False,\n", + " 'asleep': False,\n", + " 'alon': False,\n", + " 'shower': False,\n", + " 'spend': False,\n", + " 'rest': False,\n", + " 'either': False,\n", + " 'pay': False,\n", + " 'interest': False,\n", + " 'awak': False,\n", + " 'wear': False,\n", + " 'cuz': False,\n", + " 'photo': False,\n", + " 'visit': False,\n", + " 'smile': False,\n", + " 'fine': False,\n", + " 'mind': False,\n", + " 'internet': False,\n", + " 'album': False,\n", + " 'instead': False,\n", + " 'rememb': False,\n", + " 'cat': False,\n", + " 'catch': False,\n", + " 'btw': False,\n", + " 'email': False,\n", + " 'ah': False,\n", + " 'set': False,\n", + " 'link': False,\n", + " 'onlin': False,\n", + " 'red': False,\n", + " 'add': False,\n", + " 'kind': False,\n", + " 'mileycyru': False,\n", + " 'train': False,\n", + " 'hell': False,\n", + " 'sign': False,\n", + " 'agre': False,\n", + " 'half': False,\n", + " 'close': False,\n", + " 'sunni': False,\n", + " 'bout': False,\n", + " 'june': False,\n", + " 'ice': False,\n", + " 'fix': False,\n", + " 'top': False,\n", + " 'ladi': False,\n", + " 'min': False,\n", + " 'season': False,\n", + " 'wont': False,\n", + " 'lmao': False,\n", + " 'bought': False,\n", + " 'hand': False,\n", + " 'ride': False,\n", + " 'dress': False,\n", + " 'annoy': False,\n", + " 'decid': False,\n", + " 'short': False,\n", + " 'blip': False,\n", + " 'ipod': False,\n", + " 'wed': False,\n", + " 'kill': False,\n", + " 'cut': False,\n", + " 'nap': False,\n", + " 'broke': False,\n", + " 'co': False,\n", + " 'second': False,\n", + " 'scare': False,\n", + " 'card': False,\n", + " 'join': False,\n", + " 'cook': False,\n", + " 'fm': False,\n", + " 'holiday': False,\n", + " 'togeth': False,\n", + " 'sometim': False,\n", + " 'youtub': False,\n", + " 'order': False,\n", + " 'soo': False,\n", + " 'offici': False,\n", + " 'weird': False,\n", + " 'band': False,\n", + " 'nite': False,\n", + " 'page': False,\n", + " 'high': False,\n", + " 'sore': False,\n", + " 'problem': False,\n", + " 'stori': False,\n", + " 'tommcfli': False,\n", + " 'save': False,\n", + " 'reason': False,\n", + " 'nd': False,\n", + " 'laptop': False,\n", + " 'side': False,\n", + " 'graduat': False,\n", + " 'yea': False,\n", + " 'crap': False,\n", + " 'told': False,\n", + " 'question': False,\n", + " 'goe': False,\n", + " 'forget': False,\n", + " 'throat': False,\n", + " 'congrat': False,\n", + " 'homework': False,\n", + " 'relax': False,\n", + " 'hmm': False,\n", + " 'shirt': False,\n", + " 'sim': False,\n", + " 'bye': False,\n", + " 'stuck': False,\n", + " 'tuesday': False,\n", + " 'til': False,\n", + " 'pack': False,\n", + " 'figur': False,\n", + " 'gym': False,\n", + " 'air': False,\n", + " 'breakfast': False,\n", + " 'broken': False,\n", + " 'past': False,\n", + " 'ive': False,\n", + " 'worst': False,\n", + " 'award': False,\n", + " 'citi': False,\n", + " 'park': False,\n", + " 'dear': False,\n", + " 'laugh': False,\n", + " 'sigh': False,\n", + " 'download': False,\n", + " 'perfect': False,\n", + " 'le': False,\n", + " 'definit': False,\n", + " 'afternoon': False,\n", + " 'celebr': False,\n", + " 'mad': False,\n", + " 'david': False,\n", + " 'sing': False,\n", + " 'answer': False,\n", + " 'slow': False,\n", + " 'differ': False,\n", + " 'fast': False,\n", + " 'cours': False,\n", + " 'hungri': False,\n", + " 'church': False,\n", + " 'lie': False,\n", + " 'fb': False,\n", + " 'luv': False,\n", + " 'chocol': False,\n", + " 'chanc': False,\n", + " 'goin': False,\n", + " 'record': False,\n", + " 'comment': False,\n", + " 'travel': False,\n", + " 'dm': False,\n", + " 'uk': False,\n", + " 'moon': False,\n", + " 'complet': False,\n", + " 'sleepi': False,\n", + " 'foot': False,\n", + " 'ddlovato': False,\n", + " 'mr': False,\n", + " 'tea': False,\n", + " 'manag': False,\n", + " 'fli': False,\n", + " 'easi': False,\n", + " 'store': False,\n", + " 'math': False,\n", + " 'depress': False,\n", + " 'jealou': False,\n", + " 'line': False,\n", + " 'cake': False,\n", + " 'tour': False,\n", + " 'freak': False,\n", + " 'dead': False,\n", + " 'understand': False,\n", + " 'worth': False,\n", + " 'team': False,\n", + " 'plu': False,\n", + " 'box': False,\n", + " 'longer': False,\n", + " 'account': False,\n", + " 'ppl': False,\n", + " 'water': False,\n", + " 'moment': False,\n", + " 'support': False,\n", + " 'fell': False,\n", + " 'mood': False,\n", + " 'sadli': False,\n", + " 'messag': False,\n", + " 'rather': False,\n", + " 'woman': False,\n", + " 'black': False,\n", + " 'upset': False,\n", + " 'yep': False,\n", + " 'version': False,\n", + " 'three': False,\n", + " 'episod': False,\n", + " 'radio': False,\n", + " 'unfortun': False,\n", + " 'except': False,\n", + " 'mum': False,\n", + " 'cream': False,\n", + " 'met': False,\n", + " 'chat': False,\n", + " 'begin': False,\n", + " 'shot': False,\n", + " 'mac': False,\n", + " 'cousin': False,\n", + " 'camp': False,\n", + " 'share': False,\n", + " 'app': False,\n", + " 'load': False,\n", + " 'appar': False,\n", + " 'shame': False,\n", + " 'leg': False,\n", + " 'beer': False,\n", + " 'jona': False,\n", + " 'ad': False,\n", + " 'colleg': False,\n", + " 'kick': False,\n", + " 'count': False,\n", + " 'coupl': False,\n", + " 'felt': False,\n", + " 'point': False,\n", + " 'nope': False,\n", + " 'bum': False,\n", + " 'ach': False,\n", + " 'due': False,\n", + " 'via': False,\n", + " 'stress': False,\n", + " 'hr': False,\n", + " 'flight': False,\n", + " 'father': False,\n", + " 'babe': False,\n", + " 'hubbi': False,\n", + " 'design': False,\n", + " 'ate': False,\n", + " 'blue': False,\n", + " 'bike': False,\n", + " 'bitch': False,\n", + " 'absolut': False,\n", + " 'town': False,\n", + " 'upload': False,\n", + " 'bless': False,\n", + " 'xd': False,\n", + " 'burn': False,\n", + " 'club': False,\n", + " 'lay': False,\n", + " 'special': False,\n", + " 'flu': False,\n", + " 'fantast': False,\n", + " 'pool': False,\n", + " 'warm': False,\n", + " 'bet': False,\n", + " 'son': False,\n", + " 'lazi': False,\n", + " 'soooo': False,\n", + " 'ear': False,\n", + " 'followfriday': False,\n", + " 'power': False,\n", + " 'forev': False,\n", + " 'smell': False,\n", + " 'lone': False,\n", + " 'date': False,\n", + " 'twit': False,\n", + " 'vacat': False,\n", + " 'wit': False,\n", + " 'john': False,\n", + " 'miley': False,\n", + " 'swim': False,\n", + " 'sort': False,\n", + " 'cheer': False,\n", + " 'wors': False,\n", + " 'hotel': False,\n", + " 'bodi': False,\n", + " 'crash': False,\n", + " 'usual': False,\n", + " 'practic': False,\n", + " 'wtf': False,\n", + " 'knew': False,\n", + " 'alright': False,\n", + " 'type': False,\n", + " 'yo': False,\n", + " 'horribl': False,\n", + " 'plane': False,\n", + " 'road': False,\n", + " 'green': False,\n", + " 'meant': False,\n", + " 'yummi': False,\n", + " 'realiz': False,\n", + " 'myspac': False,\n", + " 'english': False,\n", + " 'ju': False,\n", + " 'shoot': False,\n", + " 'connect': False,\n", + " 'websit': False,\n", + " 'cover': False,\n", + " 'dvd': False,\n", + " 'product': False,\n", + " 'juli': False,\n", + " 'safe': False,\n", + " 'ahhh': False,\n", + " 'note': False,\n", + " 'thx': False,\n", + " 'beat': False,\n", + " 'pull': False,\n", + " 'woo': False,\n", + " 'parent': False,\n", + " 'sent': False,\n", + " 'present': False,\n", + " 'art': False,\n", + " 'prob': False,\n", + " 'cd': False,\n", + " 'appl': False,\n", + " 'film': False,\n", + " 'bbq': False,\n", + " 'fact': False,\n", + " 'stomach': False,\n", + " 'drop': False,\n", + " 'chees': False,\n", + " 'yup': False,\n", + " 'sunshin': False,\n", + " 'child': False,\n", + " 'tom': False,\n", + " 'sat': False,\n", + " 'remind': False,\n", + " 'histori': False,\n", + " 'sold': False,\n", + " 'paper': False,\n", + " 'bird': False,\n", + " 'addict': False,\n", + " 'tear': False,\n", + " 'doctor': False,\n", + " 'doesnt': False,\n", + " 'ahh': False,\n", + " 'step': False,\n", + " 'inspir': False,\n", + " 'star': False,\n", + " 'fair': False,\n", + " 'disappoint': False,\n", + " 'terribl': False,\n", + " 'mtv': False,\n", + " 'da': False,\n", + " 'spent': False,\n", + " 'white': False,\n", + " 'especi': False,\n", + " 'pink': False,\n", + " 'garden': False,\n", + " 'lose': False,\n", + " 'interview': False,\n", + " 'laker': False,\n", + " 'idk': False,\n", + " 'shoe': False,\n", + " 'allow': False,\n", + " 'alot': False,\n", + " 'tummi': False,\n", + " 'appreci': False,\n", + " 'block': False,\n", + " 'goodby': False,\n", + " 'watchin': False,\n", + " 'dark': False,\n", + " 'hahah': False,\n", + " 'bar': False,\n", + " 'door': False,\n", + " 'bb': False,\n", + " 'gd': False,\n", + " 'london': False,\n", + " 'cancel': False,\n", + " 'project': False,\n", + " 'surpris': False,\n", + " 'promis': False,\n", + " 'stand': False,\n", + " 'yr': False,\n", + " 'angel': False,\n", + " 'cooki': False,\n", + " 'chill': False,\n", + " 'land': False,\n", + " 'dure': False,\n", + " 'event': False,\n", + " 'havent': False,\n", + " 'storm': False,\n", + " 'background': False,\n", + " 'airport': False,\n", + " 'si': False,\n", + " 'edit': False,\n", + " 'mention': False,\n", + " 'number': False,\n", + " 'futur': False,\n", + " 'becom': False,\n", + " 'possibl': False,\n", + " 'revis': False,\n", + " 'ouch': False,\n", + " 'extra': False,\n", + " 'slept': False,\n", + " 'light': False,\n", + " 'window': False,\n", + " 'age': False,\n", + " 'boyfriend': False,\n", + " 'act': False,\n", + " 'small': False,\n", + " 'bu': False,\n", + " 'gave': False,\n", + " 'demi': False,\n", + " 'current': False,\n", + " 'mommi': False,\n", + " 'mess': False,\n", + " 'uh': False,\n", + " 'channel': False,\n", + " 'search': False,\n", + " 'sale': False,\n", + " 'price': False,\n", + " 'fri': False,\n", + " 'touch': False,\n", + " 'ball': False,\n", + " 'chicken': False,\n", + " 'finger': False,\n", + " 'huh': False,\n", + " 'fight': False,\n", + " 'mail': False,\n", + " 'front': False,\n", + " 'web': False,\n", + " 'voic': False,\n", + " 'piss': False,\n", + " 'daddi': False,\n", + " 'eh': False,\n", + " 'chip': False,\n", + " 'recommend': False,\n", + " 'wine': False,\n", + " 'tr': False,\n", + " 'puppi': False,\n", + " 'street': False,\n", + " 'scari': False,\n", + " 'arm': False,\n", + " 'hmmm': False,\n", + " 'expect': False,\n", + " 'twilight': False,\n", + " 'kitti': False,\n", + " 'round': False,\n", + " 'bug': False,\n", + " 'pizza': False,\n", + " 'itun': False,\n", + " 'king': False,\n", + " 'huge': False,\n", + " 'coz': False,\n", + " 'color': False,\n", + " 'kati': False,\n", + " 'confus': False,\n", + " 'speak': False,\n", + " 'tip': False,\n", + " 'xxx': False,\n", + " 'jonasbroth': False,\n", + " 'bag': False,\n", + " 'nick': False,\n", + " 'xo': False,\n", + " 'ny': False,\n", + " 'kate': False,\n", + " 'caught': False,\n", + " 'hospit': False,\n", + " 'deserv': False,\n", + " 'gay': False,\n", + " 'xoxo': False,\n", + " 'nobodi': False,\n", + " 'bday': False,\n", + " 'wast': False,\n", + " 'servic': False,\n", + " 'invit': False,\n", + " 'id': False,\n", + " 'hill': False,\n", + " 'offer': False,\n", + " 'low': False,\n", + " 'along': False,\n", + " 'thursday': False,\n", + " 'kiss': False,\n", + " 'jb': False,\n", + " 'exhaust': False,\n", + " 'drunk': False,\n", + " 'impress': False,\n", + " 'rip': False,\n", + " 'code': False,\n", + " 'paint': False,\n", + " 'em': False,\n", + " 'matter': False,\n", + " 'dunno': False,\n", + " 'jon': False,\n", + " 'mile': False,\n", + " 'wednesday': False,\n", + " 'blood': False,\n", + " 'ahead': False,\n", + " 'googl': False,\n", + " 'avail': False,\n", + " 'england': False,\n", + " 'case': False,\n", + " 'exactli': False,\n", + " 'hun': False,\n", + " 'wash': False,\n", + " 'magic': False,\n", + " 'delet': False,\n", + " 'tast': False,\n", + " 'wet': False,\n", + " 'notic': False,\n", + " 'proud': False,\n", + " 'jame': False,\n", + " 'fire': False,\n", + " 'continu': False,\n", + " 'non': False,\n", + " 'countri': False,\n", + " 'vip': False,\n", + " 'taylor': False,\n", + " 'earlier': False,\n", + " 'chri': False,\n", + " 'kno': False,\n", + " 'ruin': False,\n", + " 'direct': False,\n", + " 'arriv': False,\n", + " 'heat': False,\n", + " 'prepar': False,\n", + " 'fit': False,\n", + " 'pray': False,\n", + " 'doubt': False,\n", + " 'hold': False,\n", + " 'race': False,\n", + " 'everybodi': False,\n", + " 'fav': False,\n", + " 'daughter': False,\n", + " 'track': False,\n", + " 'honey': False,\n", + " 'troubl': False,\n", + " 'taken': False,\n", + " 'brain': False,\n", + " 'tune': False,\n", + " 'consid': False,\n", + " 'bro': False,\n", + " 'net': False,\n", + " 'joe': False,\n", + " 'husband': False,\n", + " 'sky': False,\n", + " 'shut': False,\n", + " 'tonit': False,\n", + " 'normal': False,\n", + " 'gorgeou': False,\n", + " 'apart': False,\n", + " 'ff': False,\n", + " 'choic': False,\n", + " 'etc': False,\n", + " 'jordanknight': False,\n", + " 'somewher': False,\n", + " 'behind': False,\n", + " 'lesson': False,\n", + " 'fill': False,\n", + " 'camera': False,\n", + " 'nail': False,\n", + " 'nah': False,\n", + " 'cross': False,\n", + " 'couldnt': False,\n", + " 'ohh': False,\n", + " 'delici': False,\n", + " 'instal': False,\n", + " 'secret': False,\n", + " 'screen': False,\n", + " 'marri': False,\n", + " 'forc': False,\n", + " 'hangov': False,\n", + " 'issu': False,\n", + " 'paul': False,\n", + " 'darn': False,\n", + " 'profil': False,\n", + " 'group': False,\n", + " 'suggest': False,\n", + " 'aint': False,\n", + " 'memori': False,\n", + " 'tweep': False,\n", + " 'spam': False,\n", + " 'winter': False,\n", + " 'smoke': False,\n", + " 'sam': False,\n", + " 'compani': False,\n", + " 'ryan': False,\n", + " 'self': False,\n", + " 'hannah': False,\n", + " 'bath': False,\n", + " 'prefer': False,\n", + " 'sweeti': False,\n", + " 'death': False,\n", + " 'blow': False,\n", + " 'shall': False,\n", + " 'screw': False,\n", + " 'often': False,\n", + " 'dat': False,\n", + " 'system': False,\n", + " 'disney': False,\n", + " 'state': False,\n", + " 'sa': False,\n", + " 'result': False,\n", + " 'peac': False,\n", + " 'bunch': False,\n", + " 'creat': False,\n", + " 'american': False,\n", + " 'sushi': False,\n", + " 'adam': False,\n", + " 'nyc': False,\n", + " 'ooh': False,\n", + " 'crappi': False,\n", + " 'mo': False,\n", + " 'experi': False,\n", + " 'dang': False,\n", + " ...},\n", + " 0)" + ] + }, + "execution_count": 82, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def create_feature_set(document, selected_words):\n", + " document_words = set(document)\n", + " features = {}\n", + " for word in selected_words:\n", + " features[word] = (word in document_words)\n", + " \n", + " return features\n", + "feature_set = []\n", + "for index, row in data.iterrows():\n", + " document = row['text_processed']\n", + " sentiment = row['target']\n", + " features = create_feature_set(document, selected_words)\n", + " feature_set.append((features, sentiment))\n", + "\n", + "for features, sentiment in feature_set:\n", + " data = data.sample(10000)\n", + "features, sentiment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# From here forward optional " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Building and Traininng Naive Bayes Model\n", + "\n", + " In this step you will split your feature set into a training and a test set. Then you will create a Bayes classifier instance using `nltk.NaiveBayesClassifier.train` ([example](https://www.nltk.org/book/ch06.html)) to train with the training dataset.\n", + "\n", + "After training the model, call `classifier.show_most_informative_features()` to inspect the most important features. The output will look like:\n", + "\n", + "```\n", + "Most Informative Features\n", + "\t snow = True False : True = 34.3 : 1.0\n", + "\t easter = True False : True = 26.2 : 1.0\n", + "\t headach = True False : True = 20.9 : 1.0\n", + "\t argh = True False : True = 17.6 : 1.0\n", + "\tunfortun = True False : True = 16.9 : 1.0\n", + "\t jona = True True : False = 16.2 : 1.0\n", + "\t ach = True False : True = 14.9 : 1.0\n", + "\t sad = True False : True = 13.0 : 1.0\n", + "\t parent = True False : True = 12.9 : 1.0\n", + "\t spring = True False : True = 12.7 : 1.0\n", + "```\n", + "\n", + "The [following video](https://www.youtube.com/watch?v=rISOsUaTrO4) will help you complete this step. The source code in this video can be found [here](https://pythonprogramming.net/naive-bayes-classifier-nltk-tutorial/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Building and Training NB](nb-model-building.jpg)](https://www.youtube.com/watch?v=rISOsUaTrO4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Testing Naive Bayes Model\n", + "\n", + "Now we'll test our classifier with the test dataset. This is done by calling `nltk.classify.accuracy(classifier, test)`.\n", + "\n", + "As mentioned in one of the tutorial videos, a Naive Bayes model is considered OK if your accuracy score is over 0.6. If your accuracy score is over 0.7, you've done a great job!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bonus Question 1: Improve Model Performance\n", + "\n", + "If you are still not exhausted so far and want to dig deeper, try to improve your classifier performance. There are many aspects you can dig into, for example:\n", + "\n", + "* Improve stemming and lemmatization. Inspect your bag of words and the most important features. Are there any words you should furuther remove from analysis? You can append these words to further remove to the stop words list.\n", + "\n", + "* Remember we only used the top 5,000 features to build model? Try using different numbers of top features. The bottom line is to use as few features as you can without compromising your model performance. The fewer features you select into your model, the faster your model is trained. Then you can use a larger sample size to improve your model accuracy score." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bonus Question 2: Machine Learning Pipeline\n", + "\n", + "In a new Jupyter Notebook, combine all your codes into a function (or a class). Your new function will execute the complete machine learning pipeline job by receiving the dataset location and output the classifier. This will allow you to use your function to predict the sentiment of any tweet in real time. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bonus Question 3: Apache Spark\n", + "\n", + "If you have completed the Apache Spark advanced topic lab, what you can do is to migrate your pipeline from local to a Databricks Notebook. Share your notebook with your instructor and classmates to show off your achievements!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/challenge-2.ipynb b/your-code/challenge-2.ipynb deleted file mode 100644 index 6b0e116..0000000 --- a/your-code/challenge-2.ipynb +++ /dev/null @@ -1,320 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Challenge 2: Sentiment Analysis\n", - "\n", - "In this challenge we will learn sentiment analysis and practice performing sentiment analysis on Twitter tweets. \n", - "\n", - "## Introduction\n", - "\n", - "Sentiment analysis is to *systematically identify, extract, quantify, and study affective states and subjective information* based on texts ([reference](https://en.wikipedia.org/wiki/Sentiment_analysis)). In simple words, it's to understand whether a person is happy or unhappy in producing the piece of text. Why we (or rather, companies) care about sentiment in texts? It's because by understanding the sentiments in texts, we will be able to know if our customers are happy or unhappy about our products and services. If they are unhappy, the subsequent action is to figure out what have caused the unhappiness and make improvements.\n", - "\n", - "Basic sentiment analysis only understands the *positive* or *negative* (sometimes *neutral* too) polarities of the sentiment. More advanced sentiment analysis will also consider dimensions such as agreement, subjectivity, confidence, irony, and so on. In this challenge we will conduct the basic positive vs negative sentiment analysis based on real Twitter tweets.\n", - "\n", - "NLTK comes with a [sentiment analysis package](https://www.nltk.org/api/nltk.sentiment.html). This package is great for dummies to perform sentiment analysis because it requires only the textual data to make predictions. For example:\n", - "\n", - "```python\n", - ">>> from nltk.sentiment.vader import SentimentIntensityAnalyzer\n", - ">>> txt = \"Ironhack is a Global Tech School ranked num 2 worldwide. 
", - "
", - "Our mission is to help people transform their careers and join a thriving community of tech professionals that love what they do.\"\n", - ">>> analyzer = SentimentIntensityAnalyzer()\n", - ">>> analyzer.polarity_scores(txt)\n", - "{'neg': 0.0, 'neu': 0.741, 'pos': 0.259, 'compound': 0.8442}\n", - "```\n", - "\n", - "In this challenge, however, you will not use NLTK's sentiment analysis package because in your Machine Learning training in the past 2 weeks you have learned how to make predictions more accurate than that. The [tweets data](https://www.kaggle.com/kazanova/sentiment140) we will be using today are already coded for the positive/negative sentiment. You will be able to use the Naïve Bayes classifier you learned in the lesson to predict the sentiment of tweets based on the labels." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Conducting Sentiment Analysis\n", - "\n", - "### Loading and Exploring Data\n", - "\n", - "The dataset we'll be using today is located on Kaggle (https://www.kaggle.com/kazanova/sentiment140). Once you have downloaded and imported the dataset, it you will need to define the columns names: df.columns = ['target','id','date','flag','user','text']\n", - "\n", - "*Notes:* \n", - "\n", - "* The dataset is huuuuge (1.6m tweets). When you develop your data analysis codes, you can sample a subset of the data (e.g. 20k records) so that you will save a lot of time when you test your codes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Prepare Textual Data for Sentiment Analysis\n", - "\n", - "Now, apply the functions you have written in Challenge 1 to your whole data set. These functions include:\n", - "\n", - "* `clean_up()`\n", - "\n", - "* `tokenize()`\n", - "\n", - "* `stem_and_lemmatize()`\n", - "\n", - "* `remove_stopwords()`\n", - "\n", - "Create a new column called `text_processed` in the dataframe to contain the processed data. At the end, your `text_processed` column should contain lists of word tokens that are cleaned up. Your data should look like below:\n", - "\n", - "![Processed Data](data-cleaning-results.png)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Creating Bag of Words\n", - "\n", - "The purpose of this step is to create a [bag of words](https://en.wikipedia.org/wiki/Bag-of-words_model) from the processed data. The bag of words contains all the unique words in your whole text body (a.k.a. *corpus*) with the number of occurrence of each word. It will allow you to understand which words are the most important features across the whole corpus.\n", - "\n", - "Also, you can imagine you will have a massive set of words. The less important words (i.e. those of very low number of occurrence) do not contribute much to the sentiment. Therefore, you only need to use the most important words to build your feature set in the next step. In our case, we will use the top 5,000 words with the highest frequency to build the features.\n", - "\n", - "In the cell below, combine all the words in `text_processed` and calculate the frequency distribution of all words. A convenient library to calculate the term frequency distribution is NLTK's `FreqDist` class ([documentation](https://www.nltk.org/api/nltk.html#module-nltk.probability)). Then select the top 5,000 words from the frequency distribution." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Building Features\n", - "\n", - "Now let's build the features. Using the top 5,000 words, create a 2-dimensional matrix to record whether each of those words is contained in each document (tweet). Then you also have an output column to indicate whether the sentiment in each tweet is positive. For example, assuming your bag of words has 5 items (`['one', 'two', 'three', 'four', 'five']`) out of 4 documents (`['A', 'B', 'C', 'D']`), your feature set is essentially:\n", - "\n", - "| Doc | one | two | three | four | five | is_positive |\n", - "|---|---|---|---|---|---|---|\n", - "| A | True | False | False | True | False | True |\n", - "| B | False | False | False | True | True | False |\n", - "| C | False | True | False | False | False | True |\n", - "| D | True | False | False | False | True | False|\n", - "\n", - "However, because the `nltk.NaiveBayesClassifier.train` class we will use in the next step does not work with Pandas dataframe, the structure of your feature set should be converted to the Python list looking like below:\n", - "\n", - "```python\n", - "[\n", - "\t({\n", - "\t\t'one': True,\n", - "\t\t'two': False,\n", - "\t\t'three': False,\n", - "\t\t'four': True,\n", - "\t\t'five': False\n", - "\t}, True),\n", - "\t({\n", - "\t\t'one': False,\n", - "\t\t'two': False,\n", - "\t\t'three': False,\n", - "\t\t'four': True,\n", - "\t\t'five': True\n", - "\t}, False),\n", - "\t({\n", - "\t\t'one': False,\n", - "\t\t'two': True,\n", - "\t\t'three': False,\n", - "\t\t'four': False,\n", - "\t\t'five': False\n", - "\t}, True),\n", - "\t({\n", - "\t\t'one': True,\n", - "\t\t'two': False,\n", - "\t\t'three': False,\n", - "\t\t'four': False,\n", - "\t\t'five': True\n", - "\t}, False)\n", - "]\n", - "```\n", - "\n", - "To help you in this step, watch the [following video](https://www.youtube.com/watch?v=-vVskDsHcVc) to learn how to build the feature set with Python and NLTK. The source code in this video can be found [here](https://pythonprogramming.net/words-as-features-nltk-tutorial/)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Building Features](building-features.jpg)](https://www.youtube.com/watch?v=-vVskDsHcVc)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Building and Traininng Naive Bayes Model\n", - "\n", - "In this step you will split your feature set into a training and a test set. Then you will create a Bayes classifier instance using `nltk.NaiveBayesClassifier.train` ([example](https://www.nltk.org/book/ch06.html)) to train with the training dataset.\n", - "\n", - "After training the model, call `classifier.show_most_informative_features()` to inspect the most important features. The output will look like:\n", - "\n", - "```\n", - "Most Informative Features\n", - "\t snow = True False : True = 34.3 : 1.0\n", - "\t easter = True False : True = 26.2 : 1.0\n", - "\t headach = True False : True = 20.9 : 1.0\n", - "\t argh = True False : True = 17.6 : 1.0\n", - "\tunfortun = True False : True = 16.9 : 1.0\n", - "\t jona = True True : False = 16.2 : 1.0\n", - "\t ach = True False : True = 14.9 : 1.0\n", - "\t sad = True False : True = 13.0 : 1.0\n", - "\t parent = True False : True = 12.9 : 1.0\n", - "\t spring = True False : True = 12.7 : 1.0\n", - "```\n", - "\n", - "The [following video](https://www.youtube.com/watch?v=rISOsUaTrO4) will help you complete this step. The source code in this video can be found [here](https://pythonprogramming.net/naive-bayes-classifier-nltk-tutorial/)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Building and Training NB](nb-model-building.jpg)](https://www.youtube.com/watch?v=rISOsUaTrO4)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Testing Naive Bayes Model\n", - "\n", - "Now we'll test our classifier with the test dataset. This is done by calling `nltk.classify.accuracy(classifier, test)`.\n", - "\n", - "As mentioned in one of the tutorial videos, a Naive Bayes model is considered OK if your accuracy score is over 0.6. If your accuracy score is over 0.7, you've done a great job!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bonus Question 1: Improve Model Performance\n", - "\n", - "If you are still not exhausted so far and want to dig deeper, try to improve your classifier performance. There are many aspects you can dig into, for example:\n", - "\n", - "* Improve stemming and lemmatization. Inspect your bag of words and the most important features. Are there any words you should furuther remove from analysis? You can append these words to further remove to the stop words list.\n", - "\n", - "* Remember we only used the top 5,000 features to build model? Try using different numbers of top features. The bottom line is to use as few features as you can without compromising your model performance. The fewer features you select into your model, the faster your model is trained. Then you can use a larger sample size to improve your model accuracy score." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bonus Question 2: Machine Learning Pipeline\n", - "\n", - "In a new Jupyter Notebook, combine all your codes into a function (or a class). Your new function will execute the complete machine learning pipeline job by receiving the dataset location and output the classifier. This will allow you to use your function to predict the sentiment of any tweet in real time. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bonus Question 3: Apache Spark\n", - "\n", - "If you have completed the Apache Spark advanced topic lab, what you can do is to migrate your pipeline from local to a Databricks Notebook. Share your notebook with your instructor and classmates to show off your achievements!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}