Skip to content

[lab-nlp] - Sebastian Linares #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Astrobert0 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[lab-nlp] - Sebastian Linares #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added your-code/__pycache__/cleaning_functions.cpython-38.pyc
View file
Open in desktop
Binary file not shown.
140 changes: 123 additions & 17 deletions your-code/challenge-1.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,27 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"metadata": {},
"outputs": [],
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"' s Q website is ironhack '"
]
},
"metadata": {},
"execution_count": 9
}
],
"source": [
"import re\n",
"\n",
"string = \"@Ironhack's-#Q website 776-is http://ironhack.com [(2018)]\\\")\"\n",
"\n",
"def clean_up(s):\n",
" rules_list= ['@\\w+','[^a-z, A-Z]', 'http', 'com']\n",
" \"\"\"\n",
" Cleans up numbers, URLs, and special characters from a string.\n",
"\n",
Expand All @@ -79,7 +95,12 @@
"\n",
" Returns:\n",
" A string that has been cleaned up.\n",
" \"\"\""
" \"\"\"\n",
" for rule in rules_list:\n",
" s = re.sub(rule,' ',s)\n",
" return s\n",
"cleaned = clean_up(string)\n",
"cleaned"
]
},
{
Expand All @@ -101,10 +122,23 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {},
"outputs": [],
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"['s', 'Q', 'website', 'is', 'ironhack']"
]
},
"metadata": {},
"execution_count": 2
}
],
"source": [
"import nltk\n",
"\n",
"def tokenize(s):\n",
" \"\"\"\n",
" Tokenize a string.\n",
Expand All @@ -114,7 +148,10 @@
"\n",
" Returns:\n",
" A list of words as the result of tokenization.\n",
" \"\"\""
" \"\"\"\n",
" return nltk.word_tokenize(s)\n",
"token = tokenize(cleaned)\n",
"token"
]
},
{
Expand Down Expand Up @@ -145,20 +182,59 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"from nltk.stem import WordNetLemmatizer\n",
"from nltk.stem import PorterStemmer\n",
"'''nltk.download('wordnet')\n",
"'''\n",
"\n",
"\n",
"def stem_and_lemmatize(l):\n",
" \"\"\"\n",
" lemmatizer = WordNetLemmatizer()\n",
" stemporter = PorterStemmer()\n",
"\n",
" \"\"\"=\n",
" Perform stemming and lemmatization on a list of words.\n",
"\n",
" Args:\n",
" l: A list of strings.\n",
"\n",
" Returns:\n",
" A list of strings after being stemmed and lemmatized.\n",
" \"\"\""
" \"\"\"\n",
" stemandlemmalist=[]\n",
" \n",
"\n",
" for word in l:\n",
" stemmed = stemporter.stem(word)\n",
" stemandlemma = lemmatizer.lemmatize(stemmed)\n",
" stemandlemmalist.append(stemandlemma)\n",
" return stemandlemmalist\n",
"\n",
"stemandlemma = stem_and_lemmatize(token)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"['s', 'q', 'websit', 'is', 'ironhack']"
]
},
"metadata": {},
"execution_count": 9
}
],
"source": [
"stemandlemma"
]
},
{
Expand All @@ -176,10 +252,32 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 14,
"metadata": {},
"outputs": [],
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"[nltk_data] Downloading package stopwords to\n[nltk_data] C:\\Users\\Sebas!\\AppData\\Roaming\\nltk_data...\n[nltk_data] Package stopwords is already up-to-date!\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"['q', 'websit', 'ironhack']"
]
},
"metadata": {},
"execution_count": 14
}
],
"source": [
"from nltk.corpus import stopwords\n",
"'''nltk.download('stopwords')\n",
"'''\n",
"\n",
"def remove_stopwords(l):\n",
" \"\"\"\n",
" Remove English stopwords from a list of strings.\n",
Expand All @@ -189,7 +287,16 @@
"\n",
" Returns:\n",
" A list of strings after stop words are removed.\n",
" \"\"\""
" \"\"\"\n",
" stop_words = set(stopwords.words('english'))\n",
" \n",
" for words in l:\n",
" if words in stop_words:\n",
" l.remove(words)\n",
" return l\n",
"\n",
"filtered = remove_stopwords(stemed)\n",
"filtered"
]
},
{
Expand All @@ -204,9 +311,8 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
"name": "python385jvsc74a57bd09efc80705562ef6f8028ba9c07828938c290468cbec0ebcf2b44f68ee94d478d",
"display_name": "Python 3.8.5 64-bit"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -218,9 +324,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
}
Loading