diff --git a/your-code/main.ipynb b/your-code/main.ipynb
index 4128c1e..85bb336 100755
--- a/your-code/main.ipynb
+++ b/your-code/main.ipynb
@@ -12,11 +12,19 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 251,
"metadata": {},
"outputs": [],
"source": [
- "# import numpy and pandas\n"
+ "# import numpy and pandas\n",
+ "import numpy as np \n",
+ "import pandas as pd\n",
+ "from scipy.stats import trim_mean, mode, skew, gaussian_kde, pearsonr, spearmanr, beta\n",
+ "from statsmodels.stats.weightstats import ztest as ztest\n",
+ "from scipy.stats import ttest_ind, norm, t\n",
+ "from scipy.stats import f_oneway\n",
+ "from scipy.stats import sem\n",
+ "from statsmodels.stats.proportion import proportions_ztest as proportions_ztest\n"
]
},
{
@@ -30,7 +38,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@@ -48,11 +56,130 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 11,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Name | \n",
+ " Job Titles | \n",
+ " Department | \n",
+ " Full or Part-Time | \n",
+ " Salary or Hourly | \n",
+ " Typical Hours | \n",
+ " Annual Salary | \n",
+ " Hourly Rate | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " AARON, JEFFERY M | \n",
+ " SERGEANT | \n",
+ " POLICE | \n",
+ " F | \n",
+ " Salary | \n",
+ " NaN | \n",
+ " 101442.0 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " AARON, KARINA | \n",
+ " POLICE OFFICER (ASSIGNED AS DETECTIVE) | \n",
+ " POLICE | \n",
+ " F | \n",
+ " Salary | \n",
+ " NaN | \n",
+ " 94122.0 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " AARON, KIMBERLEI R | \n",
+ " CHIEF CONTRACT EXPEDITER | \n",
+ " GENERAL SERVICES | \n",
+ " F | \n",
+ " Salary | \n",
+ " NaN | \n",
+ " 101592.0 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " ABAD JR, VICENTE M | \n",
+ " CIVIL ENGINEER IV | \n",
+ " WATER MGMNT | \n",
+ " F | \n",
+ " Salary | \n",
+ " NaN | \n",
+ " 110064.0 | \n",
+ " NaN | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " ABASCAL, REECE E | \n",
+ " TRAFFIC CONTROL AIDE-HOURLY | \n",
+ " OEMC | \n",
+ " P | \n",
+ " Hourly | \n",
+ " 20.0 | \n",
+ " NaN | \n",
+ " 19.86 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " Name Job Titles \\\n",
+ "0 AARON, JEFFERY M SERGEANT \n",
+ "1 AARON, KARINA POLICE OFFICER (ASSIGNED AS DETECTIVE) \n",
+ "2 AARON, KIMBERLEI R CHIEF CONTRACT EXPEDITER \n",
+ "3 ABAD JR, VICENTE M CIVIL ENGINEER IV \n",
+ "4 ABASCAL, REECE E TRAFFIC CONTROL AIDE-HOURLY \n",
+ "\n",
+ " Department Full or Part-Time Salary or Hourly Typical Hours \\\n",
+ "0 POLICE F Salary NaN \n",
+ "1 POLICE F Salary NaN \n",
+ "2 GENERAL SERVICES F Salary NaN \n",
+ "3 WATER MGMNT F Salary NaN \n",
+ "4 OEMC P Hourly 20.0 \n",
+ "\n",
+ " Annual Salary Hourly Rate \n",
+ "0 101442.0 NaN \n",
+ "1 94122.0 NaN \n",
+ "2 101592.0 NaN \n",
+ "3 110064.0 NaN \n",
+ "4 NaN 19.86 "
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "# Your code here:\n"
+ "salaries.head()"
]
},
{
@@ -64,11 +191,32 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 48,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Name 0\n",
+ "Job Titles 0\n",
+ "Department 0\n",
+ "Full or Part-Time 0\n",
+ "Salary or Hourly 0\n",
+ "Typical Hours 25161\n",
+ "Annual Salary 8022\n",
+ "Hourly Rate 25161\n",
+ "dtype: int64"
+ ]
+ },
+ "execution_count": 48,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "# Your code here:\n"
+ "\n",
+ "salaries.isna().sum()\n",
+ "# Isues in Typical Hours, Hourly Rate, and Annual Salary \n"
]
},
{
@@ -80,12 +228,24 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 58,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Salaried: 25161 \n",
+ " Hourly: 8022\n"
+ ]
+ }
+ ],
"source": [
- "# Your code here:\n",
- "\n"
+ "Salary= len(salaries[salaries['Salary or Hourly']== 'Salary'])\n",
+ "\n",
+ "Hourly= len(salaries[salaries['Salary or Hourly']== 'Hourly'])\n",
+ "\n",
+ "print(f'Salaried: {Salary} \\n Hourly: {Hourly}')\n"
]
},
{
@@ -104,11 +264,229 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 72,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Name | \n",
+ "
\n",
+ " \n",
+ " | Department | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | ADMIN HEARNG | \n",
+ " 39 | \n",
+ "
\n",
+ " \n",
+ " | ANIMAL CONTRL | \n",
+ " 81 | \n",
+ "
\n",
+ " \n",
+ " | AVIATION | \n",
+ " 1629 | \n",
+ "
\n",
+ " \n",
+ " | BOARD OF ELECTION | \n",
+ " 107 | \n",
+ "
\n",
+ " \n",
+ " | BOARD OF ETHICS | \n",
+ " 8 | \n",
+ "
\n",
+ " \n",
+ " | BUDGET & MGMT | \n",
+ " 46 | \n",
+ "
\n",
+ " \n",
+ " | BUILDINGS | \n",
+ " 269 | \n",
+ "
\n",
+ " \n",
+ " | BUSINESS AFFAIRS | \n",
+ " 171 | \n",
+ "
\n",
+ " \n",
+ " | CITY CLERK | \n",
+ " 84 | \n",
+ "
\n",
+ " \n",
+ " | CITY COUNCIL | \n",
+ " 411 | \n",
+ "
\n",
+ " \n",
+ " | COMMUNITY DEVELOPMENT | \n",
+ " 207 | \n",
+ "
\n",
+ " \n",
+ " | COPA | \n",
+ " 116 | \n",
+ "
\n",
+ " \n",
+ " | CULTURAL AFFAIRS | \n",
+ " 65 | \n",
+ "
\n",
+ " \n",
+ " | DISABILITIES | \n",
+ " 28 | \n",
+ "
\n",
+ " \n",
+ " | DoIT | \n",
+ " 99 | \n",
+ "
\n",
+ " \n",
+ " | FAMILY & SUPPORT | \n",
+ " 615 | \n",
+ "
\n",
+ " \n",
+ " | FINANCE | \n",
+ " 560 | \n",
+ "
\n",
+ " \n",
+ " | FIRE | \n",
+ " 4641 | \n",
+ "
\n",
+ " \n",
+ " | GENERAL SERVICES | \n",
+ " 980 | \n",
+ "
\n",
+ " \n",
+ " | HEALTH | \n",
+ " 488 | \n",
+ "
\n",
+ " \n",
+ " | HUMAN RELATIONS | \n",
+ " 16 | \n",
+ "
\n",
+ " \n",
+ " | HUMAN RESOURCES | \n",
+ " 79 | \n",
+ "
\n",
+ " \n",
+ " | INSPECTOR GEN | \n",
+ " 87 | \n",
+ "
\n",
+ " \n",
+ " | LAW | \n",
+ " 407 | \n",
+ "
\n",
+ " \n",
+ " | LICENSE APPL COMM | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " | MAYOR'S OFFICE | \n",
+ " 85 | \n",
+ "
\n",
+ " \n",
+ " | OEMC | \n",
+ " 2102 | \n",
+ "
\n",
+ " \n",
+ " | POLICE | \n",
+ " 13414 | \n",
+ "
\n",
+ " \n",
+ " | POLICE BOARD | \n",
+ " 2 | \n",
+ "
\n",
+ " \n",
+ " | PROCUREMENT | \n",
+ " 92 | \n",
+ "
\n",
+ " \n",
+ " | PUBLIC LIBRARY | \n",
+ " 1015 | \n",
+ "
\n",
+ " \n",
+ " | STREETS & SAN | \n",
+ " 2198 | \n",
+ "
\n",
+ " \n",
+ " | TRANSPORTN | \n",
+ " 1140 | \n",
+ "
\n",
+ " \n",
+ " | TREASURER | \n",
+ " 22 | \n",
+ "
\n",
+ " \n",
+ " | WATER MGMNT | \n",
+ " 1879 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " Name\n",
+ "Department \n",
+ "ADMIN HEARNG 39\n",
+ "ANIMAL CONTRL 81\n",
+ "AVIATION 1629\n",
+ "BOARD OF ELECTION 107\n",
+ "BOARD OF ETHICS 8\n",
+ "BUDGET & MGMT 46\n",
+ "BUILDINGS 269\n",
+ "BUSINESS AFFAIRS 171\n",
+ "CITY CLERK 84\n",
+ "CITY COUNCIL 411\n",
+ "COMMUNITY DEVELOPMENT 207\n",
+ "COPA 116\n",
+ "CULTURAL AFFAIRS 65\n",
+ "DISABILITIES 28\n",
+ "DoIT 99\n",
+ "FAMILY & SUPPORT 615\n",
+ "FINANCE 560\n",
+ "FIRE 4641\n",
+ "GENERAL SERVICES 980\n",
+ "HEALTH 488\n",
+ "HUMAN RELATIONS 16\n",
+ "HUMAN RESOURCES 79\n",
+ "INSPECTOR GEN 87\n",
+ "LAW 407\n",
+ "LICENSE APPL COMM 1\n",
+ "MAYOR'S OFFICE 85\n",
+ "OEMC 2102\n",
+ "POLICE 13414\n",
+ "POLICE BOARD 2\n",
+ "PROCUREMENT 92\n",
+ "PUBLIC LIBRARY 1015\n",
+ "STREETS & SAN 2198\n",
+ "TRANSPORTN 1140\n",
+ "TREASURER 22\n",
+ "WATER MGMNT 1879"
+ ]
+ },
+ "execution_count": 72,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "# Your code here:\n"
+ "salaries[['Name', 'Department']].groupby('Department').count()"
]
},
{
@@ -122,17 +500,70 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 81,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/opt/anaconda3/envs/Ironhack/lib/python3.7/site-packages/pandas/core/frame.py:4913: SettingWithCopyWarning: \n",
+ "A value is trying to be set on a copy of a slice from a DataFrame\n",
+ "\n",
+ "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+ " errors=errors,\n"
+ ]
+ }
+ ],
+ "source": [
+ "hourly= salaries[salaries['Salary or Hourly']== 'Hourly']\n",
+ "hourly.drop('Annual Salary', inplace= True, axis=1)\n",
+ "hourly.reset_index(drop= True, inplace= True)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 133,
"metadata": {},
"outputs": [],
"source": [
- "# Your code here:\n"
+ "hourlyrate= hourly['Hourly Rate']\n",
+ "data = np.random.normal(hourlyrate.mean(), pop_std, n)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 219,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "p-value: 1.8228873859286195e-94\n",
+ "Rejection of H0. Keep the alternative: wage of all hourly workers is significantly different from $30/hr\n"
+ ]
+ }
+ ],
+ "source": [
+ "hourlyrate.dropna(inplace = True)\n",
+ "\n",
+ "statistic, p_value = ztest(hourlyrate, value = 30, alternative=\"two-sided\")\n",
+ "print(f'p-value: {p_value}')\n",
+ "\n",
+ "if p_value < 0.05:\n",
+ " print(\"Rejection of H0. Keep the alternative: wage of all hourly workers is significantly different from $30/hr\")\n",
+ "if p_value > 0.05:\n",
+ " print(\"We don't have enough evidence to reject H0\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
+ "\n",
"We are also curious about salaries in the police force. The chief of police in Chicago claimed in a press briefing that salaries this year are higher than last year's mean of $86000/year a year for all salaried employees. Test this one sided hypothesis using a 95% confidence interval.\n",
"\n",
"Hint: A one tailed test has a p-value that is half of the two tailed p-value. If our hypothesis is greater than, then to reject, the test statistic must also be positive."
@@ -140,11 +571,39 @@
},
{
"cell_type": "code",
- "execution_count": 8,
- "metadata": {},
+ "execution_count": 215,
+ "metadata": {
+ "scrolled": true
+ },
"outputs": [],
"source": [
- "# Your code here:\n"
+ "police=salaries[salaries[\"Department\"] == \"POLICE\"]\n",
+ "poSalary= police['Annual Salary']\n",
+ "poSalary.dropna(inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 221,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "p-value: 0.002056169057509571\n",
+ "Rejection of H0. Keep the alternative: salaries this year are higher than last year's mean of $86000/year a year for all salaried employees\n"
+ ]
+ }
+ ],
+ "source": [
+ "\n",
+ "statistic, p_value = ztest(poSalary, value = 86000, alternative=\"two-sided\")\n",
+ "print(f'p-value: {p_value}')\n",
+ "if p_value < 0.05:\n",
+ " print(\"Rejection of H0. Keep the alternative: salaries this year are higher than last year's mean of $86000/year a year for all salaried employees\")\n",
+ "if p_value > 0.05:\n",
+ " print(\"We don't have enough evidence to reject H0\")\n"
]
},
{
@@ -156,11 +615,15 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 241,
"metadata": {},
"outputs": [],
"source": [
- "# Your code here:\n"
+ "depts= pd.crosstab(index = salaries['Department'], columns = salaries['Salary or Hourly'])\n",
+ "depts[depts.index== 'STREETS & SAN']\n",
+ "\n",
+ "StS= salaries[(salaries['Salary or Hourly']== 'Hourly') & (salaries['Department']== 'STREETS & SAN')]\n",
+ "shr= StS['Hourly Rate']"
]
},
{
@@ -172,11 +635,29 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 244,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "p-value: 5.478855641943434e-22\n",
+ "Rejection of H0. Keep the alternative: salaries this year are higher than last year's mean of $86000/year a year for all salaried employees\n"
+ ]
+ }
+ ],
"source": [
- "# Your code here:\n"
+ "# STREETS & SAN\n",
+ "\n",
+ "statistic, p_value = ztest(shr, value = 35, alternative=\"smaller\")\n",
+ "print(f'p-value: {p_value}')\n",
+ "if p_value < 0.05:\n",
+ " print(\"Rejection of H0. Keep the alternative: hourly wage is less than $35/hour\")\n",
+ "if p_value > 0.05:\n",
+ " print(\"We don't have enough evidence to reject H0\")\n",
+ " \n",
+ " "
]
},
{
@@ -200,11 +681,24 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 248,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(32.52345834488425, 33.05365708767623)"
+ ]
+ },
+ "execution_count": 248,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "# Your code here:\n"
+ "sem_hourly = sem(hourlyrate, nan_policy='omit')\n",
+ "\n",
+ "t.interval(0.95, df = len(hourlyrate)-1, loc=hourlyrate.mean(), scale=sem_hourly)"
]
},
{
@@ -216,11 +710,23 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 250,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(86177.05631531784, 86795.77269094894)"
+ ]
+ },
+ "execution_count": 250,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "# Your code here:\n"
+ "sem_poSalary= sem(poSalary, nan_policy='omit')\n",
+ "t.interval(0.95, df = len(poSalary)-1, loc=poSalary.mean(), scale=sem_poSalary)"
]
},
{
@@ -238,17 +744,51 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 259,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Hourly workers in the City of Chicago is significantly different from 25%\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "0.0004481127249057967"
+ ]
+ },
+ "execution_count": 259,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "# Your code here:\n"
+ "count= len(hourly)\n",
+ "nobs= len(salaries)\n",
+ "\n",
+ "statistic, p_value = proportions_ztest(count, nobs, value=0.25, alternative='two-sided')\n",
+ "if p_value < 0.05:\n",
+ " print(f\"Hourly workers in the City of Chicago is significantly different from 25%\")\n",
+ "if p_value > 0.05:\n",
+ " print(f\"We don't have enough evidence to reject H0\")\n",
+ " \n",
+ "p_value"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -262,7 +802,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.3"
+ "version": "3.7.13"
}
},
"nbformat": 4,