From e5537d41bf2f9dd9a71f8ae2c78f0654be75a9c8 Mon Sep 17 00:00:00 2001 From: Muriel Berkenstock Date: Mon, 7 Nov 2022 18:32:46 +0100 Subject: [PATCH] s --- your-code/main.ipynb | 614 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 577 insertions(+), 37 deletions(-) diff --git a/your-code/main.ipynb b/your-code/main.ipynb index 4128c1e..85bb336 100755 --- a/your-code/main.ipynb +++ b/your-code/main.ipynb @@ -12,11 +12,19 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 251, "metadata": {}, "outputs": [], "source": [ - "# import numpy and pandas\n" + "# import numpy and pandas\n", + "import numpy as np \n", + "import pandas as pd\n", + "from scipy.stats import trim_mean, mode, skew, gaussian_kde, pearsonr, spearmanr, beta\n", + "from statsmodels.stats.weightstats import ztest as ztest\n", + "from scipy.stats import ttest_ind, norm, t\n", + "from scipy.stats import f_oneway\n", + "from scipy.stats import sem\n", + "from statsmodels.stats.proportion import proportions_ztest as proportions_ztest\n" ] }, { @@ -30,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -48,11 +56,130 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
NameJob TitlesDepartmentFull or Part-TimeSalary or HourlyTypical HoursAnnual SalaryHourly Rate
0AARON, JEFFERY MSERGEANTPOLICEFSalaryNaN101442.0NaN
1AARON, KARINAPOLICE OFFICER (ASSIGNED AS DETECTIVE)POLICEFSalaryNaN94122.0NaN
2AARON, KIMBERLEI RCHIEF CONTRACT EXPEDITERGENERAL SERVICESFSalaryNaN101592.0NaN
3ABAD JR, VICENTE MCIVIL ENGINEER IVWATER MGMNTFSalaryNaN110064.0NaN
4ABASCAL, REECE ETRAFFIC CONTROL AIDE-HOURLYOEMCPHourly20.0NaN19.86
\n", + "
" + ], + "text/plain": [ + " Name Job Titles \\\n", + "0 AARON, JEFFERY M SERGEANT \n", + "1 AARON, KARINA POLICE OFFICER (ASSIGNED AS DETECTIVE) \n", + "2 AARON, KIMBERLEI R CHIEF CONTRACT EXPEDITER \n", + "3 ABAD JR, VICENTE M CIVIL ENGINEER IV \n", + "4 ABASCAL, REECE E TRAFFIC CONTROL AIDE-HOURLY \n", + "\n", + " Department Full or Part-Time Salary or Hourly Typical Hours \\\n", + "0 POLICE F Salary NaN \n", + "1 POLICE F Salary NaN \n", + "2 GENERAL SERVICES F Salary NaN \n", + "3 WATER MGMNT F Salary NaN \n", + "4 OEMC P Hourly 20.0 \n", + "\n", + " Annual Salary Hourly Rate \n", + "0 101442.0 NaN \n", + "1 94122.0 NaN \n", + "2 101592.0 NaN \n", + "3 110064.0 NaN \n", + "4 NaN 19.86 " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Your code here:\n" + "salaries.head()" ] }, { @@ -64,11 +191,32 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 48, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Name 0\n", + "Job Titles 0\n", + "Department 0\n", + "Full or Part-Time 0\n", + "Salary or Hourly 0\n", + "Typical Hours 25161\n", + "Annual Salary 8022\n", + "Hourly Rate 25161\n", + "dtype: int64" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Your code here:\n" + "\n", + "salaries.isna().sum()\n", + "# Isues in Typical Hours, Hourly Rate, and Annual Salary \n" ] }, { @@ -80,12 +228,24 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 58, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Salaried: 25161 \n", + " Hourly: 8022\n" + ] + } + ], "source": [ - "# Your code here:\n", - "\n" + "Salary= len(salaries[salaries['Salary or Hourly']== 'Salary'])\n", + "\n", + "Hourly= len(salaries[salaries['Salary or Hourly']== 'Hourly'])\n", + "\n", + "print(f'Salaried: {Salary} \\n Hourly: {Hourly}')\n" ] }, { @@ -104,11 +264,229 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 72, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Name
Department
ADMIN HEARNG39
ANIMAL CONTRL81
AVIATION1629
BOARD OF ELECTION107
BOARD OF ETHICS8
BUDGET & MGMT46
BUILDINGS269
BUSINESS AFFAIRS171
CITY CLERK84
CITY COUNCIL411
COMMUNITY DEVELOPMENT207
COPA116
CULTURAL AFFAIRS65
DISABILITIES28
DoIT99
FAMILY & SUPPORT615
FINANCE560
FIRE4641
GENERAL SERVICES980
HEALTH488
HUMAN RELATIONS16
HUMAN RESOURCES79
INSPECTOR GEN87
LAW407
LICENSE APPL COMM1
MAYOR'S OFFICE85
OEMC2102
POLICE13414
POLICE BOARD2
PROCUREMENT92
PUBLIC LIBRARY1015
STREETS & SAN2198
TRANSPORTN1140
TREASURER22
WATER MGMNT1879
\n", + "
" + ], + "text/plain": [ + " Name\n", + "Department \n", + "ADMIN HEARNG 39\n", + "ANIMAL CONTRL 81\n", + "AVIATION 1629\n", + "BOARD OF ELECTION 107\n", + "BOARD OF ETHICS 8\n", + "BUDGET & MGMT 46\n", + "BUILDINGS 269\n", + "BUSINESS AFFAIRS 171\n", + "CITY CLERK 84\n", + "CITY COUNCIL 411\n", + "COMMUNITY DEVELOPMENT 207\n", + "COPA 116\n", + "CULTURAL AFFAIRS 65\n", + "DISABILITIES 28\n", + "DoIT 99\n", + "FAMILY & SUPPORT 615\n", + "FINANCE 560\n", + "FIRE 4641\n", + "GENERAL SERVICES 980\n", + "HEALTH 488\n", + "HUMAN RELATIONS 16\n", + "HUMAN RESOURCES 79\n", + "INSPECTOR GEN 87\n", + "LAW 407\n", + "LICENSE APPL COMM 1\n", + "MAYOR'S OFFICE 85\n", + "OEMC 2102\n", + "POLICE 13414\n", + "POLICE BOARD 2\n", + "PROCUREMENT 92\n", + "PUBLIC LIBRARY 1015\n", + "STREETS & SAN 2198\n", + "TRANSPORTN 1140\n", + "TREASURER 22\n", + "WATER MGMNT 1879" + ] + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Your code here:\n" + "salaries[['Name', 'Department']].groupby('Department').count()" ] }, { @@ -122,17 +500,70 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 81, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/Ironhack/lib/python3.7/site-packages/pandas/core/frame.py:4913: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " errors=errors,\n" + ] + } + ], + "source": [ + "hourly= salaries[salaries['Salary or Hourly']== 'Hourly']\n", + "hourly.drop('Annual Salary', inplace= True, axis=1)\n", + "hourly.reset_index(drop= True, inplace= True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 133, "metadata": {}, "outputs": [], "source": [ - "# Your code here:\n" + "hourlyrate= hourly['Hourly Rate']\n", + "data = np.random.normal(hourlyrate.mean(), pop_std, n)" + ] + }, + { + "cell_type": "code", + "execution_count": 219, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "p-value: 1.8228873859286195e-94\n", + "Rejection of H0. Keep the alternative: wage of all hourly workers is significantly different from $30/hr\n" + ] + } + ], + "source": [ + "hourlyrate.dropna(inplace = True)\n", + "\n", + "statistic, p_value = ztest(hourlyrate, value = 30, alternative=\"two-sided\")\n", + "print(f'p-value: {p_value}')\n", + "\n", + "if p_value < 0.05:\n", + " print(\"Rejection of H0. Keep the alternative: wage of all hourly workers is significantly different from $30/hr\")\n", + "if p_value > 0.05:\n", + " print(\"We don't have enough evidence to reject H0\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ + "\n", "We are also curious about salaries in the police force. The chief of police in Chicago claimed in a press briefing that salaries this year are higher than last year's mean of $86000/year a year for all salaried employees. Test this one sided hypothesis using a 95% confidence interval.\n", "\n", "Hint: A one tailed test has a p-value that is half of the two tailed p-value. If our hypothesis is greater than, then to reject, the test statistic must also be positive." @@ -140,11 +571,39 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": {}, + "execution_count": 215, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ - "# Your code here:\n" + "police=salaries[salaries[\"Department\"] == \"POLICE\"]\n", + "poSalary= police['Annual Salary']\n", + "poSalary.dropna(inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 221, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "p-value: 0.002056169057509571\n", + "Rejection of H0. Keep the alternative: salaries this year are higher than last year's mean of $86000/year a year for all salaried employees\n" + ] + } + ], + "source": [ + "\n", + "statistic, p_value = ztest(poSalary, value = 86000, alternative=\"two-sided\")\n", + "print(f'p-value: {p_value}')\n", + "if p_value < 0.05:\n", + " print(\"Rejection of H0. Keep the alternative: salaries this year are higher than last year's mean of $86000/year a year for all salaried employees\")\n", + "if p_value > 0.05:\n", + " print(\"We don't have enough evidence to reject H0\")\n" ] }, { @@ -156,11 +615,15 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 241, "metadata": {}, "outputs": [], "source": [ - "# Your code here:\n" + "depts= pd.crosstab(index = salaries['Department'], columns = salaries['Salary or Hourly'])\n", + "depts[depts.index== 'STREETS & SAN']\n", + "\n", + "StS= salaries[(salaries['Salary or Hourly']== 'Hourly') & (salaries['Department']== 'STREETS & SAN')]\n", + "shr= StS['Hourly Rate']" ] }, { @@ -172,11 +635,29 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 244, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "p-value: 5.478855641943434e-22\n", + "Rejection of H0. Keep the alternative: salaries this year are higher than last year's mean of $86000/year a year for all salaried employees\n" + ] + } + ], "source": [ - "# Your code here:\n" + "# STREETS & SAN\n", + "\n", + "statistic, p_value = ztest(shr, value = 35, alternative=\"smaller\")\n", + "print(f'p-value: {p_value}')\n", + "if p_value < 0.05:\n", + " print(\"Rejection of H0. Keep the alternative: hourly wage is less than $35/hour\")\n", + "if p_value > 0.05:\n", + " print(\"We don't have enough evidence to reject H0\")\n", + " \n", + " " ] }, { @@ -200,11 +681,24 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 248, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(32.52345834488425, 33.05365708767623)" + ] + }, + "execution_count": 248, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Your code here:\n" + "sem_hourly = sem(hourlyrate, nan_policy='omit')\n", + "\n", + "t.interval(0.95, df = len(hourlyrate)-1, loc=hourlyrate.mean(), scale=sem_hourly)" ] }, { @@ -216,11 +710,23 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 250, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(86177.05631531784, 86795.77269094894)" + ] + }, + "execution_count": 250, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Your code here:\n" + "sem_poSalary= sem(poSalary, nan_policy='omit')\n", + "t.interval(0.95, df = len(poSalary)-1, loc=poSalary.mean(), scale=sem_poSalary)" ] }, { @@ -238,17 +744,51 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 259, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hourly workers in the City of Chicago is significantly different from 25%\n" + ] + }, + { + "data": { + "text/plain": [ + "0.0004481127249057967" + ] + }, + "execution_count": 259, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Your code here:\n" + "count= len(hourly)\n", + "nobs= len(salaries)\n", + "\n", + "statistic, p_value = proportions_ztest(count, nobs, value=0.25, alternative='two-sided')\n", + "if p_value < 0.05:\n", + " print(f\"Hourly workers in the City of Chicago is significantly different from 25%\")\n", + "if p_value > 0.05:\n", + " print(f\"We don't have enough evidence to reject H0\")\n", + " \n", + "p_value" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -262,7 +802,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.13" } }, "nbformat": 4,