From 08e6175a63244fa8313329c90bce40246581d548 Mon Sep 17 00:00:00 2001 From: ZiqiangWang0130 <46898160+ZiqiangWang0130@users.noreply.github.com> Date: Fri, 3 Dec 2021 01:55:37 -0500 Subject: [PATCH 1/2] Add files via upload --- feature_selection_with_SelectKBest_with_data_Norm.ipynb | 1 + 1 file changed, 1 insertion(+) create mode 100644 feature_selection_with_SelectKBest_with_data_Norm.ipynb diff --git a/feature_selection_with_SelectKBest_with_data_Norm.ipynb b/feature_selection_with_SelectKBest_with_data_Norm.ipynb new file mode 100644 index 0000000..fff37be --- /dev/null +++ b/feature_selection_with_SelectKBest_with_data_Norm.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.10"},"colab":{"name":"feature_selection_with_SelectKBest_with_data_Norm.ipynb","provenance":[],"collapsed_sections":[]}},"cells":[{"cell_type":"code","metadata":{"id":"5T5tLCBgHYK-","executionInfo":{"status":"ok","timestamp":1638487394403,"user_tz":300,"elapsed":13860,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["import pandas as pd \n","import seaborn as sns\n","import numpy as np\n","import matplotlib.pyplot as plt\n","import scipy\n","from scipy.spatial.distance import cdist\n","import sklearn \n","from sklearn.cluster import KMeans\n","from sklearn.model_selection import train_test_split\n","from sklearn.feature_selection import mutual_info_classif, SelectKBest\n","import xgboost as xgb\n","from xgboost import XGBClassifier, plot_importance\n","\n","#data = pd.read_csv('train_data.csv') # load original dataset \n","\n","# split dataset into 75/25 train test split \n","#train, test = sklearn.model_selection.train_test_split(data, test_size=.25, random_state=42, shuffle=True, stratify=data['Label'])\n","#train.to_csv('split_train_data.csv', index = False) # save train data for future use \n","#test.to_csv('split_test_data.csv', index = False) # save test data for future use \n","\n","train_split = pd.read_csv('split_train_data.csv')\n","\n","\n"],"execution_count":15,"outputs":[]},{"cell_type":"code","metadata":{"id":"mJXtj0sV1Xn8"},"source":["corr = train_split.corr(method='pearson')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QFa1oX3e1bL8","executionInfo":{"status":"ok","timestamp":1638394045375,"user_tz":300,"elapsed":152,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"3a41b2bf-974c-4a3c-f9f2-42369e6c21bb"},"source":["print(corr.iloc[:,0])"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["G26 1.000000\n","G26_Target Sequence_in_SMILES_perc -0.658644\n","G26_SMILES_in_Target Sequence_perc -0.633055\n","G26_ARRO 0.169460\n","G26_SMILES_base 0.317016\n"," ... \n","G10_fd_Target Sequence_base 0.159265\n","G10_std_SMILES_dist 0.430174\n","G10_std_Target Sequence_dist 0.111489\n","KIBA 0.201958\n","Label 0.152989\n","Name: G26, Length: 338, dtype: float64\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MSAEiUx80PS0","executionInfo":{"status":"ok","timestamp":1638394074040,"user_tz":300,"elapsed":192,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"1d30dd87-a8a5-4a9f-de9a-19263bbf85c4"},"source":["print(data.shape)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["(109479, 338)\n"]}]},{"cell_type":"code","metadata":{"id":"qNZtNRk4nCcn","executionInfo":{"status":"ok","timestamp":1638487443917,"user_tz":300,"elapsed":595,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["train_split = train_split.drop('KIBA', axis = 1)\n","train, test = sklearn.model_selection.train_test_split(train_split, test_size=.25, random_state=42, shuffle=True, stratify=train_split['Label'])"],"execution_count":16,"outputs":[]},{"cell_type":"code","metadata":{"id":"o1GnGYd8juC6","executionInfo":{"status":"ok","timestamp":1638487489284,"user_tz":300,"elapsed":605,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["from sklearn.feature_selection import f_classif"],"execution_count":19,"outputs":[]},{"cell_type":"code","metadata":{"id":"0_vbHgv9whZm"},"source":["from sklearn.feature_selection import f_regression"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"rJSFsqtRxHXU"},"source":["from sklearn.feature_selection import mutual_info_regression"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"q30ZDm8cROai","executionInfo":{"status":"ok","timestamp":1638487455646,"user_tz":300,"elapsed":267,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"ae63a960-1c2f-4375-f056-ce9fbfb594da"},"source":["print(train_split.shape)"],"execution_count":17,"outputs":[{"output_type":"stream","name":"stdout","text":["(82109, 337)\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2aWu9FPVnATV","executionInfo":{"status":"ok","timestamp":1638489928271,"user_tz":300,"elapsed":309,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"34ed06cb-b02c-4ad9-b251-28ba25532aa1"},"source":["print(train)"],"execution_count":27,"outputs":[{"output_type":"stream","name":"stdout","text":[" G26 ... Label\n","62386 5.771787 ... False\n","63350 5.051778 ... False\n","8881 5.648391 ... False\n","15956 5.139381 ... False\n","56441 5.467752 ... False\n","... ... ... ...\n","17916 5.220579 ... False\n","8252 5.163831 ... True\n","59884 5.036404 ... False\n","39791 6.872042 ... False\n","6867 5.609244 ... True\n","\n","[61581 rows x 337 columns]\n"]}]},{"cell_type":"code","metadata":{"id":"KaNiSiKxHYLC","executionInfo":{"status":"ok","timestamp":1638487492177,"user_tz":300,"elapsed":460,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["# Feature Selection \n","X = train.drop('Label', axis = 1)\n","y = train['Label']\n","\n","k_best = SelectKBest(f_classif, k=100) # might want to update feature selection method / change k val\n","train_fs = k_best.fit(X, y)\n","cols = k_best.get_support(indices=True)\n","train_fs = train.iloc[:,cols]"],"execution_count":20,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MhjJCc0KxiP1","executionInfo":{"status":"ok","timestamp":1638486707049,"user_tz":300,"elapsed":32010,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"4e16e189-2c47-4a28-cc04-67c293f98066"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"CLhYxCbMbDJa","executionInfo":{"status":"ok","timestamp":1638487328440,"user_tz":300,"elapsed":278,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"e0c63250-5748-461d-8aed-87284faae87b"},"source":["%cd ../"],"execution_count":14,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/drive/MyDrive/SYSC5405/Project\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Emqj471mhiVd","executionInfo":{"status":"ok","timestamp":1638487506098,"user_tz":300,"elapsed":271,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"c5c641ed-a426-4b64-afc8-c7fba621e186"},"source":["print(cols)"],"execution_count":21,"outputs":[{"output_type":"stream","name":"stdout","text":["[ 0 10 13 14 15 16 18 20 22 23 24 25 26 27 42 43 48 49\n"," 50 52 54 56 57 58 60 62 64 65 66 67 68 69 70 71 76 78\n"," 140 141 142 146 148 149 150 151 152 153 196 203 210 211 218 220 222 224\n"," 225 232 234 238 239 244 246 248 250 252 253 258 259 260 262 264 266 267\n"," 268 274 275 276 278 279 280 281 286 287 288 292 295 300 308 309 314 315\n"," 316 318 320 321 322 323 328 330 332 334]\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"i5B05ma4h3tB","executionInfo":{"status":"ok","timestamp":1638490454217,"user_tz":300,"elapsed":460,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"870faa31-e124-4c56-933c-c9db78d55b03"},"source":["print(train_fs)"],"execution_count":29,"outputs":[{"output_type":"stream","name":"stdout","text":[" G26 G26_fd_SMILES_base ... G10_fd_SMILES_base G10_std_SMILES_dist\n","62386 5.771787 0.076806 ... -0.039719 -0.656573\n","63350 5.051778 -0.079144 ... -0.036837 -0.573908\n","8881 5.648391 0.050248 ... -0.007045 0.467020\n","15956 5.139381 -0.028619 ... 0.067213 2.927458\n","56441 5.467752 -0.015724 ... 0.004252 0.826105\n","... ... ... ... ... ...\n","17916 5.220579 0.020019 ... 0.084854 3.423547\n","8252 5.163831 -0.124667 ... 0.000754 0.752722\n","59884 5.036404 -0.023823 ... -0.043870 -0.802460\n","39791 6.872042 0.221234 ... 0.075078 3.385024\n","6867 5.609244 0.045507 ... -0.051585 -0.988755\n","\n","[61581 rows x 100 columns]\n"]}]},{"cell_type":"code","metadata":{"id":"9NCZ2UJuHYLD","executionInfo":{"status":"ok","timestamp":1638490483566,"user_tz":300,"elapsed":875,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["# Outlier Removal \n","\n","# box plot pre-OR ?\n","preOL_mean = train_fs.mean()\n","preOL_med = train_fs.median()\n","\n","Q1 = train_fs.quantile(0.25)\n","Q3 = train_fs.quantile(0.75)\n","\n","IQR = Q3-Q1\n","\n","# this sets outliers to Nan\n","# train_or=train_fs[~((train_fs<(Q1-1.5*IQR)) | (train_fs>(Q3+1.5*IQR)))]\n","\n","# this relaces outliers with the corresponding bound \n","train_or = train_fs[(train_fs>(Q1-1.5*IQR))]\n","train_or = train_or.fillna(Q1-1.5*IQR)\n","train_or = train_or[(train_fs<(Q3+1.5*IQR))]\n","train_or = train_or.fillna(Q3+1.5*IQR)\n","\n","postOL_mean = train_or.mean()\n","postOL_med = train_or.median()\n","# box plot post OR ?\n","\n","OLdescriptor_frames = [preOL_mean, preOL_med, postOL_mean, postOL_med]\n","OLdescriptor_names = ['preOL_mean', 'preOL_med', 'postOL_mean', 'postOL_med']\n","\n","OLdescriptor = pd.concat(OLdescriptor_frames, axis = 1)\n","OLdescriptor.columns = OLdescriptor_names\n","#print(OLdescriptor)"],"execution_count":30,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"eRYZgpBIhCni","executionInfo":{"status":"ok","timestamp":1638491712519,"user_tz":300,"elapsed":272,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"4e1042f3-489c-42e2-9f61-79fa5e518565"},"source":["print(train_or)"],"execution_count":49,"outputs":[{"output_type":"stream","name":"stdout","text":[" G26 G26_fd_SMILES_base ... G10_fd_SMILES_base G10_std_SMILES_dist\n","62386 5.771787 0.076806 ... -0.039719 -0.656573\n","63350 5.051778 -0.079144 ... -0.036837 -0.573908\n","8881 5.648391 0.050248 ... -0.007045 0.467020\n","15956 5.139381 -0.028619 ... 0.067213 2.927458\n","56441 5.467752 -0.015724 ... 0.004252 0.826105\n","... ... ... ... ... ...\n","17916 5.220579 0.020019 ... 0.084854 3.423547\n","8252 5.163831 -0.124667 ... 0.000754 0.752722\n","59884 5.036404 -0.023823 ... -0.043870 -0.802460\n","39791 6.175706 0.104837 ... 0.075078 3.385024\n","6867 5.609244 0.045507 ... -0.051585 -0.988755\n","\n","[61581 rows x 100 columns]\n"]}]},{"cell_type":"code","metadata":{"id":"hf_AHYSmpM08","executionInfo":{"status":"ok","timestamp":1638490510937,"user_tz":300,"elapsed":256,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["from sklearn import preprocessing"],"execution_count":32,"outputs":[]},{"cell_type":"code","metadata":{"id":"zumDLJOY0mCp","executionInfo":{"status":"ok","timestamp":1638493590311,"user_tz":300,"elapsed":300,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["train_norm = train_or.values"],"execution_count":65,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"lp9AUKnp0uNw","executionInfo":{"status":"ok","timestamp":1638493601353,"user_tz":300,"elapsed":293,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"dcc40ca3-7d90-4608-a12b-84d38dc6746e"},"source":["print(train_norm)"],"execution_count":66,"outputs":[{"output_type":"stream","name":"stdout","text":["[[ 5.77178669 0.0768065 2.17615677 ... -0.69013272 -0.03971912\n"," -0.65657302]\n"," [ 5.05177784 -0.07914385 -1.10831977 ... -0.6508944 -0.0368367\n"," -0.57390836]\n"," [ 5.64839125 0.05024753 0.67865392 ... -0.03981535 -0.00704541\n"," 0.46701991]\n"," ...\n"," [ 5.03640413 -0.02382321 -1.74964522 ... -0.76514714 -0.04386997\n"," -0.80245971]\n"," [ 6.17570639 0.10483736 0.60116087 ... 0.1084824 0.07507801\n"," 3.3850238 ]\n"," [ 5.60924435 0.04550669 0.69142603 ... -0.80957877 -0.05158534\n"," -0.98875548]]\n"]}]},{"cell_type":"code","metadata":{"id":"pOCIE4Uf1MU4","executionInfo":{"status":"ok","timestamp":1638493656327,"user_tz":300,"elapsed":277,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["min_max_scaler = preprocessing.MinMaxScaler()"],"execution_count":67,"outputs":[]},{"cell_type":"code","metadata":{"id":"4lqoo8ot1R2L","executionInfo":{"status":"ok","timestamp":1638493683010,"user_tz":300,"elapsed":270,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["x_scaled = min_max_scaler.fit_transform(train_norm)"],"execution_count":68,"outputs":[]},{"cell_type":"code","metadata":{"id":"xPkpRwB21Y-a","executionInfo":{"status":"ok","timestamp":1638494007914,"user_tz":300,"elapsed":291,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["df = pd.DataFrame(x_scaled, columns=train_or.columns)"],"execution_count":72,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"odlVY0ap1d7u","executionInfo":{"status":"ok","timestamp":1638494011276,"user_tz":300,"elapsed":277,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"d4612449-d07a-4a9e-f44c-26c191f788c1"},"source":["print(df)"],"execution_count":73,"outputs":[{"output_type":"stream","name":"stdout","text":[" G26 G26_fd_SMILES_base ... G10_fd_SMILES_base G10_std_SMILES_dist\n","0 0.697589 0.889907 ... 0.348102 0.323693\n","1 0.158524 0.277403 ... 0.361077 0.335169\n","2 0.605204 0.785595 ... 0.495174 0.479687\n","3 0.224112 0.475842 ... 0.829430 0.821281\n","4 0.469960 0.526489 ... 0.546028 0.529540\n","... ... ... ... ... ...\n","61576 0.284904 0.666870 ... 0.908838 0.890156\n","61577 0.242418 0.098607 ... 0.530282 0.519352\n","61578 0.147014 0.494678 ... 0.329418 0.303438\n","61579 1.000000 1.000000 ... 0.864832 0.884808\n","61580 0.575895 0.766975 ... 0.294689 0.277574\n","\n","[61581 rows x 100 columns]\n"]}]},{"cell_type":"code","metadata":{"id":"klCz1EqK23gz"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ryhQ7cffpR1B","executionInfo":{"status":"ok","timestamp":1638492862878,"user_tz":300,"elapsed":295,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["train_norm = preprocessing.Normalizer().fit(train_or)"],"execution_count":61,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DLqEQBWkpl4q","executionInfo":{"status":"ok","timestamp":1638492893855,"user_tz":300,"elapsed":265,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"e14a77a8-020a-499d-db57-348d43563578"},"source":["print(train_norm.transform(train_or))"],"execution_count":63,"outputs":[{"output_type":"stream","name":"stdout","text":["[[ 0.19753808 0.00262868 0.07447847 ... -0.02361963 -0.00135938\n"," -0.02247106]\n"," [ 0.16845295 -0.00263907 -0.03695723 ... -0.02170426 -0.00122833\n"," -0.01913714]\n"," [ 0.18715873 0.00166495 0.02248711 ... -0.00131928 -0.00023345\n"," 0.01547465]\n"," ...\n"," [ 0.16881012 -0.00079851 -0.05864458 ... -0.02564619 -0.00147043\n"," -0.02689683]\n"," [ 0.18062656 0.00306627 0.0175827 ... 0.00317288 0.00219588\n"," 0.0990049 ]\n"," [ 0.18628563 0.0015113 0.02296258 ... -0.02688649 -0.00171317\n"," -0.03283703]]\n"]}]},{"cell_type":"code","metadata":{"id":"S8xXZwqNuMBL","executionInfo":{"status":"ok","timestamp":1638498686328,"user_tz":300,"elapsed":290,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}}},"source":["scaled_df = pd.DataFrame(train_norm, columns=train_or.columns)"],"execution_count":74,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4SwtNsxhId7i","executionInfo":{"status":"ok","timestamp":1638498703797,"user_tz":300,"elapsed":292,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"89f44258-9f40-4883-b492-30c0b53f18d0"},"source":["print(scaled_df)"],"execution_count":75,"outputs":[{"output_type":"stream","name":"stdout","text":[" G26 G26_fd_SMILES_base ... G10_fd_SMILES_base G10_std_SMILES_dist\n","0 5.771787 0.076806 ... -0.039719 -0.656573\n","1 5.051778 -0.079144 ... -0.036837 -0.573908\n","2 5.648391 0.050248 ... -0.007045 0.467020\n","3 5.139381 -0.028619 ... 0.067213 2.927458\n","4 5.467752 -0.015724 ... 0.004252 0.826105\n","... ... ... ... ... ...\n","61576 5.220579 0.020019 ... 0.084854 3.423547\n","61577 5.163831 -0.124667 ... 0.000754 0.752722\n","61578 5.036404 -0.023823 ... -0.043870 -0.802460\n","61579 6.175706 0.104837 ... 0.075078 3.385024\n","61580 5.609244 0.045507 ... -0.051585 -0.988755\n","\n","[61581 rows x 100 columns]\n"]}]},{"cell_type":"code","metadata":{"id":"9R0GKCvAHYLE","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1638487684934,"user_tz":300,"elapsed":626,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"1a666db2-b161-40ad-81d7-7911a8e7946a"},"source":["# Normalization \n","\n","# min max normalization \n","train_mmnorm = train_or\n","for column in train_mmnorm.columns:\n"," col = train_mmnorm[column]\n"," num = col - col.min()\n"," denom = col.max()-col.min()\n"," train_mmnorm[column] = num/denom\n","print(train_mmnorm)\n","\n","# z-score normalization \n","#train_znorm = train_or\n","#for column in train_znorm.columns:\n","# col = train_znorm[column]\n","# num = col - col.mean()\n","# denom = col.std()\n","# train_znorm[column] = num/denom\n","\n","#print(train_znorm)"],"execution_count":23,"outputs":[{"output_type":"stream","name":"stdout","text":[" G26 G26_fd_SMILES_base ... G10_fd_SMILES_base G10_std_SMILES_dist\n","62386 0.697589 0.889907 ... 0.348102 0.323693\n","63350 0.158524 0.277403 ... 0.361077 0.335169\n","8881 0.605204 0.785595 ... 0.495174 0.479687\n","15956 0.224112 0.475842 ... 0.829430 0.821281\n","56441 0.469960 0.526489 ... 0.546028 0.529540\n","... ... ... ... ... ...\n","17916 0.284904 0.666870 ... 0.908838 0.890156\n","8252 0.242418 0.098607 ... 0.530282 0.519352\n","59884 0.147014 0.494678 ... 0.329418 0.303438\n","39791 1.000000 1.000000 ... 0.864832 0.884808\n","6867 0.575895 0.766975 ... 0.294689 0.277574\n","\n","[61581 rows x 100 columns]\n"]}]},{"cell_type":"code","metadata":{"id":"lF4ctUetHYLG"},"source":["# k means classifier \n","k_means = KMeans(init='k-means++', n_clusters=3, n_init=10) # n_clusters based on elbow plots! \n","km_classiier = k_means.fit(train_mmnorm)\n","\n","km_train = k_means.predict(train_mmnorm)\n","\n","train_cluster = train_mmnorm\n","train_cluster['Cluster'] = list(km_train)\n","train_cluster['Label'] = y # if outliers are handeled differently (dropped) this will break "],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"x244IfunHYLI"},"source":["# for test set \n","X = test.drop('Label', axis = 1)\n","y = test['Label']\n","\n","# feature selection fron train\n","test_fs = k_best.fit(X, y)\n","test_fs = train.iloc[:,cols]\n","\n","# outlier replacement using train set \n","#test_or = test_fs[(test_fs>(Q1-1.5*IQR))]\n","#test_or = test_or.fillna(Q1-1.5*IQR)\n","#test_or = test_or[(test_fs<(Q3+1.5*IQR))]\n","#test_or = test_or.fillna(Q3+1.5*IQR)\n","\n","# normalization?\n","# create test_mmnorm\n","\n","km_test = k_means.predict(test_mmnorm)\n","\n","test_cluster = test_mmnorm\n","test_cluster['Cluster'] = list(km_test)\n","test_cluster['Label'] = y\n","\n","cluster1 = test_cluster[test_cluster['Cluster']==0]\n","yc1 = cluster1['Label']\n","Xc1 = cluster1.drop(['Label', 'Cluster'], axis = 1)\n","xgb_c1test = xgb.DMatrix(Xc1, label=yc1)\n","\n","xgb_c1model.predict() \n","#etccccc"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2QeZPEMhHYLJ"},"source":[""],"execution_count":null,"outputs":[]}]} \ No newline at end of file From f82e54ed2c994ceb25d67a00fb7d449ec4e8ef62 Mon Sep 17 00:00:00 2001 From: ZiqiangWang0130 <46898160+ZiqiangWang0130@users.noreply.github.com> Date: Fri, 3 Dec 2021 01:57:53 -0500 Subject: [PATCH 2/2] Add files via upload --- select_feature_using_SPEC.ipynb | 1 + 1 file changed, 1 insertion(+) create mode 100644 select_feature_using_SPEC.ipynb diff --git a/select_feature_using_SPEC.ipynb b/select_feature_using_SPEC.ipynb new file mode 100644 index 0000000..0befca7 --- /dev/null +++ b/select_feature_using_SPEC.ipynb @@ -0,0 +1 @@ +{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"select_feature_using_SPEC.ipynb","provenance":[],"toc_visible":true,"mount_file_id":"1GEzDDTnBnMh75ry3voVsahydH2P9KKnR","authorship_tag":"ABX9TyMOZ1VPEFJu2kuCiV+tuad9"},"kernelspec":{"name":"python2","display_name":"Python 2"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"m6WNKn_e4Q7j","executionInfo":{"status":"ok","timestamp":1638428020145,"user_tz":300,"elapsed":168,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"39d4021d-492b-4a8f-e8c3-416f0ae24b16"},"source":["!ls"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["drive sample_data\n"]}]},{"cell_type":"code","metadata":{"id":"KIP52Hzb4UP0"},"source":["import pandas as pd"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"iUC_QAfs4W16"},"source":["data = pd.read_csv('../data/train_data.csv')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"gHWYHaIH4kz2"},"source":["from skfeature.function.similarity_based import SPEC"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"xgBMqW5y4n74"},"source":["import sklearn"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"IVgbNRTS4qN0"},"source":["from sklearn.model_selection import train_test_split"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"p9aeXlDm4sFS"},"source":["train, test = sklearn.model_selection.train_test_split(data, test_size=.25, random_state=42, shuffle=True, stratify=data['Label'])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"WEDGRjzd44HO"},"source":["train.to_csv('../data/split_train_data.csv', index=False)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GULGyyTE5F9J"},"source":["test.to_csv('../data/split_test_data.csv', index=False)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"dlW50z9_5YW_"},"source":["train_split = pd.read_csv('../data/split_train_data.csv')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"_J915zd-5d1t"},"source":["train_split = train_split.drop('KIBA', axis=1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nlA_uXom5gpA"},"source":["train, test = sklearn.model_selection.train_test_split(train_split, test_size=.25, random_state=42, shuffle=True, stratify=train_split['Label'])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"br7iXxUm5pwG"},"source":["X = train.drop('Label', axis=1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ejumjVLU5qlE"},"source":["y = train['Label']"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fciRjdGq5sdO"},"source":["kwargs = {'style': 0}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nPtJERSd5ucu","colab":{"base_uri":"https://localhost:8080/","height":175},"executionInfo":{"status":"error","timestamp":1638427827185,"user_tz":300,"elapsed":179,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"343920bc-abfe-441e-9cd9-7b270686b30a"},"source":["score = SPEC.spec(X, **kwargs)"],"execution_count":null,"outputs":[{"output_type":"error","ename":"NameError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSPEC\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mNameError\u001b[0m: name 'SPEC' is not defined"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6PvMkRnw6E2p","executionInfo":{"status":"ok","timestamp":1638427816832,"user_tz":300,"elapsed":322,"user":{"displayName":"Ziqiang Wang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiweiPeMwOjHmnxkMvA-huMaqUyaI1DrHObBMhHog=s64","userId":"15738722149016886899"}},"outputId":"9e799e98-4dd2-45d6-aad7-8b15d7f1933d"},"source":["!ls"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["drive sample_data\n"]}]}]} \ No newline at end of file