{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.5"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":13.131641,"end_time":"2024-01-04T21:22:40.769486","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-01-04T21:22:27.637845","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"c42fd27d","cell_type":"code","source":"import pandas as pd\ndf = pd.read_csv(\"train.csv\")\ndf","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:31.029474Z","iopub.status.busy":"2024-01-04T21:22:31.028851Z","iopub.status.idle":"2024-01-04T21:22:31.680985Z","shell.execute_reply":"2024-01-04T21:22:31.679875Z"},"papermill":{"duration":0.670639,"end_time":"2024-01-04T21:22:31.68369","exception":false,"start_time":"2024-01-04T21:22:31.013051","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"31198698","cell_type":"code","source":"# Checking the no. of rows and columns in our dataset\ndf.shape","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:31.712728Z","iopub.status.busy":"2024-01-04T21:22:31.711975Z","iopub.status.idle":"2024-01-04T21:22:31.71921Z","shell.execute_reply":"2024-01-04T21:22:31.71814Z"},"papermill":{"duration":0.024294,"end_time":"2024-01-04T21:22:31.721506","exception":false,"start_time":"2024-01-04T21:22:31.697212","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"c2c1b96c","cell_type":"code","source":"# Checking the no. of unique values in each dataset\ndf.nunique()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:31.750552Z","iopub.status.busy":"2024-01-04T21:22:31.749676Z","iopub.status.idle":"2024-01-04T21:22:31.78697Z","shell.execute_reply":"2024-01-04T21:22:31.785846Z"},"papermill":{"duration":0.054935,"end_time":"2024-01-04T21:22:31.789565","exception":false,"start_time":"2024-01-04T21:22:31.73463","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"46fb20e0","cell_type":"code","source":"# Null Values in Dataset\ndf.isnull().sum()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:31.819641Z","iopub.status.busy":"2024-01-04T21:22:31.818747Z","iopub.status.idle":"2024-01-04T21:22:31.836518Z","shell.execute_reply":"2024-01-04T21:22:31.835304Z"},"papermill":{"duration":0.036011,"end_time":"2024-01-04T21:22:31.839181","exception":false,"start_time":"2024-01-04T21:22:31.80317","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5165cb74","cell_type":"code","source":"df=df.drop(\"density\" , axis=1)\ndf.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"29034116","cell_type":"code","source":"df=df.drop(\"BIRADS\" , axis=1)\ndf.head()","metadata":{},"outputs":[],"execution_count":null},{"id":"e6297c70","cell_type":"code","source":"# Checking the datatype of each column\ndf.info()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:31.870496Z","iopub.status.busy":"2024-01-04T21:22:31.870014Z","iopub.status.idle":"2024-01-04T21:22:31.903429Z","shell.execute_reply":"2024-01-04T21:22:31.901811Z"},"papermill":{"duration":0.052808,"end_time":"2024-01-04T21:22:31.906363","exception":false,"start_time":"2024-01-04T21:22:31.853555","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1da93597","cell_type":"code","source":"# Extracting age column\ndf.iloc[:,5:6]","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:31.936865Z","iopub.status.busy":"2024-01-04T21:22:31.936389Z","iopub.status.idle":"2024-01-04T21:22:31.953151Z","shell.execute_reply":"2024-01-04T21:22:31.951967Z"},"papermill":{"duration":0.03534,"end_time":"2024-01-04T21:22:31.955986","exception":false,"start_time":"2024-01-04T21:22:31.920646","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"67709ae3","cell_type":"code","source":"# check for duplicated data in dataset\ndf.duplicated().sum()","metadata":{},"outputs":[],"execution_count":null},{"id":"ca240780","cell_type":"code","source":"import numpy as np\nfrom sklearn.impute import SimpleImputer\n# Handling Null Values using SimpleImputer\nimputer = SimpleImputer(missing_values=np.nan,strategy=\"mean\")\nimputer.fit(df.iloc[:,5:6].values)\ndf.iloc[:,5:6] = imputer.transform(df.iloc[:,5:6].values)","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:31.985762Z","iopub.status.busy":"2024-01-04T21:22:31.985291Z","iopub.status.idle":"2024-01-04T21:22:33.484743Z","shell.execute_reply":"2024-01-04T21:22:33.483482Z"},"papermill":{"duration":1.517723,"end_time":"2024-01-04T21:22:33.487699","exception":false,"start_time":"2024-01-04T21:22:31.969976","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e851ec23","cell_type":"code","source":"# Changing the datatype\n# Convert image_id and patient_id to string\ndf[\"image_id\"] = df[\"image_id\"].astype(\"str\")\ndf[\"patient_id\"] = df[\"patient_id\"].astype(\"str\")","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:33.5899Z","iopub.status.busy":"2024-01-04T21:22:33.589421Z","iopub.status.idle":"2024-01-04T21:22:33.647062Z","shell.execute_reply":"2024-01-04T21:22:33.646084Z"},"papermill":{"duration":0.075899,"end_time":"2024-01-04T21:22:33.649763","exception":false,"start_time":"2024-01-04T21:22:33.573864","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"392751ca","cell_type":"markdown","source":"*** -0.2594312653716567** suggests a moderate negative correlation \n* Considering the fact that :\n    1. BIRADS has huge number of null values.\n    2. Its not greatly impacting the cancer prediction\n- We are dropping the column ","metadata":{"execution":{"iopub.execute_input":"2024-01-04T14:31:05.717637Z","iopub.status.busy":"2024-01-04T14:31:05.717199Z","iopub.status.idle":"2024-01-04T14:31:05.72598Z","shell.execute_reply":"2024-01-04T14:31:05.724644Z","shell.execute_reply.started":"2024-01-04T14:31:05.717603Z"},"papermill":{"duration":0.013516,"end_time":"2024-01-04T21:22:33.740782","exception":false,"start_time":"2024-01-04T21:22:33.727266","status":"completed"},"tags":[]}},{"id":"63ad73a6","cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:33.922483Z","iopub.status.busy":"2024-01-04T21:22:33.921077Z","iopub.status.idle":"2024-01-04T21:22:33.943623Z","shell.execute_reply":"2024-01-04T21:22:33.942787Z"},"papermill":{"duration":0.040947,"end_time":"2024-01-04T21:22:33.945756","exception":false,"start_time":"2024-01-04T21:22:33.904809","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"f17888e2","cell_type":"markdown","source":"**Cramer's V**\n* Cramér's V is particularly useful when dealing with categorical variables and assessing the strength of association between them.\n* We use the following technique to find the correlation between 'density' and 'cancer' column in our dataset after dropping the null values.","metadata":{"papermill":{"duration":0.014442,"end_time":"2024-01-04T21:22:33.974752","exception":false,"start_time":"2024-01-04T21:22:33.96031","status":"completed"},"tags":[]}},{"id":"016068ef","cell_type":"markdown","source":"* **0.021332267953364684** suggest a weak correlation \n* Considering the fact that :\n    1. BIRADS has huge number of null values.\n    2. Its not greatly impacting the cancer prediction\n- We are dropping the column ","metadata":{"papermill":{"duration":0.014411,"end_time":"2024-01-04T21:22:34.065409","exception":false,"start_time":"2024-01-04T21:22:34.050998","status":"completed"},"tags":[]}},{"id":"68d60112","cell_type":"code","source":"# Dropping unrelated column\ndf.drop(\"site_id\",axis=1,inplace=True)\ndf.head()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:34.143218Z","iopub.status.busy":"2024-01-04T21:22:34.142556Z","iopub.status.idle":"2024-01-04T21:22:34.153967Z","shell.execute_reply":"2024-01-04T21:22:34.152742Z"},"papermill":{"duration":0.030297,"end_time":"2024-01-04T21:22:34.156711","exception":false,"start_time":"2024-01-04T21:22:34.126414","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b1a8480d","cell_type":"code","source":"df.isna().sum()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:34.189485Z","iopub.status.busy":"2024-01-04T21:22:34.189021Z","iopub.status.idle":"2024-01-04T21:22:34.21103Z","shell.execute_reply":"2024-01-04T21:22:34.210289Z"},"papermill":{"duration":0.040189,"end_time":"2024-01-04T21:22:34.213013","exception":false,"start_time":"2024-01-04T21:22:34.172824","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2f2c2e3e","cell_type":"code","source":"# Correlation between various columns in a dataset\nnumeric_df = df.select_dtypes(include=['number'])\nnumeric_df.corr()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:34.245169Z","iopub.status.busy":"2024-01-04T21:22:34.24393Z","iopub.status.idle":"2024-01-04T21:22:34.268755Z","shell.execute_reply":"2024-01-04T21:22:34.267561Z"},"papermill":{"duration":0.043534,"end_time":"2024-01-04T21:22:34.271317","exception":false,"start_time":"2024-01-04T21:22:34.227783","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"964669af","cell_type":"code","source":"import seaborn as sns\nsns.heatmap(numeric_df.corr())","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:34.30401Z","iopub.status.busy":"2024-01-04T21:22:34.303477Z","iopub.status.idle":"2024-01-04T21:22:34.964527Z","shell.execute_reply":"2024-01-04T21:22:34.963341Z"},"papermill":{"duration":0.680514,"end_time":"2024-01-04T21:22:34.967108","exception":false,"start_time":"2024-01-04T21:22:34.286594","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4c11b52b","cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.subplot(1,2,1)\nsns.distplot(df[\"age\"]) \n\nplt.subplot(1,2,2)\nsns.boxplot(df[\"machine_id\"])  ","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:35.000735Z","iopub.status.busy":"2024-01-04T21:22:35.000276Z","iopub.status.idle":"2024-01-04T21:22:35.765331Z","shell.execute_reply":"2024-01-04T21:22:35.764304Z"},"papermill":{"duration":0.784714,"end_time":"2024-01-04T21:22:35.767735","exception":false,"start_time":"2024-01-04T21:22:34.983021","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"130acfa3","cell_type":"code","source":"# Checking for outlier in the dataset\ndf.describe()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:35.803736Z","iopub.status.busy":"2024-01-04T21:22:35.803291Z","iopub.status.idle":"2024-01-04T21:22:35.83738Z","shell.execute_reply":"2024-01-04T21:22:35.836122Z"},"papermill":{"duration":0.054596,"end_time":"2024-01-04T21:22:35.839657","exception":false,"start_time":"2024-01-04T21:22:35.785061","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e1b22843","cell_type":"code","source":"df['age'].value_counts()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:35.875959Z","iopub.status.busy":"2024-01-04T21:22:35.875443Z","iopub.status.idle":"2024-01-04T21:22:35.8873Z","shell.execute_reply":"2024-01-04T21:22:35.886137Z"},"papermill":{"duration":0.033174,"end_time":"2024-01-04T21:22:35.889553","exception":false,"start_time":"2024-01-04T21:22:35.856379","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"0e4ed08f","cell_type":"code","source":"sns.distplot(df['age'])","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:35.924864Z","iopub.status.busy":"2024-01-04T21:22:35.92438Z","iopub.status.idle":"2024-01-04T21:22:36.549881Z","shell.execute_reply":"2024-01-04T21:22:36.548556Z"},"papermill":{"duration":0.646632,"end_time":"2024-01-04T21:22:36.552598","exception":false,"start_time":"2024-01-04T21:22:35.905966","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"d73954d4","cell_type":"markdown","source":"## Detecting and Handling Outlier","metadata":{"papermill":{"duration":0.01685,"end_time":"2024-01-04T21:22:36.586855","exception":false,"start_time":"2024-01-04T21:22:36.570005","status":"completed"},"tags":[]}},{"id":"6789e316","cell_type":"code","source":"sns.boxplot(df['age'])","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:36.626074Z","iopub.status.busy":"2024-01-04T21:22:36.625202Z","iopub.status.idle":"2024-01-04T21:22:36.801568Z","shell.execute_reply":"2024-01-04T21:22:36.800252Z"},"papermill":{"duration":0.200347,"end_time":"2024-01-04T21:22:36.804562","exception":false,"start_time":"2024-01-04T21:22:36.604215","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b4fa9c57","cell_type":"code","source":"Q1 = df['age'].quantile(0.25)\nQ3 = df['age'].quantile(0.75)\nIQR = Q3 - Q1\nmin_value = Q1-1.5*IQR\nmax_value = Q3+1.5*IQR\nprint(\"min_value\",min_value,\" \",\"max_value=\",max_value)","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:36.843732Z","iopub.status.busy":"2024-01-04T21:22:36.843202Z","iopub.status.idle":"2024-01-04T21:22:36.85452Z","shell.execute_reply":"2024-01-04T21:22:36.853166Z"},"papermill":{"duration":0.034406,"end_time":"2024-01-04T21:22:36.857048","exception":false,"start_time":"2024-01-04T21:22:36.822642","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"9245f516","cell_type":"code","source":"df[(df['age']<min_value) | (df['age']>max_value)]","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:36.894908Z","iopub.status.busy":"2024-01-04T21:22:36.894451Z","iopub.status.idle":"2024-01-04T21:22:36.916844Z","shell.execute_reply":"2024-01-04T21:22:36.915517Z"},"papermill":{"duration":0.044491,"end_time":"2024-01-04T21:22:36.919403","exception":false,"start_time":"2024-01-04T21:22:36.874912","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"74232e4b","cell_type":"code","source":"df[df['age']==89]","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:36.959292Z","iopub.status.busy":"2024-01-04T21:22:36.958819Z","iopub.status.idle":"2024-01-04T21:22:36.979412Z","shell.execute_reply":"2024-01-04T21:22:36.978178Z"},"papermill":{"duration":0.044311,"end_time":"2024-01-04T21:22:36.981809","exception":false,"start_time":"2024-01-04T21:22:36.937498","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"a27c29ee","cell_type":"code","source":"df[(df['age']<29)].shape","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:37.021139Z","iopub.status.busy":"2024-01-04T21:22:37.020695Z","iopub.status.idle":"2024-01-04T21:22:37.029031Z","shell.execute_reply":"2024-01-04T21:22:37.028001Z"},"papermill":{"duration":0.030204,"end_time":"2024-01-04T21:22:37.031211","exception":false,"start_time":"2024-01-04T21:22:37.001007","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5a15efc3","cell_type":"markdown","source":"We can observe that there are 3 outliers of age **26,28 and 89** also people of age 26 & 28 are not likely to get affected by cancer","metadata":{"papermill":{"duration":0.018063,"end_time":"2024-01-04T21:22:37.067489","exception":false,"start_time":"2024-01-04T21:22:37.049426","status":"completed"},"tags":[]}},{"id":"bead3cf5","cell_type":"code","source":"df.groupby(\"age\")['cancer'].sum().sort_values(ascending=False)","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:37.106446Z","iopub.status.busy":"2024-01-04T21:22:37.1056Z","iopub.status.idle":"2024-01-04T21:22:37.117174Z","shell.execute_reply":"2024-01-04T21:22:37.116135Z"},"papermill":{"duration":0.03327,"end_time":"2024-01-04T21:22:37.119426","exception":false,"start_time":"2024-01-04T21:22:37.086156","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"24cd52fc","cell_type":"markdown","source":"#### We define bins to create groups for people of each age group","metadata":{"papermill":{"duration":0.018126,"end_time":"2024-01-04T21:22:37.156949","exception":false,"start_time":"2024-01-04T21:22:37.138823","status":"completed"},"tags":[]}},{"id":"c1300665","cell_type":"code","source":"bins=[20,30,40,50,60,70,80,90]\ndf['age-group'] =  pd.cut(df['age'], bins=bins)\nsns.countplot(x='age-group', data=df)\nplt.xlabel('Age Group')\nplt.ylabel('Count')\nplt.title('Distribution of Age Groups')\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:37.197193Z","iopub.status.busy":"2024-01-04T21:22:37.196767Z","iopub.status.idle":"2024-01-04T21:22:37.42244Z","shell.execute_reply":"2024-01-04T21:22:37.421131Z"},"papermill":{"duration":0.24892,"end_time":"2024-01-04T21:22:37.425194","exception":false,"start_time":"2024-01-04T21:22:37.176274","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"895a6092","cell_type":"markdown","source":"#### The age group of people is mostly **'40-70'**","metadata":{"papermill":{"duration":0.018637,"end_time":"2024-01-04T21:22:37.463451","exception":false,"start_time":"2024-01-04T21:22:37.444814","status":"completed"},"tags":[]}},{"id":"550a0c30","cell_type":"code","source":"df.groupby('age')['cancer'].sum()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:37.504271Z","iopub.status.busy":"2024-01-04T21:22:37.50298Z","iopub.status.idle":"2024-01-04T21:22:37.515765Z","shell.execute_reply":"2024-01-04T21:22:37.514526Z"},"papermill":{"duration":0.035831,"end_time":"2024-01-04T21:22:37.518054","exception":false,"start_time":"2024-01-04T21:22:37.482223","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e3947489","cell_type":"code","source":"# Removing people of age group less than 29 from our dataset\ndf = df[(df[\"age\"]>min_value)]","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:37.558604Z","iopub.status.busy":"2024-01-04T21:22:37.558126Z","iopub.status.idle":"2024-01-04T21:22:37.572433Z","shell.execute_reply":"2024-01-04T21:22:37.571478Z"},"papermill":{"duration":0.037717,"end_time":"2024-01-04T21:22:37.57511","exception":false,"start_time":"2024-01-04T21:22:37.537393","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1883a583","cell_type":"code","source":"grouped_df = df.groupby('age-group')['cancer'].count().reset_index()\ngrouped_df.plot(x='age-group',y='cancer',kind='bar')","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:37.616032Z","iopub.status.busy":"2024-01-04T21:22:37.615532Z","iopub.status.idle":"2024-01-04T21:22:37.880042Z","shell.execute_reply":"2024-01-04T21:22:37.878798Z"},"papermill":{"duration":0.288335,"end_time":"2024-01-04T21:22:37.882926","exception":false,"start_time":"2024-01-04T21:22:37.594591","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"d468096f","cell_type":"code","source":"df.groupby(\"age-group\")[\"cancer\"].count().sort_values(ascending=False)","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:37.925063Z","iopub.status.busy":"2024-01-04T21:22:37.924581Z","iopub.status.idle":"2024-01-04T21:22:37.937809Z","shell.execute_reply":"2024-01-04T21:22:37.936717Z"},"papermill":{"duration":0.037236,"end_time":"2024-01-04T21:22:37.940009","exception":false,"start_time":"2024-01-04T21:22:37.902773","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"ea4c9663","cell_type":"markdown","source":"#### We conclude that people of age group **40-70** are more likely to get affected by cancer","metadata":{"papermill":{"duration":0.019357,"end_time":"2024-01-04T21:22:37.979246","exception":false,"start_time":"2024-01-04T21:22:37.959889","status":"completed"},"tags":[]}},{"id":"decbbd21","cell_type":"code","source":"df.groupby(\"implant\")[\"cancer\"].count().sort_values(ascending=False)","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.022569Z","iopub.status.busy":"2024-01-04T21:22:38.022054Z","iopub.status.idle":"2024-01-04T21:22:38.033225Z","shell.execute_reply":"2024-01-04T21:22:38.031845Z"},"papermill":{"duration":0.035511,"end_time":"2024-01-04T21:22:38.035612","exception":false,"start_time":"2024-01-04T21:22:38.000101","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"923a4ef9","cell_type":"code","source":"df.groupby('implant')['cancer'].count().plot(kind='bar')","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.078912Z","iopub.status.busy":"2024-01-04T21:22:38.078427Z","iopub.status.idle":"2024-01-04T21:22:38.249668Z","shell.execute_reply":"2024-01-04T21:22:38.248496Z"},"papermill":{"duration":0.195891,"end_time":"2024-01-04T21:22:38.252446","exception":false,"start_time":"2024-01-04T21:22:38.056555","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b46992f0","cell_type":"code","source":"# No of patients not having implant but cancer positive\nlen(df[(df['implant']==0) & (df['cancer']==1)])","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.295184Z","iopub.status.busy":"2024-01-04T21:22:38.293928Z","iopub.status.idle":"2024-01-04T21:22:38.304563Z","shell.execute_reply":"2024-01-04T21:22:38.303503Z"},"papermill":{"duration":0.034307,"end_time":"2024-01-04T21:22:38.306772","exception":false,"start_time":"2024-01-04T21:22:38.272465","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4c300ded","cell_type":"code","source":"# No of patients having implant and cancer positive\nlen(df[(df['implant']==1) & (df['cancer']==1)])","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.350123Z","iopub.status.busy":"2024-01-04T21:22:38.3497Z","iopub.status.idle":"2024-01-04T21:22:38.358125Z","shell.execute_reply":"2024-01-04T21:22:38.357076Z"},"papermill":{"duration":0.033086,"end_time":"2024-01-04T21:22:38.360126","exception":false,"start_time":"2024-01-04T21:22:38.32704","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"04df5bba","cell_type":"markdown","source":"* We notice that in relation to breast **implant** our data is highly imbalanced because **53200** patients in our training dataset didnot had an implant while only **1477** patients had implant\n* **13 out of 1477** patients who had implant is positive for cancer\n* **1145 out of 53200** patients who did not had implant is negative for cancer","metadata":{"papermill":{"duration":0.020841,"end_time":"2024-01-04T21:22:38.402338","exception":false,"start_time":"2024-01-04T21:22:38.381497","status":"completed"},"tags":[]}},{"id":"b67ee3b9","cell_type":"code","source":"df.groupby(\"biopsy\")[\"cancer\"].count().sort_values(ascending=False)","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.446595Z","iopub.status.busy":"2024-01-04T21:22:38.445795Z","iopub.status.idle":"2024-01-04T21:22:38.456779Z","shell.execute_reply":"2024-01-04T21:22:38.455473Z"},"papermill":{"duration":0.036051,"end_time":"2024-01-04T21:22:38.459302","exception":false,"start_time":"2024-01-04T21:22:38.423251","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"ac2e4f80","cell_type":"code","source":"# No of patients who had biopsy and also tested positive for cancer\nlen(df[(df['cancer']==1) & (df['biopsy']==1)])","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.503491Z","iopub.status.busy":"2024-01-04T21:22:38.501998Z","iopub.status.idle":"2024-01-04T21:22:38.512349Z","shell.execute_reply":"2024-01-04T21:22:38.511309Z"},"papermill":{"duration":0.034577,"end_time":"2024-01-04T21:22:38.514451","exception":false,"start_time":"2024-01-04T21:22:38.479874","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"128a9d30","cell_type":"code","source":"sns.countplot(x='biopsy', hue='cancer', data=df)\nplt.xlabel('Biopsy')\nplt.ylabel('Count')\nplt.title('Biopsy vs. Cancer')\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.557178Z","iopub.status.busy":"2024-01-04T21:22:38.556164Z","iopub.status.idle":"2024-01-04T21:22:38.756891Z","shell.execute_reply":"2024-01-04T21:22:38.755637Z"},"papermill":{"duration":0.225067,"end_time":"2024-01-04T21:22:38.759661","exception":false,"start_time":"2024-01-04T21:22:38.534594","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"067da643","cell_type":"markdown","source":"### We conclude that all patients tested positive for cancer had biopsy","metadata":{"papermill":{"duration":0.021856,"end_time":"2024-01-04T21:22:38.802833","exception":false,"start_time":"2024-01-04T21:22:38.780977","status":"completed"},"tags":[]}},{"id":"921e0ef7","cell_type":"code","source":"df.groupby(\"invasive\")[\"cancer\"].count().sort_values(ascending=False)","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.849311Z","iopub.status.busy":"2024-01-04T21:22:38.848433Z","iopub.status.idle":"2024-01-04T21:22:38.860946Z","shell.execute_reply":"2024-01-04T21:22:38.859653Z"},"papermill":{"duration":0.039043,"end_time":"2024-01-04T21:22:38.863616","exception":false,"start_time":"2024-01-04T21:22:38.824573","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"ac6f6ce1","cell_type":"code","source":"len(df[(df['invasive']==1) & (df['cancer']==1)])","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.909453Z","iopub.status.busy":"2024-01-04T21:22:38.908135Z","iopub.status.idle":"2024-01-04T21:22:38.919302Z","shell.execute_reply":"2024-01-04T21:22:38.918016Z"},"papermill":{"duration":0.036492,"end_time":"2024-01-04T21:22:38.92158","exception":false,"start_time":"2024-01-04T21:22:38.885088","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"c12dc37b","cell_type":"code","source":"len(df[(df['invasive']==0) & (df['cancer']==1)])","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:38.966746Z","iopub.status.busy":"2024-01-04T21:22:38.965612Z","iopub.status.idle":"2024-01-04T21:22:38.975465Z","shell.execute_reply":"2024-01-04T21:22:38.974331Z"},"papermill":{"duration":0.034505,"end_time":"2024-01-04T21:22:38.977621","exception":false,"start_time":"2024-01-04T21:22:38.943116","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"a95b9bf5","cell_type":"code","source":"sns.countplot(x='invasive', hue='cancer', data=df)","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:39.023092Z","iopub.status.busy":"2024-01-04T21:22:39.022335Z","iopub.status.idle":"2024-01-04T21:22:39.233342Z","shell.execute_reply":"2024-01-04T21:22:39.232Z"},"papermill":{"duration":0.236918,"end_time":"2024-01-04T21:22:39.236017","exception":false,"start_time":"2024-01-04T21:22:38.999099","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"3718bcbf","cell_type":"markdown","source":"We conclude:\n* **340 out of 53859** patients having cancer pos has invasive=False\n* **818 out of 818** patients having cancer pos has invasive=True\ni.e in all the patients having cancer,invasive= True","metadata":{"papermill":{"duration":0.021895,"end_time":"2024-01-04T21:22:39.280367","exception":false,"start_time":"2024-01-04T21:22:39.258472","status":"completed"},"tags":[]}},{"id":"f88c1f96","cell_type":"code","source":"# There are 11907 unique patients in the dataset\nlen(df['patient_id'].unique())","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:39.326748Z","iopub.status.busy":"2024-01-04T21:22:39.326311Z","iopub.status.idle":"2024-01-04T21:22:39.337661Z","shell.execute_reply":"2024-01-04T21:22:39.336598Z"},"papermill":{"duration":0.037372,"end_time":"2024-01-04T21:22:39.339593","exception":false,"start_time":"2024-01-04T21:22:39.302221","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b46da525","cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:39.387025Z","iopub.status.busy":"2024-01-04T21:22:39.385845Z","iopub.status.idle":"2024-01-04T21:22:39.392893Z","shell.execute_reply":"2024-01-04T21:22:39.392122Z"},"papermill":{"duration":0.033098,"end_time":"2024-01-04T21:22:39.394791","exception":false,"start_time":"2024-01-04T21:22:39.361693","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"15c10464","cell_type":"code","source":"# Group columns having same patient_id\npatient_summary = df.groupby('patient_id')['image_id'].count().sort_values(ascending=False)\npatient_summary","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:39.442632Z","iopub.status.busy":"2024-01-04T21:22:39.441334Z","iopub.status.idle":"2024-01-04T21:22:39.467138Z","shell.execute_reply":"2024-01-04T21:22:39.466178Z"},"papermill":{"duration":0.052175,"end_time":"2024-01-04T21:22:39.469634","exception":false,"start_time":"2024-01-04T21:22:39.417459","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b2a7c174","cell_type":"code","source":"# There are no duplicate images in our dataset\nlen(df['image_id'].unique())","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:39.516317Z","iopub.status.busy":"2024-01-04T21:22:39.515605Z","iopub.status.idle":"2024-01-04T21:22:39.531614Z","shell.execute_reply":"2024-01-04T21:22:39.530435Z"},"papermill":{"duration":0.041945,"end_time":"2024-01-04T21:22:39.534081","exception":false,"start_time":"2024-01-04T21:22:39.492136","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"ea7ced8c","cell_type":"code","source":"cancer_per_patient = df.groupby(\"patient_id\")['cancer'].max().values\nnegative_count = (cancer_per_patient==0).sum()\npositive_count = (cancer_per_patient==1).sum()\nprint(f'There are {negative_count} no.of unique patients negative with cancer.')\nprint(f'There are {positive_count} no.of unique patients positive with cancer.')","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:39.581206Z","iopub.status.busy":"2024-01-04T21:22:39.580473Z","iopub.status.idle":"2024-01-04T21:22:39.603288Z","shell.execute_reply":"2024-01-04T21:22:39.601945Z"},"papermill":{"duration":0.049742,"end_time":"2024-01-04T21:22:39.60613","exception":false,"start_time":"2024-01-04T21:22:39.556388","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"390a7e39","cell_type":"markdown","source":"**cancer_per_patient**, contains the maximum 'cancer' value for each patient. If a patient has at least one record with 'cancer' equal to 1, the maximum value for that patient will be 1. If all entries for a patient are 0, the maximum value will be 0.","metadata":{"papermill":{"duration":0.022223,"end_time":"2024-01-04T21:22:39.651123","exception":false,"start_time":"2024-01-04T21:22:39.6289","status":"completed"},"tags":[]}},{"id":"3df82ad8","cell_type":"code","source":"# Train-Test Split\ndf_train = pd.read_csv(\"train.csv\")\ndf_test = pd.read_csv(\"test.csv\")\nX = df_train.drop('cancer', axis=1)  \ny = df_train['cancer']  ","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:39.697934Z","iopub.status.busy":"2024-01-04T21:22:39.697198Z","iopub.status.idle":"2024-01-04T21:22:39.813069Z","shell.execute_reply":"2024-01-04T21:22:39.811947Z"},"papermill":{"duration":0.142539,"end_time":"2024-01-04T21:22:39.8161","exception":false,"start_time":"2024-01-04T21:22:39.673561","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2721c737","cell_type":"code","source":"df_test","metadata":{"execution":{"iopub.execute_input":"2024-01-04T21:22:39.865133Z","iopub.status.busy":"2024-01-04T21:22:39.86471Z","iopub.status.idle":"2024-01-04T21:22:39.878635Z","shell.execute_reply":"2024-01-04T21:22:39.87745Z"},"papermill":{"duration":0.041046,"end_time":"2024-01-04T21:22:39.880916","exception":false,"start_time":"2024-01-04T21:22:39.83987","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"3b9dd5d7","cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}