{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## INITIAL OBSERVATIONS FORM THE DATASET\n- All images are in dicom(Digital Imaging and Communications in Medicine) format.\n- total patients in train set are 11914.\n- usually patient will have 4 images but not always.\n- cancer, biopsy, invasive, BIRADS, difficult_negative_case are only provided for train dataset.\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport plotly.express as px\nimport numpy as np                                               \nimport seaborn as sns\nimport matplotlib.pyplot as plt                             ","metadata":{"execution":{"iopub.status.busy":"2022-11-29T05:24:41.290033Z","iopub.execute_input":"2022-11-29T05:24:41.290343Z","iopub.status.idle":"2022-11-29T05:24:41.591715Z","shell.execute_reply.started":"2022-11-29T05:24:41.290317Z","shell.execute_reply":"2022-11-29T05:24:41.590845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-29T05:24:43.970537Z","iopub.execute_input":"2022-11-29T05:24:43.970881Z","iopub.status.idle":"2022-11-29T05:24:44.023989Z","shell.execute_reply.started":"2022-11-29T05:24:43.970854Z","shell.execute_reply":"2022-11-29T05:24:44.02296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T05:24:45.963109Z","iopub.execute_input":"2022-11-29T05:24:45.963484Z","iopub.status.idle":"2022-11-29T05:24:45.978949Z","shell.execute_reply.started":"2022-11-29T05:24:45.963459Z","shell.execute_reply":"2022-11-29T05:24:45.978075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T05:35:39.465696Z","iopub.execute_input":"2022-11-29T05:35:39.466079Z","iopub.status.idle":"2022-11-29T05:35:39.488126Z","shell.execute_reply.started":"2022-11-29T05:35:39.466049Z","shell.execute_reply":"2022-11-29T05:35:39.487341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T05:36:57.926924Z","iopub.execute_input":"2022-11-29T05:36:57.927283Z","iopub.status.idle":"2022-11-29T05:36:57.942766Z","shell.execute_reply.started":"2022-11-29T05:36:57.927254Z","shell.execute_reply":"2022-11-29T05:36:57.941713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 37 missing values in age\n- 28420 missing values in BIRADS column\n- 25236 missing values in denisity column","metadata":{}},{"cell_type":"code","source":"df['cancer'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T05:56:54.739372Z","iopub.execute_input":"2022-11-29T05:56:54.739745Z","iopub.status.idle":"2022-11-29T05:56:54.749707Z","shell.execute_reply.started":"2022-11-29T05:56:54.739716Z","shell.execute_reply":"2022-11-29T05:56:54.748464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### the classes are highly imbalanced we need to take care while we are training the model.","metadata":{}},{"cell_type":"code","source":"df.groupby(['machine_id', 'cancer']).size().reset_index().pivot(columns='machine_id', index='cancer', values=0)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T06:03:20.689253Z","iopub.execute_input":"2022-11-29T06:03:20.690289Z","iopub.status.idle":"2022-11-29T06:03:20.714858Z","shell.execute_reply.started":"2022-11-29T06:03:20.690226Z","shell.execute_reply":"2022-11-29T06:03:20.713831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x ='machine_id',hue = \"cancer\",data = df)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T05:38:47.614664Z","iopub.execute_input":"2022-11-29T05:38:47.615056Z","iopub.status.idle":"2022-11-29T05:38:47.82864Z","shell.execute_reply.started":"2022-11-29T05:38:47.615024Z","shell.execute_reply":"2022-11-29T05:38:47.827801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### most of the patients images are taken from machine id 49 ","metadata":{}},{"cell_type":"code","source":"df.groupby(['density', 'cancer']).size().reset_index().pivot(columns='density', index='cancer', values=0)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T06:04:52.694373Z","iopub.execute_input":"2022-11-29T06:04:52.694756Z","iopub.status.idle":"2022-11-29T06:04:52.713909Z","shell.execute_reply.started":"2022-11-29T06:04:52.694727Z","shell.execute_reply":"2022-11-29T06:04:52.712761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### most of the patients have  mediun(B,C) breast tissue denisity ","metadata":{}},{"cell_type":"code","source":"sns.countplot(x ='density',hue = \"cancer\",data = df)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T05:31:08.001734Z","iopub.execute_input":"2022-11-29T05:31:08.002088Z","iopub.status.idle":"2022-11-29T05:31:08.196813Z","shell.execute_reply.started":"2022-11-29T05:31:08.002058Z","shell.execute_reply":"2022-11-29T05:31:08.19587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" conditions = [(df['age'] <= 40),\n          (df['age'] > 40) & (df['age'] <=50 ),\n          (df['age'] > 50) & (df['age'] <= 60),\n          (df['age'] > 60) & (df['age'] <= 70),\n          (df['age'] > 70) & (df['age'] <= 80),\n        (df['age'] > 80) & (df['age'] <= 90),\n              (df['age'] >= 90)]\n\n      # create a list of the values we want to assign for each condition\nvalues = [\"l_40\",\"40_50\", '50_60', '60_70',\"70_80\",\"80_90\",\"g_90\"]\n\n      # create a new column and use np.select to assign values to it using our lists as arguments\ndf['age_range'] = np.select(conditions, values)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T06:37:00.480324Z","iopub.execute_input":"2022-11-29T06:37:00.480752Z","iopub.status.idle":"2022-11-29T06:37:00.503643Z","shell.execute_reply.started":"2022-11-29T06:37:00.480716Z","shell.execute_reply":"2022-11-29T06:37:00.501721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x ='age_range',hue = \"cancer\",data = df)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T06:37:04.763336Z","iopub.execute_input":"2022-11-29T06:37:04.763718Z","iopub.status.idle":"2022-11-29T06:37:05.027734Z","shell.execute_reply.started":"2022-11-29T06:37:04.763687Z","shell.execute_reply":"2022-11-29T06:37:05.026893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(['age_range', 'cancer']).size().reset_index().pivot(columns='age_range', index='cancer', values=0)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T06:38:00.244736Z","iopub.execute_input":"2022-11-29T06:38:00.245129Z","iopub.status.idle":"2022-11-29T06:38:00.271025Z","shell.execute_reply.started":"2022-11-29T06:38:00.245095Z","shell.execute_reply":"2022-11-29T06:38:00.269556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### patients with age in range 60-70 are have high chance of breast cancer.","metadata":{}},{"cell_type":"code","source":"df.groupby(['implant', 'cancer']).size().reset_index().pivot(columns='implant', index='cancer', values=0)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T06:41:24.016137Z","iopub.execute_input":"2022-11-29T06:41:24.016562Z","iopub.status.idle":"2022-11-29T06:41:24.03505Z","shell.execute_reply.started":"2022-11-29T06:41:24.016526Z","shell.execute_reply":"2022-11-29T06:41:24.033943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(['view', 'cancer']).size().reset_index().pivot(columns='view', index='cancer', values=0)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T06:42:43.462234Z","iopub.execute_input":"2022-11-29T06:42:43.462644Z","iopub.status.idle":"2022-11-29T06:42:43.491Z","shell.execute_reply.started":"2022-11-29T06:42:43.46261Z","shell.execute_reply":"2022-11-29T06:42:43.48934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### the orientation of images are mostly in CC nad MLO","metadata":{}},{"cell_type":"code","source":"df.groupby(['laterality', 'cancer']).size().reset_index().pivot(columns='laterality', index='cancer', values=0)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T06:44:31.383175Z","iopub.execute_input":"2022-11-29T06:44:31.383575Z","iopub.status.idle":"2022-11-29T06:44:31.405016Z","shell.execute_reply.started":"2022-11-29T06:44:31.383541Z","shell.execute_reply":"2022-11-29T06:44:31.403882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"11914-df.groupby(['patient_id', 'cancer']).size().reset_index().pivot(columns='cancer', index='patient_id', values=0)[1].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T06:58:51.862966Z","iopub.execute_input":"2022-11-29T06:58:51.863356Z","iopub.status.idle":"2022-11-29T06:58:51.885189Z","shell.execute_reply.started":"2022-11-29T06:58:51.863324Z","shell.execute_reply":"2022-11-29T06:58:51.884172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 487 patients have breast cancer","metadata":{}},{"cell_type":"markdown","source":"# few resourses which can help: \n- https://pubs.rsna.org/doi/full/10.1148/radiol.2018181371\n- https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6278839/","metadata":{}}]}