{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-12T03:43:40.723406Z","iopub.execute_input":"2022-12-12T03:43:40.724461Z","iopub.status.idle":"2022-12-12T03:44:06.499031Z","shell.execute_reply.started":"2022-12-12T03:43:40.724311Z","shell.execute_reply":"2022-12-12T03:44:06.497817Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing the training dataset","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:06.500756Z","iopub.execute_input":"2022-12-12T03:44:06.501684Z","iopub.status.idle":"2022-12-12T03:44:06.6423Z","shell.execute_reply.started":"2022-12-12T03:44:06.501644Z","shell.execute_reply":"2022-12-12T03:44:06.640982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The shape of the training dataset\ntrain_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:06.643601Z","iopub.execute_input":"2022-12-12T03:44:06.643939Z","iopub.status.idle":"2022-12-12T03:44:06.651064Z","shell.execute_reply.started":"2022-12-12T03:44:06.643903Z","shell.execute_reply":"2022-12-12T03:44:06.649771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Percentage of null elements in each column","metadata":{}},{"cell_type":"code","source":"(train_df.isnull().sum()/train_df.shape[0])*100","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:06.654682Z","iopub.execute_input":"2022-12-12T03:44:06.655948Z","iopub.status.idle":"2022-12-12T03:44:06.678376Z","shell.execute_reply.started":"2022-12-12T03:44:06.655907Z","shell.execute_reply":"2022-12-12T03:44:06.676994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The columns **BIRADS** and **machine_id** has highest percentage of null elements","metadata":{}},{"cell_type":"markdown","source":"# Exploring the site_id column\n**site_id** - ID code for the source hospital","metadata":{}},{"cell_type":"code","source":"train_df[\"site_id\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:06.679735Z","iopub.execute_input":"2022-12-12T03:44:06.680872Z","iopub.status.idle":"2022-12-12T03:44:06.695046Z","shell.execute_reply.started":"2022-12-12T03:44:06.68083Z","shell.execute_reply":"2022-12-12T03:44:06.693676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There two source hospitals in the training dataset\n\nNow looking into the distribution of the source hospital ","metadata":{}},{"cell_type":"code","source":"(train_df[\"site_id\"].value_counts()/train_df[\"site_id\"].shape[0])\\\n.plot(kind=\"bar\",rot=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:06.696785Z","iopub.execute_input":"2022-12-12T03:44:06.697282Z","iopub.status.idle":"2022-12-12T03:44:06.882894Z","shell.execute_reply.started":"2022-12-12T03:44:06.697236Z","shell.execute_reply":"2022-12-12T03:44:06.88161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the samples are from the source hospital 1","metadata":{}},{"cell_type":"markdown","source":"# Exploring the patient_id column\n**patient_id** - ID code for the patient.","metadata":{}},{"cell_type":"code","source":"# Number of patients in the training dataset\ntrain_df[\"patient_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:06.884996Z","iopub.execute_input":"2022-12-12T03:44:06.885507Z","iopub.status.idle":"2022-12-12T03:44:06.895769Z","shell.execute_reply.started":"2022-12-12T03:44:06.885459Z","shell.execute_reply":"2022-12-12T03:44:06.894457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We look into the distribution of number of images taken for each patient","metadata":{}},{"cell_type":"code","source":"train_df[\"patient_id\"].value_counts().unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:06.897076Z","iopub.execute_input":"2022-12-12T03:44:06.897471Z","iopub.status.idle":"2022-12-12T03:44:06.907599Z","shell.execute_reply.started":"2022-12-12T03:44:06.89743Z","shell.execute_reply":"2022-12-12T03:44:06.906414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_bins = 10\nn, bins, patches = plt.hist(train_df[\"patient_id\"].value_counts(), \\\n                            bins=np.arange(4, 14)-0.5, edgecolor='black')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:06.908828Z","iopub.execute_input":"2022-12-12T03:44:06.909173Z","iopub.status.idle":"2022-12-12T03:44:07.055215Z","shell.execute_reply.started":"2022-12-12T03:44:06.909143Z","shell.execute_reply":"2022-12-12T03:44:07.054298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above graph it is quite expected that most of the patients have 4 images taken because the default for a screening exam is to capture two views per breast.","metadata":{}},{"cell_type":"code","source":"train_df[\"image_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.060235Z","iopub.execute_input":"2022-12-12T03:44:07.060884Z","iopub.status.idle":"2022-12-12T03:44:07.071623Z","shell.execute_reply.started":"2022-12-12T03:44:07.060833Z","shell.execute_reply":"2022-12-12T03:44:07.070315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring the laterality column\n**laterality** - Whether the image is of the left or right breast.","metadata":{}},{"cell_type":"code","source":"# Unique values in the laterality column\ntrain_df[\"laterality\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.073227Z","iopub.execute_input":"2022-12-12T03:44:07.074063Z","iopub.status.idle":"2022-12-12T03:44:07.085867Z","shell.execute_reply.started":"2022-12-12T03:44:07.074015Z","shell.execute_reply":"2022-12-12T03:44:07.084563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"laterality\"].value_counts()/train_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.088266Z","iopub.execute_input":"2022-12-12T03:44:07.089068Z","iopub.status.idle":"2022-12-12T03:44:07.101625Z","shell.execute_reply.started":"2022-12-12T03:44:07.08903Z","shell.execute_reply":"2022-12-12T03:44:07.100169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There almost equal percentage of photos clicked for left and right breasts ","metadata":{}},{"cell_type":"markdown","source":"# Exploring the view column\n**view** - The orientation of the image. The default for a screening exam is to capture two views per breast.","metadata":{}},{"cell_type":"code","source":"# Number unique values in the laterality column\ntrain_df[\"view\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.103169Z","iopub.execute_input":"2022-12-12T03:44:07.103618Z","iopub.status.idle":"2022-12-12T03:44:07.118629Z","shell.execute_reply.started":"2022-12-12T03:44:07.103583Z","shell.execute_reply":"2022-12-12T03:44:07.117414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"view\"].value_counts()/train_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.120114Z","iopub.execute_input":"2022-12-12T03:44:07.120571Z","iopub.status.idle":"2022-12-12T03:44:07.134999Z","shell.execute_reply.started":"2022-12-12T03:44:07.120513Z","shell.execute_reply":"2022-12-12T03:44:07.13395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observe that most of the images are clicked in MLO (mediolateral oblique) and CC (cranial caudal). The MLO view is taken from the center of the chest outward, while the CC view is taken from above the breast. For more information click [here](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6113143/#:~:text=In%20screening%20digital%20mammography%2C%20each,taken%20from%20above%20the%20breast.)","metadata":{}},{"cell_type":"markdown","source":"# Exploring the age column\n**age** - The patient's age in years.","metadata":{}},{"cell_type":"code","source":"train_df[\"age\"].plot.hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.136899Z","iopub.execute_input":"2022-12-12T03:44:07.137607Z","iopub.status.idle":"2022-12-12T03:44:07.492416Z","shell.execute_reply.started":"2022-12-12T03:44:07.137569Z","shell.execute_reply":"2022-12-12T03:44:07.491368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"age\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.494283Z","iopub.execute_input":"2022-12-12T03:44:07.494813Z","iopub.status.idle":"2022-12-12T03:44:07.504152Z","shell.execute_reply.started":"2022-12-12T03:44:07.494765Z","shell.execute_reply":"2022-12-12T03:44:07.502755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The mean age of the patients are around 58 years.","metadata":{}},{"cell_type":"markdown","source":"# Exploring the implant category\n**implant** - Whether or not the patient had breast implants. Site 1 only provides breast implant information at the patient level, not at the breast level.","metadata":{}},{"cell_type":"code","source":"train_df[\"implant\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.505213Z","iopub.execute_input":"2022-12-12T03:44:07.505611Z","iopub.status.idle":"2022-12-12T03:44:07.514077Z","shell.execute_reply.started":"2022-12-12T03:44:07.505579Z","shell.execute_reply":"2022-12-12T03:44:07.512805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_df[\"implant\"].value_counts()/train_df[\"implant\"].shape[0])\\\n.plot(kind=\"bar\",rot=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.515332Z","iopub.execute_input":"2022-12-12T03:44:07.515806Z","iopub.status.idle":"2022-12-12T03:44:07.764207Z","shell.execute_reply.started":"2022-12-12T03:44:07.51577Z","shell.execute_reply":"2022-12-12T03:44:07.763139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the patients does not have any implant.","metadata":{}},{"cell_type":"markdown","source":"# Exploring the density column\n**density** - A rating for how dense the breast tissue is, with A being the least dense and D being the most dense. Extremely dense tissue can make diagnosis more difficult. *Only provided for train*.","metadata":{}},{"cell_type":"code","source":"train_df[\"density\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.765862Z","iopub.execute_input":"2022-12-12T03:44:07.768331Z","iopub.status.idle":"2022-12-12T03:44:07.779579Z","shell.execute_reply.started":"2022-12-12T03:44:07.768275Z","shell.execute_reply":"2022-12-12T03:44:07.778196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_df[\"density\"].value_counts()/train_df[\"density\"].shape[0])\\\n.plot(kind=\"bar\",rot=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.78105Z","iopub.execute_input":"2022-12-12T03:44:07.782028Z","iopub.status.idle":"2022-12-12T03:44:07.924649Z","shell.execute_reply.started":"2022-12-12T03:44:07.781977Z","shell.execute_reply":"2022-12-12T03:44:07.923342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the patient have tissue density type B and C.","metadata":{}},{"cell_type":"markdown","source":"# Exploring the machine_id column\n**machine_id** - An ID code for the imaging device","metadata":{}},{"cell_type":"code","source":"train_df[\"machine_id\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.926247Z","iopub.execute_input":"2022-12-12T03:44:07.9275Z","iopub.status.idle":"2022-12-12T03:44:07.938872Z","shell.execute_reply.started":"2022-12-12T03:44:07.927442Z","shell.execute_reply":"2022-12-12T03:44:07.937682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_df[\"machine_id\"].value_counts()/train_df[\"machine_id\"].shape[0])\\\n.plot(kind=\"bar\",rot=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:07.940441Z","iopub.execute_input":"2022-12-12T03:44:07.941321Z","iopub.status.idle":"2022-12-12T03:44:08.119907Z","shell.execute_reply.started":"2022-12-12T03:44:07.941274Z","shell.execute_reply":"2022-12-12T03:44:08.118556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring the cancer column\n**cancer** - Whether or not the breast was positive for cancer. The target value. *Only provided for train*.","metadata":{}},{"cell_type":"code","source":"train_df[\"cancer\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.121606Z","iopub.execute_input":"2022-12-12T03:44:08.122402Z","iopub.status.idle":"2022-12-12T03:44:08.129864Z","shell.execute_reply.started":"2022-12-12T03:44:08.122335Z","shell.execute_reply":"2022-12-12T03:44:08.12895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_df[\"cancer\"].value_counts()/train_df[\"cancer\"].shape[0])\\\n.plot(kind=\"bar\",rot=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.130969Z","iopub.execute_input":"2022-12-12T03:44:08.132116Z","iopub.status.idle":"2022-12-12T03:44:08.272489Z","shell.execute_reply.started":"2022-12-12T03:44:08.13208Z","shell.execute_reply":"2022-12-12T03:44:08.271258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the patients in this dataset does not have cancer.\n![alt text](https://media.tenor.com/RRwrXR8TkpYAAAAC/thank-god-robert-downey-jr.gif)","metadata":{}},{"cell_type":"code","source":"train_df[\"cancer\"].value_counts()/train_df[\"cancer\"].shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.27427Z","iopub.execute_input":"2022-12-12T03:44:08.275691Z","iopub.status.idle":"2022-12-12T03:44:08.288374Z","shell.execute_reply.started":"2022-12-12T03:44:08.275644Z","shell.execute_reply":"2022-12-12T03:44:08.287088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looking into the age distribution of patients in this dataset who are detected with cancer","metadata":{}},{"cell_type":"code","source":"train_df[train_df[\"cancer\"]==1][\"age\"].plot.hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.289881Z","iopub.execute_input":"2022-12-12T03:44:08.290601Z","iopub.status.idle":"2022-12-12T03:44:08.583696Z","shell.execute_reply.started":"2022-12-12T03:44:08.290553Z","shell.execute_reply":"2022-12-12T03:44:08.582126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df[\"cancer\"]==1][\"age\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.585442Z","iopub.execute_input":"2022-12-12T03:44:08.585943Z","iopub.status.idle":"2022-12-12T03:44:08.596092Z","shell.execute_reply.started":"2022-12-12T03:44:08.585893Z","shell.execute_reply":"2022-12-12T03:44:08.594982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Mean age of patients of the detected cancer are around 63 years.","metadata":{}},{"cell_type":"markdown","source":"# Exploring the biopsy column\n**biopsy** - Whether or not a follow-up biopsy was performed on the breast. *Only provided for train*.","metadata":{}},{"cell_type":"code","source":"train_df[\"biopsy\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.602182Z","iopub.execute_input":"2022-12-12T03:44:08.602725Z","iopub.status.idle":"2022-12-12T03:44:08.611705Z","shell.execute_reply.started":"2022-12-12T03:44:08.602686Z","shell.execute_reply":"2022-12-12T03:44:08.610103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_df[\"biopsy\"].value_counts()/train_df[\"biopsy\"].shape[0])\\\n.plot(kind=\"bar\",rot=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.613578Z","iopub.execute_input":"2022-12-12T03:44:08.614093Z","iopub.status.idle":"2022-12-12T03:44:08.805595Z","shell.execute_reply.started":"2022-12-12T03:44:08.614056Z","shell.execute_reply":"2022-12-12T03:44:08.804431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pct = lambda x: 100 * x/x.sum()\ntrain_df[[\"cancer\", \"biopsy\"]].groupby([\"cancer\", \"biopsy\"]).size().to_frame(\"Count\")\\\n.groupby(\"cancer\").apply(pct)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T05:32:14.289934Z","iopub.execute_input":"2022-12-12T05:32:14.293315Z","iopub.status.idle":"2022-12-12T05:32:14.367631Z","shell.execute_reply.started":"2022-12-12T05:32:14.293254Z","shell.execute_reply":"2022-12-12T05:32:14.366698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From above we observe that about 3.4% of the patients who were declared negative cancer result, were sent for biopsy. ","metadata":{}},{"cell_type":"markdown","source":"# Exploring the invasive column\n**invasive** - If the breast is positive for cancer, whether or not the cancer proved to be invasive. *Only provided for train*.","metadata":{}},{"cell_type":"code","source":"train_df[\"invasive\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.829514Z","iopub.execute_input":"2022-12-12T03:44:08.830209Z","iopub.status.idle":"2022-12-12T03:44:08.839714Z","shell.execute_reply.started":"2022-12-12T03:44:08.830163Z","shell.execute_reply":"2022-12-12T03:44:08.83852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_df[\"invasive\"].value_counts()/train_df[\"invasive\"].shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.84138Z","iopub.execute_input":"2022-12-12T03:44:08.842506Z","iopub.status.idle":"2022-12-12T03:44:08.856079Z","shell.execute_reply.started":"2022-12-12T03:44:08.842465Z","shell.execute_reply":"2022-12-12T03:44:08.854733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"cancer\", \"invasive\"]].groupby([\"cancer\", \"invasive\"]).size().to_frame(\"Count\")\\\n.groupby(\"cancer\").apply(pct)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T05:47:56.001442Z","iopub.execute_input":"2022-12-12T05:47:56.004317Z","iopub.status.idle":"2022-12-12T05:47:56.076487Z","shell.execute_reply.started":"2022-12-12T05:47:56.004074Z","shell.execute_reply":"2022-12-12T05:47:56.072675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Out of the patients who were detected cancer, 70% of them have invasive cancer.","metadata":{}},{"cell_type":"code","source":"train_df[[\"cancer\", \"BIRADS\"]].groupby([\"cancer\", \"BIRADS\"]).size()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.877195Z","iopub.execute_input":"2022-12-12T03:44:08.877585Z","iopub.status.idle":"2022-12-12T03:44:08.894236Z","shell.execute_reply.started":"2022-12-12T03:44:08.877548Z","shell.execute_reply":"2022-12-12T03:44:08.893342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring the difficult_negative_case column\n**difficult_negative_case** - True if the case was unusually difficult. *Only provided for train*.","metadata":{}},{"cell_type":"code","source":"train_df[\"difficult_negative_case\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.895642Z","iopub.execute_input":"2022-12-12T03:44:08.896012Z","iopub.status.idle":"2022-12-12T03:44:08.905791Z","shell.execute_reply.started":"2022-12-12T03:44:08.895979Z","shell.execute_reply":"2022-12-12T03:44:08.904679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"density\", \\\n          \"difficult_negative_case\"]].groupby([\"density\", \\\n                                               \"difficult_negative_case\"]).size().to_frame(\"Count\")\\\n.groupby(\"density\").apply(pct)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T05:56:38.473759Z","iopub.execute_input":"2022-12-12T05:56:38.474321Z","iopub.status.idle":"2022-12-12T05:56:38.511903Z","shell.execute_reply.started":"2022-12-12T05:56:38.474275Z","shell.execute_reply":"2022-12-12T05:56:38.510595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"cancer\", \\\n          \"difficult_negative_case\"]].groupby([\"cancer\", \\\n                                               \"difficult_negative_case\"]).size().to_frame(\"Count\")\\\n.groupby(\"cancer\").apply(pct)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T05:55:34.834078Z","iopub.execute_input":"2022-12-12T05:55:34.835319Z","iopub.status.idle":"2022-12-12T05:55:34.870495Z","shell.execute_reply.started":"2022-12-12T05:55:34.835265Z","shell.execute_reply":"2022-12-12T05:55:34.869185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring the dicom format files of the training dataset\nTo this end, we use the library `pydicom` for exploring the image files","metadata":{}},{"cell_type":"code","source":"import pydicom\n# Currently importing a single file\nds = pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10006/1459541791.dcm\")\n# The number of pixels in each frame\nrows = ds.Rows\ncolumns = ds.Columns\nprint(rows, columns)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:08.948866Z","iopub.execute_input":"2022-12-12T03:44:08.949963Z","iopub.status.idle":"2022-12-12T03:44:09.147475Z","shell.execute_reply.started":"2022-12-12T03:44:08.949908Z","shell.execute_reply":"2022-12-12T03:44:09.146107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Looking into the file meta data\nds.file_meta","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:09.149153Z","iopub.execute_input":"2022-12-12T03:44:09.150441Z","iopub.status.idle":"2022-12-12T03:44:09.160017Z","shell.execute_reply.started":"2022-12-12T03:44:09.150384Z","shell.execute_reply":"2022-12-12T03:44:09.158776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# displaying the image\nplt.imshow(ds.pixel_array)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:09.161641Z","iopub.execute_input":"2022-12-12T03:44:09.161991Z","iopub.status.idle":"2022-12-12T03:44:12.702135Z","shell.execute_reply.started":"2022-12-12T03:44:09.161961Z","shell.execute_reply":"2022-12-12T03:44:12.700828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observe most parts of the image has lot of background, and breast occupies only a small portion of the image. As stated in notebook given [here](https://www.kaggle.com/code/radek1/eda-training-a-fast-ai-model-submission), most likely be a very good idea to find ways of cropping out the portions of the image htat do not contain any information! The divider line can serve an important role here.","metadata":{}},{"cell_type":"code","source":"# Tpe of view of the above image\ntrain_df.loc[(train_df[\"patient_id\"]==10006) & \\\n             (train_df[\"image_id\"]==1459541791), [\"view\", \"cancer\"]]","metadata":{"execution":{"iopub.status.busy":"2022-12-12T06:23:55.111403Z","iopub.execute_input":"2022-12-12T06:23:55.111895Z","iopub.status.idle":"2022-12-12T06:23:55.126403Z","shell.execute_reply.started":"2022-12-12T06:23:55.111854Z","shell.execute_reply":"2022-12-12T06:23:55.124816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above image is the MLO view, which reflects more of the breast in the upper-outer quadrant, giving the best view of the lateral side of the breast, which statistically is the most common place for pathological changes.","metadata":{}},{"cell_type":"code","source":"# The size of the above image in MBs\nos.path.getsize(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10006/1459541791.dcm\")/10**6","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:12.703979Z","iopub.execute_input":"2022-12-12T03:44:12.704348Z","iopub.status.idle":"2022-12-12T03:44:12.712899Z","shell.execute_reply.started":"2022-12-12T03:44:12.704315Z","shell.execute_reply":"2022-12-12T03:44:12.711219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating a column `image_destination`, which will store the destination of the training image folders","metadata":{}},{"cell_type":"code","source":"train_df[\"image_destination\"] = \"/kaggle/input/rsna-breast-cancer-detection/train_images/\" + \\\ntrain_df[\"patient_id\"].astype(str) + \"/\" + train_df[\"image_id\"].astype(str) + \".dcm\"\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:12.714433Z","iopub.execute_input":"2022-12-12T03:44:12.714813Z","iopub.status.idle":"2022-12-12T03:44:12.846395Z","shell.execute_reply.started":"2022-12-12T03:44:12.714779Z","shell.execute_reply":"2022-12-12T03:44:12.845155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculating the Height, width and Size (in MBs) for all the dicom images.","metadata":{}},{"cell_type":"code","source":"for idx in train_df.index.tolist():\n    destination = train_df.loc[idx, \"image_destination\"]\n    ds = pydicom.dcmread(destination)\n    # The number of pixels in each frame\n    rows = ds.Rows\n    columns = ds.Columns\n    mbs = os.path.getsize(destination)/10**6\n    train_df.loc[idx, \"Height\"] = rows\n    train_df.loc[idx, \"Width\"] = columns\n    train_df.loc[idx, \"Size\"] = mbs\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T03:44:12.847913Z","iopub.execute_input":"2022-12-12T03:44:12.848568Z","iopub.status.idle":"2022-12-12T04:34:10.697308Z","shell.execute_reply.started":"2022-12-12T03:44:12.848524Z","shell.execute_reply":"2022-12-12T04:34:10.694069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring the Height column","metadata":{}},{"cell_type":"code","source":"sorted(train_df[\"Height\"].unique())","metadata":{"execution":{"iopub.status.busy":"2022-12-12T06:37:22.795246Z","iopub.execute_input":"2022-12-12T06:37:22.796018Z","iopub.status.idle":"2022-12-12T06:37:22.804753Z","shell.execute_reply.started":"2022-12-12T06:37:22.795972Z","shell.execute_reply":"2022-12-12T06:37:22.803656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"Height\"].value_counts().plot(kind=\"bar\", rot=0, figsize=(15, 10))","metadata":{"execution":{"iopub.status.busy":"2022-12-12T06:31:38.586376Z","iopub.execute_input":"2022-12-12T06:31:38.586864Z","iopub.status.idle":"2022-12-12T06:31:38.852256Z","shell.execute_reply.started":"2022-12-12T06:31:38.586826Z","shell.execute_reply":"2022-12-12T06:31:38.849188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From above we observe that Height of the images are varying and most of them has height of 4096.","metadata":{}},{"cell_type":"markdown","source":"# Exploring the Width column","metadata":{}},{"cell_type":"code","source":"sorted(train_df[\"Width\"].unique())","metadata":{"execution":{"iopub.status.busy":"2022-12-12T06:37:08.900563Z","iopub.execute_input":"2022-12-12T06:37:08.901056Z","iopub.status.idle":"2022-12-12T06:37:08.911148Z","shell.execute_reply.started":"2022-12-12T06:37:08.901021Z","shell.execute_reply":"2022-12-12T06:37:08.909784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"Width\"].value_counts().plot(kind=\"bar\", rot=0, figsize=(15, 10))","metadata":{"execution":{"iopub.status.busy":"2022-12-12T06:37:50.865084Z","iopub.execute_input":"2022-12-12T06:37:50.86743Z","iopub.status.idle":"2022-12-12T06:37:51.251252Z","shell.execute_reply.started":"2022-12-12T06:37:50.867303Z","shell.execute_reply":"2022-12-12T06:37:51.248978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From above we observe that Width of the images are varying and most of them has height of 3328.","metadata":{}},{"cell_type":"markdown","source":"# Exploring the Size column","metadata":{}},{"cell_type":"code","source":"min(train_df[\"Size\"]), max(train_df[\"Size\"])","metadata":{"execution":{"iopub.status.busy":"2022-12-12T06:40:14.135554Z","iopub.execute_input":"2022-12-12T06:40:14.136018Z","iopub.status.idle":"2022-12-12T06:40:14.155262Z","shell.execute_reply.started":"2022-12-12T06:40:14.135977Z","shell.execute_reply":"2022-12-12T06:40:14.153622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Minimum image size is 0.32 MB and maximum size of an image is 23 MB","metadata":{}},{"cell_type":"code","source":"train_df[\"Size\"].plot.hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T06:39:47.015075Z","iopub.execute_input":"2022-12-12T06:39:47.016249Z","iopub.status.idle":"2022-12-12T06:39:47.316556Z","shell.execute_reply.started":"2022-12-12T06:39:47.016202Z","shell.execute_reply":"2022-12-12T06:39:47.315328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"Size\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T06:41:20.399827Z","iopub.execute_input":"2022-12-12T06:41:20.401264Z","iopub.status.idle":"2022-12-12T06:41:20.410697Z","shell.execute_reply.started":"2022-12-12T06:41:20.401217Z","shell.execute_reply":"2022-12-12T06:41:20.409502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"On an average the size of the images are around 5.75 MB","metadata":{}},{"cell_type":"code","source":"train_df.to_csv(\"rsna_training_modified_dataset.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T06:48:29.814914Z","iopub.execute_input":"2022-12-12T06:48:29.815475Z","iopub.status.idle":"2022-12-12T06:48:30.293997Z","shell.execute_reply.started":"2022-12-12T06:48:29.815432Z","shell.execute_reply":"2022-12-12T06:48:30.292596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}