{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA Breast Cancer Detection - EDA\n\n# Imports\n## Packages","metadata":{}},{"cell_type":"code","source":"!pip install -U pylibjpeg pylibjpeg-openjpeg pylibjpeg-libjpeg pydicom python-gdcm","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:08.896688Z","iopub.execute_input":"2022-12-22T15:20:08.897112Z","iopub.status.idle":"2022-12-22T15:20:28.292863Z","shell.execute_reply.started":"2022-12-22T15:20:08.89708Z","shell.execute_reply":"2022-12-22T15:20:28.291401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport pandas as pd\n\nimport pydicom\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.29546Z","iopub.execute_input":"2022-12-22T15:20:28.295855Z","iopub.status.idle":"2022-12-22T15:20:28.302168Z","shell.execute_reply.started":"2022-12-22T15:20:28.295813Z","shell.execute_reply":"2022-12-22T15:20:28.30081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Options","metadata":{}},{"cell_type":"code","source":"sns.set_theme(style=\"ticks\")","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.303632Z","iopub.execute_input":"2022-12-22T15:20:28.304014Z","iopub.status.idle":"2022-12-22T15:20:28.318128Z","shell.execute_reply.started":"2022-12-22T15:20:28.303949Z","shell.execute_reply":"2022-12-22T15:20:28.316761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_data = '/kaggle/input/rsna-breast-cancer-detection'\nimage_path = os.path.join(path_data, 'train_images')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.321754Z","iopub.execute_input":"2022-12-22T15:20:28.322251Z","iopub.status.idle":"2022-12-22T15:20:28.330876Z","shell.execute_reply.started":"2022-12-22T15:20:28.322213Z","shell.execute_reply":"2022-12-22T15:20:28.329591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Datasets","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(path_data, 'train.csv'))","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.333119Z","iopub.execute_input":"2022-12-22T15:20:28.333583Z","iopub.status.idle":"2022-12-22T15:20:28.418414Z","shell.execute_reply.started":"2022-12-22T15:20:28.333546Z","shell.execute_reply":"2022-12-22T15:20:28.41721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyze\n\n## Overall analysis","metadata":{}},{"cell_type":"code","source":"df_train.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.419967Z","iopub.execute_input":"2022-12-22T15:20:28.420297Z","iopub.status.idle":"2022-12-22T15:20:28.442663Z","shell.execute_reply.started":"2022-12-22T15:20:28.420266Z","shell.execute_reply":"2022-12-22T15:20:28.441423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.444266Z","iopub.execute_input":"2022-12-22T15:20:28.444709Z","iopub.status.idle":"2022-12-22T15:20:28.452166Z","shell.execute_reply.started":"2022-12-22T15:20:28.444664Z","shell.execute_reply":"2022-12-22T15:20:28.450974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.453709Z","iopub.execute_input":"2022-12-22T15:20:28.454169Z","iopub.status.idle":"2022-12-22T15:20:28.491007Z","shell.execute_reply.started":"2022-12-22T15:20:28.454123Z","shell.execute_reply":"2022-12-22T15:20:28.489683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, we have some missing values for the features *age*, *BIRADS* and *density*.","metadata":{}},{"cell_type":"code","source":"df_train[['age']].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.492502Z","iopub.execute_input":"2022-12-22T15:20:28.492877Z","iopub.status.idle":"2022-12-22T15:20:28.515601Z","shell.execute_reply.started":"2022-12-22T15:20:28.492839Z","shell.execute_reply":"2022-12-22T15:20:28.514342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[['laterality', 'view', 'density', 'difficult_negative_case']].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.52081Z","iopub.execute_input":"2022-12-22T15:20:28.52158Z","iopub.status.idle":"2022-12-22T15:20:28.56259Z","shell.execute_reply.started":"2022-12-22T15:20:28.521536Z","shell.execute_reply":"2022-12-22T15:20:28.561347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of unique id sites:', df_train.site_id.nunique())\nprint('Number of unique id patients:', df_train.patient_id.nunique())\nprint('Number of unique id images:', df_train.image_id.nunique())\nprint('Number of unique id machines:', df_train.machine_id.nunique())\ndf_train[['site_id', 'patient_id', 'image_id', 'machine_id']].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.564425Z","iopub.execute_input":"2022-12-22T15:20:28.564952Z","iopub.status.idle":"2022-12-22T15:20:28.607215Z","shell.execute_reply.started":"2022-12-22T15:20:28.564874Z","shell.execute_reply":"2022-12-22T15:20:28.605907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Patients\n\nWe analyze patients","metadata":{}},{"cell_type":"code","source":"print('The number of different patients is:', df_train.patient_id.nunique())","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.608723Z","iopub.execute_input":"2022-12-22T15:20:28.609204Z","iopub.status.idle":"2022-12-22T15:20:28.61722Z","shell.execute_reply.started":"2022-12-22T15:20:28.609157Z","shell.execute_reply":"2022-12-22T15:20:28.615798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby(by='patient_id').age.nunique().unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.619031Z","iopub.execute_input":"2022-12-22T15:20:28.619552Z","iopub.status.idle":"2022-12-22T15:20:28.642374Z","shell.execute_reply.started":"2022-12-22T15:20:28.619502Z","shell.execute_reply":"2022-12-22T15:20:28.641124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that each patient has a unique age (or unknown)","metadata":{}},{"cell_type":"code","source":"_, ax = plt.subplots(figsize=(18, 8))\ndf_patient_unique = df_train[~(df_train.age.isna())].groupby(by='patient_id').age.agg(pd.Series.mode)\n\nsns.histplot(df_patient_unique, bins=60)\nplt.title('Distribution of the age of patients')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:28.643634Z","iopub.execute_input":"2022-12-22T15:20:28.644125Z","iopub.status.idle":"2022-12-22T15:20:30.557584Z","shell.execute_reply.started":"2022-12-22T15:20:28.644077Z","shell.execute_reply":"2022-12-22T15:20:30.556278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Images","metadata":{}},{"cell_type":"code","source":"df_image_per_patient = df_train.groupby(by='patient_id').image_id.count().reset_index(drop=False).groupby(by='image_id').count().reset_index(drop=False)\ndf_image_per_patient.columns = ['n_images', 'count_patients']\n# df_image_per_patient\n_, ax = plt.subplots(figsize=(12, 8))\nsns.barplot(data=df_image_per_patient, x='n_images', y='count_patients', ax=ax)\nax.set_title('Distribution of the number of images per patient')\nax.set_xlabel('Number of images')\nax.set_ylabel('Count of patients')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:30.559004Z","iopub.execute_input":"2022-12-22T15:20:30.559371Z","iopub.status.idle":"2022-12-22T15:20:30.853843Z","shell.execute_reply.started":"2022-12-22T15:20:30.559339Z","shell.execute_reply":"2022-12-22T15:20:30.852659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax = plt.subplots(2,2,figsize=(16,10))\nsns.countplot(data=df_train, x='laterality', ax=ax[0,0], palette='Greens_r')\nsns.countplot(data=df_train, x='view', ax=ax[0,1], palette='Reds_r')\nsns.countplot(data=df_train, x='implant', ax=ax[1,0], palette='Blues_r')\nsns.countplot(data=df_train, x='density', ax=ax[1,1], palette='Purples_r');\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:30.855534Z","iopub.execute_input":"2022-12-22T15:20:30.85932Z","iopub.status.idle":"2022-12-22T15:20:31.456153Z","shell.execute_reply.started":"2022-12-22T15:20:30.859263Z","shell.execute_reply":"2022-12-22T15:20:31.454871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Target","metadata":{}},{"cell_type":"code","source":"df_train.cancer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:31.457916Z","iopub.execute_input":"2022-12-22T15:20:31.458625Z","iopub.status.idle":"2022-12-22T15:20:31.469261Z","shell.execute_reply.started":"2022-12-22T15:20:31.458584Z","shell.execute_reply":"2022-12-22T15:20:31.467689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, our dataset is **highly imbalanced**. It should be taken into account when preparing the dataset and training models.","metadata":{}},{"cell_type":"code","source":"sns.pairplot(df_train[['age', 'biopsy', 'invasive', 'BIRADS', 'cancer']], hue='cancer')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:20:31.471117Z","iopub.execute_input":"2022-12-22T15:20:31.471486Z","iopub.status.idle":"2022-12-22T15:21:09.524817Z","shell.execute_reply.started":"2022-12-22T15:20:31.471454Z","shell.execute_reply":"2022-12-22T15:21:09.523987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.violinplot(data=df_train, x='cancer', y='age')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:21:09.526155Z","iopub.execute_input":"2022-12-22T15:21:09.526907Z","iopub.status.idle":"2022-12-22T15:21:09.927729Z","shell.execute_reply.started":"2022-12-22T15:21:09.52687Z","shell.execute_reply":"2022-12-22T15:21:09.926728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Distribution of the parameter invasive of the cancer:')\ndf_train[df_train.cancer == 1].invasive.value_counts() / df_train[df_train.cancer == 1].shape[0] * 100","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:21:09.929105Z","iopub.execute_input":"2022-12-22T15:21:09.930272Z","iopub.status.idle":"2022-12-22T15:21:09.943968Z","shell.execute_reply.started":"2022-12-22T15:21:09.930231Z","shell.execute_reply":"2022-12-22T15:21:09.942731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot some images","metadata":{}},{"cell_type":"code","source":"def get_images_one_patient(path_to_images: str, patient_id: str):\n    \"\"\"Get all images of one patient\n    \n    Args:\n        path_to_images: Path to the images repository\n        patient_id: Id of the patient to extract images\n    \"\"\"\n    patient_path_image = os.path.join(path_to_images, str(patient_id))\n    slices = [pydicom.dcmread(os.path.join(patient_path_image, file), force=True) for file in os.listdir(patient_path_image)]\n    return slices\n\n\ndef plot_all_scans_one_patient(scans_patient: list) -> None:\n    \"\"\"\n    \"\"\"\n    n_rows = len(scans_patient) // 2 + len(scans_patient)%2\n    _, ax = plt.subplots(n_rows, 2, figsize=(12, 5 * n_rows))\n    ax = ax.flatten()\n    for idx in range(len(scans_patient)):\n        ax[idx].imshow(scans_patient[idx].pixel_array)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:21:09.945393Z","iopub.execute_input":"2022-12-22T15:21:09.945727Z","iopub.status.idle":"2022-12-22T15:21:09.957866Z","shell.execute_reply.started":"2022-12-22T15:21:09.945694Z","shell.execute_reply":"2022-12-22T15:21:09.956722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scans_patient_wtht_cancer = get_images_one_patient(image_path, patient_id=df_train[df_train.cancer == 0].patient_id.unique()[0])\nplot_all_scans_one_patient(scans_patient_wtht_cancer)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:21:09.959809Z","iopub.execute_input":"2022-12-22T15:21:09.96092Z","iopub.status.idle":"2022-12-22T15:21:21.046257Z","shell.execute_reply.started":"2022-12-22T15:21:09.960832Z","shell.execute_reply":"2022-12-22T15:21:21.044787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scans_patient_w_cancer = get_images_one_patient(image_path, patient_id=df_train[df_train.cancer == 1].patient_id.unique()[0])\n# plot_all_scans_one_patient(scans_patient_w_cancer)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T15:23:22.472272Z","iopub.execute_input":"2022-12-22T15:23:22.472706Z","iopub.status.idle":"2022-12-22T15:23:22.53614Z","shell.execute_reply.started":"2022-12-22T15:23:22.472673Z","shell.execute_reply":"2022-12-22T15:23:22.534867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}