{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Mammography Breast Cancer Detection","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"%matplotlib inline\n\nimport os\nimport glob\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nPATH_DATASET = \"/kaggle/input/rsna-breast-cancer-detection\"","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:22.195315Z","iopub.execute_input":"2022-12-02T08:16:22.19601Z","iopub.status.idle":"2022-12-02T08:16:22.226077Z","shell.execute_reply.started":"2022-12-02T08:16:22.195879Z","shell.execute_reply":"2022-12-02T08:16:22.224728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore provided dataset","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(PATH_DATASET, \"train.csv\"))\ndisplay(df_train.head())\nprint(f'cases: {len(df_train)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:22.229316Z","iopub.execute_input":"2022-12-02T08:16:22.229751Z","iopub.status.idle":"2022-12-02T08:16:22.409278Z","shell.execute_reply.started":"2022-12-02T08:16:22.22969Z","shell.execute_reply":"2022-12-02T08:16:22.407451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = df_train.groupby(\"patient_id\").size().hist(bins=10)\nax.set_xlabel(\"nb scans per patiemt\")\nax.set_ylabel(\"nb of patients\")","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:22.411589Z","iopub.execute_input":"2022-12-02T08:16:22.412162Z","iopub.status.idle":"2022-12-02T08:16:22.742484Z","shell.execute_reply.started":"2022-12-02T08:16:22.412108Z","shell.execute_reply":"2022-12-02T08:16:22.740949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visual metadata\n\n- **site_id**: ID code for the source hospital.\n- **laterality**: Whether the image is of the left or right breast.\n- **view**: The orientation of the image. The default for a screening exam is to capture two views per breast.\n- **implant**: Whether or not the patient had breast implants. Site 1 only provides breast implant information at the patient level, not at the breast level.\n- **density**: A rating for how dense the breast tissue is, with A being the least dense and D being the most dense. Extremely dense tissue can make diagnosis more difficult.\n- **machine_id**: An ID code for the imaging device.\n- **cancer**: Whether or not the breast was positive for cancer. The target value. Only provided for train.\n- **biopsy**: Whether or not a follow-up biopsy was performed on the breast. Only provided for train.\n- **invasive**: If the breast is positive for cancer, whether or not the cancer proved to be invasive. Only provided for train.\n- **BIRADS**: 0 if the breast required follow-up, 1 if the breast was rated as negative for cancer, and 2 if the breast was rated as normal. Only provided for train.\n- **difficult_negative_case**: True if the case was unusually difficult. Only provided for train.","metadata":{}},{"cell_type":"code","source":"cols = [\"site_id\", \"laterality\", \"view\", \"implant\", \"density\", \"machine_id\",\n        \"cancer\", \"biopsy\", \"invasive\", \"BIRADS\", \"difficult_negative_case\"]\nnb_rows = int(np.ceil(len(cols) / 2))\n_, axarr = plt.subplots(ncols=2, nrows=nb_rows, figsize=(8, 4 * nb_rows))\nfor i, col in enumerate(cols):\n    df_train[col].value_counts().plot.pie(ax=axarr[i // 2, i % 2], autopct=\"%.1f%%\")","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:22.744613Z","iopub.execute_input":"2022-12-02T08:16:22.745188Z","iopub.status.idle":"2022-12-02T08:16:24.421663Z","shell.execute_reply.started":"2022-12-02T08:16:22.745132Z","shell.execute_reply":"2022-12-02T08:16:24.420596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Simple patient stat","metadata":{}},{"cell_type":"code","source":"df_train_patients = df_train.groupby(\"patient_id\").max()\ndisplay(df_train_patients.head())\nprint(f'cases: {len(df_train_patients)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:24.424601Z","iopub.execute_input":"2022-12-02T08:16:24.42555Z","iopub.status.idle":"2022-12-02T08:16:27.817407Z","shell.execute_reply.started":"2022-12-02T08:16:24.425493Z","shell.execute_reply":"2022-12-02T08:16:27.816194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\"implant\", \"cancer\", \"biopsy\", \"invasive\", \"BIRADS\", \"difficult_negative_case\"]\nnb_rows = int(np.ceil(len(cols) / 2))\n_, axarr = plt.subplots(ncols=2, nrows=nb_rows, figsize=(8, 4 * nb_rows))\nfor i, col in enumerate(cols):\n    df_train_patients[col].value_counts().plot.pie(ax=axarr[i // 2, i % 2], autopct=\"%.1f%%\")","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:27.819403Z","iopub.execute_input":"2022-12-02T08:16:27.819948Z","iopub.status.idle":"2022-12-02T08:16:28.424566Z","shell.execute_reply.started":"2022-12-02T08:16:27.819884Z","shell.execute_reply":"2022-12-02T08:16:28.422588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Patient / age\n\nverify that each patirnt went to screening oonly once / only at one age","metadata":{}},{"cell_type":"code","source":"patient_ages = [{\"patient_id\": pid, \"ages\": len(dfg[\"age\"].unique())} for pid, dfg in df_train.groupby(\"patient_id\")]\npd.DataFrame(patient_ages)[\"ages\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:28.427544Z","iopub.execute_input":"2022-12-02T08:16:28.42845Z","iopub.status.idle":"2022-12-02T08:16:30.19774Z","shell.execute_reply.started":"2022-12-02T08:16:28.428371Z","shell.execute_reply":"2022-12-02T08:16:30.196539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_patients[\"age\"].hist(bins=45)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:30.199249Z","iopub.execute_input":"2022-12-02T08:16:30.199617Z","iopub.status.idle":"2022-12-02T08:16:30.51613Z","shell.execute_reply.started":"2022-12-02T08:16:30.199584Z","shell.execute_reply":"2022-12-02T08:16:30.515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_patients.groupby(\"implant\")[\"age\"].hist(bins=45, legend=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:30.518279Z","iopub.execute_input":"2022-12-02T08:16:30.518889Z","iopub.status.idle":"2022-12-02T08:16:31.022009Z","shell.execute_reply.started":"2022-12-02T08:16:30.518848Z","shell.execute_reply":"2022-12-02T08:16:31.020774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Age / cancer\n\nsee distribution of cancer per age","metadata":{}},{"cell_type":"code","source":"df_train_patients.groupby(\"cancer\")[\"age\"].hist(bins=45, legend=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:31.023683Z","iopub.execute_input":"2022-12-02T08:16:31.024986Z","iopub.status.idle":"2022-12-02T08:16:31.576359Z","shell.execute_reply.started":"2022-12-02T08:16:31.024921Z","shell.execute_reply":"2022-12-02T08:16:31.575237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_patients.groupby(\"biopsy\")[\"age\"].hist(bins=45, legend=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:31.577735Z","iopub.execute_input":"2022-12-02T08:16:31.578679Z","iopub.status.idle":"2022-12-02T08:16:32.090934Z","shell.execute_reply.started":"2022-12-02T08:16:31.578631Z","shell.execute_reply":"2022-12-02T08:16:32.088328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Implant / cancer\n\nCheck if there is some corelation","metadata":{}},{"cell_type":"code","source":"df_train_patients[[\"cancer\", \"implant\"]].corr()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:32.092978Z","iopub.execute_input":"2022-12-02T08:16:32.093489Z","iopub.status.idle":"2022-12-02T08:16:32.108438Z","shell.execute_reply.started":"2022-12-02T08:16:32.093444Z","shell.execute_reply":"2022-12-02T08:16:32.107171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, axarr = plt.subplots(ncols=2, figsize=(8, 4))\nfor i, (implant, dfg) in enumerate(df_train_patients.groupby(\"implant\")):\n    dfg[\"cancer\"].value_counts().plot.pie(ax=axarr[i], autopct=\"%.1f%%\")\n    axarr[i].set_title(f\"implant: {implant}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:32.110272Z","iopub.execute_input":"2022-12-02T08:16:32.111888Z","iopub.status.idle":"2022-12-02T08:16:32.341073Z","shell.execute_reply.started":"2022-12-02T08:16:32.111768Z","shell.execute_reply":"2022-12-02T08:16:32.339838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploring images","metadata":{}},{"cell_type":"code","source":"!pip install -qU \"python-gdcm\" pydicom pylibjpeg \"opencv-python-headless\" -f /kaggle/input/pydicom-package --no-index\n!pip list | grep dicom","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-12-02T08:24:08.471555Z","iopub.execute_input":"2022-12-02T08:24:08.472093Z","iopub.status.idle":"2022-12-02T08:24:44.804284Z","shell.execute_reply.started":"2022-12-02T08:24:08.472044Z","shell.execute_reply":"2022-12-02T08:24:44.80292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nfrom tqdm.auto import tqdm\n\nls_image = glob.glob(os.path.join(PATH_DATASET, \"train_images\", \"*\", \"*.dcm\"))\nprint(f\"found images: {len(ls_image)}\")\n\n# sizes = []\n# for p_img in tqdm(ls_image):\n#     dicom = pydicom.dcmread(p_img)\n#     im_size = dicom.pixel_array.shape\n#     sizes.append({\"width\": im_size[1], \"height\": im_size[0]})","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:16:52.883626Z","iopub.execute_input":"2022-12-02T08:16:52.884215Z","iopub.status.idle":"2022-12-02T08:17:24.305495Z","shell.execute_reply.started":"2022-12-02T08:16:52.88415Z","shell.execute_reply":"2022-12-02T08:17:24.304501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load sample images","metadata":{}},{"cell_type":"code","source":"dicom = pydicom.dcmread(ls_image[0])\nprint(dicom)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:17:24.306868Z","iopub.execute_input":"2022-12-02T08:17:24.30789Z","iopub.status.idle":"2022-12-02T08:17:24.411497Z","shell.execute_reply.started":"2022-12-02T08:17:24.307847Z","shell.execute_reply":"2022-12-02T08:17:24.410018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nfrom PIL import Image\nfrom pydicom.pixel_data_handlers import apply_voi_lut\n\nnb_spls = 8\n_, axarr = plt.subplots(ncols=2, nrows=nb_spls, figsize=(10, 6 * nb_spls))\nfor j, (cancer, dfg) in enumerate(df_train.groupby(\"cancer\")):\n    for i, (_, row) in enumerate(dfg.sample(nb_spls).iterrows()):\n        dicom_path = os.path.join(\n            PATH_DATASET, \"train_images\", str(row[\"patient_id\"]), str(row[\"image_id\"]) + \".dcm\")\n        dicom = pydicom.dcmread(dicom_path)\n        img = apply_voi_lut(dicom.pixel_array, dicom)\n        print(img.shape, img.min(), img.max())\n        img = (img.astype(float) - img.min()) / (img.max() - img.min())\n        axarr[i, j].imshow(img, cmap=\"gray\")\n        axarr[i, j].set_title(f\"cancer: {row['cancer']}\")\n        axarr[i, j].set_axis_off()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:17:24.413607Z","iopub.execute_input":"2022-12-02T08:17:24.414093Z","iopub.status.idle":"2022-12-02T08:18:12.064817Z","shell.execute_reply.started":"2022-12-02T08:17:24.414048Z","shell.execute_reply":"2022-12-02T08:18:12.063321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fake predictions","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv(os.path.join(PATH_DATASET, \"test.csv\"))\ndisplay(df_test.head())\nprint(f'cases: {len(df_test)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:18:12.067091Z","iopub.execute_input":"2022-12-02T08:18:12.067601Z","iopub.status.idle":"2022-12-02T08:18:12.095186Z","shell.execute_reply.started":"2022-12-02T08:18:12.067554Z","shell.execute_reply":"2022-12-02T08:18:12.093804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv(os.path.join(PATH_DATASET, \"sample_submission.csv\"))\ndisplay(df_sub.head())\nprint(f'cases: {len(df_sub)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:18:12.096764Z","iopub.execute_input":"2022-12-02T08:18:12.097147Z","iopub.status.idle":"2022-12-02T08:18:12.114553Z","shell.execute_reply.started":"2022-12-02T08:18:12.097112Z","shell.execute_reply":"2022-12-02T08:18:12.113131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Stats for laterality\n\nalso prediction is aggregated per patient and laterality, compute the stats","metadata":{}},{"cell_type":"code","source":"df_train[\"prediction_id\"] = df_train.apply(lambda r: f\"{r['patient_id']}_{r['laterality']}\", axis=1)\ndf_train_stat = df_train.groupby(\"prediction_id\").max()\ndisplay(df_train_stat.head())","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:18:12.116213Z","iopub.execute_input":"2022-12-02T08:18:12.117184Z","iopub.status.idle":"2022-12-02T08:18:19.361686Z","shell.execute_reply.started":"2022-12-02T08:18:12.117146Z","shell.execute_reply":"2022-12-02T08:18:19.36013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat = {}\n_, axarr = plt.subplots(ncols=2, figsize=(8, 4))\nfor i, (laterality, dfg) in enumerate(df_train_stat.groupby(\"laterality\")):\n    dfg[\"cancer\"].value_counts().plot.pie(ax=axarr[i], autopct=\"%.1f%%\")\n    axarr[i].set_title(f\"laterality: {laterality}\")\n    stat[laterality] = dfg[\"cancer\"].sum() / float(len(dfg))\n\nprint(stat)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:18:19.363686Z","iopub.execute_input":"2022-12-02T08:18:19.364334Z","iopub.status.idle":"2022-12-02T08:18:19.621059Z","shell.execute_reply.started":"2022-12-02T08:18:19.364277Z","shell.execute_reply":"2022-12-02T08:18:19.619039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate predictions","metadata":{}},{"cell_type":"code","source":"preds = []\nfor _, row in df_test.iterrows():\n    img_path = os.path.join(\n        PATH_DATASET, \"test_images\", str(row[\"patient_id\"]), f'{row[\"image_id\"]}.dcm')\n    if not os.path.isfile(img_path):\n        print(f\"missing: {img_path}\")\n    preds.append({\n        \"prediction_id\": row[\"prediction_id\"],\n        \"cancer\": stat[row[\"laterality\"]]\n    })\n\ndf_preds = pd.DataFrame(preds)\ndisplay(df_preds.head())","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:18:19.623756Z","iopub.execute_input":"2022-12-02T08:18:19.625072Z","iopub.status.idle":"2022-12-02T08:18:19.666802Z","shell.execute_reply.started":"2022-12-02T08:18:19.624994Z","shell.execute_reply":"2022-12-02T08:18:19.665354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_preds = df_preds.groupby('prediction_id').max()\ndf_preds[\"cancer\"].to_csv(\"submission.csv\")\n\n! head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-12-02T08:18:19.668299Z","iopub.execute_input":"2022-12-02T08:18:19.668934Z","iopub.status.idle":"2022-12-02T08:18:20.796803Z","shell.execute_reply.started":"2022-12-02T08:18:19.668889Z","shell.execute_reply":"2022-12-02T08:18:20.795213Z"},"trusted":true},"execution_count":null,"outputs":[]}]}