{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Implementation","metadata":{"papermill":{"duration":0.018117,"end_time":"2021-04-27T06:21:03.600608","exception":false,"start_time":"2021-04-27T06:21:03.582491","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib\nimport pydicom as dicom\nimport cv2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"papermill":{"duration":0.356846,"end_time":"2021-04-27T06:21:03.974273","exception":false,"start_time":"2021-04-27T06:21:03.617427","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:28.312056Z","iopub.execute_input":"2022-11-07T06:51:28.312564Z","iopub.status.idle":"2022-11-07T06:51:28.712413Z","shell.execute_reply.started":"2022-11-07T06:51:28.31243Z","shell.execute_reply":"2022-11-07T06:51:28.711453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/'\nos.listdir(path)","metadata":{"papermill":{"duration":0.031405,"end_time":"2021-04-27T06:21:04.022828","exception":false,"start_time":"2021-04-27T06:21:03.991423","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:28.714434Z","iopub.execute_input":"2022-11-07T06:51:28.71524Z","iopub.status.idle":"2022-11-07T06:51:28.724553Z","shell.execute_reply.started":"2022-11-07T06:51:28.715201Z","shell.execute_reply":"2022-11-07T06:51:28.723516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(path+'train.csv')\nsamp_subm = pd.read_csv(path+'sample_submission.csv')","metadata":{"papermill":{"duration":0.192716,"end_time":"2021-04-27T06:21:04.233326","exception":false,"start_time":"2021-04-27T06:21:04.04061","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:28.72607Z","iopub.execute_input":"2022-11-07T06:51:28.726445Z","iopub.status.idle":"2022-11-07T06:51:28.90561Z","shell.execute_reply.started":"2022-11-07T06:51:28.726409Z","shell.execute_reply":"2022-11-07T06:51:28.904445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Exploratory Data Analysis","metadata":{"papermill":{"duration":0.017311,"end_time":"2021-04-27T06:21:04.268424","exception":false,"start_time":"2021-04-27T06:21:04.251113","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print('Number train samples:', len(train_data.index))\nprint('Number test samples:', len(samp_subm.index))","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.026636,"end_time":"2021-04-27T06:21:04.313","exception":false,"start_time":"2021-04-27T06:21:04.286364","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:28.908517Z","iopub.execute_input":"2022-11-07T06:51:28.909255Z","iopub.status.idle":"2022-11-07T06:51:28.915599Z","shell.execute_reply.started":"2022-11-07T06:51:28.909212Z","shell.execute_reply":"2022-11-07T06:51:28.914549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(12, 4))\nx = train_data['class_name'].value_counts().keys()\ny = train_data['class_name'].value_counts().values\nax.bar(x, y)\nax.set_xticklabels(x, rotation=90)\nax.set_title('Distribution of the labels')\nplt.grid()\nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.335829,"end_time":"2021-04-27T06:21:04.667061","exception":false,"start_time":"2021-04-27T06:21:04.331232","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:28.917329Z","iopub.execute_input":"2022-11-07T06:51:28.918018Z","iopub.status.idle":"2022-11-07T06:51:29.230572Z","shell.execute_reply.started":"2022-11-07T06:51:28.917979Z","shell.execute_reply":"2022-11-07T06:51:29.229551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see the dataset is inbalanced.","metadata":{"papermill":{"duration":0.019017,"end_time":"2021-04-27T06:21:04.705424","exception":false,"start_time":"2021-04-27T06:21:04.686407","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Read DICOM Files\nidnum = 2\nimage_id = train_data.loc[idnum, 'image_id']\ndata_file = dicom.dcmread(path+'train/'+image_id+'.dicom')\nimg = data_file.pixel_array","metadata":{"papermill":{"duration":2.138335,"end_time":"2021-04-27T06:21:06.863202","exception":false,"start_time":"2021-04-27T06:21:04.724867","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:29.232229Z","iopub.execute_input":"2022-11-07T06:51:29.232897Z","iopub.status.idle":"2022-11-07T06:51:30.305631Z","shell.execute_reply.started":"2022-11-07T06:51:29.232853Z","shell.execute_reply":"2022-11-07T06:51:30.304552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data_file)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.030583,"end_time":"2021-04-27T06:21:06.914239","exception":false,"start_time":"2021-04-27T06:21:06.883656","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:30.307012Z","iopub.execute_input":"2022-11-07T06:51:30.307742Z","iopub.status.idle":"2022-11-07T06:51:30.314826Z","shell.execute_reply.started":"2022-11-07T06:51:30.3077Z","shell.execute_reply":"2022-11-07T06:51:30.3127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Image shape:', img.shape)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.029433,"end_time":"2021-04-27T06:21:06.963799","exception":false,"start_time":"2021-04-27T06:21:06.934366","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:30.31642Z","iopub.execute_input":"2022-11-07T06:51:30.317135Z","iopub.status.idle":"2022-11-07T06:51:30.325626Z","shell.execute_reply.started":"2022-11-07T06:51:30.317096Z","shell.execute_reply":"2022-11-07T06:51:30.324608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox = [train_data.loc[idnum, 'x_min'],\n        train_data.loc[idnum, 'y_min'],\n        train_data.loc[idnum, 'x_max'],\n        train_data.loc[idnum, 'y_max']]\nfig, ax = plt.subplots(1, 1, figsize=(20, 4))\nax.imshow(img, cmap='gray')\np = matplotlib.patches.Rectangle((bbox[0], bbox[1]),\n                                 bbox[2]-bbox[0],\n                                 bbox[3]-bbox[1],\n                                 ec='r', fc='none', lw=2.)\nax.add_patch(p)\nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.674205,"end_time":"2021-04-27T06:21:07.660012","exception":false,"start_time":"2021-04-27T06:21:06.985807","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:30.327336Z","iopub.execute_input":"2022-11-07T06:51:30.327972Z","iopub.status.idle":"2022-11-07T06:51:30.926381Z","shell.execute_reply.started":"2022-11-07T06:51:30.327936Z","shell.execute_reply":"2022-11-07T06:51:30.925485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_train_data(idx_list):\n    fig, axs = plt.subplots(1, 3, figsize=(15, 10))\n    fig.subplots_adjust(hspace = .1, wspace=.1)\n    axs = axs.ravel()\n    for i in range(3):\n        image_id = train_data.loc[idx_list[i], 'image_id']\n        data_file = dicom.dcmread(path+'train/'+image_id+'.dicom')\n        img = data_file.pixel_array\n        axs[i].imshow(img, cmap='gray')\n        axs[i].set_title(train_data.loc[idx_list[i], 'class_name'])\n        axs[i].set_xticklabels([])\n        axs[i].set_yticklabels([])\n        if train_data.loc[idx_list[i], 'class_name'] != 'No finding':\n            bbox = [train_data.loc[idx_list[i], 'x_min'],\n                    train_data.loc[idx_list[i], 'y_min'],\n                    train_data.loc[idx_list[i], 'x_max'],\n                    train_data.loc[idx_list[i], 'y_max']]\n            p = matplotlib.patches.Rectangle((bbox[0], bbox[1]),\n                                             bbox[2]-bbox[0],\n                                             bbox[3]-bbox[1],\n                                             ec='r', fc='none', lw=2.)\n            axs[i].add_patch(p)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.037565,"end_time":"2021-04-27T06:21:07.720091","exception":false,"start_time":"2021-04-27T06:21:07.682526","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:30.929856Z","iopub.execute_input":"2022-11-07T06:51:30.930158Z","iopub.status.idle":"2022-11-07T06:51:30.939916Z","shell.execute_reply.started":"2022-11-07T06:51:30.93013Z","shell.execute_reply":"2022-11-07T06:51:30.938791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for num in range(15):\n    idx_list = train_data[train_data['class_id']==num][0:3].index.values\n    plot_train_data(idx_list)","metadata":{"papermill":{"duration":87.614843,"end_time":"2021-04-27T06:22:35.35828","exception":false,"start_time":"2021-04-27T06:21:07.743437","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:51:30.941562Z","iopub.execute_input":"2022-11-07T06:51:30.941916Z","iopub.status.idle":"2022-11-07T06:52:43.514025Z","shell.execute_reply.started":"2022-11-07T06:51:30.941882Z","shell.execute_reply":"2022-11-07T06:52:43.512974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6. Results\n","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.121659,"end_time":"2021-04-27T06:22:35.618786","exception":false,"start_time":"2021-04-27T06:22:35.497127","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#samp_subm.to_csv('submission1.csv', index=False)","metadata":{"papermill":{"duration":0.128367,"end_time":"2021-04-27T06:22:35.86499","exception":false,"start_time":"2021-04-27T06:22:35.736623","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:52:43.515562Z","iopub.execute_input":"2022-11-07T06:52:43.516189Z","iopub.status.idle":"2022-11-07T06:52:43.520793Z","shell.execute_reply.started":"2022-11-07T06:52:43.516149Z","shell.execute_reply":"2022-11-07T06:52:43.519645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_2class = pd.read_csv(\"../input/vinbigdata-2class-prediction/2-cls test pred.csv\")\nlow_threshold = 0.001\nhigh_threshold = 0.87\npred_2class","metadata":{"papermill":{"duration":0.172873,"end_time":"2021-04-27T06:22:36.1577","exception":false,"start_time":"2021-04-27T06:22:35.984827","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:52:43.52254Z","iopub.execute_input":"2022-11-07T06:52:43.522944Z","iopub.status.idle":"2022-11-07T06:52:43.55741Z","shell.execute_reply.started":"2022-11-07T06:52:43.522904Z","shell.execute_reply":"2022-11-07T06:52:43.556486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NORMAL = \"14 1 0 0 1 1\"\n\npred_det_df = pd.read_csv(\"../input/vinbigdatastack/submission_postprocessed.csv\")\nn_normal_before = len(pred_det_df.query(\"PredictionString == @NORMAL\"))\nmerged_df = pd.merge(pred_det_df, pred_2class, on=\"image_id\", how=\"left\")\n\n\nif \"target\" in merged_df.columns:\n    merged_df[\"class0\"] = 1 - merged_df[\"target\"]\n\nc0, c1, c2 = 0, 0, 0\nfor i in range(len(merged_df)):\n    p0 = merged_df.loc[i, \"class0\"]\n    if p0 < low_threshold:\n\n        c0 += 1\n    elif low_threshold <= p0 and p0 < high_threshold:\n\n        merged_df.loc[i, \"PredictionString\"] += f\" 14 {p0} 0 0 1 1\"\n        c1 += 1\n    else:\n\n        merged_df.loc[i, \"PredictionString\"] = NORMAL\n        c2 += 1","metadata":{"papermill":{"duration":1.175345,"end_time":"2021-04-27T06:22:37.451261","exception":false,"start_time":"2021-04-27T06:22:36.275916","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:52:43.558746Z","iopub.execute_input":"2022-11-07T06:52:43.559224Z","iopub.status.idle":"2022-11-07T06:52:44.498594Z","shell.execute_reply.started":"2022-11-07T06:52:43.559187Z","shell.execute_reply":"2022-11-07T06:52:44.497616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_normal_after = len(merged_df.query(\"PredictionString == @NORMAL\"))\nprint(\n    f\"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}\"\n)\nprint(f\"Keep {c0} Add {c1} Replace {c2}\")\nsubmission_filepath = str(\"submission.csv\")\nsubmission_df = merged_df[[\"image_id\", \"PredictionString\"]]\nsubmission_df.to_csv(submission_filepath, index=False)\nprint(f\"Saved to {submission_filepath}\")","metadata":{"papermill":{"duration":0.166437,"end_time":"2021-04-27T06:22:37.737694","exception":false,"start_time":"2021-04-27T06:22:37.571257","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T06:52:44.500116Z","iopub.execute_input":"2022-11-07T06:52:44.500481Z","iopub.status.idle":"2022-11-07T06:52:44.52438Z","shell.execute_reply.started":"2022-11-07T06:52:44.500441Z","shell.execute_reply":"2022-11-07T06:52:44.523355Z"},"trusted":true},"execution_count":null,"outputs":[]}]}