{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Implementation","metadata":{"papermill":{"duration":0.005349,"end_time":"2022-11-07T06:53:19.208488","exception":false,"start_time":"2022-11-07T06:53:19.203139","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib\nimport pydicom as dicom\nimport cv2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"papermill":{"duration":0.390818,"end_time":"2022-11-07T06:53:19.603605","exception":false,"start_time":"2022-11-07T06:53:19.212787","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:16.814072Z","iopub.execute_input":"2022-11-07T07:54:16.814597Z","iopub.status.idle":"2022-11-07T07:54:17.229378Z","shell.execute_reply.started":"2022-11-07T07:54:16.814497Z","shell.execute_reply":"2022-11-07T07:54:17.227589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/'\nos.listdir(path)","metadata":{"papermill":{"duration":0.023226,"end_time":"2022-11-07T06:53:19.633974","exception":false,"start_time":"2022-11-07T06:53:19.610748","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:17.232472Z","iopub.execute_input":"2022-11-07T07:54:17.233924Z","iopub.status.idle":"2022-11-07T07:54:17.246568Z","shell.execute_reply.started":"2022-11-07T07:54:17.233864Z","shell.execute_reply":"2022-11-07T07:54:17.244798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(path+'train.csv')\nsamp_subm = pd.read_csv(path+'sample_submission.csv')","metadata":{"papermill":{"duration":0.191451,"end_time":"2022-11-07T06:53:19.831874","exception":false,"start_time":"2022-11-07T06:53:19.640423","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:17.248928Z","iopub.execute_input":"2022-11-07T07:54:17.24967Z","iopub.status.idle":"2022-11-07T07:54:17.521077Z","shell.execute_reply.started":"2022-11-07T07:54:17.24962Z","shell.execute_reply":"2022-11-07T07:54:17.519723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Exploratory Data Analysis","metadata":{"papermill":{"duration":0.003963,"end_time":"2022-11-07T06:53:19.840276","exception":false,"start_time":"2022-11-07T06:53:19.836313","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print('Number train samples:', len(train_data.index))\nprint('Number test samples:', len(samp_subm.index))","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.014181,"end_time":"2022-11-07T06:53:19.858775","exception":false,"start_time":"2022-11-07T06:53:19.844594","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:17.525116Z","iopub.execute_input":"2022-11-07T07:54:17.525657Z","iopub.status.idle":"2022-11-07T07:54:17.538539Z","shell.execute_reply.started":"2022-11-07T07:54:17.525587Z","shell.execute_reply":"2022-11-07T07:54:17.536553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(12, 4))\nx = train_data['class_name'].value_counts().keys()\ny = train_data['class_name'].value_counts().values\nax.bar(x, y)\nax.set_xticklabels(x, rotation=90)\nax.set_title('Distribution of the labels')\nplt.grid()\nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.318034,"end_time":"2022-11-07T06:53:20.181055","exception":false,"start_time":"2022-11-07T06:53:19.863021","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:17.540581Z","iopub.execute_input":"2022-11-07T07:54:17.541381Z","iopub.status.idle":"2022-11-07T07:54:17.900349Z","shell.execute_reply.started":"2022-11-07T07:54:17.541341Z","shell.execute_reply":"2022-11-07T07:54:17.89898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see the dataset is inbalanced.","metadata":{"papermill":{"duration":0.004613,"end_time":"2022-11-07T06:53:20.190552","exception":false,"start_time":"2022-11-07T06:53:20.185939","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Read DICOM Files\nidnum = 2\nimage_id = train_data.loc[idnum, 'image_id']\ndata_file = dicom.dcmread(path+'train/'+image_id+'.dicom')\nimg = data_file.pixel_array","metadata":{"papermill":{"duration":1.098937,"end_time":"2022-11-07T06:53:21.294533","exception":false,"start_time":"2022-11-07T06:53:20.195596","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:17.902375Z","iopub.execute_input":"2022-11-07T07:54:17.902843Z","iopub.status.idle":"2022-11-07T07:54:19.427456Z","shell.execute_reply.started":"2022-11-07T07:54:17.902797Z","shell.execute_reply":"2022-11-07T07:54:19.426013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data_file)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.016598,"end_time":"2022-11-07T06:53:21.316252","exception":false,"start_time":"2022-11-07T06:53:21.299654","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:19.429484Z","iopub.execute_input":"2022-11-07T07:54:19.430018Z","iopub.status.idle":"2022-11-07T07:54:19.43966Z","shell.execute_reply.started":"2022-11-07T07:54:19.429962Z","shell.execute_reply":"2022-11-07T07:54:19.43839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Image shape:', img.shape)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.013512,"end_time":"2022-11-07T06:53:21.334456","exception":false,"start_time":"2022-11-07T06:53:21.320944","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:19.441477Z","iopub.execute_input":"2022-11-07T07:54:19.442923Z","iopub.status.idle":"2022-11-07T07:54:19.467238Z","shell.execute_reply.started":"2022-11-07T07:54:19.442862Z","shell.execute_reply":"2022-11-07T07:54:19.464684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox = [train_data.loc[idnum, 'x_min'],\n        train_data.loc[idnum, 'y_min'],\n        train_data.loc[idnum, 'x_max'],\n        train_data.loc[idnum, 'y_max']]\nfig, ax = plt.subplots(1, 1, figsize=(20, 4))\nax.imshow(img, cmap='gray')\np = matplotlib.patches.Rectangle((bbox[0], bbox[1]),\n                                 bbox[2]-bbox[0],\n                                 bbox[3]-bbox[1],\n                                 ec='r', fc='none', lw=2.)\nax.add_patch(p)\nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.617214,"end_time":"2022-11-07T06:53:21.957043","exception":false,"start_time":"2022-11-07T06:53:21.339829","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:19.468941Z","iopub.execute_input":"2022-11-07T07:54:19.469755Z","iopub.status.idle":"2022-11-07T07:54:20.285474Z","shell.execute_reply.started":"2022-11-07T07:54:19.469705Z","shell.execute_reply":"2022-11-07T07:54:20.28381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_train_data(idx_list):\n    fig, axs = plt.subplots(1, 3, figsize=(15, 10))\n    fig.subplots_adjust(hspace = .1, wspace=.1)\n    axs = axs.ravel()\n    for i in range(3):\n        image_id = train_data.loc[idx_list[i], 'image_id']\n        data_file = dicom.dcmread(path+'train/'+image_id+'.dicom')\n        img = data_file.pixel_array\n        axs[i].imshow(img, cmap='gray')\n        axs[i].set_title(train_data.loc[idx_list[i], 'class_name'])\n        axs[i].set_xticklabels([])\n        axs[i].set_yticklabels([])\n        if train_data.loc[idx_list[i], 'class_name'] != 'No finding':\n            bbox = [train_data.loc[idx_list[i], 'x_min'],\n                    train_data.loc[idx_list[i], 'y_min'],\n                    train_data.loc[idx_list[i], 'x_max'],\n                    train_data.loc[idx_list[i], 'y_max']]\n            p = matplotlib.patches.Rectangle((bbox[0], bbox[1]),\n                                             bbox[2]-bbox[0],\n                                             bbox[3]-bbox[1],\n                                             ec='r', fc='none', lw=2.)\n            axs[i].add_patch(p)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.018781,"end_time":"2022-11-07T06:53:21.981327","exception":false,"start_time":"2022-11-07T06:53:21.962546","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:20.29053Z","iopub.execute_input":"2022-11-07T07:54:20.291056Z","iopub.status.idle":"2022-11-07T07:54:20.305085Z","shell.execute_reply.started":"2022-11-07T07:54:20.291006Z","shell.execute_reply":"2022-11-07T07:54:20.303531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for num in range(15):\n    idx_list = train_data[train_data['class_id']==num][0:3].index.values\n    plot_train_data(idx_list)","metadata":{"papermill":{"duration":72.572366,"end_time":"2022-11-07T06:54:34.55886","exception":false,"start_time":"2022-11-07T06:53:21.986494","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:54:20.307156Z","iopub.execute_input":"2022-11-07T07:54:20.307625Z","iopub.status.idle":"2022-11-07T07:55:51.991586Z","shell.execute_reply.started":"2022-11-07T07:54:20.307587Z","shell.execute_reply":"2022-11-07T07:55:51.99012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6. Results\n","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.044721,"end_time":"2022-11-07T06:54:34.649728","exception":false,"start_time":"2022-11-07T06:54:34.605007","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#samp_subm.to_csv('submission1.csv', index=False)","metadata":{"papermill":{"duration":0.052663,"end_time":"2022-11-07T06:54:34.747692","exception":false,"start_time":"2022-11-07T06:54:34.695029","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:55:51.99315Z","iopub.execute_input":"2022-11-07T07:55:51.99351Z","iopub.status.idle":"2022-11-07T07:55:52.000637Z","shell.execute_reply.started":"2022-11-07T07:55:51.993477Z","shell.execute_reply":"2022-11-07T07:55:51.999091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_2class = pd.read_csv(\"../input/vinbigdata-2class-prediction/2-cls test pred.csv\")\nlow_threshold = 0.001\nhigh_threshold = 0.87\npred_2class","metadata":{"papermill":{"duration":0.07528,"end_time":"2022-11-07T06:54:34.867331","exception":false,"start_time":"2022-11-07T06:54:34.792051","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:55:52.002639Z","iopub.execute_input":"2022-11-07T07:55:52.003461Z","iopub.status.idle":"2022-11-07T07:55:52.046693Z","shell.execute_reply.started":"2022-11-07T07:55:52.003405Z","shell.execute_reply":"2022-11-07T07:55:52.045068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NORMAL = \"14 1 0 0 1 1\"\n\npred_det_df = pd.read_csv(\"../input/vinbigdata-chest-xray-models/submission_postprocessed.csv\")\nn_normal_before = len(pred_det_df.query(\"PredictionString == @NORMAL\"))\nmerged_df = pd.merge(pred_det_df, pred_2class, on=\"image_id\", how=\"left\")\n\n\nif \"target\" in merged_df.columns:\n    merged_df[\"class0\"] = 1 - merged_df[\"target\"]\n\nc0, c1, c2 = 0, 0, 0\nfor i in range(len(merged_df)):\n    p0 = merged_df.loc[i, \"class0\"]\n    if p0 < low_threshold:\n\n        c0 += 1\n    elif low_threshold <= p0 and p0 < high_threshold:\n\n        merged_df.loc[i, \"PredictionString\"] += f\" 14 {p0} 0 0 1 1\"\n        c1 += 1\n    else:\n\n        merged_df.loc[i, \"PredictionString\"] = NORMAL\n        c2 += 1","metadata":{"papermill":{"duration":1.155233,"end_time":"2022-11-07T06:54:36.069522","exception":false,"start_time":"2022-11-07T06:54:34.914289","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:58:37.683981Z","iopub.execute_input":"2022-11-07T07:58:37.685078Z","iopub.status.idle":"2022-11-07T07:58:38.825218Z","shell.execute_reply.started":"2022-11-07T07:58:37.685002Z","shell.execute_reply":"2022-11-07T07:58:38.823449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_normal_after = len(merged_df.query(\"PredictionString == @NORMAL\"))\nprint(\n    f\"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}\"\n)\nprint(f\"Keep {c0} Add {c1} Replace {c2}\")\nsubmission_filepath = str(\"submission.csv\")\nsubmission_df = merged_df[[\"image_id\", \"PredictionString\"]]\nsubmission_df.to_csv(submission_filepath, index=False)\nprint(f\"Saved to {submission_filepath}\")","metadata":{"papermill":{"duration":0.127118,"end_time":"2022-11-07T06:54:36.286094","exception":false,"start_time":"2022-11-07T06:54:36.158976","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-07T07:58:49.975536Z","iopub.execute_input":"2022-11-07T07:58:49.976217Z","iopub.status.idle":"2022-11-07T07:58:50.014872Z","shell.execute_reply.started":"2022-11-07T07:58:49.976159Z","shell.execute_reply":"2022-11-07T07:58:50.013655Z"},"trusted":true},"execution_count":null,"outputs":[]}]}