{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"d2fed37a-8e84-4440-b1e7-258145a3d390","_cell_guid":"a31a56da-3bce-4351-a301-1762d89013e1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-12-23T10:51:00.21047Z","iopub.execute_input":"2021-12-23T10:51:00.210829Z","iopub.status.idle":"2021-12-23T10:51:21.140821Z","shell.execute_reply.started":"2021-12-23T10:51:00.210727Z","shell.execute_reply":"2021-12-23T10:51:21.139736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conda install Pillow","metadata":{"_uuid":"af727422-b3f0-4c5d-9373-3833c58df9c7","_cell_guid":"11dff78e-2164-4a6d-8d4d-78d39ae2f0f7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-12-22T14:38:55.504302Z","iopub.execute_input":"2021-12-22T14:38:55.50457Z","iopub.status.idle":"2021-12-22T14:39:44.447116Z","shell.execute_reply.started":"2021-12-22T14:38:55.504541Z","shell.execute_reply":"2021-12-22T14:39:44.446155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(widhts)","metadata":{"execution":{"iopub.status.busy":"2021-12-22T19:58:14.638927Z","iopub.execute_input":"2021-12-22T19:58:14.639218Z","iopub.status.idle":"2021-12-22T19:58:14.644964Z","shell.execute_reply.started":"2021-12-22T19:58:14.639189Z","shell.execute_reply":"2021-12-22T19:58:14.644332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(widhts)\nprint(heights)","metadata":{"execution":{"iopub.status.busy":"2021-12-22T20:12:22.276749Z","iopub.execute_input":"2021-12-22T20:12:22.277469Z","iopub.status.idle":"2021-12-22T20:12:22.302695Z","shell.execute_reply.started":"2021-12-22T20:12:22.277432Z","shell.execute_reply":"2021-12-22T20:12:22.301808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport pydicom as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\ntrain_df=pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\nprint(train_df.head())\ndimensions_df=pd.read_csv('../input/train-meta/train_meta.csv')\ndef temporary(id):\n    row = dimensions_df[dimensions_df['image_id']==id]\n    return row\n# train_df['image_width'] = train_df.apply(lambda row: temporary(row.image_id).dim0, axis =1)\n# train_df['image_height'] = train_df.apply(lambda row: temporary(row.image_id).dim1, axis =1)\n# train_df.to_csv('/kaggle/working/final_train_df.csv',index=False)\nimage_ids=train_df[\"image_id\"].to_list()\nwidhts=[]\nheights=[]\nfor id in image_ids:\n    row=temporary(id)\n    widht=row[\"dim0\"].tolist()\n    height=row[\"dim1\"].tolist()\n    widhts.append(widht[0])\n    heights.append(height[0])\n# print(image_ids)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-23T10:51:21.143501Z","iopub.execute_input":"2021-12-23T10:51:21.143849Z","iopub.status.idle":"2021-12-23T10:54:15.456373Z","shell.execute_reply.started":"2021-12-23T10:51:21.143817Z","shell.execute_reply":"2021-12-23T10:54:15.455491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame.from_dict({'width': heights, 'height': widhts})","metadata":{"execution":{"iopub.status.busy":"2021-12-23T10:56:20.794098Z","iopub.execute_input":"2021-12-23T10:56:20.794632Z","iopub.status.idle":"2021-12-23T10:56:20.858078Z","shell.execute_reply.started":"2021-12-23T10:56:20.794598Z","shell.execute_reply":"2021-12-23T10:56:20.857192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-23T10:56:24.913372Z","iopub.execute_input":"2021-12-23T10:56:24.913685Z","iopub.status.idle":"2021-12-23T10:56:24.9291Z","shell.execute_reply.started":"2021-12-23T10:56:24.913653Z","shell.execute_reply":"2021-12-23T10:56:24.928126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-22T20:13:01.332313Z","iopub.execute_input":"2021-12-22T20:13:01.333026Z","iopub.status.idle":"2021-12-22T20:13:01.339018Z","shell.execute_reply.started":"2021-12-22T20:13:01.332976Z","shell.execute_reply":"2021-12-22T20:13:01.338206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape\n","metadata":{"execution":{"iopub.status.busy":"2021-12-22T20:13:14.102283Z","iopub.execute_input":"2021-12-22T20:13:14.103208Z","iopub.status.idle":"2021-12-22T20:13:14.108092Z","shell.execute_reply.started":"2021-12-22T20:13:14.103166Z","shell.execute_reply":"2021-12-22T20:13:14.1075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.concat([train_df, df], axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-23T10:56:31.123124Z","iopub.execute_input":"2021-12-23T10:56:31.12347Z","iopub.status.idle":"2021-12-23T10:56:31.134569Z","shell.execute_reply.started":"2021-12-23T10:56:31.123411Z","shell.execute_reply":"2021-12-23T10:56:31.133564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids","metadata":{"execution":{"iopub.status.busy":"2021-12-22T20:16:19.377245Z","iopub.execute_input":"2021-12-22T20:16:19.37789Z","iopub.status.idle":"2021-12-22T20:16:19.401547Z","shell.execute_reply.started":"2021-12-22T20:16:19.377852Z","shell.execute_reply":"2021-12-22T20:16:19.400495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-23T10:56:39.427242Z","iopub.execute_input":"2021-12-23T10:56:39.427699Z","iopub.status.idle":"2021-12-23T10:56:39.446488Z","shell.execute_reply.started":"2021-12-23T10:56:39.427661Z","shell.execute_reply":"2021-12-23T10:56:39.445754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-22T20:18:08.707881Z","iopub.execute_input":"2021-12-22T20:18:08.708162Z","iopub.status.idle":"2021-12-22T20:18:08.714886Z","shell.execute_reply.started":"2021-12-22T20:18:08.708133Z","shell.execute_reply":"2021-12-22T20:18:08.713916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.to_csv('final_train_df_correct.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-23T10:56:52.839089Z","iopub.execute_input":"2021-12-23T10:56:52.839824Z","iopub.status.idle":"2021-12-23T10:56:53.347907Z","shell.execute_reply.started":"2021-12-23T10:56:52.839791Z","shell.execute_reply":"2021-12-23T10:56:53.346886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-22T19:10:04.057538Z","iopub.execute_input":"2021-12-22T19:10:04.058164Z","iopub.status.idle":"2021-12-22T19:10:04.139454Z","shell.execute_reply.started":"2021-12-22T19:10:04.058057Z","shell.execute_reply":"2021-12-22T19:10:04.138241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pydicom as dicom\nimport matplotlib.patches as ptc\nfrom glob import glob\nimport shutil, os\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom PIL import Image","metadata":{"_uuid":"982f3e24-8223-441d-98ac-1fd665dba71e","_cell_guid":"17851823-82c9-46bf-bda6-bd6dc3aeeb8c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-12-22T15:31:40.577271Z","iopub.execute_input":"2021-12-22T15:31:40.57765Z","iopub.status.idle":"2021-12-22T15:31:40.583905Z","shell.execute_reply.started":"2021-12-22T15:31:40.577591Z","shell.execute_reply":"2021-12-22T15:31:40.582985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.read_csv('/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\ntrain_df.head()","metadata":{"_uuid":"a7d9bf24-b12b-418e-b00c-a6e957a4b454","_cell_guid":"0bee783d-6498-44a9-af29-fb3aa7861c26","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-12-22T15:31:40.586477Z","iopub.execute_input":"2021-12-22T15:31:40.587735Z","iopub.status.idle":"2021-12-22T15:31:40.760573Z","shell.execute_reply.started":"2021-12-22T15:31:40.587682Z","shell.execute_reply":"2021-12-22T15:31:40.759588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = train_df[train_df['image_id']==\"9a5094b2563a1ef3ff50dc5c7ff71345\"]","metadata":{"execution":{"iopub.status.busy":"2021-12-22T15:34:33.969723Z","iopub.execute_input":"2021-12-22T15:34:33.970027Z","iopub.status.idle":"2021-12-22T15:34:33.991405Z","shell.execute_reply.started":"2021-12-22T15:34:33.969993Z","shell.execute_reply":"2021-12-22T15:34:33.990355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row","metadata":{"execution":{"iopub.status.busy":"2021-12-22T15:41:39.445752Z","iopub.execute_input":"2021-12-22T15:41:39.446056Z","iopub.status.idle":"2021-12-22T15:41:39.468101Z","shell.execute_reply.started":"2021-12-22T15:41:39.44602Z","shell.execute_reply":"2021-12-22T15:41:39.467097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row[\"temp\"]=","metadata":{"execution":{"iopub.status.busy":"2021-12-22T15:35:59.652727Z","iopub.execute_input":"2021-12-22T15:35:59.653013Z","iopub.status.idle":"2021-12-22T15:35:59.658967Z","shell.execute_reply.started":"2021-12-22T15:35:59.652981Z","shell.execute_reply":"2021-12-22T15:35:59.658039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.nunique().to_frame().rename(columns={0:\"Unique Values\"}).style.background_gradient(cmap=\"plasma\")","metadata":{"_uuid":"7a39e87b-cc9a-4940-b19f-ff19005509bd","_cell_guid":"efa59d22-ec60-44b0-85cc-8b4f81b6082e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-12-22T14:40:15.411619Z","iopub.execute_input":"2021-12-22T14:40:15.411931Z","iopub.status.idle":"2021-12-22T14:40:15.526347Z","shell.execute_reply.started":"2021-12-22T14:40:15.41189Z","shell.execute_reply":"2021-12-22T14:40:15.525502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(26, 8))\nsns.countplot(x=\"class_name\", data=train_df)\nplt.title(\"Class Name Distribution\")\nplt.show()","metadata":{"_uuid":"5b1e22b7-b11f-4bb7-96cb-39194ae3d92f","_cell_guid":"48fcaeda-dae9-43a0-b7c0-e0b699c9794c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-12-22T14:40:20.767174Z","iopub.execute_input":"2021-12-22T14:40:20.767959Z","iopub.status.idle":"2021-12-22T14:40:21.180977Z","shell.execute_reply.started":"2021-12-22T14:40:20.76792Z","shell.execute_reply":"2021-12-22T14:40:21.180095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nsns.countplot(x=\"class_id\", data=train_df)\nplt.title(\"Class ID Distribution\")\nplt.show()","metadata":{"_uuid":"5a9c7116-58bc-417b-88f5-92d46cf88874","_cell_guid":"1dc1e749-b5ac-477a-8fae-79f45d5fcfce","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-12-22T14:41:07.102881Z","iopub.execute_input":"2021-12-22T14:41:07.103549Z","iopub.status.idle":"2021-12-22T14:41:07.3638Z","shell.execute_reply.started":"2021-12-22T14:41:07.103512Z","shell.execute_reply":"2021-12-22T14:41:07.362904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train\"\ntest_dir = \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/test\"\ntrain_files = os.listdir(train_dir)\ntest_files = os.listdir(test_dir)","metadata":{"_uuid":"b9e03e2c-df35-4822-a214-4978788f4305","_cell_guid":"ea5376b4-c4bf-4d4d-b82c-6d1d2a7af3e0","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.571272Z","iopub.execute_input":"2021-12-17T08:24:41.573399Z","iopub.status.idle":"2021-12-17T08:24:41.593333Z","shell.execute_reply.started":"2021-12-17T08:24:41.573359Z","shell.execute_reply":"2021-12-17T08:24:41.592721Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of train images: \", len(train_files))\nprint(\"Number of test images: \", len(test_files))","metadata":{"_uuid":"09d6d263-95ad-4e6e-9091-178e5d69c207","_cell_guid":"eea6102a-24bc-4a5f-aeed-314bcc1b8902","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.596785Z","iopub.execute_input":"2021-12-17T08:24:41.598969Z","iopub.status.idle":"2021-12-17T08:24:41.607832Z","shell.execute_reply.started":"2021-12-17T08:24:41.598931Z","shell.execute_reply":"2021-12-17T08:24:41.607079Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image_path'] = \"/kaggle/working/train_imgs/\"+train_df.image_id+\".png\"\ntrain_df[\"image_path\"][0]","metadata":{"_uuid":"5f3d7588-f885-4b07-82b3-cc3978289e12","_cell_guid":"e24b1b1f-5ed6-44ce-947b-1ac422bb4e91","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.613732Z","iopub.execute_input":"2021-12-17T08:24:41.614115Z","iopub.status.idle":"2021-12-17T08:24:41.653839Z","shell.execute_reply.started":"2021-12-17T08:24:41.61408Z","shell.execute_reply":"2021-12-17T08:24:41.65308Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = train_df[train_df.class_id!=14].reset_index(drop = True)\n# train_df.shape","metadata":{"_uuid":"c8d00d70-82c7-4c97-96f2-c74f6603090d","_cell_guid":"73a1816e-cd9c-4817-a6ab-bc6111a4a9cb","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.657214Z","iopub.execute_input":"2021-12-17T08:24:41.657639Z","iopub.status.idle":"2021-12-17T08:24:41.669243Z","shell.execute_reply.started":"2021-12-17T08:24:41.657573Z","shell.execute_reply":"2021-12-17T08:24:41.66609Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"_uuid":"ccb25da8-20e1-4151-93be-7ee43f6e6c16","_cell_guid":"76fa9f2c-8e14-4d34-ad89-56b0617a08e1","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.672671Z","iopub.execute_input":"2021-12-17T08:24:41.674664Z","iopub.status.idle":"2021-12-17T08:24:41.699814Z","shell.execute_reply.started":"2021-12-17T08:24:41.674629Z","shell.execute_reply":"2021-12-17T08:24:41.699161Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_dicom_df(fn):\n    _ = dicom.read_file(os.path.join(train_dir, fn))\n    pass","metadata":{"_uuid":"c97f7287-e676-47f4-bda7-a337f1beb2ba","_cell_guid":"20862584-043a-49fa-becb-689852ef299d","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.701381Z","iopub.execute_input":"2021-12-17T08:24:41.701612Z","iopub.status.idle":"2021-12-17T08:24:41.705692Z","shell.execute_reply.started":"2021-12-17T08:24:41.70158Z","shell.execute_reply":"2021-12-17T08:24:41.704814Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_pixel_array(data,fn):\n    plt.figure()\n    plt.imshow(data, cmap=plt.cm.bone)\n    path=\"/kaggle/working/train_imgs/\"\n    fn_new = fn[:len(fn) - 6]\n    plt.savefig(path + fn_new +\".png\")","metadata":{"_uuid":"08834b56-1a29-4b2f-8259-a8195bc32a99","_cell_guid":"a06b5816-4773-46d2-890c-0f9813ed0bb1","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.707242Z","iopub.execute_input":"2021-12-17T08:24:41.707757Z","iopub.status.idle":"2021-12-17T08:24:41.714705Z","shell.execute_reply.started":"2021-12-17T08:24:41.707721Z","shell.execute_reply":"2021-12-17T08:24:41.713758Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/train_imgs', exist_ok = True)","metadata":{"_uuid":"75214f62-c7d0-45ac-9b02-e8ca84577990","_cell_guid":"f6b33298-f179-43ad-9989-262b3c8e0aac","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.716169Z","iopub.execute_input":"2021-12-17T08:24:41.716431Z","iopub.status.idle":"2021-12-17T08:24:41.723557Z","shell.execute_reply.started":"2021-12-17T08:24:41.716397Z","shell.execute_reply":"2021-12-17T08:24:41.722789Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_pixel_array(data,fn):\n    plt.figure()\n    plt.imshow(data, cmap=plt.cm.bone)\n    plt.show()\n\n\n# ref kernel: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\ndef read_xray(path, voi_lut=True, fix_monochrome=True):\n    dcm_data = dicom.read_file(path)\n    \n    def show_dcm_info(data):\n        print(\"Gender :\", data.PatientSex)\n        if 'PixelData' in data:\n            rows = int(data.Rows)\n            cols = int(data.Columns)\n            print(\"Image size : {rows:d} x {cols:d}, {size:d} bytes\".format(\n                rows=rows, cols=cols, size=len(data.PixelData)))\n            if 'PixelSpacing' in data:\n                print(\"Pixel spacing :\", data.PixelSpacing)\n    \n    show_dcm_info(dcm_data)\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dcm_data.pixel_array, dcm_data)\n    else:\n        data = dcm_data.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dcm_data.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"_uuid":"047cf43a-9148-4360-90c8-1d129c259439","_cell_guid":"7de4b1cf-c7b1-43bc-986b-99f4d7c25498","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.72464Z","iopub.execute_input":"2021-12-17T08:24:41.725468Z","iopub.status.idle":"2021-12-17T08:24:41.737697Z","shell.execute_reply.started":"2021-12-17T08:24:41.725433Z","shell.execute_reply":"2021-12-17T08:24:41.73697Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_xray2(path, voi_lut=True, fix_monochrome=True):\n    dcm_data = dicom.read_file(path)\n    new_image = dcm_data.pixel_array.astype(float)\n    scaled_image = (np.maximum(new_image, 0) / new_image.max()) * 255.0\n    scaled_image = np.uint8(scaled_image) \n    final_image = Image.fromarray(scaled_image)\n    return final_image","metadata":{"_uuid":"41e21f62-8668-4535-8445-b0063145dd13","_cell_guid":"18b63cbd-034e-40ee-be72-c8d7636ffbd4","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.739391Z","iopub.execute_input":"2021-12-17T08:24:41.739654Z","iopub.status.idle":"2021-12-17T08:24:41.748546Z","shell.execute_reply.started":"2021-12-17T08:24:41.739604Z","shell.execute_reply":"2021-12-17T08:24:41.747739Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(3000):\n    fn = train_files[i]\n    file_path = os.path.join(train_dir, fn)\n    data = read_xray2(file_path)\n    path=\"/kaggle/working/train_imgs/\"\n    fn_new = fn[:len(fn) - 6]\n    data.save(path + fn_new +\".png\")","metadata":{"_uuid":"3d98a867-6b0c-4d94-8093-636608353979","_cell_guid":"d86923f8-f2bc-4b9b-b348-f291fcfea887","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T08:24:41.749852Z","iopub.execute_input":"2021-12-17T08:24:41.750341Z","iopub.status.idle":"2021-12-17T10:29:16.865499Z","shell.execute_reply.started":"2021-12-17T08:24:41.750296Z","shell.execute_reply":"2021-12-17T10:29:16.864658Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Examining train images...\")\nfor _ in range(5):\n    fn = train_files[np.random.randint(0, len(train_files))]\n    file_path = os.path.join(train_dir, fn)\n    data = read_xray(file_path)\n    plot_pixel_array(data,fn)","metadata":{"_uuid":"8e4e3ca3-1850-48f1-8aa3-fa7ee516490f","_cell_guid":"7793e0ce-b3df-4a4e-a539-ed70ff9cb217","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:16.86699Z","iopub.execute_input":"2021-12-17T10:29:16.86727Z","iopub.status.idle":"2021-12-17T10:29:27.026732Z","shell.execute_reply.started":"2021-12-17T10:29:16.867236Z","shell.execute_reply":"2021-12-17T10:29:27.026073Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for _ in range(10):\n    idx = np.random.randint(0, len(train_files))\n    img_id = train_df.loc[idx, 'image_id']\n    img = read_xray(os.path.join(train_dir, img_id+\".dicom\"))\n    plt.figure(figsize=(8, 14))\n    plt.imshow(img, cmap='gray')\n    plt.title(train_df.loc[idx, 'class_name'])\n    \n    if train_df.loc[idx, 'class_name'] != 'No finding':\n        bbox = [train_df.loc[idx, 'x_min'],\n                train_df.loc[idx, 'y_min'],\n                train_df.loc[idx, 'x_max'],\n                train_df.loc[idx, 'y_max']]\n        \n        patch = ptc.Rectangle((bbox[0], bbox[1]),\n                              bbox[2]-bbox[0],\n                              bbox[3]-bbox[1],\n                              ec='r', fc='none', lw=2.)\n        ax = plt.gca()\n        ax.add_patch(patch)","metadata":{"_uuid":"5963ec41-3e4f-418d-bfa8-fd22a2170306","_cell_guid":"c81cf16c-d5f3-41a6-91ee-788a8751d2a7","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:27.028145Z","iopub.execute_input":"2021-12-17T10:29:27.028396Z","iopub.status.idle":"2021-12-17T10:29:50.413753Z","shell.execute_reply.started":"2021-12-17T10:29:27.028361Z","shell.execute_reply":"2021-12-17T10:29:50.41316Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_files=os.listdir(\"/kaggle/working/train_imgs\")","metadata":{"_uuid":"0235d2b8-7d1a-4424-ad23-4b0f335a4c2f","_cell_guid":"8e6493a2-4850-4f9e-a9d0-c8387c9cd414","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:50.414979Z","iopub.execute_input":"2021-12-17T10:29:50.415477Z","iopub.status.idle":"2021-12-17T10:29:50.422316Z","shell.execute_reply.started":"2021-12-17T10:29:50.415438Z","shell.execute_reply":"2021-12-17T10:29:50.421634Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(1):\n#     idx = 10850\n#     img_id = train_df.loc[idx, 'image_id']\n#     path=\"/kaggle/working/train_imgs/\"\n#     img = os.path.join(path, img_id+\".png\")\n#     img=plt.imread(img)\n#     plt.figure(figsize=(8, 14))\n#     plt.imshow(img, cmap='gray')\n#     plt.title(train_df.loc[idx, 'class_name'])\n    \n#     if train_df.loc[idx, 'class_name'] != 'No finding':\n#         bbox = [train_df.loc[idx, 'x_min'],\n#                 train_df.loc[idx, 'y_min'],\n#                 train_df.loc[idx, 'x_max'],\n#                 train_df.loc[idx, 'y_max']]\n        \n#         patch = ptc.Rectangle((bbox[0], bbox[1]),\n#                               bbox[2]-bbox[0],\n#                               bbox[3]-bbox[1],\n#                               ec='r', fc='none', lw=2.)\n#         ax = plt.gca()\n#         ax.add_patch(patch)","metadata":{"_uuid":"59f96021-8c43-4189-8261-e1e1bbe130d7","_cell_guid":"30242501-decc-4fde-9177-1b559c623c01","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:50.423896Z","iopub.execute_input":"2021-12-17T10:29:50.424374Z","iopub.status.idle":"2021-12-17T10:29:50.431489Z","shell.execute_reply.started":"2021-12-17T10:29:50.424328Z","shell.execute_reply":"2021-12-17T10:29:50.430609Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"5da383c7-13a6-48f3-be03-40ac701c9477","_cell_guid":"6fc44034-293c-42eb-abe8-e1db15ae700a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_ids, class_names = list(zip(*set(zip(train_df.class_id, train_df.class_name))))\nclasses = list(np.array(class_names)[np.argsort(class_ids)])\nclasses = list(map(lambda x: str(x), classes))\nclasses","metadata":{"_uuid":"1757e03c-d5fc-4931-a5e5-d57be79b850d","_cell_guid":"f9d874ae-9e16-48d8-8e38-379b1b2762ab","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:50.432583Z","iopub.execute_input":"2021-12-17T10:29:50.432786Z","iopub.status.idle":"2021-12-17T10:29:50.462835Z","shell.execute_reply.started":"2021-12-17T10:29:50.432751Z","shell.execute_reply":"2021-12-17T10:29:50.462171Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp=train_df[\"image_id\"].to_list()","metadata":{"_uuid":"756d014c-56a7-4206-afa7-8c160a801e3b","_cell_guid":"64cbc334-307a-4e56-8ebc-7d94899b9948","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:50.464175Z","iopub.execute_input":"2021-12-17T10:29:50.464594Z","iopub.status.idle":"2021-12-17T10:29:50.474981Z","shell.execute_reply.started":"2021-12-17T10:29:50.464558Z","shell.execute_reply":"2021-12-17T10:29:50.474318Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp2=[]\nfor i in range(3000):\n    fn=train_files[i]\n    temp2.append(fn[:len(fn)-6])\n    \ntemp_df=train_df[train_df[\"image_id\"].isin(temp2)]","metadata":{"_uuid":"7d2a7917-1ae4-452e-b110-ce79e29fda1a","_cell_guid":"1ad5e241-d8af-464d-a966-dc8e7a471d0d","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:50.476473Z","iopub.execute_input":"2021-12-17T10:29:50.476752Z","iopub.status.idle":"2021-12-17T10:29:50.504487Z","shell.execute_reply.started":"2021-12-17T10:29:50.476712Z","shell.execute_reply":"2021-12-17T10:29:50.503832Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_df","metadata":{"_uuid":"7ea7866e-7878-4e06-9faf-ff35414a0f53","_cell_guid":"adbb89f6-b14e-46ca-a6aa-23f0f6575c09","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:50.506162Z","iopub.execute_input":"2021-12-17T10:29:50.506351Z","iopub.status.idle":"2021-12-17T10:29:50.535049Z","shell.execute_reply.started":"2021-12-17T10:29:50.50632Z","shell.execute_reply":"2021-12-17T10:29:50.534338Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_df = temp_df[temp_df.class_id!=14].reset_index(drop = True)","metadata":{"_uuid":"e7644a34-8f15-46fb-a516-b28de966c15c","_cell_guid":"f32eb5cc-3672-45d4-a279-452e64b68c66","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:50.53642Z","iopub.execute_input":"2021-12-17T10:29:50.536669Z","iopub.status.idle":"2021-12-17T10:29:50.543279Z","shell.execute_reply.started":"2021-12-17T10:29:50.536627Z","shell.execute_reply":"2021-12-17T10:29:50.542429Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_df['image_width'] = temp_df.apply(lambda row: Image.open(row.image_path).width, axis =1)\ntemp_df['image_height'] = temp_df.apply(lambda row: Image.open(row.image_path).height, axis =1)\n\ntemp_df['x_min'] = temp_df.apply(lambda row: (row.x_min)/row.image_width, axis =1)\ntemp_df['y_min'] = temp_df.apply(lambda row: (row.y_min)/row.image_height, axis =1)\n\ntemp_df['x_max'] = temp_df.apply(lambda row: (row.x_max)/row.image_width, axis =1)\ntemp_df['y_max'] = temp_df.apply(lambda row: (row.y_max)/row.image_height, axis =1)\n\ntemp_df['x_mid'] = temp_df.apply(lambda row: (row.x_max+row.x_min)/2, axis =1)\ntemp_df['y_mid'] = temp_df.apply(lambda row: (row.y_max+row.y_min)/2, axis =1)\n\ntemp_df['w'] = temp_df.apply(lambda row: (row.x_max-row.x_min), axis =1)\ntemp_df['h'] = temp_df.apply(lambda row: (row.y_max-row.y_min), axis =1)\n\ntemp_df['area'] = temp_df['w']*temp_df['h']\ntemp_df.head()\n\nfeatures = [\"image_id\",\"class_id\",'x_min', 'y_min', 'x_max', 'y_max', 'x_mid', 'y_mid', 'w', 'h', 'area']\nX = temp_df[features]\ny = temp_df['class_id']\nX.shape, y.shape","metadata":{"_uuid":"e84f2d43-11a7-4c79-86ab-25ab1e8b863c","_cell_guid":"ac9a1cae-78b9-4fc4-b615-ab0927eda65e","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:50.548978Z","iopub.execute_input":"2021-12-17T10:29:50.549183Z","iopub.status.idle":"2021-12-17T10:29:54.031612Z","shell.execute_reply.started":"2021-12-17T10:29:50.549159Z","shell.execute_reply":"2021-12-17T10:29:54.030891Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"_uuid":"32ee0a3a-d686-4acf-bd6b-ee872d2b1251","_cell_guid":"8ded86f6-edc9-4926-acf8-8d974e5b2d03","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.032777Z","iopub.execute_input":"2021-12-17T10:29:54.033027Z","iopub.status.idle":"2021-12-17T10:29:54.046726Z","shell.execute_reply.started":"2021-12-17T10:29:54.032994Z","shell.execute_reply":"2021-12-17T10:29:54.045676Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"_uuid":"7e1fb10f-933c-4bdc-9bc4-a5d7c623d97f","_cell_guid":"2dd06169-7eb1-430d-878e-60864bb402ec","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.053413Z","iopub.execute_input":"2021-12-17T10:29:54.054756Z","iopub.status.idle":"2021-12-17T10:29:54.09227Z","shell.execute_reply.started":"2021-12-17T10:29:54.054714Z","shell.execute_reply":"2021-12-17T10:29:54.090107Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.describe()","metadata":{"_uuid":"7df4e50c-c955-4905-9572-48dd1fc27739","_cell_guid":"498d3702-2f1c-480c-8846-26f9e034eb16","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.096744Z","iopub.execute_input":"2021-12-17T10:29:54.098217Z","iopub.status.idle":"2021-12-17T10:29:54.163176Z","shell.execute_reply.started":"2021-12-17T10:29:54.098161Z","shell.execute_reply":"2021-12-17T10:29:54.161743Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp6=X_train.image_id.unique()\nlen(temp6)","metadata":{"_uuid":"8a1d26cc-3638-433b-8025-9d6cb05bec71","_cell_guid":"c5a5ac87-bbed-4917-af90-83cfe1041d2a","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.167987Z","iopub.execute_input":"2021-12-17T10:29:54.168409Z","iopub.status.idle":"2021-12-17T10:29:54.184304Z","shell.execute_reply.started":"2021-12-17T10:29:54.168368Z","shell.execute_reply":"2021-12-17T10:29:54.183389Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_val","metadata":{"_uuid":"6c7fdb6d-5870-4b31-9ae9-d0af5a70d929","_cell_guid":"292f3458-ce15-4514-ab0b-e741112b5cdd","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.188717Z","iopub.execute_input":"2021-12-17T10:29:54.190984Z","iopub.status.idle":"2021-12-17T10:29:54.223201Z","shell.execute_reply.started":"2021-12-17T10:29:54.190942Z","shell.execute_reply":"2021-12-17T10:29:54.22134Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"_uuid":"24b259ce-f1bc-4598-bc15-fd2bd8fdee8a","_cell_guid":"5d4d256a-5e66-46dd-8a90-5268cf721ad1","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.224262Z","iopub.execute_input":"2021-12-17T10:29:54.224569Z","iopub.status.idle":"2021-12-17T10:29:54.232727Z","shell.execute_reply.started":"2021-12-17T10:29:54.224528Z","shell.execute_reply":"2021-12-17T10:29:54.231341Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val","metadata":{"_uuid":"d6f1439a-888f-4c8f-a20e-8177aeee227b","_cell_guid":"d3815fef-36cb-4bdb-b3ea-8aba9a091c38","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.235551Z","iopub.execute_input":"2021-12-17T10:29:54.236433Z","iopub.status.idle":"2021-12-17T10:29:54.252456Z","shell.execute_reply.started":"2021-12-17T10:29:54.236398Z","shell.execute_reply":"2021-12-17T10:29:54.25145Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files   = []\nval_files += list(\"/kaggle/working/train_imgs/\"+X_val[\"image_id\"]+\".png\")\ntrain_files += list(\"/kaggle/working/train_imgs/\"+X_train[\"image_id\"]+\".png\")\nlen(train_files), len(val_files)","metadata":{"_uuid":"84f5457f-a072-48e6-b795-5d7e44514a7f","_cell_guid":"9229698f-90e0-402b-9f95-7286929e9c5c","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.257686Z","iopub.execute_input":"2021-12-17T10:29:54.258105Z","iopub.status.idle":"2021-12-17T10:29:54.280652Z","shell.execute_reply.started":"2021-12-17T10:29:54.258061Z","shell.execute_reply":"2021-12-17T10:29:54.279889Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files","metadata":{"_uuid":"ec61f7be-d0ba-401a-8fe8-2d6ca787be67","_cell_guid":"3b706501-4d33-4489-9031-acb5598e4bfd","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.284213Z","iopub.execute_input":"2021-12-17T10:29:54.286675Z","iopub.status.idle":"2021-12-17T10:29:54.328753Z","shell.execute_reply.started":"2021-12-17T10:29:54.286636Z","shell.execute_reply":"2021-12-17T10:29:54.328024Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_files","metadata":{"_uuid":"c6660cba-b2e7-4106-9314-0a9b071b6bcf","_cell_guid":"ae84ac0c-bde1-4448-9175-e22cb3f9309d","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.332567Z","iopub.execute_input":"2021-12-17T10:29:54.334947Z","iopub.status.idle":"2021-12-17T10:29:54.374475Z","shell.execute_reply.started":"2021-12-17T10:29:54.334885Z","shell.execute_reply":"2021-12-17T10:29:54.37386Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/vinbigdata/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/val', exist_ok = True)","metadata":{"_uuid":"d2666918-99ec-464a-a3f1-34fbeeea7db5","_cell_guid":"7b557e39-6dc9-4b8d-b293-73748619fd00","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.378279Z","iopub.execute_input":"2021-12-17T10:29:54.380554Z","iopub.status.idle":"2021-12-17T10:29:54.387638Z","shell.execute_reply.started":"2021-12-17T10:29:54.380516Z","shell.execute_reply":"2021-12-17T10:29:54.386842Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil as sh","metadata":{"_uuid":"88ae090a-47cf-40f4-8778-d5e83a5ad23d","_cell_guid":"f75bc15c-00f5-435b-bc27-49a0007f19dd","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.392162Z","iopub.execute_input":"2021-12-17T10:29:54.394508Z","iopub.status.idle":"2021-12-17T10:29:54.39989Z","shell.execute_reply.started":"2021-12-17T10:29:54.394471Z","shell.execute_reply":"2021-12-17T10:29:54.399083Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    for train_img in tqdm(X_train.image_id.unique()):\n        with open(f'/kaggle/working/vinbigdata/labels/train/{train_img}.txt', 'w+') as f:\n            row = X_train[X_train['image_id']==train_img]\\\n            [['class_id', 'x_mid', 'y_mid', 'w', 'h']].values\n#             row[:, 1:] /= SIZE\n            row = row.astype('str')\n            for box in range(len(row)):\n                text = ' '.join(row[box])\n                f.write(text)\n                f.write('\\n')\n        sh.copy(f'/kaggle/working/train_imgs/{train_img}.png', \n                f'/kaggle/working/vinbigdata/images/train/{train_img}.png')\n        \n    for val_img in tqdm(X_val.image_id.unique()):\n        with open(f'/kaggle/working/vinbigdata/labels/val/{val_img}.txt', 'w+') as f:\n            row = X_val[X_val['image_id']==val_img]\\\n            [['class_id', 'x_mid', 'y_mid', 'w', 'h']].values\n#             row[:, 1:] /= SIZE\n            row = row.astype('str')\n            for box in range(len(row)):\n                text = ' '.join(row[box])\n                f.write(text)\n                f.write('\\n')\n        sh.copy(f'/kaggle/working/train_imgs/{val_img}.png', \n                f'/kaggle/working/vinbigdata/images/val/{val_img}.png')","metadata":{"_uuid":"f742a3bc-ada8-4a7e-b51b-8202cade63d1","_cell_guid":"575cef89-1b64-4a52-83de-7d96f0b54c7c","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:29:54.402663Z","iopub.execute_input":"2021-12-17T10:29:54.403364Z","iopub.status.idle":"2021-12-17T10:30:30.638384Z","shell.execute_reply.started":"2021-12-17T10:29:54.403328Z","shell.execute_reply":"2021-12-17T10:30:30.637696Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes=classes[:-1]","metadata":{"_uuid":"e216a064-7b01-44ed-8bbc-e7fedcaf8efc","_cell_guid":"d0320f6b-e231-41e6-83a2-175bb0b19326","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:30:30.639812Z","iopub.execute_input":"2021-12-17T10:30:30.640291Z","iopub.status.idle":"2021-12-17T10:30:30.670819Z","shell.execute_reply.started":"2021-12-17T10:30:30.640247Z","shell.execute_reply":"2021-12-17T10:30:30.668186Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes","metadata":{"_uuid":"8cf0dadb-19c3-474d-9034-44c20b1f91cb","_cell_guid":"30f28246-c64b-4d8e-bdea-40c3674b292c","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:30:30.681465Z","iopub.execute_input":"2021-12-17T10:30:30.682876Z","iopub.status.idle":"2021-12-17T10:30:35.535275Z","shell.execute_reply.started":"2021-12-17T10:30:30.682828Z","shell.execute_reply":"2021-12-17T10:30:35.534486Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/ultralytics/yolov5  # clone\n%cd yolov5\n%pip install -qr requirements.txt  # install\n\nimport torch\nfrom yolov5 import utils\ndisplay = utils.notebook_init()  # checks","metadata":{"_uuid":"7849f4d8-fde5-4380-9383-584893afdc48","_cell_guid":"0ed80b53-d54f-4498-a5a2-62dacca9fa60","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:30:35.536294Z","iopub.execute_input":"2021-12-17T10:30:35.537047Z","iopub.status.idle":"2021-12-17T10:31:02.684379Z","shell.execute_reply.started":"2021-12-17T10:30:35.536996Z","shell.execute_reply":"2021-12-17T10:31:02.683509Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sh.copy('/kaggle/input/yaml-file-chest/chest_vinbig.yaml','/kaggle/working/yolov5/data/chest_vinbig.yaml')","metadata":{"_uuid":"fe7529ce-c310-4163-acec-310511d1425e","_cell_guid":"afc27d58-2a46-4865-ba61-3813899fcf7b","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:31:02.686711Z","iopub.execute_input":"2021-12-17T10:31:02.68719Z","iopub.status.idle":"2021-12-17T10:31:02.691241Z","shell.execute_reply.started":"2021-12-17T10:31:02.687147Z","shell.execute_reply":"2021-12-17T10:31:02.690419Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/vinbigdata/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/vinbigdata/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 14,\n    names = classes\n    )\n\nwith open(join( cwd , 'vinbigdata.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'vinbigdata.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"_uuid":"4258d5c8-d75a-468f-8069-12a8385fdd79","_cell_guid":"5b6882c4-94de-4f0a-a887-6183e59a0bcd","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:31:02.692685Z","iopub.execute_input":"2021-12-17T10:31:02.6932Z","iopub.status.idle":"2021-12-17T10:31:02.715224Z","shell.execute_reply.started":"2021-12-17T10:31:02.693158Z","shell.execute_reply":"2021-12-17T10:31:02.714474Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/yolov5')\n# %pip install -qr requirements.txt # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{"_uuid":"1b9af68a-3c1b-43f7-81a1-96eedc95fadc","_cell_guid":"5b05d011-7987-49fd-bc81-03fba1be8bc7","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:31:02.716537Z","iopub.execute_input":"2021-12-17T10:31:02.716788Z","iopub.status.idle":"2021-12-17T10:31:02.723715Z","shell.execute_reply.started":"2021-12-17T10:31:02.716755Z","shell.execute_reply":"2021-12-17T10:31:02.722752Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights yolov5s.pt --img 640 --conf 0.25 --source data/images/\nImage(filename='runs/detect/exp/zidane.jpg', width=600)","metadata":{"_uuid":"b8540943-ddf9-4f08-8cf4-4625c216a9af","_cell_guid":"37a63796-ee79-4814-a7de-bee32b2af9f3","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:31:02.72523Z","iopub.execute_input":"2021-12-17T10:31:02.72573Z","iopub.status.idle":"2021-12-17T10:31:19.606491Z","shell.execute_reply.started":"2021-12-17T10:31:02.725691Z","shell.execute_reply":"2021-12-17T10:31:19.605071Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python train.py --img 640 --batch 16 --epochs 30 --data /kaggle/working/vinbigdata.yaml --weights yolov5x.pt --cache","metadata":{"_uuid":"09567261-c6d1-4aa7-b8e3-06914757a4be","_cell_guid":"6ae4da37-5a17-4ae7-a72c-ca21dc9d6c48","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T10:31:19.608364Z","iopub.execute_input":"2021-12-17T10:31:19.6089Z","iopub.status.idle":"2021-12-17T12:05:02.394295Z","shell.execute_reply.started":"2021-12-17T10:31:19.60886Z","shell.execute_reply":"2021-12-17T12:05:02.392249Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch0.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch1.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('runs/train/exp/train_batch2.jpg'))","metadata":{"_uuid":"a27d4b5c-cd60-4472-b175-b1497f6c9edf","_cell_guid":"886e07b3-5806-4100-a10a-41fb811e62fa","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T12:05:02.397362Z","iopub.execute_input":"2021-12-17T12:05:02.397684Z","iopub.status.idle":"2021-12-17T12:05:05.752001Z","shell.execute_reply.started":"2021-12-17T12:05:02.397628Z","shell.execute_reply":"2021-12-17T12:05:05.751106Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.axis('off')\nplt.imshow(plt.imread('runs/train/exp/results.png'));","metadata":{"_uuid":"b0266a69-27f3-480c-8137-c59a759eb234","_cell_guid":"fb3243fa-6f42-4766-a7c6-f728217c516f","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T12:05:05.753525Z","iopub.execute_input":"2021-12-17T12:05:05.753795Z","iopub.status.idle":"2021-12-17T12:05:06.855618Z","shell.execute_reply.started":"2021-12-17T12:05:05.753759Z","shell.execute_reply":"2021-12-17T12:05:06.854904Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights 'runs/train/exp/weights/best.pt'\\\n--img 640\\\n--conf 0.15\\\n--iou 0.5\\\n--source /kaggle/working/vinbigdata/images/val\\\n--exist-ok","metadata":{"_uuid":"15d0d7be-d97c-4fb7-8e3d-80d23251c1ed","_cell_guid":"8cffd778-9b55-4448-9b99-cee3a5ebcc52","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T12:05:06.857177Z","iopub.execute_input":"2021-12-17T12:05:06.857636Z","iopub.status.idle":"2021-12-17T12:09:53.065921Z","shell.execute_reply.started":"2021-12-17T12:05:06.857599Z","shell.execute_reply":"2021-12-17T12:09:53.06493Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nimport numpy as np\nimport random\nimport cv2\nfrom glob import glob\nfrom tqdm import tqdm\n\nfiles = glob('runs/detect/exp/*')\nfor _ in range(3):\n    row = 4\n    col = 4\n    grid_files = random.sample(files, row*col)\n    images     = []\n    for image_path in tqdm(grid_files):\n        img          = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n        images.append(img)\n\n    fig = plt.figure(figsize=(col*5, row*5))\n    grid = ImageGrid(fig, 111,  # similar to subplot(111)\n                     nrows_ncols=(col, row),  # creates 2x2 grid of axes\n                     axes_pad=0.05,  # pad between axes in inch.\n                     )\n\n    for ax, im in zip(grid, images):\n        # Iterating over the grid returns the Axes.\n        ax.imshow(im)\n        ax.set_xticks([])\n        ax.set_yticks([])\n    plt.show()","metadata":{"_uuid":"9eda80f8-f30a-4bf2-95b3-cb709323fe6b","_cell_guid":"72ecb124-f058-495a-a8bb-1e7eb7c98b00","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T12:09:53.068367Z","iopub.execute_input":"2021-12-17T12:09:53.068677Z","iopub.status.idle":"2021-12-17T12:10:33.896425Z","shell.execute_reply.started":"2021-12-17T12:09:53.068635Z","shell.execute_reply":"2021-12-17T12:10:33.894105Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r data_final.zip /kaggle/working","metadata":{"_uuid":"d8a53a6e-6d56-4c26-8bb6-81669762279a","_cell_guid":"5a38e0fa-2596-446d-b244-9f18c0882b9c","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T12:57:37.779774Z","iopub.execute_input":"2021-12-17T12:57:37.780407Z","iopub.status.idle":"2021-12-17T12:58:57.589539Z","shell.execute_reply.started":"2021-12-17T12:57:37.780368Z","shell.execute_reply":"2021-12-17T12:58:57.588656Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r /kaggle/working/yolo_trained.zip /kaggle/working/yolov5","metadata":{"_uuid":"59eb3f65-8d9e-4970-9d7a-1f22d8d621fa","_cell_guid":"f0df3977-e064-4689-8ae7-bc6048323462","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T13:19:46.296427Z","iopub.execute_input":"2021-12-17T13:19:46.297174Z","iopub.status.idle":"2021-12-17T13:25:22.490671Z","shell.execute_reply.started":"2021-12-17T13:19:46.29713Z","shell.execute_reply":"2021-12-17T13:25:22.489795Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm -r /kaggle/working/yolov5/yolo_trained.zip","metadata":{"_uuid":"00b23909-c50d-480f-a5c9-03f42e6a9682","_cell_guid":"78921bab-6911-4cf8-8076-9411d0aef44c","collapsed":false,"execution":{"iopub.status.busy":"2021-12-17T13:18:18.014751Z","iopub.execute_input":"2021-12-17T13:18:18.016463Z","iopub.status.idle":"2021-12-17T13:18:18.734203Z","shell.execute_reply.started":"2021-12-17T13:18:18.016417Z","shell.execute_reply":"2021-12-17T13:18:18.733228Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}