{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Version\n* `v13`: Fold4\n* `v12`: Fold3\n* `v10`: Fold2\n* `v09`: Fold1\n* `v03`: Fold0","metadata":{}},{"cell_type":"code","source":"!pip install --upgrade seaborn","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":9.633907,"end_time":"2021-01-01T09:44:53.448657","exception":false,"start_time":"2021-01-01T09:44:43.81475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:32.96231Z","iopub.execute_input":"2022-03-30T17:18:32.962743Z","iopub.status.idle":"2022-03-30T17:18:43.330336Z","shell.execute_reply.started":"2022-03-30T17:18:32.962709Z","shell.execute_reply":"2022-03-30T17:18:43.329235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns","metadata":{"papermill":{"duration":0.926929,"end_time":"2021-01-01T09:44:54.403588","exception":false,"start_time":"2021-01-01T09:44:53.476659","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:43.333872Z","iopub.execute_input":"2022-03-30T17:18:43.334423Z","iopub.status.idle":"2022-03-30T17:18:44.43204Z","shell.execute_reply.started":"2022-03-30T17:18:43.334363Z","shell.execute_reply":"2022-03-30T17:18:44.431112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dim = 512 #512, 256, 'original'\nfold = 4","metadata":{"execution":{"iopub.status.busy":"2022-03-30T17:18:44.433484Z","iopub.execute_input":"2022-03-30T17:18:44.433788Z","iopub.status.idle":"2022-03-30T17:18:44.438876Z","shell.execute_reply.started":"2022-03-30T17:18:44.433752Z","shell.execute_reply":"2022-03-30T17:18:44.437961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f'../input/vinbigdata-{dim}-image-dataset/vinbigdata/train.csv')\ntrain_df.head()","metadata":{"papermill":{"duration":0.262045,"end_time":"2021-01-01T09:44:54.691965","exception":false,"start_time":"2021-01-01T09:44:54.42992","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:44.440446Z","iopub.execute_input":"2022-03-30T17:18:44.440757Z","iopub.status.idle":"2022-03-30T17:18:44.666825Z","shell.execute_reply.started":"2022-03-30T17:18:44.440726Z","shell.execute_reply":"2022-03-30T17:18:44.665844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image_path'] = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/train/'+train_df.image_id+('.png' if dim!='original' else '.jpg')\ntrain_df.head()","metadata":{"papermill":{"duration":0.086788,"end_time":"2021-01-01T09:44:54.805857","exception":false,"start_time":"2021-01-01T09:44:54.719069","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:44.669518Z","iopub.execute_input":"2022-03-30T17:18:44.669843Z","iopub.status.idle":"2022-03-30T17:18:44.750256Z","shell.execute_reply.started":"2022-03-30T17:18:44.669809Z","shell.execute_reply":"2022-03-30T17:18:44.749127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Only 14 Class","metadata":{"papermill":{"duration":0.027478,"end_time":"2021-01-01T09:44:54.861374","exception":false,"start_time":"2021-01-01T09:44:54.833896","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df = train_df[train_df.class_id!=14].reset_index(drop = True)","metadata":{"papermill":{"duration":0.05543,"end_time":"2021-01-01T09:44:54.944088","exception":false,"start_time":"2021-01-01T09:44:54.888658","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:45.196354Z","iopub.execute_input":"2022-03-30T17:18:45.196818Z","iopub.status.idle":"2022-03-30T17:18:45.23588Z","shell.execute_reply.started":"2022-03-30T17:18:45.196776Z","shell.execute_reply":"2022-03-30T17:18:45.234921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pre-Processing","metadata":{"papermill":{"duration":0.027303,"end_time":"2021-01-01T09:44:54.999199","exception":false,"start_time":"2021-01-01T09:44:54.971896","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df['x_min'] = train_df.apply(lambda row: (row.x_min)/row.width, axis =1)\ntrain_df['y_min'] = train_df.apply(lambda row: (row.y_min)/row.height, axis =1)\n\ntrain_df['x_max'] = train_df.apply(lambda row: (row.x_max)/row.width, axis =1)\ntrain_df['y_max'] = train_df.apply(lambda row: (row.y_max)/row.height, axis =1)\n\ntrain_df['x_mid'] = train_df.apply(lambda row: (row.x_max+row.x_min)/2, axis =1)\ntrain_df['y_mid'] = train_df.apply(lambda row: (row.y_max+row.y_min)/2, axis =1)\n\ntrain_df['w'] = train_df.apply(lambda row: (row.x_max-row.x_min), axis =1)\ntrain_df['h'] = train_df.apply(lambda row: (row.y_max-row.y_min), axis =1)\n\ntrain_df['area'] = train_df['w']*train_df['h']\ntrain_df.head()","metadata":{"papermill":{"duration":7.821668,"end_time":"2021-01-01T09:45:02.854149","exception":false,"start_time":"2021-01-01T09:44:55.032481","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:46.230435Z","iopub.execute_input":"2022-03-30T17:18:46.231122Z","iopub.status.idle":"2022-03-30T17:18:54.985043Z","shell.execute_reply.started":"2022-03-30T17:18:46.231079Z","shell.execute_reply":"2022-03-30T17:18:54.983859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['x_min', 'y_min', 'x_max', 'y_max', 'x_mid', 'y_mid', 'w', 'h', 'area']\nX = train_df[features]\ny = train_df['class_id']\nX.shape, y.shape","metadata":{"papermill":{"duration":0.040387,"end_time":"2021-01-01T09:45:02.923416","exception":false,"start_time":"2021-01-01T09:45:02.883029","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:54.987126Z","iopub.execute_input":"2022-03-30T17:18:54.987476Z","iopub.status.idle":"2022-03-30T17:18:54.999221Z","shell.execute_reply.started":"2022-03-30T17:18:54.987434Z","shell.execute_reply":"2022-03-30T17:18:54.998289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_ids, class_names = list(zip(*set(zip(train_df.class_id, train_df.class_name))))\nclasses = list(np.array(class_names)[np.argsort(class_ids)])\nclasses = list(map(lambda x: str(x), classes))\nclasses","metadata":{"papermill":{"duration":0.050418,"end_time":"2021-01-01T09:45:03.002944","exception":false,"start_time":"2021-01-01T09:45:02.952526","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:55.000666Z","iopub.execute_input":"2022-03-30T17:18:55.000974Z","iopub.status.idle":"2022-03-30T17:18:55.028486Z","shell.execute_reply.started":"2022-03-30T17:18:55.000945Z","shell.execute_reply":"2022-03-30T17:18:55.027231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split","metadata":{"papermill":{"duration":0.052492,"end_time":"2021-01-01T09:47:56.110766","exception":false,"start_time":"2021-01-01T09:47:56.058274","status":"completed"},"tags":[]}},{"cell_type":"code","source":"gkf  = GroupKFold(n_splits = 5)\ntrain_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups = train_df.image_id.tolist())):\n    train_df.loc[val_idx, 'fold'] = fold\ntrain_df.head()","metadata":{"papermill":{"duration":0.134603,"end_time":"2021-01-01T09:47:56.297774","exception":false,"start_time":"2021-01-01T09:47:56.163171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:55.902334Z","iopub.execute_input":"2022-03-30T17:18:55.902789Z","iopub.status.idle":"2022-03-30T17:18:55.999865Z","shell.execute_reply.started":"2022-03-30T17:18:55.902749Z","shell.execute_reply":"2022-03-30T17:18:55.998614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files   = []\nval_files += list(train_df[train_df.fold==fold].image_path.unique())\ntrain_files += list(train_df[train_df.fold!=fold].image_path.unique())\nlen(train_files), len(val_files)","metadata":{"papermill":{"duration":0.086817,"end_time":"2021-01-01T09:47:56.443789","exception":false,"start_time":"2021-01-01T09:47:56.356972","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:18:59.172415Z","iopub.execute_input":"2022-03-30T17:18:59.172858Z","iopub.status.idle":"2022-03-30T17:18:59.206053Z","shell.execute_reply.started":"2022-03-30T17:18:59.172815Z","shell.execute_reply":"2022-03-30T17:18:59.20518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Copying Files","metadata":{"papermill":{"duration":0.083752,"end_time":"2021-01-01T09:47:56.584924","exception":false,"start_time":"2021-01-01T09:47:56.501172","status":"completed"},"tags":[]}},{"cell_type":"code","source":"os.makedirs('/kaggle/working/vinbigdata/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/vinbigdata-yolo-labels-dataset/labels'\nfor file in tqdm(train_files):\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/train')\n    \nfor file in tqdm(val_files):\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/val')","metadata":{"papermill":{"duration":124.654777,"end_time":"2021-01-01T09:50:01.331041","exception":false,"start_time":"2021-01-01T09:47:56.676264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:19:01.091006Z","iopub.execute_input":"2022-03-30T17:19:01.091415Z","iopub.status.idle":"2022-03-30T17:19:48.729268Z","shell.execute_reply.started":"2022-03-30T17:19:01.091381Z","shell.execute_reply":"2022-03-30T17:19:48.728092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-03-30T17:21:16.711317Z","iopub.execute_input":"2022-03-30T17:21:16.711766Z","iopub.status.idle":"2022-03-30T17:21:18.970223Z","shell.execute_reply.started":"2022-03-30T17:21:16.711728Z","shell.execute_reply":"2022-03-30T17:21:18.969093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Class Name","metadata":{"papermill":{"duration":0.068822,"end_time":"2021-01-01T09:50:01.458337","exception":false,"start_time":"2021-01-01T09:50:01.389515","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pwd","metadata":{"execution":{"iopub.status.busy":"2022-03-30T17:20:40.79131Z","iopub.execute_input":"2022-03-30T17:20:40.791755Z","iopub.status.idle":"2022-03-30T17:20:41.544182Z","shell.execute_reply.started":"2022-03-30T17:20:40.791716Z","shell.execute_reply":"2022-03-30T17:20:41.542813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# [YOLOv5](https://github.com/ultralytics/yolov5)\n![](https://user-images.githubusercontent.com/26833433/98699617-a1595a00-2377-11eb-8145-fc674eb9b1a7.jpg)\n![](https://user-images.githubusercontent.com/26833433/90187293-6773ba00-dd6e-11ea-8f90-cd94afc0427f.png)","metadata":{"papermill":{"duration":0.056257,"end_time":"2021-01-01T09:50:01.716608","exception":false,"start_time":"2021-01-01T09:50:01.660351","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# YOLOv5 Stuff","metadata":{"papermill":{"duration":0.055699,"end_time":"2021-01-01T09:50:01.82747","exception":false,"start_time":"2021-01-01T09:50:01.771771","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/vinbigdata/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/vinbigdata/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 14,\n    names = classes\n    )\n\nwith open(join( cwd , 'vinbigdata.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'vinbigdata.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"papermill":{"duration":0.113001,"end_time":"2021-01-01T09:50:01.996448","exception":false,"start_time":"2021-01-01T09:50:01.883447","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-30T17:21:27.296597Z","iopub.execute_input":"2022-03-30T17:21:27.297093Z","iopub.status.idle":"2022-03-30T17:21:27.387753Z","shell.execute_reply.started":"2022-03-30T17:21:27.29704Z","shell.execute_reply":"2022-03-30T17:21:27.38657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}