{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is a fork of https://www.kaggle.com/awsaf49/vinbigdata-cxr-ad-yolov5-14-class-train","metadata":{}},{"cell_type":"code","source":"!pip install --upgrade seaborn","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.execute_input":"2021-01-01T09:44:43.843489Z","iopub.status.busy":"2021-01-01T09:44:43.842712Z","iopub.status.idle":"2021-01-01T09:44:53.448523Z","shell.execute_reply":"2021-01-01T09:44:53.447971Z"},"papermill":{"duration":9.633907,"end_time":"2021-01-01T09:44:53.448657","exception":false,"start_time":"2021-01-01T09:44:43.81475","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns","metadata":{"execution":{"iopub.execute_input":"2021-01-01T09:44:53.50848Z","iopub.status.busy":"2021-01-01T09:44:53.50769Z","iopub.status.idle":"2021-01-01T09:44:54.403472Z","shell.execute_reply":"2021-01-01T09:44:54.402433Z"},"papermill":{"duration":0.926929,"end_time":"2021-01-01T09:44:54.403588","exception":false,"start_time":"2021-01-01T09:44:53.476659","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dim = 512 \nfold = 4 # We need to change these when we are changing the folds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f'../input/vinbigdata-{dim}-image-dataset/vinbigdata/train.csv')\ntrain_df['image_path'] = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/train/'+train_df.image_id+('.png' if dim!='original' else '.jpg')\ntrain_df = train_df[train_df.class_id!=14].reset_index(drop = True)","metadata":{"execution":{"iopub.execute_input":"2021-01-01T09:44:54.468231Z","iopub.status.busy":"2021-01-01T09:44:54.467706Z","iopub.status.idle":"2021-01-01T09:44:54.690894Z","shell.execute_reply":"2021-01-01T09:44:54.69183Z"},"papermill":{"duration":0.262045,"end_time":"2021-01-01T09:44:54.691965","exception":false,"start_time":"2021-01-01T09:44:54.42992","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gkf  = GroupKFold(n_splits = 5)\ntrain_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups = train_df.image_id.tolist())):\n    train_df.loc[val_idx, 'fold'] = fold\ntrain_df.head()","metadata":{"execution":{"iopub.execute_input":"2021-01-01T09:47:56.234311Z","iopub.status.busy":"2021-01-01T09:47:56.233425Z","iopub.status.idle":"2021-01-01T09:47:56.297206Z","shell.execute_reply":"2021-01-01T09:47:56.297652Z"},"papermill":{"duration":0.134603,"end_time":"2021-01-01T09:47:56.297774","exception":false,"start_time":"2021-01-01T09:47:56.163171","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files   = []\nval_files += list(train_df[train_df.fold==fold].image_path.unique())\ntrain_files += list(train_df[train_df.fold!=fold].image_path.unique())\nlen(train_files), len(val_files)","metadata":{"execution":{"iopub.execute_input":"2021-01-01T09:47:56.420454Z","iopub.status.busy":"2021-01-01T09:47:56.419355Z","iopub.status.idle":"2021-01-01T09:47:56.443157Z","shell.execute_reply":"2021-01-01T09:47:56.443661Z"},"papermill":{"duration":0.086817,"end_time":"2021-01-01T09:47:56.443789","exception":false,"start_time":"2021-01-01T09:47:56.356972","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/vinbigdata/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/vinbigdata-yolo-labels-dataset/labels'\nfor file in tqdm(train_files):\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/train')\n    \nfor file in tqdm(val_files):\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/val')","metadata":{"execution":{"iopub.execute_input":"2021-01-01T09:47:56.779532Z","iopub.status.busy":"2021-01-01T09:47:56.778609Z","iopub.status.idle":"2021-01-01T09:50:01.33093Z","shell.execute_reply":"2021-01-01T09:50:01.330431Z"},"papermill":{"duration":124.654777,"end_time":"2021-01-01T09:50:01.331041","exception":false,"start_time":"2021-01-01T09:47:56.676264","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_ids, class_names = list(zip(*set(zip(train_df.class_id, train_df.class_name))))\nclasses = list(np.array(class_names)[np.argsort(class_ids)])\nclasses = list(map(lambda x: str(x), classes))\nclasses","metadata":{"execution":{"iopub.execute_input":"2021-01-01T09:50:01.595454Z","iopub.status.busy":"2021-01-01T09:50:01.594289Z","iopub.status.idle":"2021-01-01T09:50:01.60074Z","shell.execute_reply":"2021-01-01T09:50:01.601395Z"},"papermill":{"duration":0.082234,"end_time":"2021-01-01T09:50:01.601574","exception":false,"start_time":"2021-01-01T09:50:01.51934","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/vinbigdata/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/vinbigdata/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 14,\n    names = classes\n    )\n\nwith open(join( cwd , 'vinbigdata.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'vinbigdata.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"execution":{"iopub.execute_input":"2021-01-01T09:50:01.950234Z","iopub.status.busy":"2021-01-01T09:50:01.949285Z","iopub.status.idle":"2021-01-01T09:50:01.995866Z","shell.execute_reply":"2021-01-01T09:50:01.996316Z"},"papermill":{"duration":0.113001,"end_time":"2021-01-01T09:50:01.996448","exception":false,"start_time":"2021-01-01T09:50:01.883447","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/ultralytics/yolov5\n# !git clone https://github.com/ultralytics/yolov5  # clone repo\n# %cd yolov5\nshutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5')\n# %pip install -qr requirements.txt # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2021-01-01T09:50:02.170487Z","iopub.status.busy":"2021-01-01T09:50:02.169672Z","iopub.status.idle":"2021-01-01T09:50:08.782533Z","shell.execute_reply":"2021-01-01T09:50:08.783883Z"},"papermill":{"duration":6.702428,"end_time":"2021-01-01T09:50:08.784153","exception":false,"start_time":"2021-01-01T09:50:02.081725","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !WANDB_MODE=\"dryrun\" python train.py --img 640 --batch 16 --epochs 3 --data coco128.yaml --weights yolov5s.pt --nosave --cache \n!WANDB_MODE=\"dryrun\" python train.py --img 512 --batch 16 --epochs 30 --data /kaggle/working/vinbigdata.yaml --weights yolov5x.pt --cache","metadata":{"execution":{"iopub.execute_input":"2021-01-01T09:50:19.916161Z","iopub.status.busy":"2021-01-01T09:50:19.915216Z","iopub.status.idle":"2021-01-01T15:22:16.288743Z","shell.execute_reply":"2021-01-01T15:22:16.289579Z"},"papermill":{"duration":19916.498298,"end_time":"2021-01-01T15:22:16.289734","exception":false,"start_time":"2021-01-01T09:50:19.791436","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree('/kaggle/working/vinbigdata')\nshutil.rmtree('runs/detect')\nfor file in (glob('runs/train/exp/**/*.png', recursive = True)+glob('runs/train/exp/**/*.jpg', recursive = True)):\n    os.remove(file)","metadata":{"execution":{"iopub.execute_input":"2021-01-01T15:24:21.60059Z","iopub.status.busy":"2021-01-01T15:24:21.599598Z","iopub.status.idle":"2021-01-01T15:24:22.413063Z","shell.execute_reply":"2021-01-01T15:24:22.411761Z"},"papermill":{"duration":5.709202,"end_time":"2021-01-01T15:24:22.413173","exception":false,"start_time":"2021-01-01T15:24:16.703971","status":"completed"},"tags":[],"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]}]}