{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade seaborn","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns\nimport torch\nfrom IPython.display import Image, clear_output  # Hiển thị ảnh\n\ndim = 512 #512, 256, 'original'\nfold = 4","metadata":{"execution":{"iopub.status.busy":"2022-05-23T15:03:32.231122Z","iopub.execute_input":"2022-05-23T15:03:32.231486Z","iopub.status.idle":"2022-05-23T15:03:33.611229Z","shell.execute_reply.started":"2022-05-23T15:03:32.231392Z","shell.execute_reply":"2022-05-23T15:03:33.61035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(f'../input/vinbigdata-{dim}-image-dataset/vinbigdata/train.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['image_path'] = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/train/'+data.image_id+('.png' if dim!='original' else '.jpg')\ndata.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tiền xử lý dữ liệu","metadata":{}},{"cell_type":"code","source":"data = data[data.class_id!=14].reset_index(drop = True)  #chỉ lấy 14 lớp,xóa lớp no finding","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['x_min'] = data.apply(lambda row: (row.x_min)/row.width, axis =1)\ndata['y_min'] = data.apply(lambda row: (row.y_min)/row.height, axis =1)\n\ndata['x_max'] = data.apply(lambda row: (row.x_max)/row.width, axis =1)\ndata['y_max'] = data.apply(lambda row: (row.y_max)/row.height, axis =1)\n\ndata['x_mid'] = data.apply(lambda row: (row.x_max+row.x_min)/2, axis =1)\ndata['y_mid'] = data.apply(lambda row: (row.y_max+row.y_min)/2, axis =1)\n\ndata['w'] = data.apply(lambda row: (row.x_max-row.x_min), axis =1)\ndata['h'] = data.apply(lambda row: (row.y_max-row.y_min), axis =1)\n\ndata['area'] = data['w']*data['h']\ndata.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['x_min', 'y_min', 'x_max', 'y_max', 'x_mid', 'y_mid', 'w', 'h', 'area']\nX = data[features]\ny = data['class_id']\nX.shape, y.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ['Aortic enlargement',\n 'Atelectasis',\n 'Calcification',\n 'Cardiomegaly',\n 'Consolidation',\n 'ILD',\n 'Infiltration',\n 'Lung Opacity',\n 'Nodule/Mass',\n 'Other lesion',\n 'Pleural effusion',\n 'Pleural thickening',\n 'Pneumothorax',\n 'Pulmonary fibrosis']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# t-SNE Visualization","metadata":{}},{"cell_type":"code","source":"%%time\nfrom sklearn.manifold import TSNE\n\ntsne = TSNE(n_components = 2, perplexity = 40, random_state=1, n_iter=5000)\ndata_X = X\ndata_y = y.loc[data_X.index]\nembs = tsne.fit_transform(data_X)\n# Thêm vào khung dữ liệu để thuận tiện\nplot_x = embs[:, 0]\nplot_y = embs[:, 1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize = (15, 15))\nplt.axis('off')\nscatter = plt.scatter(plot_x, plot_y, marker = 'o',s = 50, c=data_y.tolist(), alpha= 0.5,cmap='viridis')\nplt.legend(handles=scatter.legend_elements()[0], labels=classes)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Chia tỉ lệ","metadata":{}},{"cell_type":"code","source":"gkf  = GroupKFold(n_splits = 5)\ndata['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(data, groups = data.image_id.tolist())):\n    data.loc[val_idx, 'fold'] = fold\ndata.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files   = []\nval_files += list(data[data.fold==fold].image_path.unique())\ntrain_files += list(data[data.fold!=fold].image_path.unique())\nlen(train_files), len(val_files)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tệp bản sao","metadata":{}},{"cell_type":"code","source":"os.makedirs('/kaggle/working/vinbigdata/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/vinbigdata-yolo-labels-dataset/labels'\n#tổng hợp các ảnh trùng thành 1\nfor file in tqdm(train_files):\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/train')\n    \nfor file in tqdm(val_files):\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/val')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# YOLOv5 Stuff","metadata":{}},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/vinbigdata/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/vinbigdata/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 14,\n    names = classes\n    )\n\nwith open(join( cwd , 'vinbigdata.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'vinbigdata.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5')\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python train.py --img 640 --batch 16 --epochs 30 --data /kaggle/working/vinbigdata.yaml --weights yolov5x.pt --cache","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trực quan hóa dự đoán","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(3, 2, figsize = (2*5,3*5), constrained_layout = True)\nfor row in range(3):\n    ax[row][0].imshow(plt.imread(f'runs/train/exp/test_batch{row}_labels.jpg'))\n    ax[row][0].set_xticks([])\n    ax[row][0].set_yticks([])\n    ax[row][0].set_title(f'runs/train/exp/test_batch{row}_labels.jpg - Vùng tin tưởng', fontsize = 12)\n    \n    ax[row][1].imshow(plt.imread(f'runs/train/exp/test_batch{row}_pred.jpg'))\n    ax[row][1].set_xticks([])\n    ax[row][1].set_yticks([])\n    ax[row][1].set_title(f'runs/train/exp/test_batch{row}_pred.jpg - Hộp dự đoán', fontsize = 12)","metadata":{},"execution_count":null,"outputs":[]}]}