{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Quick data analysis with YOLOv5 at a glance\n\n\nThis code is based on [VinBigData-CXR-AD YOLOv5 14 Class [infer]](https://www.kaggle.com/awsaf49/vinbigdata-cxr-ad-yolov5-14-class-infer) and [VBD Chest X-ray Abnormalities Detection | EDA](https://www.kaggle.com/mrutyunjaybiswal/vbd-chest-x-ray-abnormalities-detection-eda)\n\n\nThe Goal of notebook is to analyze data with YOLOv5."},{"metadata":{},"cell_type":"markdown","source":"> Please upvote if this notebook was helpful to you. Thank you:)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dim = 512 #1024, 256, 'original'\ntest_dir = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/test'\nweights_dir = '/kaggle/input/vinbigdata-cxr-ad-yolov5-14-class-train/yolov5/runs/train/exp/weights/best.pt'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(f'../input/vinbigdata-{dim}-image-dataset/vinbigdata/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Number of data"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train_df.image_id.nunique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> number of normal data"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train_df[train_df['class_id']==14].image_id.nunique()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(26, 8))\nsns.countplot(x=\"class_name\", data=train_df)\nplt.title(\"Class Name Distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['image_path'] = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/train/'+train_df.image_id+('.png' if dim!='original' else '.jpg')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_df = train_df[train_df.class_id!=14].reset_index(drop = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> The average number of disease in patient"},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_df)/train_df.image_id.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold = 4\ngkf  = GroupKFold(n_splits = 5)\ntrain_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups = train_df.image_id.tolist())):\n    train_df.loc[val_idx, 'fold'] = fold\nval_df = train_df[train_df['fold']==4]\nval_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = []\nval_files   = []\nval_files += list(train_df[train_df.fold==fold].image_path.unique())\ntrain_files += list(train_df[train_df.fold!=fold].image_path.unique())\nlen(train_files), len(val_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"os.makedirs('/kaggle/working/vinbigdata/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/vinbigdata-yolo-labels-dataset/labels'\nfor file in train_files:\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/train')\n    \nfor file in val_files:\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/val')\n    \nval_dir = f'/kaggle/working/vinbigdata/images/val'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# YOLOv5 Set up"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"shutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5') # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true,"_kg_hide-input":false,"_kg_hide-output":true},"cell_type":"code","source":"!python detect.py --weights $weights_dir\\\n--img 640\\\n--conf 0.15\\\n--iou 0.4\\\n--source $val_dir\\\n--save-txt --save-conf --exist-ok","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nimport numpy as np\nimport random\nimport cv2\nfrom glob import glob\nfrom tqdm import tqdm\n\nfiles = glob('runs/detect/exp/*png')\nfor _ in range(1):\n    row = 4\n    col = 4\n    grid_files = random.sample(files, row*col)\n    images     = []\n    for image_path in tqdm(grid_files):\n        img          = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)\n        images.append(img)\n\n    fig = plt.figure(figsize=(col*5, row*5))\n    grid = ImageGrid(fig, 111,  # similar to subplot(111)\n                     nrows_ncols=(col, row),  # creates 2x2 grid of axes\n                     axes_pad=0.05,  # pad between axes in inch.\n                     )\n\n    for ax, im in zip(grid, images):\n        # Iterating over the grid returns the Axes.\n        ax.imshow(im)\n        ax.set_xticks([])\n        ax.set_yticks([])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def yolo2voc(image_height, image_width, bboxes):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n    \n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n    \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    \n    return bboxes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Distribution"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"image_ids = []\nPredictionStrings = []\nclasses = []\nscores = []\nx_min = []\ny_min = []\nx_max = []\ny_max = []\n\n\nfor file_path in glob('runs/detect/exp/labels/*txt'):\n    image_id = file_path.split('/')[-1].split('.')[0]\n    w, h = val_df.loc[val_df.image_id==image_id,['width', 'height']].values[0]\n    f = open(file_path, 'r')\n    data = np.array(f.read().replace('\\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)\n    data = data[:, [0, 5, 1, 2, 3, 4]]\n    bboxes = list(np.concatenate((data[:, :2], np.round(yolo2voc(h, w, data[:, 2:]))), axis =1).reshape(-1))#.astype(str))\n    for i in range(len(bboxes)//6):\n        image_ids.append(image_id)\n        classes.append(int(bboxes[i*6]))\n        scores.append(int(bboxes[i*6+1]))\n        x_min.append(int(bboxes[i*6+2]))\n        y_min.append(int(bboxes[i*6+3]))\n        x_max.append(int(bboxes[i*6+4]))\n        y_max.append(int(bboxes[i*6+5]))\n#         bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]\n#     image_ids.append(image_id)\n#     PredictionStrings.append(' '.join(bboxes))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"pred_df = pd.DataFrame({'image_id':image_ids,\n                        'classes':classes,\n                        'scores':scores,\n                        'x_min':x_min,\n                        'y_min':y_min,\n                        'x_max':x_max,\n                        'y_max':y_max\n                       })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"cids = val_df.class_name.value_counts().sort_index()\n\nfig = plt.figure(figsize=(10,10))\nax = cids.plot.bar()\nplt.title(\"Correct Distribution of validation dataset\")\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class_ids, class_names = list(zip(*set(zip(train_df.class_id, train_df.class_name))))\nclasses = list(np.array(class_names)[np.argsort(class_ids)])\nclasses = list(map(lambda x: str(x), classes))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"cids = pred_df.classes.value_counts().sort_index()#.to_frame()\ncids.index = classes\n\nfig = plt.figure(figsize=(10,10))\nax = cids.plot.bar()\nplt.title(\"Predicted Distribution\")\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Discussion\n\n* More than 70% of data is normal data\n\n* The average number of disease in patient's X-ray image is about 8.2\n\n* There is a difference between distirbution of prediction and ground truth. It will get worse if we include normal data\n\n* Some diseases appear only in certain location in the image."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}