{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is a fork of https://www.kaggle.com/awsaf49/vinbigdata-cxr-ad-yolov5-14-class-infer","metadata":{}},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns, gc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dim = 512 #1024, 256, 'original'\ntest_dir = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/test.csv')\ntest_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# YOLOv5 Stuff","metadata":{}},{"cell_type":"code","source":"shutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5') # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def yolo2voc(image_height, image_width, bboxes):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n    \n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n    \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    \n    return bboxes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1,6):\n    torch.cuda.empty_cache()\n    gc.collect()\n    WEIGHT_FILE = f'/kaggle/input/resnet50d-096-stage-2/FOLD{i}.pt'    \n    !python detect.py --weights $WEIGHT_FILE --img 512 --conf 0.15 --iou 0.5 --source $test_dir --save-txt --save-conf --exist-ok\n    image_ids = []\n    PredictionStrings = []\n\n    for file_path in glob('runs/detect/exp/labels/*txt'):\n        image_id = file_path.split('/')[-1].split('.')[0]\n        w, h = test_df.loc[test_df.image_id==image_id,['width', 'height']].values[0]\n        f = open(file_path, 'r')\n        data = np.array(f.read().replace('\\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)\n        data = data[:, [0, 5, 1, 2, 3, 4]]\n        bboxes = list(np.concatenate((data[:, :2], yolo2voc(h, w, data[:, 2:])), axis =1).reshape(-1).astype(str))\n        for idx in range(len(bboxes)):\n            bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]\n        image_ids.append(image_id)\n        PredictionStrings.append(' '.join(bboxes))\n        os.remove(file_path)\n\n    pred_df = pd.DataFrame({'image_id':image_ids,\n                            'PredictionString':PredictionStrings})\n    sub_df = pd.merge(test_df, pred_df, on = 'image_id', how = 'left').fillna(\"14 1 0 0 1 1\")\n    sub_df = sub_df[['image_id', 'PredictionString']]\n    sub_df.to_csv(f'/kaggle/working/Fold_{i}_cleaned.csv',index = False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1,6):\n    torch.cuda.empty_cache()\n    gc.collect()\n    WEIGHT_FILE = f'/kaggle/input/resnet50d-096-stage-2/FOLD{i}.pt'    \n    !python detect.py --weights $WEIGHT_FILE --img 512 --conf 0.00001 --iou 0.0 --source $test_dir --save-txt --save-conf --exist-ok\n    image_ids = []\n    PredictionStrings = []\n\n    for file_path in glob('runs/detect/exp/labels/*txt'):\n        image_id = file_path.split('/')[-1].split('.')[0]\n        w, h = test_df.loc[test_df.image_id==image_id,['width', 'height']].values[0]\n        f = open(file_path, 'r')\n        data = np.array(f.read().replace('\\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)\n        data = data[:, [0, 5, 1, 2, 3, 4]]\n        bboxes = list(np.concatenate((data[:, :2], yolo2voc(h, w, data[:, 2:])), axis =1).reshape(-1).astype(str))\n        for idx in range(len(bboxes)):\n            bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]\n        image_ids.append(image_id)\n        PredictionStrings.append(' '.join(bboxes))\n        os.remove(file_path)\n\n    pred_df = pd.DataFrame({'image_id':image_ids,\n                            'PredictionString':PredictionStrings})\n    sub_df = pd.merge(test_df, pred_df, on = 'image_id', how = 'left').fillna(\"14 1 0 0 1 1\")\n    sub_df = sub_df[['image_id', 'PredictionString']]\n    sub_df.to_csv(f'/kaggle/working/Fold_{i}.csv',index = False)","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1,6):\n    torch.cuda.empty_cache()\n    gc.collect()\n    WEIGHT_FILE = f'/kaggle/input/resnet50d-096-stage-2/FOLD{i}.pt'    \n    !python detect.py --weights $WEIGHT_FILE --img 512 --conf 0.15 --iou 0.5 --source $test_dir --save-txt --save-conf --exist-ok --augment\n    image_ids = []\n    PredictionStrings = []\n\n    for file_path in glob('runs/detect/exp/labels/*txt'):\n        image_id = file_path.split('/')[-1].split('.')[0]\n        w, h = test_df.loc[test_df.image_id==image_id,['width', 'height']].values[0]\n        f = open(file_path, 'r')\n        data = np.array(f.read().replace('\\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)\n        data = data[:, [0, 5, 1, 2, 3, 4]]\n        bboxes = list(np.concatenate((data[:, :2], yolo2voc(h, w, data[:, 2:])), axis =1).reshape(-1).astype(str))\n        for idx in range(len(bboxes)):\n            bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]\n        image_ids.append(image_id)\n        PredictionStrings.append(' '.join(bboxes))\n        os.remove(file_path)\n\n    pred_df = pd.DataFrame({'image_id':image_ids,\n                            'PredictionString':PredictionStrings})\n    sub_df = pd.merge(test_df, pred_df, on = 'image_id', how = 'left').fillna(\"14 1 0 0 1 1\")\n    sub_df = sub_df[['image_id', 'PredictionString']]\n    sub_df.to_csv(f'/kaggle/working/Fold_{i}_TTA.csv',index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# torch.cuda.empty_cache()\n# gc.collect()\n# WEIGHT_FILE = f'/kaggle/input/resnet50d-096-stage-2/FOLD5.pt'    \n# !python detect.py --weights $WEIGHT_FILE --img 512 --conf 0.0 --iou 0.0 --source $test_dir --save-txt --save-conf --exist-ok --augment\n# image_ids = []\n# PredictionStrings = []\n\n# for file_path in glob('runs/detect/exp/labels/*txt'):\n#     image_id = file_path.split('/')[-1].split('.')[0]\n#     w, h = test_df.loc[test_df.image_id==image_id,['width', 'height']].values[0]\n#     f = open(file_path, 'r')\n#     data = np.array(f.read().replace('\\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)\n#     data = data[:, [0, 5, 1, 2, 3, 4]]\n#     bboxes = list(np.concatenate((data[:, :2], yolo2voc(h, w, data[:, 2:])), axis =1).reshape(-1).astype(str))\n#     for idx in range(len(bboxes)):\n#         bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]\n#     image_ids.append(image_id)\n#     PredictionStrings.append(' '.join(bboxes))\n\n# pred_df = pd.DataFrame({'image_id':image_ids,\n#                         'PredictionString':PredictionStrings})\n# sub_df = pd.merge(test_df, pred_df, on = 'image_id', how = 'left').fillna(\"14 1 0 0 1 1\")\n# sub_df = sub_df[['image_id', 'PredictionString']]\n# sub_df.to_csv(f'/kaggle/working/Fold_5_uncleaned.csv',index = False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree('/kaggle/working/yolov5')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}