{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -r ../input/yolov5/yolov5/requirements.txt\n!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import sys\nsys.path.extend(['../input/effdet/',\n                 '../input/iterstrat/',\n                 '../input/weightedboxfusion/',\n                 '../input/timm-nfnet/',\n                 '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master/'])\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport seaborn as sns\nfrom pathlib import Path\nimport json\nimport re\nimport logging\nimport gc\nimport random\nimport warnings\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\nimport os\nfrom glob import glob\nfrom joblib import Parallel, delayed\nimport shutil as sh\nfrom itertools import product\nfrom collections import OrderedDict\nimport plotly\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.offline as pyo\nfrom ml_stratifiers import MultilabelStratifiedKFold\n\nimport torchvision.transforms as transforms\nimport albumentations as al\nfrom albumentations import ImageOnlyTransform\nfrom albumentations.pytorch import ToTensorV2, ToTensor\nfrom albumentations.core.transforms_interface import DualTransform, ImageOnlyTransform\n\nimport cv2\nimport pydicom\nfrom IPython.display import display, Image\nfrom sklearn.model_selection import train_test_split, KFold, GroupKFold, StratifiedKFold\nimport torch\nfrom torch.nn import functional as f\nimport torch.nn as nn\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\nfrom torch.utils.data import Dataset, DataLoader, sampler\nimport timm\nfrom efficientnet_pytorch import EfficientNet\n\npd.options.display.max_columns = None\nwarnings.filterwarnings('ignore')\n\nlogging.basicConfig(format='%(asctime)s +++ %(message)s',\n                    datefmt='%d-%m-%y %H:%M:%S', level=logging.INFO)\nlogger = logging.getLogger()\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nlogger.info(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"os.environ[\"WANDB_API_KEY\"] = '8f435998b1a6f9a4e59bfaef1deed81c1362a97d'\nos.environ[\"WANDB_MODE\"] = \"dryrun\"\n\nMAIN_PATH = '../input/vinbigdata-chest-xray-abnormalities-detection/'\nCLASSIFIER_MAIN_PATH = '../input/efficientnet-pytorch/'\nRESIZE_1024_PATH = '../input/vinbigdata-chest-xray-resized-png-1024x1024/'\nRESIZE_512_PATH = '../input/vinbigdata/'\nTRAIN_PATH = os.path.join(MAIN_PATH, 'train.csv')\nSUB_PATH = os.path.join(MAIN_PATH, 'sample_submission.csv')\nTRAIN_DICOM_PATH = os.path.join(MAIN_PATH, 'train')\nTEST_DICOM_PATH = os.path.join(MAIN_PATH, 'test')\nTRAIN_1024_PATH = os.path.join(RESIZE_1024_PATH, 'train')\nTEST_1024_PATH = os.path.join(RESIZE_1024_PATH, 'test')\nTRAIN_512_PATH = os.path.join(RESIZE_512_PATH, 'train')\nTEST_512_PATH = os.path.join(RESIZE_512_PATH, 'test')\nTRAIN_META_PATH = os.path.join(RESIZE_1024_PATH, 'train_meta.csv')\nTEST_META_PATH = '../input/vinbigdata-testmeta/test_meta.csv'\nTEST_CLASS_PATH = '../input/vinbigdata-2class-prediction/2-cls test pred.csv'\nMODEL_WEIGHT = '../input/efficientdet/tf_efficientdet_d7_53-6d1d7a95.pth'\nSIZE = 512\nIMG_SIZE = (SIZE, SIZE)\nACCULATION = 1\nMOSAIC_RATIO = 0.4\n    \nclass GlobalConfig:\n    model_use = 'd0'\n    model_weight = '../input/efficientdet/tf_efficientdet_d0_34-f153e0cf.pth'\n    img_size = IMG_SIZE\n    fold_num = 10\n    seed = 89\n    num_workers = 12\n    batch_size = 8\n    n_epochs = 20\n    lr = 1e-2\n    verbose = 1\n    verbose_step = 1\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n#     output_path = './save/'\n    scheduler_params = dict(\n        mode='min', \n        factor=0.2,\n        patience=1,\n        threshold_mode='abs',\n        min_lr=1e-7\n    )\n    \nclass PredictConfig:\n    img_size = IMG_SIZE\n    batch_size = 16\n    model_use = 'd0'\n    model_classifier_use = 'b5'\n    weight_classifier = '../input/efficientdet/model_classifier.pth'\n#     weight_list = sorted([os.path.join('./save/', i) for i in os.listdir('./save/')])\n    score_thresh = 0.01\n    iou_thresh = 0.55\n    iou_thresh2 = 0.4\n    skip_thresh = 0.0001\n    sigma = 0.1\n    score_last = 0.06\n    score_last2 = 0.8\n    \n    \nlist_remove = [34843, 21125, 647, 18011, 2539, 22373, 12675, 7359, 20642, 5502, 19818, 5832, 28056, 28333, 20758,\n               925, 43, 2199, 4610, 21306, 16677, 1768, 17232, 1378, 24949, 30203, 31410, 87, 25318, 92, 31724,\n               118, 17687, 12605, 26157, 33875, 7000, 3730, 18776, 13225, 1109, 2161, 33627, 15500, 28633, 28152,\n               10114, 10912, 9014,  4427, 25630, 11464, 6419, 22164, 4386, 17557, 15264, 21853, 33142, 32895, 9733,\n               33010, 17493, 32128, 28802, 11658, 8841, 29557, 4802, 8591, 778, 9935, 12359, 5210, 7556, 24505, 5664,\n               28670, 27820, 19359, 9817, 7800, 32934, 34098, 27931, 16074, 27308, 30645, 31029, 35697, 6199, 27065,\n               1771, 14689, 31860, 1975, 29294, 2304, 34018, 23406, 26501, 26011, 2479, 32796, 25836, 3032, 31454,\n               32066, 19722, 15997, 6049, 9458, 11005, 23151, 24503, 35411, 18092, 23815, 30742, 33942, 34542, 7655,\n               25345, 3750, 17046, 3844, 5958, 4250, 18823, 14898, 22581, 25805, 9651, 33194, 36007, 30160, 24459,\n               10838, 16544, 31252, 8053, 28487, 6208, 25244, 8470, 10089, 24813, 14769, 34305, 34047, 23366, 8049,\n               13276, 22380, 32797, 32440, 11031, 18304, 33692, 21349, 26333, 34331, 9110, 21092, 34882, 35626, 10203,\n               25648, 30754, 29567, 33542, 15146, 26759, 20846, 22493, 33187, 22813, 30219, 14548, 14627, 20494, 28332,\n               15930, 31347, 33489, 35005, 34032, 24183, 18643, 18536, 29754, 20380, 29750, 20539, 35791, 27275, 32248]\nimage_remove = ['9c83d9f88170cd38f7bca54fe27dc48a', 'ac2a615b3861212f9a2ada6acd077fd9',\n                'f9f7feefb4bac748ff7ad313e4a78906', 'f89143595274fa6016f6eec550442af9',\n                '6c08a98e48ba72aee1b7b62e1f28e6da', 'e7a58f5647d24fc877f9cb3d051792e2',\n                '8f98e3e6e86e573a6bd32403086b3707', '43d3137e74ebd344636228e786cb91b0',\n                '575b98a9f9824d519937a776bd819cc4', 'ca6c1531a83f8ee89916ed934f8d4847',\n                '0c6a7e3c733bd4f4d89443ca16615fc6', 'ae5cec1517ab3e82c5374e4c6219a17d',\n                '064023f1ff95962a1eee46b9f05f7309', '27c831fee072b232499541b0aca58d9c',\n                '0b98b21145a9425bf3eeea4b0de425e7', '7df5c81873c74ecc40610a1ad4eb2943']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = IMG_SIZE\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)\n    \ndef Visualize_class(df, feature, title):\n    num_image = df[feature].value_counts().rename_axis(feature).reset_index(name='num_image')\n    fig = px.bar(num_image[::-1], x='num_image', y=feature, orientation='h', color='num_image')\n    fig.update_layout(\n    title={\n        'text': title,\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\n    fig.show()\n    \n    \ndef img_size(path):\n    information = pydicom.dcmread(path)\n    h, w = information.Rows, information.Columns\n    return (h, w)\n\n\ndef label_resize(org_size, img_size, *bbox):\n    x0, y0, x1, y1 = bbox\n    x0_new = int(np.round(x0*img_size[1]/org_size[1]))\n    y0_new = int(np.round(y0*img_size[0]/org_size[0]))\n    x1_new = int(np.round(x1*img_size[1]/org_size[1]))\n    y1_new = int(np.round(y1*img_size[0]/org_size[0]))\n    return x0_new, y0_new, x1_new, y1_new\n\n\ndef list_color(class_list):\n    dict_color = dict()\n    for classid in class_list:\n        dict_color[classid] = [i/256 for i in random.sample(range(256), 3)]\n    \n    return dict_color\n\n\ndef split_df(df):\n    kf = MultilabelStratifiedKFold(n_splits=GlobalConfig.fold_num,\n                                   shuffle=True, random_state=GlobalConfig.seed)\n    df['id'] = df.index\n    annot_pivot = pd.pivot_table(df, index='image_id', columns='class_id',\n                                 values='id', fill_value=0, aggfunc='count') \\\n    .reset_index().rename_axis(None, axis=1)\n    for fold, (train_idx, val_idx) in enumerate(kf.split(annot_pivot,\n                                                         annot_pivot.iloc[:, 1:train_abnormal['class_id'].nunique()])):\n        annot_pivot[f'fold_{fold}'] = 0\n        annot_pivot.loc[val_idx, f'fold_{fold}'] = 1\n    return annot_pivot\n    \n    \ndef display_image(df, list_image, num_image=1, is_dicom_file=True):\n    \n    dict_color = list_color(range(15))\n    list_abnormal = [i for i in df['class_name'].unique() if i!='No finding']\n    for abnormal in list_abnormal:\n        abnormal_df = df[df['class_name']==abnormal].reset_index(drop=True)\n        abnormal_random = np.random.choice(abnormal_df['image_id'].unique(), num_image)\n        for abnormal_img in abnormal_random:\n            images = abnormal_df[abnormal_df['image_id']==abnormal_img].reset_index(drop=True)\n            fig, ax = plt.subplots(1, figsize=(15, 15))\n            img_path = [i for i in list_image if abnormal_img in i][0]\n            if is_dicom_file:\n                information = pydicom.dcmread(img_path)\n                img = information.pixel_array\n            else:\n                img = cv2.imread(img_path)\n            ax.imshow(img, plt.cm.bone)\n            for idx, image in images.iterrows():\n                bbox = [image.x_min, image.y_min, image.x_max, image.y_max]\n                if is_dicom_file:\n                    x_min, y_min, x_max, y_max = bbox\n                else:\n                    org_size = image[['h', 'w']].values\n                    x_min, y_min, x_max, y_max = label_resize(org_size, IMG_SIZE, *bbox)\n                class_name, class_id = image.class_name, image.class_id\n                rect = patches.Rectangle((x_min, y_min), x_max-x_min, y_max-y_min,\n                                         linewidth=1, edgecolor=dict_color[class_id], facecolor='none')\n                ax.add_patch(rect)\n                plt.text(x_min, y_min, class_name, fontsize=15, color='red')\n\n            plt.title(abnormal_img) \n            plt.show()\n            \ndef display_image_test(df, size_df, list_image, num_image=3):\n    \n    dict_color = list_color(range(15))\n    image_row_random = np.random.choice(len(df), num_image, replace=(len(df)<num_image))\n    for image_idx in image_row_random:\n        image_id, pred = df.loc[image_idx, 'image_id'], df.loc[image_idx, 'PredictionString']\n        org_size = size_df[size_df['image_id']==image_id][['h', 'w']].values[0].tolist()\n        fig, ax = plt.subplots(1, figsize=(15, 15))\n        img_path = [i for i in list_image if image_id in i][0]\n        img = cv2.imread(img_path)\n        ax.imshow(img, plt.cm.bone)\n        if pred != '14 1 0 0 1 1':\n            list_pred = pred.split(' ')\n            for box_idx in range(len(list_pred)//6):\n                bbox = map(int, list_pred[6*box_idx+2:6*box_idx+6])\n                x_min, y_min, x_max, y_max = label_resize(org_size, IMG_SIZE, *bbox)\n                class_name, score = int(list_pred[6*box_idx]), float(list_pred[6*box_idx+1])\n                rect = patches.Rectangle((x_min, y_min), x_max-x_min, y_max-y_min,\n                                         linewidth=1, edgecolor=dict_color[class_name], facecolor='none')\n                ax.add_patch(rect)\n                plt.text(x_min, y_min, f'{class_name}: {score}', fontsize=15, color='red')            \n\n        plt.title(image_id) \n        plt.show()\n        \ndef ensemble_multibox(boxes, scores, labels, iou_thr, sigma,\n                      skip_box_thr, weights=None, method='wbf'):\n    if method=='nms':\n        boxes, scores, labels = nms(boxes, scores, labels,\n                                    weights=weights,\n                                    iou_thr=iou_thr)\n    elif method=='soft_nms':\n        boxes, scores, labels = soft_nms(boxes, scores, labels,\n                                         weights=weights,\n                                         sigma=sigma,\n                                         iou_thr=iou_thr,\n                                         thresh=skip_box_thr)\n    elif method=='nms_weight':\n        boxes, scores, labels = non_maximum_weighted(boxes, scores, labels,\n                                                     weights=weights,\n                                                     iou_thr=iou_thr,\n                                                     skip_box_thr=skip_box_thr)\n    elif method=='wbf':\n        boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels,\n                                                      weights=weights,\n                                                      iou_thr=iou_thr,\n                                                      skip_box_thr=skip_box_thr)\n    \n    return boxes, scores, labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read File"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_dicom_list = glob(f'{TRAIN_DICOM_PATH}/*.dicom')\ntest_dicom_list = glob(f'{TEST_DICOM_PATH}/*.dicom')\n\ntrain_list = glob(f'{TRAIN_1024_PATH}/*.png')\ntest_list = glob(f'{TEST_1024_PATH}/*.png')\nlogger.info(f'Train have {len(train_list)} file and test have {len(test_list)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nsize_df = pd.read_csv(TRAIN_META_PATH)\nsize_df.columns = ['image_id', 'h', 'w']\n\ntrain_df = pd.read_csv(TRAIN_PATH)\nif DEBUG: \n    five_percent = len(train_df) // 20\n    train_df = train_df.head(five_percent)\ntrain_df = train_df.merge(size_df, on='image_id', how='left')\ntrain_df[['x_min', 'y_min']] = train_df[['x_min', 'y_min']].fillna(0)\ntrain_df[['x_max', 'y_max']] = train_df[['x_max', 'y_max']].fillna(1)\n\ntrain_df.tail()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analyze file"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_normal = train_df[train_df['class_name']=='No finding'].reset_index(drop=True)\ntrain_normal['x_min_resize'] = 0\ntrain_normal['y_min_resize'] = 0\ntrain_normal['x_max_resize'] = 1\ntrain_normal['y_max_resize'] = 1\n\ntrain_abnormal = train_df[train_df['class_name']!='No finding'].reset_index(drop=True)\ntrain_abnormal[['x_min_resize', 'y_min_resize', 'x_max_resize', 'y_max_resize']] = train_abnormal \\\n.apply(lambda x: label_resize(x[['h', 'w']].values, IMG_SIZE, *x[['x_min', 'y_min', 'x_max', 'y_max']].values),\n       axis=1, result_type=\"expand\")\ntrain_abnormal['x_center'] = 0.5*(train_abnormal['x_min_resize'] + train_abnormal['x_max_resize'])\ntrain_abnormal['y_center'] = 0.5*(train_abnormal['y_min_resize'] + train_abnormal['y_max_resize'])\ntrain_abnormal['width'] = train_abnormal['x_max_resize'] - train_abnormal['x_min_resize']\ntrain_abnormal['height'] = train_abnormal['y_max_resize'] - train_abnormal['y_min_resize']\ntrain_abnormal['area'] = train_abnormal.apply(lambda x: (x['x_max_resize']-x['x_min_resize'])*(x['y_max_resize']-x['y_min_resize']), axis=1)\ntrain_abnormal = train_abnormal[~train_abnormal.index.isin(list_remove)].reset_index(drop=True)\n\ntrain_abnormal.tail()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Change wbf"},{"metadata":{"trusted":true},"cell_type":"code","source":"def Preprocess_wbf(df, size=SIZE, iou_thr=0.4, skip_box_thr=0.0001):\n    list_image = []\n    list_boxes = []\n    list_cls = []\n    list_h, list_w = [], []\n    new_df = pd.DataFrame()\n    for image_id in tqdm(df['image_id'].unique(), leave=False):\n        image_df = df[df['image_id']==image_id].reset_index(drop=True)\n        h, w = image_df.loc[0, ['h', 'w']].values\n        boxes = image_df[['x_min_resize', 'y_min_resize',\n                          'x_max_resize', 'y_max_resize']].values.tolist()\n        boxes = [[j/(size-1) for j in i] for i in boxes]\n        scores = [1.0]*len(boxes)\n        labels = [float(i) for i in image_df['class_id'].values]\n        boxes, scores, labels = weighted_boxes_fusion([boxes], [scores], [labels],\n                                                      weights=None,\n                                                      iou_thr=iou_thr,\n                                                      skip_box_thr=skip_box_thr)\n        list_image.extend([image_id]*len(boxes))\n        list_h.extend([h]*len(boxes))\n        list_w.extend([w]*len(boxes))\n        list_boxes.extend(boxes)\n        list_cls.extend(labels.tolist())\n    list_boxes = [[int(j*(size-1)) for j in i] for i in list_boxes]\n    new_df['image_id'] = list_image\n    new_df['class_id'] = list_cls\n    new_df['h'] = list_h\n    new_df['w'] = list_w\n    new_df['x_min_resize'], new_df['y_min_resize'], \\\n    new_df['x_max_resize'], new_df['y_max_resize'] = np.transpose(list_boxes)\n    new_df['x_center'] = 0.5*(new_df['x_min_resize'] + new_df['x_max_resize'])\n    new_df['y_center'] = 0.5*(new_df['y_min_resize'] + new_df['y_max_resize'])\n    new_df['width'] = new_df['x_max_resize'] - new_df['x_min_resize']\n    new_df['height'] = new_df['y_max_resize'] - new_df['y_min_resize']\n    new_df['area'] = new_df.apply(lambda x: (x['x_max_resize']-x['x_min_resize'])\\\n                                  *(x['y_max_resize']-x['y_min_resize']), axis=1)\n    return new_df\n\n# train_abnormal = Preprocess_wbf(train_abnormal)\n# train_abnormal.tail()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fold"},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_df(df):\n    kf = MultilabelStratifiedKFold(n_splits=GlobalConfig.fold_num, shuffle=True, random_state=GlobalConfig.seed)\n    df['id'] = df.index\n    annot_pivot = pd.pivot_table(df, index=['image_id'], columns=['class_id'],\n                                 values='id', fill_value=0, aggfunc='count') \\\n    .reset_index().rename_axis(None, axis=1)\n    for fold, (train_idx, val_idx) in enumerate(kf.split(annot_pivot,\n                                                         annot_pivot.iloc[:, 1:(1+df['class_id'].nunique())])):\n        annot_pivot[f'fold_{fold}'] = 0\n        annot_pivot.loc[val_idx, f'fold_{fold}'] = 1\n    return annot_pivot\n\nsize_df = pd.read_csv(TRAIN_META_PATH)\nsize_df.columns = ['image_id', 'h', 'w']\n\nfold_csv = split_df(train_abnormal)\nfold_csv = fold_csv.merge(size_df, on='image_id', how='left')\nfold_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_file(df, split_df, train_file, train_folder, fold):\n    \n    os.makedirs(f'{train_file}/labels/train/', exist_ok=True)\n    os.makedirs(f'{train_file}/images/train/', exist_ok=True)\n    os.makedirs(f'{train_file}/labels/val/', exist_ok=True)\n    os.makedirs(f'{train_file}/images/val/', exist_ok=True)\n    \n    list_image_train = split_df[split_df[f'fold_{fold}']==0]['image_id']    \n    train_df = df[df['image_id'].isin(list_image_train)].reset_index(drop=True)\n    val_df = df[~df['image_id'].isin(list_image_train)].reset_index(drop=True)\n    \n    for train_img in tqdm(train_df.image_id.unique()):\n        with open(f'{train_file}/labels/train/{train_img}.txt', 'w+') as f:\n            row = train_df[train_df['image_id']==train_img]\\\n            [['class_id', 'x_center', 'y_center', 'width', 'height']].values\n            row[:, 1:] /= SIZE\n            row = row.astype('str')\n            for box in range(len(row)):\n                text = ' '.join(row[box])\n                f.write(text)\n                f.write('\\n')\n        sh.copy(f'{train_folder}/{train_img}.png', \n                f'{train_file}/images/train/{train_img}.png')\n        \n    for val_img in tqdm(val_df.image_id.unique()):\n        with open(f'{train_file}/labels/val/{val_img}.txt', 'w+') as f:\n            row = val_df[val_df['image_id']==val_img]\\\n            [['class_id', 'x_center', 'y_center', 'width', 'height']].values\n            row[:, 1:] /= SIZE\n            row = row.astype('str')\n            for box in range(len(row)):\n                text = ' '.join(row[box])\n                f.write(text)\n                f.write('\\n')\n        sh.copy(f'{train_folder}/{val_img}.png', \n                f'{train_file}/images/val/{val_img}.png')\n        \ncreate_file(train_abnormal, fold_csv, './chest_yolo', TRAIN_1024_PATH, 0)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!python ../input/yolov5/yolov5/train.py --epochs 25 --batch-size 4 \\\n--cfg ../input/yolov5/yolov5/chest_yaml/yolov5x6.yaml \\\n--hyp ../input/yolov5/yolov5/data/hyp.scratch.yaml \\\n--data ../input/yolov5/yolov5/chest_yaml/chest.yaml \\\n--weights ../input/yolov5-train/runs/train/exp/weights/best.pt --img 1024","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"result = '../working/runs/train/exp/'\nfor img in os.listdir(result):\n    if 'png' in img or 'jpg' in img:\n        display(Image(filename=f'{result}/{img}', width=900))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf ../working/chesh_yolo/","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}