{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -r ../input/yolov5-official-v31-dataset/yolov5/requirements.txt\n!pip install timm","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-14T08:54:24.986346Z","iopub.execute_input":"2022-08-14T08:54:24.986677Z","iopub.status.idle":"2022-08-14T08:54:59.31619Z","shell.execute_reply.started":"2022-08-14T08:54:24.986629Z","shell.execute_reply":"2022-08-14T08:54:59.315329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd \n#显示所有列\npd.set_option('display.max_columns', None)\n#显示所有行\npd.set_option('display.max_rows',None)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:55:50.420126Z","iopub.execute_input":"2022-08-14T08:55:50.420446Z","iopub.status.idle":"2022-08-14T08:55:50.42496Z","shell.execute_reply.started":"2022-08-14T08:55:50.420415Z","shell.execute_reply":"2022-08-14T08:55:50.42406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.extend(['../input/effdet/',\n                 '../input/iterstrat/',\n                 '../input/weightedboxfusion/',\n                 '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master/'])\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport seaborn as sns\nfrom pathlib import Path\nimport json\nimport re\nimport logging\nimport gc\nimport random\nimport warnings\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\nimport os\nfrom glob import glob\nfrom joblib import Parallel, delayed\nimport shutil as sh\nfrom itertools import product\nfrom collections import OrderedDict\nimport plotly\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.offline as pyo\nfrom ml_stratifiers import MultilabelStratifiedKFold\nfrom ensemble_boxes import nms, weighted_boxes_fusion\nimport torchvision.transforms as transforms\nimport albumentations as al\nfrom albumentations import ImageOnlyTransform\nfrom albumentations.pytorch import ToTensorV2, ToTensor\nfrom albumentations.core.transforms_interface import DualTransform, ImageOnlyTransform\n\nimport cv2\nimport pydicom\nfrom IPython.display import display, Image\nfrom sklearn.model_selection import train_test_split, KFold, GroupKFold, StratifiedKFold\nimport torch\nfrom torch.nn import functional as f\nimport torch.nn as nn\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\nfrom torch.utils.data import Dataset, DataLoader, sampler\nimport timm\nfrom efficientnet_pytorch import EfficientNet\n\npd.options.display.max_columns = None\nwarnings.filterwarnings('ignore')\n\nlogging.basicConfig(format='%(asctime)s +++ %(message)s',\n                    datefmt='%d-%m-%y %H:%M:%S', level=logging.INFO)\nlogger = logging.getLogger()\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nlogger.info(device)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-08-14T08:55:54.000834Z","iopub.execute_input":"2022-08-14T08:55:54.001164Z","iopub.status.idle":"2022-08-14T08:56:00.987321Z","shell.execute_reply.started":"2022-08-14T08:55:54.001131Z","shell.execute_reply":"2022-08-14T08:56:00.986493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"os.environ[\"WANDB_API_KEY\"] = '8f435998b1a6f9a4e59bfaef1deed81c1362a97d'\nos.environ[\"WANDB_MODE\"] = \"dryrun\"\n\nMAIN_PATH = '../input/vinbigdata-chest-xray-abnormalities-detection/'\nCLASSIFIER_MAIN_PATH = '../input/efficientnet-pytorch/'\nRESIZE_1024_PATH = '../input/vinbigdata-chest-xray-resized-png-1024x1024/'\nRESIZE_512_PATH = '../input/vinbigdata/'\nTRAIN_PATH = os.path.join(MAIN_PATH, 'train.csv')\nSUB_PATH = os.path.join(MAIN_PATH, 'sample_submission.csv')\nTRAIN_DICOM_PATH = os.path.join(MAIN_PATH, 'train')\nTEST_DICOM_PATH = os.path.join(MAIN_PATH, 'test')\nTRAIN_1024_PATH = os.path.join(RESIZE_1024_PATH, 'train')\nTEST_1024_PATH = os.path.join(RESIZE_1024_PATH, 'test')\nTRAIN_512_PATH = os.path.join(RESIZE_512_PATH, 'train')\nTEST_512_PATH = os.path.join(RESIZE_512_PATH, 'test')\nTRAIN_META_PATH = os.path.join(RESIZE_1024_PATH, 'train_meta.csv')\nTEST_META_PATH = '../input/vinbigdata-testmeta/test_meta.csv'\nTEST_CLASS_PATH = '../input/vinbigdata-2class-prediction/2-cls test pred.csv'\nMODEL_WEIGHT = '../input/efficientdet/tf_efficientdet_d7_53-6d1d7a95.pth'\nSIZE = 512\nIMG_SIZE = (SIZE, SIZE)\nACCULATION = 1\nMOSAIC_RATIO = 0.4\n    \nclass GlobalConfig:\n    model_use = 'd0'\n    model_weight = '../input/efficientdet/tf_efficientdet_d0_34-f153e0cf.pth'\n    img_size = IMG_SIZE\n    fold_num = 5\n    seed = 89\n    num_workers = 12\n    batch_size = 8\n    n_epochs = 20\n    lr = 1e-2\n    verbose = 1\n    verbose_step = 1\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n#     output_path = './save/'\n    scheduler_params = dict(\n        mode='min', \n        factor=0.2,\n        patience=1,\n        threshold_mode='abs',\n        min_lr=1e-7\n    )\n    \nclass PredictConfig:\n    img_size = IMG_SIZE\n    batch_size = 16\n    model_classifier_use = 'b0'\n    weight_classifier = '../input/effdet-d5-512/model_classifier_b0_512.pth'\n#     weight_classifier = '../input/x-chest-1024-classifier/model_classifier.pth'\n    score_thresh = 0.05\n    iou_thresh = 0.4\n    iou_thresh2 = 0.1\n    iou_thresh11 = 0.0001\n    skip_thresh = 0.0001\n    sigma = 0.1\n    score_0 = 0.385\n    score_3 = 0.4\n    score_last = 0.0\n    score_last2 = 0.95\n    score_9 = 0.1\n    score_11 = 0.015\n    classification_thresh = 0.003751\n    \nlist_remove = [34843, 21125, 647, 18011, 2539, 22373, 12675, 7359, 20642, 5502, 19818, 5832, 28056, 28333, 20758,\n               925, 43, 2199, 4610, 21306, 16677, 1768, 17232, 1378, 24949, 30203, 31410, 87, 25318, 92, 31724,\n               118, 17687, 12605, 26157, 33875, 7000, 3730, 18776, 13225, 1109, 2161, 33627, 15500, 28633, 28152,\n               10114, 10912, 9014,  4427, 25630, 11464, 6419, 22164, 4386, 17557, 15264, 21853, 33142, 32895, 9733,\n               33010, 17493, 32128, 28802, 11658, 8841, 29557, 4802, 8591, 778, 9935, 12359, 5210, 7556, 24505, 5664,\n               28670, 27820, 19359, 9817, 7800, 32934, 34098, 27931, 16074, 27308, 30645, 31029, 35697, 6199, 27065,\n               1771, 14689, 31860, 1975, 29294, 2304, 34018, 23406, 26501, 26011, 2479, 32796, 25836, 3032, 31454,\n               32066, 19722, 15997, 6049, 9458, 11005, 23151, 24503, 35411, 18092, 23815, 30742, 33942, 34542, 7655,\n               25345, 3750, 17046, 3844, 5958, 4250, 18823, 14898, 22581, 25805, 9651, 33194, 36007, 30160, 24459,\n               10838, 16544, 31252, 8053, 28487, 6208, 25244, 8470, 10089, 24813, 14769, 34305, 34047, 23366, 8049,\n               13276, 22380, 32797, 32440, 11031, 18304, 33692, 21349, 26333, 34331, 9110, 21092, 34882, 35626, 10203,\n               25648, 30754, 29567, 33542, 15146, 26759, 20846, 22493, 33187, 22813, 30219, 14548, 14627, 20494, 28332,\n               15930, 31347, 33489, 35005, 34032, 24183, 18643, 18536, 29754, 20380, 29750, 20539, 35791, 27275, 32248]\nimage_remove = ['9c83d9f88170cd38f7bca54fe27dc48a', 'ac2a615b3861212f9a2ada6acd077fd9',\n                'f9f7feefb4bac748ff7ad313e4a78906', 'f89143595274fa6016f6eec550442af9',\n                '6c08a98e48ba72aee1b7b62e1f28e6da', 'e7a58f5647d24fc877f9cb3d051792e2',\n                '8f98e3e6e86e573a6bd32403086b3707', '43d3137e74ebd344636228e786cb91b0',\n                '575b98a9f9824d519937a776bd819cc4', 'ca6c1531a83f8ee89916ed934f8d4847',\n                '0c6a7e3c733bd4f4d89443ca16615fc6', 'ae5cec1517ab3e82c5374e4c6219a17d',\n                '064023f1ff95962a1eee46b9f05f7309', '27c831fee072b232499541b0aca58d9c',\n                '0b98b21145a9425bf3eeea4b0de425e7', '7df5c81873c74ecc40610a1ad4eb2943']","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2022-08-14T08:56:11.329725Z","iopub.execute_input":"2022-08-14T08:56:11.33008Z","iopub.status.idle":"2022-08-14T08:56:11.346752Z","shell.execute_reply.started":"2022-08-14T08:56:11.330029Z","shell.execute_reply":"2022-08-14T08:56:11.345706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Function","metadata":{}},{"cell_type":"code","source":"class BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = IMG_SIZE\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)\n    \ndef Visualize_class(df, feature, title):\n    num_image = df[feature].value_counts().rename_axis(feature).reset_index(name='num_image')\n    fig = px.bar(num_image[::-1], x='num_image', y=feature, orientation='h', color='num_image')\n    fig.update_layout(\n    title={\n        'text': title,\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\n    fig.show()\n    \n    \ndef img_size(path):\n    information = pydicom.dcmread(path)\n    h, w = information.Rows, information.Columns\n    return (h, w)\n\n\ndef label_resize(org_size, img_size, *bbox):\n    x0, y0, x1, y1 = bbox\n    x0_new = int(np.round(x0*img_size[1]/org_size[1]))\n    y0_new = int(np.round(y0*img_size[0]/org_size[0]))\n    x1_new = int(np.round(x1*img_size[1]/org_size[1]))\n    y1_new = int(np.round(y1*img_size[0]/org_size[0]))\n    return x0_new, y0_new, x1_new, y1_new\n\n\ndef list_color(class_list):\n    dict_color = dict()\n    for classid in class_list:\n        dict_color[classid] = [i/256 for i in random.sample(range(256), 3)]\n    \n    return dict_color\n\n\ndef split_df(df):\n    kf = MultilabelStratifiedKFold(n_splits=GlobalConfig.fold_num,\n                                   shuffle=True, random_state=GlobalConfig.seed)\n    df['id'] = df.index\n    annot_pivot = pd.pivot_table(df, index='image_id', columns='class_id',\n                                 values='id', fill_value=0, aggfunc='count') \\\n    .reset_index().rename_axis(None, axis=1)\n    for fold, (train_idx, val_idx) in enumerate(kf.split(annot_pivot,\n                                                         annot_pivot.iloc[:, 1:train_abnormal['class_id'].nunique()])):\n        annot_pivot[f'fold_{fold}'] = 0\n        annot_pivot.loc[val_idx, f'fold_{fold}'] = 1\n    return annot_pivot\n    \n    \ndef display_image(df, list_image, num_image=1, is_dicom_file=True):\n    \n    dict_color = list_color(range(15))\n    list_abnormal = [i for i in df['class_name'].unique() if i!='No finding']\n    for abnormal in list_abnormal:\n        abnormal_df = df[df['class_name']==abnormal].reset_index(drop=True)\n        abnormal_random = np.random.choice(abnormal_df['image_id'].unique(), num_image)\n        for abnormal_img in abnormal_random:\n            images = abnormal_df[abnormal_df['image_id']==abnormal_img].reset_index(drop=True)\n            fig, ax = plt.subplots(1, figsize=(15, 15))\n            img_path = [i for i in list_image if abnormal_img in i][0]\n            if is_dicom_file:\n                information = pydicom.dcmread(img_path)\n                img = information.pixel_array\n            else:\n                img = cv2.imread(img_path)\n            ax.imshow(img, plt.cm.bone)\n            for idx, image in images.iterrows():\n                bbox = [image.x_min, image.y_min, image.x_max, image.y_max]\n                if is_dicom_file:\n                    x_min, y_min, x_max, y_max = bbox\n                else:\n                    org_size = image[['h', 'w']].values\n                    x_min, y_min, x_max, y_max = label_resize(org_size, IMG_SIZE, *bbox)\n                class_name, class_id = image.class_name, image.class_id\n                rect = patches.Rectangle((x_min, y_min), x_max-x_min, y_max-y_min,\n                                         linewidth=1, edgecolor=dict_color[class_id], facecolor='none')\n                ax.add_patch(rect)\n                plt.text(x_min, y_min, class_name, fontsize=15, color='red')\n\n            plt.title(abnormal_img) \n            plt.show()\n            \ndef display_image_test(df, size_df, list_image, num_image=3):\n    \n    dict_color = list_color(range(15))\n    image_row_random = np.random.choice(len(df), num_image, replace=(len(df)<num_image))\n    for image_idx in image_row_random:\n        image_id, pred = df.loc[image_idx, 'image_id'], df.loc[image_idx, 'PredictionString']\n        org_size = size_df[size_df['image_id']==image_id][['h', 'w']].values[0].tolist()\n        fig, ax = plt.subplots(1, figsize=(15, 15))\n        img_path = [i for i in list_image if image_id in i][0]\n        img = cv2.imread(img_path)\n        ax.imshow(img, plt.cm.bone)\n        if pred != '14 1 0 0 1 1':\n            list_pred = pred.split(' ')\n            for box_idx in range(len(list_pred)//6):\n                bbox = map(int, list_pred[6*box_idx+2:6*box_idx+6])\n                x_min, y_min, x_max, y_max = label_resize(org_size, IMG_SIZE, *bbox)\n                class_name, score = int(list_pred[6*box_idx]), float(list_pred[6*box_idx+1])\n                rect = patches.Rectangle((x_min, y_min), x_max-x_min, y_max-y_min,\n                                         linewidth=1, edgecolor=dict_color[class_name], facecolor='none')\n                ax.add_patch(rect)\n                plt.text(x_min, y_min, f'{class_name}: {score}', fontsize=15, color='red')            \n\n        plt.title(image_id) \n        plt.show()\n        \ndef ensemble_multibox(boxes, scores, labels, iou_thr, sigma,\n                      skip_box_thr, weights=None, method='wbf'):\n    if method=='nms':\n        boxes, scores, labels = nms(boxes, scores, labels,\n                                    weights=weights,\n                                    iou_thr=iou_thr)\n    elif method=='soft_nms':\n        boxes, scores, labels = soft_nms(boxes, scores, labels,\n                                         weights=weights,\n                                         sigma=sigma,\n                                         iou_thr=iou_thr,\n                                         thresh=skip_box_thr)\n    elif method=='nms_weight':\n        boxes, scores, labels = non_maximum_weighted(boxes, scores, labels,\n                                                     weights=weights,\n                                                     iou_thr=iou_thr,\n                                                     skip_box_thr=skip_box_thr)\n    elif method=='wbf':\n        boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels,\n                                                      weights=weights,\n                                                      iou_thr=iou_thr,\n                                                      skip_box_thr=skip_box_thr)\n    \n    return boxes, scores, labels","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:56:17.317431Z","iopub.execute_input":"2022-08-14T08:56:17.317792Z","iopub.status.idle":"2022-08-14T08:56:17.354043Z","shell.execute_reply.started":"2022-08-14T08:56:17.317757Z","shell.execute_reply":"2022-08-14T08:56:17.352931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read File","metadata":{}},{"cell_type":"code","source":"%%time\n\ntrain_dicom_list = glob(f'{TRAIN_DICOM_PATH}/*.dicom')\ntest_dicom_list = glob(f'{TEST_DICOM_PATH}/*.dicom')\n\ntrain_list = glob(f'{TRAIN_512_PATH}/*.png')\ntest_list = glob(f'{TEST_512_PATH}/*.png')\nlogger.info(f'Train have {len(train_list)} file and test have {len(test_list)}')","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:56:27.80371Z","iopub.execute_input":"2022-08-14T08:56:27.804129Z","iopub.status.idle":"2022-08-14T08:56:30.528179Z","shell.execute_reply.started":"2022-08-14T08:56:27.804093Z","shell.execute_reply":"2022-08-14T08:56:30.527427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nsize_df = pd.read_csv(TRAIN_META_PATH)\nsize_df.columns = ['image_id', 'h', 'w']\n\ntrain_df = pd.read_csv(TRAIN_PATH)\ntrain_df = train_df.merge(size_df, on='image_id', how='left')\ntrain_df[['x_min', 'y_min']] = train_df[['x_min', 'y_min']].fillna(0)\ntrain_df[['x_max', 'y_max']] = train_df[['x_max', 'y_max']].fillna(1)\n\ntrain_df.tail()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:56:34.114499Z","iopub.execute_input":"2022-08-14T08:56:34.114864Z","iopub.status.idle":"2022-08-14T08:56:34.435406Z","shell.execute_reply.started":"2022-08-14T08:56:34.114831Z","shell.execute_reply":"2022-08-14T08:56:34.434254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyze file","metadata":{}},{"cell_type":"code","source":"Visualize_class(train_df, feature='class_name', title='Types of thoracic abnormalities')\nlogger.info(f\"Train have {train_df['class_name'].nunique()-1} types of thoracic abnormalities\")","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:57:00.788294Z","iopub.execute_input":"2022-08-14T08:57:00.788622Z","iopub.status.idle":"2022-08-14T08:57:00.858997Z","shell.execute_reply.started":"2022-08-14T08:57:00.788591Z","shell.execute_reply":"2022-08-14T08:57:00.858328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Visualize_class(train_df, feature='rad_id', title='List radiologists')\nlogger.info(f\"Train have {train_df['rad_id'].nunique()} radiologists\")","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:57:22.268306Z","iopub.execute_input":"2022-08-14T08:57:22.268649Z","iopub.status.idle":"2022-08-14T08:57:22.340097Z","shell.execute_reply.started":"2022-08-14T08:57:22.268615Z","shell.execute_reply":"2022-08-14T08:57:22.339358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_each_rad = pd.pivot_table(train_df, columns='rad_id', index='class_name',\n                                values='image_id', aggfunc='count', fill_value=0)\nplt.subplots(figsize=(15, 10))\nsns.heatmap(class_each_rad, annot=True, linewidths=1, cmap='Blues', fmt='g')\nplt.show()\n\n# R8, R9, R10 is largest\n# R1, R3, R4, R5, R6, R7 only normal case","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:57:28.321612Z","iopub.execute_input":"2022-08-14T08:57:28.321988Z","iopub.status.idle":"2022-08-14T08:57:29.907386Z","shell.execute_reply.started":"2022-08-14T08:57:28.321954Z","shell.execute_reply":"2022-08-14T08:57:29.906406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_image(train_df, train_list, is_dicom_file=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:57:49.313744Z","iopub.execute_input":"2022-08-14T08:57:49.314077Z","iopub.status.idle":"2022-08-14T08:57:56.16273Z","shell.execute_reply.started":"2022-08-14T08:57:49.31404Z","shell.execute_reply":"2022-08-14T08:57:56.161852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_normal = train_df[train_df['class_name']=='No finding'].reset_index(drop=True)\ntrain_normal['x_min_resize'] = 0\ntrain_normal['y_min_resize'] = 0\ntrain_normal['x_max_resize'] = 1\ntrain_normal['y_max_resize'] = 1\n\ntrain_abnormal = train_df[train_df['class_name']!='No finding'].reset_index(drop=True)\ntrain_abnormal[['x_min_resize', 'y_min_resize', 'x_max_resize', 'y_max_resize']] = train_abnormal \\\n.apply(lambda x: label_resize(x[['h', 'w']].values, IMG_SIZE, *x[['x_min', 'y_min', 'x_max', 'y_max']].values),\n       axis=1, result_type=\"expand\")\ntrain_abnormal['x_center'] = 0.5*(train_abnormal['x_min_resize'] + train_abnormal['x_max_resize'])\ntrain_abnormal['y_center'] = 0.5*(train_abnormal['y_min_resize'] + train_abnormal['y_max_resize'])\ntrain_abnormal['width'] = train_abnormal['x_max_resize'] - train_abnormal['x_min_resize']\ntrain_abnormal['height'] = train_abnormal['y_max_resize'] - train_abnormal['y_min_resize']\ntrain_abnormal['area'] = train_abnormal.apply(lambda x: (x['x_max_resize']-x['x_min_resize'])*(x['y_max_resize']-x['y_min_resize']), axis=1)\ntrain_abnormal = train_abnormal[~train_abnormal.index.isin(list_remove)].reset_index(drop=True)\n\ntrain_abnormal.tail()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:58:57.492817Z","iopub.execute_input":"2022-08-14T08:58:57.493176Z","iopub.status.idle":"2022-08-14T08:59:28.442503Z","shell.execute_reply.started":"2022-08-14T08:58:57.493148Z","shell.execute_reply":"2022-08-14T08:59:28.441574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Change wbf","metadata":{}},{"cell_type":"code","source":"def Preprocess_wbf(df, size=SIZE, iou_thr=0.5, skip_box_thr=0.0001):\n    list_image = []\n    list_boxes = []\n    list_cls = []\n    list_h, list_w = [], []\n    new_df = pd.DataFrame()\n    for image_id in tqdm(df['image_id'].unique(), leave=False):\n        image_df = df[df['image_id']==image_id].reset_index(drop=True)\n        h, w = image_df.loc[0, ['h', 'w']].values\n        boxes = image_df[['x_min_resize', 'y_min_resize',\n                          'x_max_resize', 'y_max_resize']].values.tolist()\n        boxes = [[j/(size-1) for j in i] for i in boxes]\n        scores = [1.0]*len(boxes)\n        labels = [float(i) for i in image_df['class_id'].values]\n        boxes, scores, labels = weighted_boxes_fusion([boxes], [scores], [labels],\n                                                      weights=None,\n                                                      iou_thr=iou_thr,\n                                                      skip_box_thr=skip_box_thr)\n        list_image.extend([image_id]*len(boxes))\n        list_h.extend([h]*len(boxes))\n        list_w.extend([w]*len(boxes))\n        list_boxes.extend(boxes)\n        list_cls.extend(labels.tolist())\n    list_boxes = [[int(j*(size-1)) for j in i] for i in list_boxes]\n    new_df['image_id'] = list_image\n    new_df['class_id'] = list_cls\n    new_df['h'] = list_h\n    new_df['w'] = list_w\n    new_df['x_min_resize'], new_df['y_min_resize'], \\\n    new_df['x_max_resize'], new_df['y_max_resize'] = np.transpose(list_boxes)\n    new_df['x_center'] = 0.5*(new_df['x_min_resize'] + new_df['x_max_resize'])\n    new_df['y_center'] = 0.5*(new_df['y_min_resize'] + new_df['y_max_resize'])\n    new_df['width'] = new_df['x_max_resize'] - new_df['x_min_resize']\n    new_df['height'] = new_df['y_max_resize'] - new_df['y_min_resize']\n    new_df['area'] = new_df.apply(lambda x: (x['x_max_resize']-x['x_min_resize'])\\\n                                  *(x['y_max_resize']-x['y_min_resize']), axis=1)\n    return new_df\n\ntrain_abnormal = Preprocess_wbf(train_abnormal)\ntrain_abnormal.tail()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T08:59:44.354072Z","iopub.execute_input":"2022-08-14T08:59:44.354479Z","iopub.status.idle":"2022-08-14T09:00:04.721623Z","shell.execute_reply.started":"2022-08-14T08:59:44.354434Z","shell.execute_reply":"2022-08-14T09:00:04.72076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fold","metadata":{}},{"cell_type":"code","source":"def split_df(df):\n    kf = MultilabelStratifiedKFold(n_splits=GlobalConfig.fold_num, shuffle=True, random_state=GlobalConfig.seed)\n    df['id'] = df.index\n    annot_pivot = pd.pivot_table(df, index=['image_id'], columns=['class_id'],\n                                 values='id', fill_value=0, aggfunc='count') \\\n    .reset_index().rename_axis(None, axis=1)\n    for fold, (train_idx, val_idx) in enumerate(kf.split(annot_pivot,\n                                                         annot_pivot.iloc[:, 1:(1+df['class_id'].nunique())])):\n        annot_pivot[f'fold_{fold}'] = 0\n        annot_pivot.loc[val_idx, f'fold_{fold}'] = 1\n    return annot_pivot\n\nsize_df = pd.read_csv(TRAIN_META_PATH)\nsize_df.columns = ['image_id', 'h', 'w']\n\nfold_csv = split_df(train_abnormal)\nfold_csv = fold_csv.merge(size_df, on='image_id', how='left')\nfold_csv.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:00:20.648341Z","iopub.execute_input":"2022-08-14T09:00:20.648704Z","iopub.status.idle":"2022-08-14T09:00:20.84054Z","shell.execute_reply.started":"2022-08-14T09:00:20.648647Z","shell.execute_reply":"2022-08-14T09:00:20.839621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Dataset","metadata":{}},{"cell_type":"code","source":"def create_file(df, split_df, train_file, train_folder, fold):\n    \n    os.makedirs(f'{train_file}/labels/train/', exist_ok=True)\n    os.makedirs(f'{train_file}/images/train/', exist_ok=True)\n    os.makedirs(f'{train_file}/labels/val/', exist_ok=True)\n    os.makedirs(f'{train_file}/images/val/', exist_ok=True)\n    \n    list_image_train = split_df[split_df[f'fold_{fold}']==0]['image_id']    \n    train_df = df[df['image_id'].isin(list_image_train)].reset_index(drop=True)\n    val_df = df[~df['image_id'].isin(list_image_train)].reset_index(drop=True)\n    \n    for train_img in tqdm(train_df.image_id.unique()):\n        with open(f'{train_file}/labels/train/{train_img}.txt', 'w+') as f:\n            row = train_df[train_df['image_id']==train_img]\\\n            [['class_id', 'x_center', 'y_center', 'width', 'height']].values\n            row[:, 1:] /= SIZE\n            row = row.astype('str')\n            for box in range(len(row)):\n                text = ' '.join(row[box])\n                f.write(text)\n                f.write('\\n')\n        sh.copy(f'{train_folder}/{train_img}.png', \n                f'{train_file}/images/train/{train_img}.png')\n        \n    for val_img in tqdm(val_df.image_id.unique()):\n        with open(f'{train_file}/labels/val/{val_img}.txt', 'w+') as f:\n            row = val_df[val_df['image_id']==val_img]\\\n            [['class_id', 'x_center', 'y_center', 'width', 'height']].values\n            row[:, 1:] /= SIZE\n            row = row.astype('str')\n            for box in range(len(row)):\n                text = ' '.join(row[box])\n                f.write(text)\n                f.write('\\n')\n        sh.copy(f'{train_folder}/{val_img}.png', \n                f'{train_file}/images/val/{val_img}.png')\n        \ncreate_file(train_abnormal, fold_csv, './chest_yolo', TRAIN_512_PATH, 0)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:00:26.599571Z","iopub.execute_input":"2022-08-14T09:00:26.599938Z","iopub.status.idle":"2022-08-14T09:01:18.71762Z","shell.execute_reply.started":"2022-08-14T09:00:26.599905Z","shell.execute_reply":"2022-08-14T09:01:18.716768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil, os\nshutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:01:21.68765Z","iopub.execute_input":"2022-08-14T09:01:21.688074Z","iopub.status.idle":"2022-08-14T09:01:22.409632Z","shell.execute_reply.started":"2022-08-14T09:01:21.688038Z","shell.execute_reply":"2022-08-14T09:01:22.408856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_ids, class_names = list(zip(*set(zip(train_df.class_id, train_df.class_name))))\nclasses = list(np.array(class_names)[np.argsort(class_ids)])\nclasses = list(map(lambda x: str(x), classes))\nclasses=classes[0:-1]","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:01:26.150982Z","iopub.execute_input":"2022-08-14T09:01:26.151319Z","iopub.status.idle":"2022-08-14T09:01:26.172231Z","shell.execute_reply.started":"2022-08-14T09:01:26.151289Z","shell.execute_reply":"2022-08-14T09:01:26.16968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(classes)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:01:31.646771Z","iopub.execute_input":"2022-08-14T09:01:31.647099Z","iopub.status.idle":"2022-08-14T09:01:31.652719Z","shell.execute_reply.started":"2022-08-14T09:01:31.647068Z","shell.execute_reply":"2022-08-14T09:01:31.65177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd = '/kaggle/working/'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('/kaggle/working/chest_yolo/images/train/*'):\n        f.write(path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('/kaggle/working/chest_yolo/images/val/*'):\n        f.write(path+'\\n')\n\ndata = dict(\n    train =  join( cwd , 'train.txt') ,\n    val   =  join( cwd , 'val.txt' ),\n    nc    = 14,\n    names = classes\n    )\n\nwith open(join( cwd , 'chest.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n\nf = open(join( cwd , 'chest.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:01:37.475075Z","iopub.execute_input":"2022-08-14T09:01:37.475407Z","iopub.status.idle":"2022-08-14T09:01:37.502473Z","shell.execute_reply.started":"2022-08-14T09:01:37.475376Z","shell.execute_reply":"2022-08-14T09:01:37.501482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python ./yolov5/train.py --epochs 6 --batch-size 4 --cfg ./yolov5/models/yolov5x.yaml \\\n--data ./chest.yaml --weights yolov5x.pt --img 512 --device 0 --multi-scale  --cache\n\n# !WANDB_MODE=\"dryrun\" python ./yolov5/train.py --epochs 1 --batch-size 4 --cfg ./yolov5/models/yolov5x.yaml \\\n# --data ./chest.yaml --weights yolov5x.pt --img 512 --device 0\n# # !python ./yolov5/train.py --epochs 60 --batch-size 4 --cfg ./chest_yaml/yolov5x.yaml \\\n# # --data ./chest_yaml/chest.yaml --weights ./chest_yaml/yolov5x.pt --img 512 --device 0","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:01:53.29913Z","iopub.execute_input":"2022-08-14T09:01:53.299459Z","iopub.status.idle":"2022-08-14T09:38:02.243983Z","shell.execute_reply.started":"2022-08-14T09:01:53.299428Z","shell.execute_reply":"2022-08-14T09:38:02.242786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a=1\n","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:38:13.834205Z","iopub.execute_input":"2022-08-14T09:38:13.834537Z","iopub.status.idle":"2022-08-14T09:38:13.839143Z","shell.execute_reply.started":"2022-08-14T09:38:13.834503Z","shell.execute_reply":"2022-08-14T09:38:13.838229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20,20))\nplt.axis('off')\nplt.imshow(plt.imread('./runs/train/exp/labels_correlogram.jpg'));","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:38:20.198412Z","iopub.execute_input":"2022-08-14T09:38:20.198756Z","iopub.status.idle":"2022-08-14T09:38:21.117353Z","shell.execute_reply.started":"2022-08-14T09:38:20.198723Z","shell.execute_reply":"2022-08-14T09:38:21.116567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20,20))\nplt.axis('off')\nplt.imshow(plt.imread('./runs/train/exp/labels.jpg'));","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:38:35.880694Z","iopub.execute_input":"2022-08-14T09:38:35.881063Z","iopub.status.idle":"2022-08-14T09:38:36.649398Z","shell.execute_reply.started":"2022-08-14T09:38:35.881031Z","shell.execute_reply":"2022-08-14T09:38:36.648606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('./runs/train/exp/train_batch0.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('./runs/train/exp/train_batch1.jpg'))\n\nplt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('./runs/train/exp/train_batch2.jpg'))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:38:55.868104Z","iopub.execute_input":"2022-08-14T09:38:55.868799Z","iopub.status.idle":"2022-08-14T09:38:57.64835Z","shell.execute_reply.started":"2022-08-14T09:38:55.868759Z","shell.execute_reply":"2022-08-14T09:38:57.647455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(3, 2, figsize = (2*5,3*5), constrained_layout = True)\nfor row in range(3):\n    ax[row][0].imshow(plt.imread(f'./runs/train/exp/test_batch{row}_labels.jpg'))\n    ax[row][0].set_xticks([])\n    ax[row][0].set_yticks([])\n    ax[row][0].set_title(f'./runs/train/exp/test_batch{row}_labels.jpg', fontsize = 12)\n    \n    ax[row][1].imshow(plt.imread(f'./runs/train/exp/test_batch{row}_pred.jpg'))\n    ax[row][1].set_xticks([])\n    ax[row][1].set_yticks([])\n    ax[row][1].set_title(f'./runs/train/exp/test_batch{row}_pred.jpg', fontsize = 12)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:39:10.096821Z","iopub.execute_input":"2022-08-14T09:39:10.097148Z","iopub.status.idle":"2022-08-14T09:39:11.613149Z","shell.execute_reply.started":"2022-08-14T09:39:10.097119Z","shell.execute_reply":"2022-08-14T09:39:11.612324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.axis('off')\nplt.imshow(plt.imread('./runs/train/exp/results.png'));","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:39:23.288493Z","iopub.execute_input":"2022-08-14T09:39:23.288866Z","iopub.status.idle":"2022-08-14T09:39:24.115971Z","shell.execute_reply.started":"2022-08-14T09:39:23.288833Z","shell.execute_reply":"2022-08-14T09:39:24.115187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.axis('off')\nplt.imshow(plt.imread('./runs/train/exp/confusion_matrix.png'));","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:39:29.350968Z","iopub.execute_input":"2022-08-14T09:39:29.351295Z","iopub.status.idle":"2022-08-14T09:39:30.889511Z","shell.execute_reply.started":"2022-08-14T09:39:29.351265Z","shell.execute_reply":"2022-08-14T09:39:30.888647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = './runs/train/exp/'\nfor img in os.listdir(result):\n    if 'png' in img or 'jpg' in img:\n        display(Image(filename=f'{result}/{img}', width=900))\n# result = '../working/runs/train/exp/'\n# for img in os.listdir(result):\n#     if 'png' in img or 'jpg' in img:\n#         display(Image(filename=f'{result}/{img}', width=900))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:39:37.007154Z","iopub.execute_input":"2022-08-14T09:39:37.007507Z","iopub.status.idle":"2022-08-14T09:39:37.109824Z","shell.execute_reply.started":"2022-08-14T09:39:37.007477Z","shell.execute_reply":"2022-08-14T09:39:37.109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model predict","metadata":{}},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:40:12.698574Z","iopub.execute_input":"2022-08-14T09:40:12.698951Z","iopub.status.idle":"2022-08-14T09:40:12.704603Z","shell.execute_reply.started":"2022-08-14T09:40:12.698918Z","shell.execute_reply":"2022-08-14T09:40:12.703628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dim = 512 #1024, 256, 'original'\ntest_dir = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/test'\n# weights_dir = '/kaggle/input/vinbigdata-cxr-ad-yolov5-14-class-train/yolov5/runs/train/exp/weights/best.pt'\nweights_dir = '/kaggle/working/runs/train/exp/weights/best.pt'","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:40:16.501187Z","iopub.execute_input":"2022-08-14T09:40:16.501516Z","iopub.status.idle":"2022-08-14T09:40:16.506701Z","shell.execute_reply.started":"2022-08-14T09:40:16.501485Z","shell.execute_reply":"2022-08-14T09:40:16.505742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/test.csv')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:40:21.767286Z","iopub.execute_input":"2022-08-14T09:40:21.767607Z","iopub.status.idle":"2022-08-14T09:40:21.803066Z","shell.execute_reply.started":"2022-08-14T09:40:21.767576Z","shell.execute_reply":"2022-08-14T09:40:21.802174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/yolov5') # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:40:25.182453Z","iopub.execute_input":"2022-08-14T09:40:25.182899Z","iopub.status.idle":"2022-08-14T09:40:25.194505Z","shell.execute_reply.started":"2022-08-14T09:40:25.182859Z","shell.execute_reply":"2022-08-14T09:40:25.192516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights $weights_dir\\\n--img 640\\\n--conf 0.15\\\n--iou 0.4\\\n--source $test_dir\\\n--save-txt --save-conf --exist-ok","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:40:30.222514Z","iopub.execute_input":"2022-08-14T09:40:30.222888Z","iopub.status.idle":"2022-08-14T09:44:41.402944Z","shell.execute_reply.started":"2022-08-14T09:40:30.222854Z","shell.execute_reply":"2022-08-14T09:44:41.401855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yolo2voc(image_height, image_width, bboxes):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n    \n    \"\"\" \n#     print('bboxes',bboxes)\n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n    \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    \n    return bboxes","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:47:23.370309Z","iopub.execute_input":"2022-08-14T09:47:23.370712Z","iopub.status.idle":"2022-08-14T09:47:23.38082Z","shell.execute_reply.started":"2022-08-14T09:47:23.370651Z","shell.execute_reply":"2022-08-14T09:47:23.379819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os \nos.getcwd()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:47:33.274453Z","iopub.execute_input":"2022-08-14T09:47:33.274828Z","iopub.status.idle":"2022-08-14T09:47:33.284132Z","shell.execute_reply.started":"2022-08-14T09:47:33.274794Z","shell.execute_reply":"2022-08-14T09:47:33.282762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns\nfrom scipy.stats import gaussian_kde\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom scipy.stats import gaussian_kde\n\ndim = 512 #512, 256, 'original'\nfold = 4\n\ntrain_df = pd.read_csv(f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/train.csv')\ntrain_df.head()\n\ntrain_df['x_min'] = train_df.apply(lambda row: (row.x_min)/row.width, axis =1)\ntrain_df['y_min'] = train_df.apply(lambda row: (row.y_min)/row.height, axis =1)\n\ntrain_df['x_max'] = train_df.apply(lambda row: (row.x_max)/row.width, axis =1)\ntrain_df['y_max'] = train_df.apply(lambda row: (row.y_max)/row.height, axis =1)\n\ntrain_df['x_mid'] = train_df.apply(lambda row: (row.x_max+row.x_min)/2, axis =1)\ntrain_df['y_mid'] = train_df.apply(lambda row: (row.y_max+row.y_min)/2, axis =1)\n\ntrain_df['w'] = train_df.apply(lambda row: (row.x_max-row.x_min), axis =1)\ntrain_df['h'] = train_df.apply(lambda row: (row.y_max-row.y_min), axis =1)\n\ntrain_df['area'] = train_df['w']*train_df['h']\ntrain_df.head()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:47:38.07639Z","iopub.execute_input":"2022-08-14T09:47:38.076809Z","iopub.status.idle":"2022-08-14T09:47:49.551429Z","shell.execute_reply.started":"2022-08-14T09:47:38.076767Z","shell.execute_reply":"2022-08-14T09:47:49.550599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids = []\nPredictionStrings = []\n\nfor file_path in tqdm(glob('runs/detect/exp/labels/*txt')):\n    image_id = file_path.split('/')[-1].split('.')[0]\n    w, h = test_df.loc[test_df.image_id==image_id,['width', 'height']].values[0]\n    f = open(file_path, 'r')\n    data = np.array(f.read().replace('\\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)\n    data = data[:, [0, 5, 1, 2, 3, 4]]\n    bboxes = list(np.round(np.concatenate((data[:, :2], np.round(yolo2voc(h, w, data[:, 2:]))), axis =1).reshape(-1), 1).astype(str))\n    for idx in range(len(bboxes)):\n        bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]\n    image_ids.append(image_id)\n    PredictionStrings.append(' '.join(bboxes))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:50:40.00339Z","iopub.execute_input":"2022-08-14T09:50:40.003756Z","iopub.status.idle":"2022-08-14T09:50:44.328306Z","shell.execute_reply.started":"2022-08-14T09:50:40.00372Z","shell.execute_reply":"2022-08-14T09:50:44.327236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.DataFrame({'image_id':image_ids,\n                        'PredictionString':PredictionStrings})\nsub_df = pd.merge(test_df, pred_df, on = 'image_id', how = 'left').fillna(\"14 1 0 0 1 1\")\nsub_df = sub_df[['image_id', 'PredictionString']]\n\nsub_df.loc[sub_df['PredictionString'] =='','PredictionString']=\"14 1 0 0 1 1\"\nsub_df.to_csv('/kaggle/working/submission_512.csv',index = False)\nsub_df.tail()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:50:58.897917Z","iopub.execute_input":"2022-08-14T09:50:58.898377Z","iopub.status.idle":"2022-08-14T09:50:59.12778Z","shell.execute_reply.started":"2022-08-14T09:50:58.898334Z","shell.execute_reply":"2022-08-14T09:50:59.125766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(sub_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T09:51:03.902456Z","iopub.execute_input":"2022-08-14T09:51:03.903022Z","iopub.status.idle":"2022-08-14T09:51:04.244767Z","shell.execute_reply.started":"2022-08-14T09:51:03.902984Z","shell.execute_reply":"2022-08-14T09:51:04.243923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/yolov5')","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:06:32.50747Z","iopub.execute_input":"2022-08-14T10:06:32.507921Z","iopub.status.idle":"2022-08-14T10:06:32.51393Z","shell.execute_reply.started":"2022-08-14T10:06:32.507885Z","shell.execute_reply":"2022-08-14T10:06:32.512566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_img_path='/kaggle/input/mydata/222.jpeg'\n","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:06:42.384351Z","iopub.execute_input":"2022-08-14T10:06:42.384713Z","iopub.status.idle":"2022-08-14T10:06:42.388923Z","shell.execute_reply.started":"2022-08-14T10:06:42.384658Z","shell.execute_reply":"2022-08-14T10:06:42.38776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights $weights_dir\\\n--img 640\\\n--conf 0.15\\\n--iou 0.4\\\n--source $pred_img_path\n#--save-txt --save-conf --exist-ok","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:06:45.001285Z","iopub.execute_input":"2022-08-14T10:06:45.001611Z","iopub.status.idle":"2022-08-14T10:06:52.583217Z","shell.execute_reply.started":"2022-08-14T10:06:45.001579Z","shell.execute_reply":"2022-08-14T10:06:52.582018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/yolov5/runs/detect/exp4/')\n!ls","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:07:17.417125Z","iopub.execute_input":"2022-08-14T10:07:17.417477Z","iopub.status.idle":"2022-08-14T10:07:18.463495Z","shell.execute_reply.started":"2022-08-14T10:07:17.417444Z","shell.execute_reply":"2022-08-14T10:07:18.46236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = '/kaggle/working/yolov5/runs/detect/exp4/'\nfor img in os.listdir(result):\n    if 'png' in img or 'jpeg' in img:\n        display(Image(filename=f'{result}/{img}', width=900))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:07:20.51222Z","iopub.execute_input":"2022-08-14T10:07:20.512577Z","iopub.status.idle":"2022-08-14T10:07:20.526502Z","shell.execute_reply.started":"2022-08-14T10:07:20.512539Z","shell.execute_reply":"2022-08-14T10:07:20.52525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (15, 15))\nplt.imshow(plt.imread('/kaggle/working/yolov5/runs/detect/exp4/222.jpeg'))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:07:30.154451Z","iopub.execute_input":"2022-08-14T10:07:30.154812Z","iopub.status.idle":"2022-08-14T10:07:30.703377Z","shell.execute_reply.started":"2022-08-14T10:07:30.154778Z","shell.execute_reply":"2022-08-14T10:07:30.702593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/yolov5')","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:07:58.054802Z","iopub.execute_input":"2022-08-14T10:07:58.055158Z","iopub.status.idle":"2022-08-14T10:07:58.059587Z","shell.execute_reply.started":"2022-08-14T10:07:58.055124Z","shell.execute_reply":"2022-08-14T10:07:58.058336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_img_path='/kaggle/input/mydata/g.png'","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:08:03.216118Z","iopub.execute_input":"2022-08-14T10:08:03.216461Z","iopub.status.idle":"2022-08-14T10:08:03.22111Z","shell.execute_reply.started":"2022-08-14T10:08:03.21642Z","shell.execute_reply":"2022-08-14T10:08:03.219837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights $weights_dir\\\n--img 640\\\n--conf 0.15\\\n--iou 0.4\\\n--source $pred_img_path\n#--save-txt --save-conf --exist-ok","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:08:07.165148Z","iopub.execute_input":"2022-08-14T10:08:07.165495Z","iopub.status.idle":"2022-08-14T10:08:13.945717Z","shell.execute_reply.started":"2022-08-14T10:08:07.165463Z","shell.execute_reply":"2022-08-14T10:08:13.944518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/yolov5/runs/detect/exp5/')\n!ls","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:08:46.539059Z","iopub.execute_input":"2022-08-14T10:08:46.539525Z","iopub.status.idle":"2022-08-14T10:08:47.578064Z","shell.execute_reply.started":"2022-08-14T10:08:46.539489Z","shell.execute_reply":"2022-08-14T10:08:47.576963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = '/kaggle/working/yolov5/runs/detect/exp5/'\nfor img in os.listdir(result):\n    if 'png' in img or 'jpeg' in img:\n        display(Image(filename=f'{result}/{img}', width=900))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:09:11.994916Z","iopub.execute_input":"2022-08-14T10:09:11.995878Z","iopub.status.idle":"2022-08-14T10:09:12.027657Z","shell.execute_reply.started":"2022-08-14T10:09:11.995816Z","shell.execute_reply":"2022-08-14T10:09:12.026797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/yolov5')","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:10:18.495287Z","iopub.execute_input":"2022-08-14T10:10:18.495621Z","iopub.status.idle":"2022-08-14T10:10:18.499649Z","shell.execute_reply.started":"2022-08-14T10:10:18.495588Z","shell.execute_reply":"2022-08-14T10:10:18.498691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_img_path='/kaggle/input/mydata/q.jpg'","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:11:14.313117Z","iopub.execute_input":"2022-08-14T10:11:14.313463Z","iopub.status.idle":"2022-08-14T10:11:14.318047Z","shell.execute_reply.started":"2022-08-14T10:11:14.313428Z","shell.execute_reply":"2022-08-14T10:11:14.316895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --weights $weights_dir\\\n--img 640\\\n--conf 0.15\\\n--iou 0.4\\\n--source $pred_img_path\n#--save-txt --save-conf --exist-ok","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:11:16.911279Z","iopub.execute_input":"2022-08-14T10:11:16.911623Z","iopub.status.idle":"2022-08-14T10:11:23.164096Z","shell.execute_reply.started":"2022-08-14T10:11:16.911588Z","shell.execute_reply":"2022-08-14T10:11:23.163104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/yolov5/runs/detect/exp7/')\n!ls","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:11:49.160455Z","iopub.execute_input":"2022-08-14T10:11:49.160815Z","iopub.status.idle":"2022-08-14T10:11:50.163175Z","shell.execute_reply.started":"2022-08-14T10:11:49.160778Z","shell.execute_reply":"2022-08-14T10:11:50.16209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = '/kaggle/working/yolov5/runs/detect/exp5/'\nfor img in os.listdir(result):\n    if 'png' in img or 'jpeg' in img:\n        display(Image(filename=f'{result}/{img}', width=900))","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:12:09.367967Z","iopub.execute_input":"2022-08-14T10:12:09.368307Z","iopub.status.idle":"2022-08-14T10:12:09.384158Z","shell.execute_reply.started":"2022-08-14T10:12:09.368274Z","shell.execute_reply":"2022-08-14T10:12:09.383341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n\n#size_df = pd.read_csv(TEST_META_PATH)\n#size_df.columns = ['image_id', 'h', 'w']\n\n#sub_df = pd.read_csv(SUB_PATH)\n#sub_df = sub_df.merge(size_df, on='image_id', how='left')\n#sub_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#!python./yolov5/detect.py --weights ./runs/train/exp/weights/best.pt --img-size 512 --conf 0.005 --source ../input/vinbigdata/test/ --iou-thres 0.45 --save-txt --save-conf\n\n#!python ../input/yolov5/yolov5/detect.py --weights ../input/yolov5-train/runs/train/exp/weights/best.pt \\\n#--img-size 512 --conf-thres 0.005 --source ../input/vinbigdata/test/ --iou-thres 0.45 \\\n#--save-txt --save-conf\n\n#python./yolov5/train.py --epochs 60 --batch-size 4 --cfg ./yolov5/models/yolov5x.yaml \\\n#--data ./chest.yaml --weights yolov5x.pt --img 512 --device 0","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def aug(file):\n#     if file=='train':\n#         return al.Compose([\n#             al.VerticalFlip(p=0.5),\n#             al.HorizontalFlip(p=0.5),\n#             al.RandomRotate90(p=0.5),\n#             al.OneOf([\n#                 al.GaussNoise(0.002, p=0.5),\n#                 al.IAAAffine(p=0.5),\n#             ], p=0.2),\n#             al.OneOf([\n#                 al.Blur(blur_limit=(3, 10), p=0.4),\n#                 al.MedianBlur(blur_limit=3, p=0.3),\n#                 al.MotionBlur(p=0.3)\n#             ], p=0.3),\n#             al.OneOf([\n#                 al.RandomBrightness(p=0.3),\n#                 al.RandomContrast(p=0.4),\n#                 al.RandomGamma(p=0.3)\n#             ], p=0.5),\n#             al.Cutout(num_holes=20, max_h_size=20, max_w_size=20, p=0.5),\n# #             al.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=45, p=0.3),\n#             al.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), p=1),\n#             ToTensorV2(p=1)\n#         ])\n\n#     elif file=='validation':\n#         return al.Compose([\n#             al.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), p=1),\n#             ToTensorV2(p=1)\n#         ])\n\n#     elif file=='test':\n#         return al.Compose([\n#             al.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), p=1),\n#             ToTensorV2(p=1)\n#         ], p=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class EfficientnetCus(nn.Module):\n    \n#     def __init__(self, model, num_class, model_weight=None, is_train=True):\n#         super(EfficientnetCus, self).__init__()\n        \n#         self.is_train = is_train\n#         self.model = timm.create_model(f'tf_efficientnet_{model}_ns', pretrained=is_train,\n#                                        in_chans=3, num_classes=num_class)\n#         if model_weight is not None:\n#             new_keys = self.model.state_dict().keys()\n#             values = torch.load(model_weight, map_location=lambda storage, loc: storage).values()\n#             self.model.load_state_dict(OrderedDict(zip(new_keys, values)))\n                \n#     def forward(self, image):\n#         if self.is_train:\n#             out = self.model(image)\n#             return out.squeeze(-1)\n#         else:\n#             vertical = image.flip(1)\n#             horizontal = image.flip(2)\n#             rotate90 = torch.rot90(image, 1, (1, 2))\n#             rotate90_ = torch.rot90(image, 1, (2, 1))\n#             out = torch.stack([image, vertical, horizontal, rotate90, rotate90_])\n#             return torch.sigmoid(self.model(out)).mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Predict_process(object):\n#     def __init__(self, device=device, config=PredictConfig):\n#         super(Predict_process, self).__init__()\n#         self.device = device\n#         self.config = config\n        \n#     def load_image(self, image_path, transforms):\n#         image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#         image = transforms(image=image)['image']\n#         return image\n        \n#     def classifier_image(self, images):\n#         model = EfficientnetCus(model=self.config.model_classifier_use, num_class=1,\n#                                 model_weight=self.config.weight_classifier, is_train=False).to(self.device)\n#         model.eval()\n#         with torch.no_grad():\n#             outputs = model(images.to(device))\n#         return outputs\n    \n#     def label_process(self, detect_result, iou_thresh, iou_thresh11):\n#         assert detect_result != ''\n#         x_center, y_center = detect_result[1::6], detect_result[2::6]\n#         w_center, h_center = detect_result[3::6], detect_result[4::6]\n#         detect_result[1::6] = [i-0.5*j for i, j in zip(x_center, w_center)]\n#         detect_result[2::6] = [i-0.5*j for i, j in zip(y_center, h_center)]\n#         detect_result[3::6] = [i+0.5*j for i, j in zip(x_center, w_center)]\n#         detect_result[4::6] = [i+0.5*j for i, j in zip(y_center, h_center)]\n#         list_new = []\n        \n#         for label_values in np.unique(detect_result[::6]):\n#             list_values = np.array([detect_result[6*idx:6*idx+6] \\\n#                                     for idx, i in enumerate(detect_result[::6]) if i==label_values])\n#             boxes = list_values[:, 1:5].tolist()\n#             scores = list_values[:, 5].tolist()\n#             labels = list_values[:, 0].tolist()\n#             if label_values in [2, 11]:\n#                 boxes, scores, labels = nms([boxes], [scores], [labels], weights=None, iou_thr=iou_thresh11)\n#             else:\n#                 boxes, scores, labels = nms([boxes], [scores], [labels], weights=None, iou_thr=iou_thresh)\n            \n#             for box in list_values:\n#                 if box[-1] in scores:\n#                     list_new.extend(box)\n#         return list_new\n        \n#     def read_label(self, label_path):\n#         with open(label_path, 'r+') as file:\n#             detect_result = file.read()\n#         if detect_result != '':\n#             detect_result = list(map(float, re.split(r'[\\n ]', detect_result)[:-1]))\n#             detect_result = self.label_process(detect_result, self.config.iou_thresh, self.config.iou_thresh11)\n#             detect_result = [int(i) if idx%6==0 else self.config.img_size[0]*i if idx%6<5 else i \n#                              for idx, i in enumerate(detect_result)]\n#         return detect_result\n        \n#     def fit(self, df, folder_image, result_txt, use_classifier=True):\n#         transforms = aug('test')\n#         all_results = []\n#         for images_id, images in tqdm(df.iterrows(), total=len(df), leave=False):\n#             if use_classifier:\n#                 image_path = os.path.join(folder_image, images.image_id+'.png')\n#                 image = self.load_image(image_path, transforms)\n#                 class_labels = self.classifier_image(image)\n#             else:\n#                 class_labels = torch.tensor([1])\n#             label_path = os.path.join(result_txt, images.image_id+'.txt')\n#             if os.path.isfile(label_path):\n#                 detect_result = self.read_label(label_path)\n#             else:\n#                 detect_result = ''\n#             result_one_image = []\n#             if detect_result != '':\n#                 img_size = [images.h, images.w]\n#                 list_label = []\n#                 for box_id in range(len(detect_result)//6)[::-1]:\n#                     label, *box, score = detect_result[6*box_id:6*box_id+6]\n#                     if class_labels.item()>=self.config.classification_thresh:\n#                         if (score > self.config.score_last) and \\\n#                         not(label in [0, 3] and label in list_label) and \\\n#                         not(label==11 and score < self.config.score_11) and \\\n#                         not(label==9 and score < self.config.score_9):\n#                             list_label.append(label)\n#                             box = label_resize(self.config.img_size, img_size, *box)\n#                             result_one_image.append(int(label))\n#                             result_one_image.append(np.round(score, 3))\n#                             result_one_image.extend([int(i) for i in box])\n#                     else:\n#                         if score > self.config.score_last2 and \\\n#                         not(label in [0, 3] and label in list_label) and \\\n#                         not(label==11 and score < self.config.score_11) and \\\n#                         not(label==9 and score < self.config.score_9):\n#                             list_label.append(label)\n#                             box = label_resize(self.config.img_size, img_size, *box)\n#                             result_one_image.append(int(label))\n#                             result_one_image.append(np.round(score, 3))\n#                             result_one_image.extend([int(i) for i in box])\n#             if len(result_one_image)==0:\n#                 all_results.append('14 1 0 0 1 1')\n#             else:\n#                 result_str = ' '.join(map(str, result_one_image)) + \\\n#                 f' 14 {class_labels.item():.3f} 0 0 1 1'\n#                 all_results.append(result_str)\n#         df['PredictionString'] = all_results\n#         df = df.drop(['h', 'w'], 1)\n        \n#         return df\n\n    \n# predict_pr = Predict_process(config=PredictConfig)\n# submission_df = predict_pr.fit(sub_df, TEST_512_PATH, '../working/runs/detect/exp/labels/')\n# submission_df.to_csv('submission.csv', index=False)\n# submission_df.head(60)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def display_image_test(df, size_df, list_image, num_image=3):\n    \n#     dict_color = list_color(range(15))\n#     image_row_random = np.random.choice(len(df), num_image, replace=(len(df)<num_image))\n#     for image_idx in image_row_random:\n#         image_id, pred = df.loc[image_idx, 'image_id'], df.loc[image_idx, 'PredictionString']\n#         org_size = size_df[size_df['image_id']==image_id][['h', 'w']].values[0].tolist()\n#         fig, ax = plt.subplots(1, figsize=(15, 15))\n#         img_path = [i for i in list_image if image_id in i][0]\n#         img = cv2.imread(img_path)\n#         ax.imshow(img, plt.cm.bone)\n#         if pred != '14 1 0 0 1 1':\n#             list_pred = pred.split(' ')\n#             for box_idx in range(len(list_pred)//6)[:-1]:\n#                 bbox = map(int, list_pred[6*box_idx+2:6*box_idx+6])\n#                 x_min, y_min, x_max, y_max = label_resize(org_size, IMG_SIZE, *bbox)\n#                 class_name, score = int(list_pred[6*box_idx]), float(list_pred[6*box_idx+1])\n#                 rect = patches.Rectangle((x_min, y_min), x_max-x_min, y_max-y_min,\n#                                          linewidth=1, edgecolor=dict_color[class_name], facecolor='none')\n#                 ax.add_patch(rect)\n#                 plt.text(x_min, y_min, f'{class_name}: {score}', fontsize=15, color='red')            \n\n#         plt.title(image_id) \n#         plt.show()\n\n# size_df = pd.read_csv(TEST_META_PATH)\n# size_df.columns = ['image_id', 'h', 'w']\n\n# image_abnormal = submission_df[submission_df['PredictionString']!='14 1 0 0 1 1'].reset_index(drop=True)\n# display_image_test(image_abnormal, size_df, test_list, num_image=20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !rm -rf ../working/runs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import  os\nprint (os.getcwd())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n# zip -r alloutput.zip ./*\nzip  -r fileName.zip  ./*","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}