{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import package and load data","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.extend(['../input/effdet/',\n                 '../input/iterstrat/',\n                 '../input/weightedboxfusion/',\n                 '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master/'])\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport seaborn as sns\nfrom pathlib import Path\nimport json\nimport re\nimport logging\nimport gc\nimport random\nimport warnings\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\nimport os\nfrom glob import glob\nfrom joblib import Parallel, delayed\nimport shutil as sh\nfrom itertools import product\nfrom collections import OrderedDict\nimport plotly\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.offline as pyo\n#from ml_stratifiers import MultilabelStratifiedKFold\n#from ensemble_boxes import nms, weighted_boxes_fusion\nimport torchvision.transforms as transforms\nimport albumentations as al\nfrom albumentations import ImageOnlyTransform\nfrom albumentations.pytorch import ToTensorV2, ToTensor\nfrom albumentations.core.transforms_interface import DualTransform, ImageOnlyTransform\n\nimport cv2\nimport pydicom\nfrom IPython.display import display, Image\nfrom sklearn.model_selection import train_test_split, KFold, GroupKFold, StratifiedKFold\nimport torch\nfrom torch.nn import functional as f\nimport torch.nn as nn\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\nfrom torch.utils.data import Dataset, DataLoader, sampler\nimport time\n#from efficientnet_pytorch import EfficientNet\n\npd.options.display.max_columns = None\nwarnings.filterwarnings('ignore')\n\nlogging.basicConfig(format='%(asctime)s +++ %(message)s',\n                    datefmt='%d-%m-%y %H:%M:%S', level=logging.INFO)\nlogger = logging.getLogger()\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nlogger.info(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. # work on data cahnge some columns","metadata":{}},{"cell_type":"code","source":"os.environ[\"WANDB_API_KEY\"] = '8f435998b1a6f9a4e59bfaef1deed81c1362a97d'\nos.environ[\"WANDB_MODE\"] = \"dryrun\"\n\nMAIN_PATH = '../input/vinbigdata-chest-xray-abnormalities-detection/'\nCLASSIFIER_MAIN_PATH = '../input/efficientnet-pytorch/'\nRESIZE_1024_PATH = '../input/vinbigdata-chest-xray-resized-png-1024x1024/'\nRESIZE_512_PATH = '../input/vinbigdata/'\nTRAIN_PATH = os.path.join(MAIN_PATH, 'train.csv')\nSUB_PATH = os.path.join(MAIN_PATH, 'sample_submission.csv')\nTRAIN_DICOM_PATH = os.path.join(MAIN_PATH, 'train')\nTEST_DICOM_PATH = os.path.join(MAIN_PATH, 'test')\nTRAIN_1024_PATH = os.path.join(RESIZE_1024_PATH, 'train')\nTEST_1024_PATH = os.path.join(RESIZE_1024_PATH, 'test')\nTRAIN_512_PATH = os.path.join(RESIZE_512_PATH, 'train')\nTEST_512_PATH = os.path.join(RESIZE_512_PATH, 'test')\nTRAIN_META_PATH = os.path.join(RESIZE_1024_PATH, 'train_meta.csv')\nTEST_META_PATH = '../input/vinbigdata-testmeta/test_meta.csv'\nTEST_CLASS_PATH = '../input/vinbigdata-2class-prediction/2-cls test pred.csv'\nMODEL_WEIGHT = '../input/efficientdet/tf_efficientdet_d7_53-6d1d7a95.pth'\nSIZE = 512\nIMG_SIZE = (SIZE, SIZE)\nACCULATION = 1\nMOSAIC_RATIO = 0.4\n    \nclass GlobalConfig:\n    model_use = 'd0'\n    model_weight = '../input/efficientdet/tf_efficientdet_d0_34-f153e0cf.pth'\n    img_size = IMG_SIZE\n    fold_num = 5\n    seed = 89\n    num_workers = 12\n    batch_size = 8\n    n_epochs = 20\n    lr = 1e-2\n    verbose = 1\n    verbose_step = 1\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n#     output_path = './save/'\n    scheduler_params = dict(\n        mode='min', \n        factor=0.2,\n        patience=1,\n        threshold_mode='abs',\n        min_lr=1e-7\n    )\n    \nclass PredictConfig:\n    img_size = IMG_SIZE\n    batch_size = 16\n    model_classifier_use = 'b0'\n    weight_classifier = '../input/effdet-d5-512/model_classifier_b0_512.pth'\n#     weight_classifier = '../input/x-chest-1024-classifier/model_classifier.pth'\n    score_thresh = 0.05\n    iou_thresh = 0.4\n    iou_thresh2 = 0.1\n    iou_thresh11 = 0.0001\n    skip_thresh = 0.0001\n    sigma = 0.1\n    score_0 = 0.385\n    score_3 = 0.4\n    score_last = 0.0\n    score_last2 = 0.95\n    score_9 = 0.1\n    score_11 = 0.015\n    classification_thresh = 0.003751\n    \nlist_remove = [34843, 21125, 647, 18011, 2539, 22373, 12675, 7359, 20642, 5502, 19818, 5832, 28056, 28333, 20758,\n               925, 43, 2199, 4610, 21306, 16677, 1768, 17232, 1378, 24949, 30203, 31410, 87, 25318, 92, 31724,\n               118, 17687, 12605, 26157, 33875, 7000, 3730, 18776, 13225, 1109, 2161, 33627, 15500, 28633, 28152,\n               10114, 10912, 9014,  4427, 25630, 11464, 6419, 22164, 4386, 17557, 15264, 21853, 33142, 32895, 9733,\n               33010, 17493, 32128, 28802, 11658, 8841, 29557, 4802, 8591, 778, 9935, 12359, 5210, 7556, 24505, 5664,\n               28670, 27820, 19359, 9817, 7800, 32934, 34098, 27931, 16074, 27308, 30645, 31029, 35697, 6199, 27065,\n               1771, 14689, 31860, 1975, 29294, 2304, 34018, 23406, 26501, 26011, 2479, 32796, 25836, 3032, 31454,\n               32066, 19722, 15997, 6049, 9458, 11005, 23151, 24503, 35411, 18092, 23815, 30742, 33942, 34542, 7655,\n               25345, 3750, 17046, 3844, 5958, 4250, 18823, 14898, 22581, 25805, 9651, 33194, 36007, 30160, 24459,\n               10838, 16544, 31252, 8053, 28487, 6208, 25244, 8470, 10089, 24813, 14769, 34305, 34047, 23366, 8049,\n               13276, 22380, 32797, 32440, 11031, 18304, 33692, 21349, 26333, 34331, 9110, 21092, 34882, 35626, 10203,\n               25648, 30754, 29567, 33542, 15146, 26759, 20846, 22493, 33187, 22813, 30219, 14548, 14627, 20494, 28332,\n               15930, 31347, 33489, 35005, 34032, 24183, 18643, 18536, 29754, 20380, 29750, 20539, 35791, 27275, 32248]\nimage_remove = ['9c83d9f88170cd38f7bca54fe27dc48a', 'ac2a615b3861212f9a2ada6acd077fd9',\n                'f9f7feefb4bac748ff7ad313e4a78906', 'f89143595274fa6016f6eec550442af9',\n                '6c08a98e48ba72aee1b7b62e1f28e6da', 'e7a58f5647d24fc877f9cb3d051792e2',\n                '8f98e3e6e86e573a6bd32403086b3707', '43d3137e74ebd344636228e786cb91b0',\n                '575b98a9f9824d519937a776bd819cc4', 'ca6c1531a83f8ee89916ed934f8d4847',\n                '0c6a7e3c733bd4f4d89443ca16615fc6', 'ae5cec1517ab3e82c5374e4c6219a17d',\n                '064023f1ff95962a1eee46b9f05f7309', '27c831fee072b232499541b0aca58d9c',\n                '0b98b21145a9425bf3eeea4b0de425e7', '7df5c81873c74ecc40610a1ad4eb2943']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Make class BasewheatTTA","metadata":{}},{"cell_type":"code","source":"class BaseWheatTTA:\n    \"\"\" author: @shonenkov \"\"\"\n    image_size = IMG_SIZE\n\n    def augment(self, image):\n        raise NotImplementedError\n    \n    def batch_augment(self, images):\n        raise NotImplementedError\n    \n    def deaugment_boxes(self, boxes):\n        raise NotImplementedError\n\nclass TTAHorizontalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n\n    def augment(self, image):\n        return image.flip(1)\n    \n    def batch_augment(self, images):\n        return images.flip(2)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]\n        return boxes\n\nclass TTAVerticalFlip(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return image.flip(2)\n    \n    def batch_augment(self, images):\n        return images.flip(3)\n    \n    def deaugment_boxes(self, boxes):\n        boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]\n        return boxes\n    \nclass TTARotate90(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    \n    def augment(self, image):\n        return torch.rot90(image, 1, (1, 2))\n\n    def batch_augment(self, images):\n        return torch.rot90(images, 1, (2, 3))\n    \n    def deaugment_boxes(self, boxes):\n        res_boxes = boxes.copy()\n        res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]\n        res_boxes[:, [1,3]] = boxes[:, [2,0]]\n        return res_boxes\n\nclass TTACompose(BaseWheatTTA):\n    \"\"\" author: @shonenkov \"\"\"\n    def __init__(self, transforms):\n        self.transforms = transforms\n        \n    def augment(self, image):\n        for transform in self.transforms:\n            image = transform.augment(image)\n        return image\n    \n    def batch_augment(self, images):\n        for transform in self.transforms:\n            images = transform.batch_augment(images)\n        return images\n    \n    def prepare_boxes(self, boxes):\n        result_boxes = boxes.copy()\n        result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)\n        result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)\n        result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)\n        result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)\n        return result_boxes\n    \n    def deaugment_boxes(self, boxes):\n        for transform in self.transforms[::-1]:\n            boxes = transform.deaugment_boxes(boxes)\n        return self.prepare_boxes(boxes)\n    \ndef Visualize_class(df, feature, title):\n    num_image = df[feature].value_counts().rename_axis(feature).reset_index(name='num_image')\n    fig = px.bar(num_image[::-1], x='num_image', y=feature, orientation='h', color='num_image')\n    fig.update_layout(\n    title={\n        'text': title,\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\n    fig.show()\n    \n    \ndef img_size(path):\n    information = pydicom.dcmread(path)\n    h, w = information.Rows, information.Columns\n    return (h, w)\n\n\ndef label_resize(org_size, img_size, *bbox):\n    x0, y0, x1, y1 = bbox\n    x0_new = int(np.round(x0*img_size[1]/org_size[1]))\n    y0_new = int(np.round(y0*img_size[0]/org_size[0]))\n    x1_new = int(np.round(x1*img_size[1]/org_size[1]))\n    y1_new = int(np.round(y1*img_size[0]/org_size[0]))\n    return x0_new, y0_new, x1_new, y1_new\n\n\ndef list_color(class_list):\n    dict_color = dict()\n    for classid in class_list:\n        dict_color[classid] = [i/256 for i in random.sample(range(256), 3)]\n    \n    return dict_color\n\n\ndef split_df(df):\n    kf = MultilabelStratifiedKFold(n_splits=GlobalConfig.fold_num,\n                                   shuffle=True, random_state=GlobalConfig.seed)\n    df['id'] = df.index\n    annot_pivot = pd.pivot_table(df, index='image_id', columns='class_id',\n                                 values='id', fill_value=0, aggfunc='count') \\\n    .reset_index().rename_axis(None, axis=1)\n    for fold, (train_idx, val_idx) in enumerate(kf.split(annot_pivot,\n                                                         annot_pivot.iloc[:, 1:train_abnormal['class_id'].nunique()])):\n        annot_pivot[f'fold_{fold}'] = 0\n        annot_pivot.loc[val_idx, f'fold_{fold}'] = 1\n    return annot_pivot\n    \n    \ndef display_image(df, list_image, num_image=1, is_dicom_file=True):\n    \n    dict_color = list_color(range(15))\n    list_abnormal = [i for i in df['class_name'].unique() if i!='No finding']\n    for abnormal in list_abnormal:\n        abnormal_df = df[df['class_name']==abnormal].reset_index(drop=True)\n        abnormal_random = np.random.choice(abnormal_df['image_id'].unique(), num_image)\n        for abnormal_img in abnormal_random:\n            images = abnormal_df[abnormal_df['image_id']==abnormal_img].reset_index(drop=True)\n            fig, ax = plt.subplots(1, figsize=(15, 15))\n            img_path = [i for i in list_image if abnormal_img in i][0]\n            if is_dicom_file:\n                information = pydicom.dcmread(img_path)\n                img = information.pixel_array\n            else:\n                img = cv2.imread(img_path)\n            ax.imshow(img, plt.cm.bone)\n            for idx, image in images.iterrows():\n                bbox = [image.x_min, image.y_min, image.x_max, image.y_max]\n                if is_dicom_file:\n                    x_min, y_min, x_max, y_max = bbox\n                else:\n                    org_size = image[['h', 'w']].values\n                    x_min, y_min, x_max, y_max = label_resize(org_size, IMG_SIZE, *bbox)\n                class_name, class_id = image.class_name, image.class_id\n                rect = patches.Rectangle((x_min, y_min), x_max-x_min, y_max-y_min,\n                                         linewidth=1, edgecolor=dict_color[class_id], facecolor='none')\n                ax.add_patch(rect)\n                plt.text(x_min, y_min, class_name, fontsize=15, color='red')\n\n            plt.title(abnormal_img) \n            plt.show()\n            \ndef display_image_test(df, size_df, list_image, num_image=3):\n    \n    dict_color = list_color(range(15))\n    image_row_random = np.random.choice(len(df), num_image, replace=(len(df)<num_image))\n    for image_idx in image_row_random:\n        image_id, pred = df.loc[image_idx, 'image_id'], df.loc[image_idx, 'PredictionString']\n        org_size = size_df[size_df['image_id']==image_id][['h', 'w']].values[0].tolist()\n        fig, ax = plt.subplots(1, figsize=(15, 15))\n        img_path = [i for i in list_image if image_id in i][0]\n        img = cv2.imread(img_path)\n        ax.imshow(img, plt.cm.bone)\n        if pred != '14 1 0 0 1 1':\n            list_pred = pred.split(' ')\n            for box_idx in range(len(list_pred)//6):\n                bbox = map(int, list_pred[6*box_idx+2:6*box_idx+6])\n                x_min, y_min, x_max, y_max = label_resize(org_size, IMG_SIZE, *bbox)\n                class_name, score = int(list_pred[6*box_idx]), float(list_pred[6*box_idx+1])\n                rect = patches.Rectangle((x_min, y_min), x_max-x_min, y_max-y_min,\n                                         linewidth=1, edgecolor=dict_color[class_name], facecolor='none')\n                ax.add_patch(rect)\n                plt.text(x_min, y_min, f'{class_name}: {score}', fontsize=15, color='red')            \n\n        plt.title(image_id) \n        plt.show()\n        \ndef ensemble_multibox(boxes, scores, labels, iou_thr, sigma,\n                      skip_box_thr, weights=None, method='wbf'):\n    if method=='nms':\n        boxes, scores, labels = nms(boxes, scores, labels,\n                                    weights=weights,\n                                    iou_thr=iou_thr)\n    elif method=='soft_nms':\n        boxes, scores, labels = soft_nms(boxes, scores, labels,\n                                         weights=weights,\n                                         sigma=sigma,\n                                         iou_thr=iou_thr,\n                                         thresh=skip_box_thr)\n    elif method=='nms_weight':\n        boxes, scores, labels = non_maximum_weighted(boxes, scores, labels,\n                                                     weights=weights,\n                                                     iou_thr=iou_thr,\n                                                     skip_box_thr=skip_box_thr)\n    elif method=='wbf':\n        boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels,\n                                                      weights=weights,\n                                                      iou_thr=iou_thr,\n                                                      skip_box_thr=skip_box_thr)\n    \n    return boxes, scores, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now Reading files","metadata":{}},{"cell_type":"code","source":"train_dicom_list = glob(f'{TRAIN_DICOM_PATH}/*.dicom')\ntest_dicom_list = glob(f'{TEST_DICOM_PATH}/*.dicom')\n\ntrain_list = glob(f'{TRAIN_512_PATH}/*.png')\ntest_list = glob(f'{TEST_512_PATH}/*.png')\nlogger.info(f'Train have {len(train_list)} file and test have {len(test_list)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_dicom_list = glob(f'{TRAIN_DICOM_PATH}/*.dicom')\ntest_dicom_list = glob(f'{TEST_DICOM_PATH}/*.dicom')\n\ntrain_list = glob(f'{TRAIN_512_PATH}/*.png')\ntest_list = glob(f'{TEST_512_PATH}/*.png')\nlogger.info(f'Train have {len(train_list)} file and test have {len(test_list)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import another pakage","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib\nimport pydicom as dicom\nimport cv2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/'\nos.listdir(path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"make a train for data","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(path+'train.csv')\nsamp_subm = pd.read_csv(path+'sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number train samples:', len(train_data.index))\nprint('Number test samples:', len(samp_subm.index))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"show the ditals for data by digram","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(12, 4))\nx = train_data['class_name'].value_counts().keys()\ny = train_data['class_name'].value_counts().values\nax.bar(x, y)\nax.set_xticklabels(x, rotation=90)\nax.set_title('Distribution of the labels')\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idnum = 2\nimage_id = train_data.loc[idnum, 'image_id']\ndata_file = dicom.dcmread(path+'train/'+image_id+'.dicom')\nimg = data_file.pixel_array","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data_file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#priint shape of img","metadata":{}},{"cell_type":"code","source":"print('Image shape:', img.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox = [train_data.loc[idnum, 'x_min'],\n        train_data.loc[idnum, 'y_min'],\n        train_data.loc[idnum, 'x_max'],\n        train_data.loc[idnum, 'y_max']]\nfig, ax = plt.subplots(1, 1, figsize=(20, 4))\nax.imshow(img, cmap='gray')\np = matplotlib.patches.Rectangle((bbox[0], bbox[1]),\n                                 bbox[2]-bbox[0],\n                                 bbox[3]-bbox[1],\n                                 ec='r', fc='none', lw=2.)\nax.add_patch(p)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"defination for plot_example","metadata":{}},{"cell_type":"code","source":"def plot_example(idx_list):\n    fig, axs = plt.subplots(1, 3, figsize=(15, 10))\n    fig.subplots_adjust(hspace = .1, wspace=.1)\n    axs = axs.ravel()\n    for i in range(3):\n        image_id = train_data.loc[idx_list[i], 'image_id']\n        data_file = dicom.dcmread(path+'train/'+image_id+'.dicom')\n        img = data_file.pixel_array\n        axs[i].imshow(img, cmap='gray')\n        axs[i].set_title(train_data.loc[idx_list[i], 'class_name'])\n        axs[i].set_xticklabels([])\n        axs[i].set_yticklabels([])\n        if train_data.loc[idx_list[i], 'class_name'] != 'No finding':\n            bbox = [train_data.loc[idx_list[i], 'x_min'],\n                    train_data.loc[idx_list[i], 'y_min'],\n                    train_data.loc[idx_list[i], 'x_max'],\n                    train_data.loc[idx_list[i], 'y_max']]\n            p = matplotlib.patches.Rectangle((bbox[0], bbox[1]),\n                                             bbox[2]-bbox[0],\n                                             bbox[3]-bbox[1],\n                                             ec='r', fc='none', lw=2.)\n            axs[i].add_patch(p)\n            \nfor num in range(15):\n    idx_list = train_data[train_data['class_id']==num][0:3].index.values\n    plot_example(idx_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"make a submition","metadata":{}},{"cell_type":"code","source":"samp_subm.to_csv('submission1.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_2class = pd.read_csv(\"../input/reddit-vaccine-myths/reddit_vm.csv\")\nlow_threshold = 0.001\nhigh_threshold = 0.87\npred_2class","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}