{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# %%\n\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport os\nimport re\nimport pydicom\nimport warnings\n\nimport time\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\n\nimport json\n\nfrom PIL import Image\nfrom torch.distributions import transforms\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\n\nimport torch\nimport torchvision\n\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.rpn import AnchorGenerator\n\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\n\nfrom matplotlib import pyplot as plt\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\n\"\"\"\n   Author: Alex Nguyen\n   Gettysburg College\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nfrom typing import Collection, List, Dict, Tuple\nimport numpy as np\nimport re\nfrom numpy.lib.type_check import imag\nimport pandas as pd\nfrom PIL import Image\nimport os\nimport random\nimport tensorflow as tf\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils\nimport collections\nimport json\nfrom sklearn.model_selection import train_test_split\n\nimport sys\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nfrom typing import List, Dict, Tuple\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport math\nimport os\nimport re\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing import image_dataset_from_directory\nimport cv2\n\ndef preprocess_img(\n        src_img: np.ndarray, \n        shape,\n        resize_method=tf.image.ResizeMethod.BILINEAR,\n        range='tanh'\n    ) -> tf.Tensor:\n    # Expect image value range 0 - 255\n\n    img = src_img\n    if len(src_img.shape) == 2:\n        img = tf.expand_dims(src_img, axis=-1)\n\n    resized = tf.image.resize(\n        img, \n        shape,\n        method=resize_method\n    )\n\n    rescaled = None\n    if range == 'tanh':\n        rescaled = tf.cast(resized, dtype=float) / 255.0\n        rescaled = (rescaled - 0.5) * 2 # range [-1, 1]\n    elif range == 'sigmoid':\n        rescaled = tf.cast(resized, dtype=float) / 255.0\n    elif range == None:\n        rescaled = tf.cast(resized, dtype=float)\n    else:\n        print(\"Wrong type!\")\n        sys.exit(1)\n\n    # Convert to BGR\n    bgr = rescaled[..., ::-1]\n    return bgr\n\ndef load_image(path):\n    return np.asarray(Image.open(path))\n\n# Ignore warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nclass BaseDataset:\n    def __init__(self, root: str, img_shape: Tuple, batch_size: int=16, steps_per_epoch: int=20):\n        self.root = root\n        self.batch_size = batch_size\n        self.steps_per_epoch = steps_per_epoch\n        self.img_shape = img_shape\n\n    def set_batch_size(self, batch_size: int):\n        self.batch_size = batch_size\n\n    def set_steps_per_epoch(self, steps_per_epoch: int):\n        self.steps_per_epoch = steps_per_epoch\n\nclass Dataset256(BaseDataset):\n    def __init__(self, root: str, img_shape: Tuple, batch_size: int=16, steps_per_epoch: int=20):\n        super().__init__(root, img_shape, batch_size=batch_size, steps_per_epoch=steps_per_epoch)\n        self.test_file = os.path.join(root, \"test.csv\")\n        self.train_file = os.path.join(root, \"train.csv\")\n        self.test_data = os.path.join(root, \"test\")\n        self.train_data = os.path.join(root, \"train\")\n        self.sample_submisison = os.path.join(root, \"sample_submission.csv\")\n    \n    def sample(self, preprocess: bool=False, **kwargs) -> np.ndarray:\n        \"\"\" Return an image that is in the train dataset\n\n        Returns:\n            np.ndarray: A numpy repr of the image. If preprocess is False,\n                an image with original shape is returned. Otherwise, it will\n                preprocess the image\n        \"\"\"\n        dir = os.listdir(self.train_data)\n        r = random.randint(0, len(dir) - 1)\n        img = load_image(os.path.join(self.train_data, dir[r]))\n        if preprocess:\n            img = preprocess_img(img, **kwargs)\n        return img\n\nclass DatasetCOCO(BaseDataset):\n    INT2LABEL = {\n        0: \"Aortic enlargement\",\n        1: \"Atelectasis\",\n        2: \"Calcification\",\n        3: \"Cardiomegaly\",\n        4: \"Consolidation\",\n        5: \"ILD\",\n        6: \"Infiltration\",\n        7: \"Lung Opacity\",\n        8: \"Nodule/Mass\",\n        9: \"Other lesion\",\n        10: \"Pleural effusion\",\n        11: \"Pleural thickening\",\n        12: \"Pneumothorax\",\n        13: \"Pulmonary fibrosis\",\n        14: \"No finding\",\n    }\n    class COCOInstance():\n        def __init__(\n            self, \n            filename: str, \n            width: float, \n            height: float\n        ) -> None:\n            if \"/\" in filename or \"\\\\\" in filename:\n                self.filename = re.findall(r\"[\\\\|\\/].*jpg$\", filename)[0][1:]\n            else:\n                self.filename = filename\n            self.width = width\n            self.height = height\n            self.boxes = []\n\n        def __str__(self):\n            return f\"File name: {self.filename}, class: {self.clss}\"\n        \n        def add_box(self, xmin, ymin, xmax, ymax, clss: int, tf_record_mode=False) -> None:\n            if tf_record_mode:\n                # TODO: Be careful!\n                # With this mode, 0 means no diseases, does it also means background? no!\n                # self.boxes.append({\"class\": (clss + 1) % 16, \"box\":[xmin, ymin, xmax, ymax]})\n                self.boxes.append({\n                    \"class\": DatasetCOCO.INT2LABEL[clss], \n                    \"box\":[\n                        min(xmin, xmax), \n                        min(ymin, ymax), \n                        max(xmin, xmax), \n                        max(ymin, ymax)\n                    ]\n                })\n            else:\n                self.boxes.append({\n                    \"class\": clss, \n                    \"box\":[\n                        min(xmin, xmax), \n                        min(ymin, ymax), \n                        max(xmin, xmax), \n                        max(ymin, ymax)\n                    ]\n                })\n\n    def __init__(self, root: str, img_shape: Tuple, batch_size: int=16, steps_per_epoch: int=20, tf_record_mode=False):\n        \"\"\" Initialization.\n        self.dataset: \n        {\n            \"train\": {id: COCOInstance},\n            \"val\": {id: COCOInstance}\n        }\n        Args:\n            root (str): [description]\n            img_shape (Tuple): [description]\n            batch_size (int, optional): [description]. Defaults to 16.\n            steps_per_epoch (int, optional): [description]. Defaults to 20.\n        \"\"\"\n        super().__init__(root, img_shape, batch_size=batch_size, steps_per_epoch=steps_per_epoch)\n        self.tf_record_mode = tf_record_mode\n        self.val_label = os.path.join(root, \"val_annotations.json\")\n        self.train_label = os.path.join(root, \"train_annotations.json\")\n        self.val_data = os.path.join(root, \"val_images\")\n        self.train_data = os.path.join(root, \"train_images\")\n        self.dataset: Dict[str, collections.defaultdict] = \\\n            {\"train\": collections.defaultdict(DatasetCOCO.COCOInstance),\n            \"val\": collections.defaultdict(DatasetCOCO.COCOInstance)}\n        self._load_data(self.train_label, \"train\")\n        self._load_data(self.val_label, \"val\")\n\n    def _load_data(self, annotation_path, key) -> None:\n        \"\"\" Load the data from the annotation files into own dataset.\n\n        Args:\n            annotation_path (path, str, etc.): Path of the annotation json file.\n            key (str): The key in the dataset dictionary. Either \"train\" or \"val\".\n        \"\"\"\n        with open(annotation_path) as f:\n            json_data = json.load(f)\n\n        # Load data and creting instances\n        for image in json_data[\"images\"]:\n            image_id = image[\"id\"]\n            if image_id not in self.dataset[key]:\n                self.dataset[key][image_id] = DatasetCOCO.COCOInstance(\n                    filename = image[\"file_name\"], \n                    width = image[\"width\"], \n                    height = image[\"height\"]\n                )\n\n        # Load bounding boxes and add to the instances.\n        for annotation in json_data[\"annotations\"]:\n            image_id = annotation[\"image_id\"]\n            self.dataset[key][image_id].add_box(\n                *annotation[\"bbox\"],\n                int(annotation[\"category_id\"]),\n                tf_record_mode=self.tf_record_mode\n            )\n\n    def export_csv(self, target_train_path: str, target_val_path: str):\n        data_train = []\n\n        # Code to append to data list here!\n        for image_id, obj in self.dataset[\"train\"].items():\n            for _dict in obj.boxes:\n                value = (\n                    obj.filename,\n                    str(obj.width),\n                    str(obj.height),\n                    str(_dict[\"class\"]),\n                    str(_dict[\"box\"][0]),\n                    str(_dict[\"box\"][1]),\n                    str(_dict[\"box\"][2]),\n                    str(_dict[\"box\"][3])\n                )\n                data_train.append(value)\n\n        data_val = []\n        for image_id, obj in self.dataset[\"val\"].items():\n            for _dict in obj.boxes:\n                value = (\n                    obj.filename,\n                    str(obj.width),\n                    str(obj.height),\n                    str(_dict[\"class\"]),\n                    str(_dict[\"box\"][0]),\n                    str(_dict[\"box\"][1]),\n                    str(_dict[\"box\"][2]),\n                    str(_dict[\"box\"][3])\n                )\n                data_val.append(value)\n\n        column_name = ['filename', 'width', 'height',\n                   'class', 'xmin', 'ymin', 'xmax', 'ymax']\n        data_train_df = pd.DataFrame(data_train, columns=column_name)\n        data_train_df.to_csv(target_train_path, index=False)\n        data_val_df = pd.DataFrame(data_val, columns=column_name)\n        data_val_df.to_csv(target_val_path, index=False)\n        return data_train_df, data_val_df\n\n    def sample(self, preprocess: bool=False, **kwargs) -> np.ndarray:\n        \"\"\" Return an image that is in the train dataset\n        Params:\n            preprocess (bool): \n            (preprocess_img **kwargs): \n        Returns:\n            np.ndarray: A numpy repr of the image. If preprocess is False,\n                an image with original shape is returned. Otherwise, it will\n                preprocess the image\n        \"\"\"\n        dir = os.listdir(self.train_data)\n        r = random.randint(0, len(dir) - 1)\n        img = load_image(os.path.join(self.train_data, dir[r]))\n        if preprocess:\n            img = preprocess_img(img, **kwargs)\n        else:\n            assert len(img.shape) == 2, \"Wrong expected image shape.\"\n            img = tf.expand_dims(img, axis=-1)\n        return img\n\n    def __str__(self):\n        len_train = len(self.dataset[\"train\"])\n        len_val = len(self.dataset[\"val\"])\n        return f\"Dataset:\\nTrain: {len_train} items.\\nVal: {len_val} items.\"\n\n\nclass DatasetCOCOPytorch(DatasetCOCO, Dataset):\n    def __init__(self, root: str, img_shape: Tuple, train_set=True, transforms=None, **kwargs):\n        super().__init__(root, img_shape, **kwargs)\n        if train_set:\n            self.dataset = self.dataset[\"train\"]\n        else:\n            self.dataset = self.dataset[\"val\"]\n        # self.indices = collections.defaultdict(int)\n        self.transforms = transforms\n        self.train_set = train_set\n    \n    def __build_path(self, img_name):\n        if self.train_set:\n            return os.path.join(self.root, \"train_images\", img_name)\n        else:\n            return os.path.join(self.root, \"val_images\", img_name)\n\n    def __len__(self):\n        return len(self.dataset)\n\n    def __getitem__(self, idx):\n        # if torch.is_tensor(idx):\n        #     idx = idx.tolist()\n        # print(\"id: \", idx)\n\n        # Since the image index in the dataset start from 0, we\n        # can assume that the index for the internal dataset be\n        # the same as the id of the image.\n        img_name = self.dataset[idx].filename\n        img_path = self.__build_path(img_name)\n        image = np.asarray(Image.open(img_path))\n\n        assert len(image.shape) == 2, \"Wrong shape length!\"\n\n        image = np.stack([image, image, image])\n        image = image.astype('float32')\n        image = image - image.min()\n        image = image / image.max()\n        image = image * 255.0\n        image = image.transpose(1,2,0)\n\n        boxes = []\n        labels = []\n        for dict_datum in self.dataset[idx].boxes:\n            boxes.append(dict_datum[\"box\"])\n            labels.append(dict_datum[\"class\"])\n        boxes = torch.as_tensor(boxes, dtype=torch.float32)\n        labels = torch.as_tensor(labels, dtype=torch.int64)\n\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        area = torch.as_tensor(area, dtype=torch.float32)\n\n        # suppose all instances are not crowd\n        iscrowd = torch.zeros((boxes.shape[0],), dtype=torch.uint8)\n\n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['image_id'] = torch.tensor([idx])\n        target['area'] = area\n        target['iscrowd'] = iscrowd\n\n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': target['boxes'],\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n\n            target['boxes'] = torch.tensor(sample['bboxes'])\n\n        if target[\"boxes\"].shape[0] == 0:\n            # Albumentation cuts the target (class 14, 1x1px in the corner)\n            target[\"boxes\"] = torch.from_numpy(np.array([[0.0, 0.0, 1.0, 1.0]]))\n            target[\"area\"] = torch.tensor([1.0], dtype=torch.float32)\n            target[\"labels\"] = torch.tensor([14], dtype=torch.int64)\n\n        # return sample\n        return image, target, idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# from core.data import DatasetCOCO, DatasetCOCOPytorch, show_img\n\npaddingSize= 0\n\n# DIR_INPUT = 'input/vinbigdata-coco-dataset-with-wbf-3x-downscaled/vinbigdata-coco-dataset-with-wbf-3x-downscaled'\n# DIR_TRAIN = f'{DIR_INPUT}/train'\n# DIR_TEST = f'{DIR_INPUT}/test'\n\n\n\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n\nnum_classes = 15 # 14 Classes + 1 background\n\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\n\n\nIMG_SHAPE = (256, 256)\n\nroot_data_folder = '/kaggle/input/vinbigdata-coco-dataset-with-wbf-3x-downscaled/vinbigdata-coco-dataset-with-wbf-3x-downscaled'\n\n\ndef get_train_transform():\n    return A.Compose([\n        A.Flip(0.5),\n        A.ShiftScaleRotate(scale_limit=0.1, rotate_limit=45, p=0.25),\n        A.LongestMaxSize(max_size=800, p=1.0),\n        # Dilation(),\n        # FasterRCNN will normalize.\n        A.Normalize(mean=(0, 0, 0), std=(1, 1, 1), max_pixel_value=255.0, p=1.0),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\n\n\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\ntrain_dataset = DatasetCOCOPytorch(root=root_data_folder, img_shape=IMG_SHAPE, transforms=get_train_transform())\n# train_dataset = DatasetCOCOPytorch(train_df, DIR_TRAIN, get_train_transform())\nvalid_dataset = DatasetCOCOPytorch(root=root_data_folder, img_shape=IMG_SHAPE, train_set=False, transforms=get_train_transform())\n\n# split the dataset in train and test set\nindices = torch.randperm(len(train_dataset)).tolist()\n# Create train and validate data loader\ntrain_data_loader = DataLoader(\n    train_dataset,\n    batch_size=4,\n    shuffle=True,\n    # num_workers=4,\n    collate_fn=collate_fn\n)\n\nvalid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=4,\n    shuffle=False,\n    # num_workers=4,\n    collate_fn=collate_fn\n)\n\n\n# # Train dataset sample\n# images, targets, image_ids = next(iter(train_data_loader))\n# images = list(image.to(device) for image in images)\n# targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n# for number in random.sample([1,2,3],3):\n#   boxes = targets[number]['boxes'].cpu().numpy().astype(np.int32)\n#   img = images[number].permute(1,2,0).cpu().numpy()\n#   labels= targets[number]['labels'].cpu().numpy().astype(np.int32)\n#   fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n#   for i in range(len(boxes)):\n#       img = cv2.rectangle(img,(boxes[i][0]+paddingSize,boxes[i][1]+paddingSize),(boxes[i][2]+paddingSize,boxes[i][3]+paddingSize),(255,0,0),2)\n#       #print(le.inverse_transform([labels[i]-1])[0])\n#       #print(label_to_name(labels[i]), (int(boxes[i][0]), int(boxes[i][1])))\n#       img = cv2.putText(img, label_to_name(labels[i]), (int(boxes[i][0]), int(boxes[i][1])), cv2.FONT_HERSHEY_TRIPLEX,1, (255,0,0), 2, cv2.LINE_AA)\n\n#   ax.set_axis_off()\n#   ax.imshow(img)\n\n\n# %%\n\nmodel.to(device)\nparams = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)\n# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.1)\n\nnum_epochs =  1 #Low epoch to save GPU time\n\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"itr = 1\nfor epoch in range(num_epochs):\n    # loss_hist.reset()\n    with tqdm(total=len(train_dataset)) as pbar:\n        for images, targets, image_ids in train_data_loader:\n            \n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n            loss_dict = model(images, targets)  \n            \n            losses = sum(loss for loss in loss_dict.values())\n            loss_value = losses.item()\n\n            optimizer.zero_grad()\n            losses.backward()\n            optimizer.step()\n\n            if itr % 50 == 0:\n                print(f\"Iteration #{itr} loss: {loss_value}\")\n\n            itr += 1\n            pbar.update(1)\n    \n    # update the learning rate\n    # if lr_scheduler is not None:\n    #     lr_scheduler.step()\n    # lossHistoryepoch.append(loss_hist.value)\n    # print(f\"Epoch #{epoch} loss: {loss_hist.value}\")  \n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}