{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# *VinBigData 2 classes classifer*\n![](https://images.medicinenet.com/images/article/main_image/chest-x-ray.jpg)","metadata":{}},{"cell_type":"markdown","source":"* This notebook is next step after this kernel :\n[https://www.kaggle.com/khaledmgamal/detectron2-vinbigdata](http://)\n\n* It takes the submission file from the above kernel and apply 2 classes classifer to classify each image as having thoracic abnormalities or normal as post processing step to decrease the false positive .      ","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nfrom pathlib import Path\nimport random\nimport sys\n\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# --- models ---\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import KFold\n\nimport torch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from dataclasses import dataclass, field\nfrom typing import Dict, Any, Tuple, Union, List\n\n\n@dataclass\nclass Flags:\n    # General\n    debug: bool = True\n    outdir: str = \"results/det\"\n    device: str = \"cuda:0\"\n\n    # Data config\n    imgdir_name: str = \"vinbigdata-chest-xray-resized-png-256x256\"\n    # split_mode: str = \"all_train\"  # all_train or valid20\n    seed: int = 111\n    target_fold: int = 0  # 0~4\n    label_smoothing: float = 0.0\n    # Model config\n    model_name: str = \"resnet18\"\n    model_mode: str = \"normal\"  # normal, cnn_fixed supported\n    # Training config\n    epoch: int = 20\n    batchsize: int = 8\n    valid_batchsize: int = 16\n    num_workers: int = 4\n    snapshot_freq: int = 5\n    scheduler_type: str = \"\"\n    scheduler_kwargs: Dict[str, Any] = field(default_factory=lambda: {})\n    scheduler_trigger: List[Union[int, str]] = field(default_factory=lambda: [1, \"iteration\"])\n    aug_kwargs: Dict[str, Dict[str, Any]] = field(default_factory=lambda: {})\n    mixup_prob: float = -1.0  # Apply mixup augmentation when positive value is set.\n\n    def update(self, param_dict: Dict) -> \"Flags\":\n        # Overwrite by `param_dict`\n        for key, value in param_dict.items():\n            if not hasattr(self, key):\n                raise ValueError(f\"[ERROR] Unexpected key for flag = {key}\")\n            setattr(self, key, value)\n        return self","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flags_dict = {\n    \"debug\": False,  # Change to True for fast debug run!\n    \"outdir\": \"results/tmp_debug\",\n    # Data\n    \"imgdir_name\": \"vinbigdata-chest-xray-resized-png-256x256\",\n    # Model\n    \"model_name\": \"resnet18\",\n    # Training\n    \"num_workers\": 4,\n    \"epoch\": 15,\n    \"batchsize\": 16,\n    \"scheduler_type\": \"CosineAnnealingWarmRestarts\",\n    \"scheduler_kwargs\": {\"T_0\": 28125},  # 15000 * 15 epoch // (batchsize=8)\n    \"scheduler_trigger\": [1, \"iteration\"],\n    \"aug_kwargs\": {\n        \"HorizontalFlip\": {\"p\": 0.5},\n        \"ShiftScaleRotate\": {\"scale_limit\": 0.15, \"rotate_limit\": 10, \"p\": 0.5},\n        \"RandomBrightnessContrast\": {\"p\": 0.5},\n        \"CoarseDropout\": {\"max_holes\": 8, \"max_height\": 25, \"max_width\": 25, \"p\": 0.5},\n        \"Blur\": {\"blur_limit\": [3, 7], \"p\": 0.5},\n        \"Downscale\": {\"scale_min\": 0.25, \"scale_max\": 0.9, \"p\": 0.3},\n        \"RandomGamma\": {\"gamma_limit\": [80, 120], \"p\": 0.6},\n    }\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dataclasses\n\n# args = parse()\nprint(\"torch\", torch.__version__)\nflags = Flags().update(flags_dict)\nprint(\"flags\", flags)\n\n\n# --- Read data ---\ninputdir = Path(\"/kaggle/input\")\ndatadir = inputdir / \"vinbigdata-chest-xray-abnormalities-detection\"\nimgdir = inputdir / flags.imgdir_name\n\n# Read in the data CSV files\ntrain = pd.read_csv(datadir / \"train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nfrom pathlib import Path\nfrom typing import Optional\n\nimport cv2\nimport numpy as np\nimport pandas as pd\n#from detectron2.structures import BoxMode\nfrom tqdm import tqdm\n\n\ndef get_vinbigdata_dicts(\n    imgdir: Path,\n    train_df: pd.DataFrame,\n    train_data_type: str = \"original\",\n    use_cache: bool = True,\n    debug: bool = True,\n    target_indices: Optional[np.ndarray] = None,\n):\n    '''\n    parameters: \n              imgdir: the path to image directory \n              train_df: the dataframe that contians the images id and the bounding boxes\n    Returns           \n               list of dict (dataset_dicts) where each dict contains following:\n\n               -file_name: file name of the image.\n               -image_id: id of the image, index is used here.\n               -height: height of the image.\n               -width: width of the image.\n               -annotation: This is the ground truth annotation data for object detection, which contains following\n               -bbox: bounding box pixel location with shape (n_boxes, 4)\n               -category_id: class label id for each bounding box, with shape (n_boxes,)\n    '''       \n    debug_str = f\"_debug{int(debug)}\"\n    train_data_type_str = f\"_{train_data_type}\"\n    cache_path = Path(\".\") / f\"dataset_dicts_cache{train_data_type_str}{debug_str}.pkl\"\n    if not use_cache or not cache_path.exists():\n        print(\"Creating data...\")\n        train_meta = pd.read_csv(imgdir / \"train_meta.csv\")\n        if debug:\n            train_meta = train_meta.iloc[:500]  # For debug....\n\n        # Load 1 image to get image size.\n        image_id = train_meta.loc[0, \"image_id\"]\n        image_path = str(imgdir / \"train\" / f\"{image_id}.png\")\n        image = cv2.imread(image_path)\n        resized_height, resized_width, ch = image.shape\n        print(f\"image shape: {image.shape}\")\n\n        dataset_dicts = []\n        for index, train_meta_row in tqdm(train_meta.iterrows(), total=len(train_meta)):\n            record = {}\n\n            image_id, height, width = train_meta_row.values\n            filename = str(imgdir / \"train\" / f\"{image_id}.png\")\n            record[\"file_name\"] = filename\n            record[\"image_id\"] = image_id\n            record[\"height\"] = resized_height\n            record[\"width\"] = resized_width\n            objs = []\n            for index2, row in train_df.query(\"image_id == @image_id\").iterrows():\n                # print(row)\n                # print(row[\"class_name\"])\n                # class_name = row[\"class_name\"]\n                class_id = row[\"class_id\"]\n                if class_id == 14:\n                    # It is \"No finding\"\n                    # This annotator does not find anything, skip.\n                    pass\n                else:\n                    # bbox_original = [int(row[\"x_min\"]), int(row[\"y_min\"]), int(row[\"x_max\"]), int(row[\"y_max\"])]\n                    h_ratio = resized_height / height\n                    w_ratio = resized_width / width\n                    bbox_resized = [\n                        int(row[\"x_min\"]) * w_ratio,\n                        int(row[\"y_min\"]) * h_ratio,\n                        int(row[\"x_max\"]) * w_ratio,\n                        int(row[\"y_max\"]) * h_ratio,\n                    ]\n                    obj = {\n                        \"bbox\": bbox_resized,\n                        #\"bbox_mode\": BoxMode.XYXY_ABS,\n                        \"category_id\": class_id,\n                    }\n                    objs.append(obj)\n            record[\"annotations\"] = objs\n            dataset_dicts.append(record)\n        with open(cache_path, mode=\"wb\") as f:\n            pickle.dump(dataset_dicts, f)\n\n    print(f\"Load from cache {cache_path}\")\n    with open(cache_path, mode=\"rb\") as f:\n        dataset_dicts = pickle.load(f)\n    if target_indices is not None:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n    return dataset_dicts\n\n############################################################################################################\n\ndef get_vinbigdata_dicts_test(\n    imgdir: Path, test_meta: pd.DataFrame, use_cache: bool = True, debug: bool = True,\n):\n    '''\n    parameters: \n              imgdir: the path to image directory \n              test_meta: the dataframe that contians the images id and the original image size\n    Returns           \n               list of dict (dataset_dicts) where each dict contains following:\n\n               -file_name: file name of the image.\n               -image_id: id of the image, index is used here.\n               -height: height of the image.\n               -width: width of the image.\n    '''     \n    \n    debug_str = f\"_debug{int(debug)}\"\n    cache_path = Path(\".\") / f\"dataset_dicts_cache_test{debug_str}.pkl\"\n    if not use_cache or not cache_path.exists():\n        print(\"Creating data...\")\n        # test_meta = pd.read_csv(imgdir / \"test_meta.csv\")\n        if debug:\n            test_meta = test_meta.iloc[:500]  # For debug....\n\n        # Load 1 image to get image size.\n        image_id = test_meta.loc[0, \"image_id\"]\n        image_path = str(imgdir / \"test\" / f\"{image_id}.png\")\n        image = cv2.imread(image_path)\n        resized_height, resized_width, ch = image.shape\n        print(f\"image shape: {image.shape}\")\n\n        dataset_dicts = []\n        for index, test_meta_row in tqdm(test_meta.iterrows(), total=len(test_meta)):\n            record = {}\n\n            image_id, height, width = test_meta_row.values\n            filename = str(imgdir / \"test\" / f\"{image_id}.png\")\n            record[\"file_name\"] = filename\n            # record[\"image_id\"] = index\n            record[\"image_id\"] = image_id\n            record[\"height\"] = resized_height\n            record[\"width\"] = resized_width\n            # objs = []\n            # record[\"annotations\"] = objs\n            dataset_dicts.append(record)\n        with open(cache_path, mode=\"wb\") as f:\n            pickle.dump(dataset_dicts, f)\n\n    print(f\"Load from cache {cache_path}\")\n    with open(cache_path, mode=\"rb\") as f:\n        dataset_dicts = pickle.load(f)\n    return dataset_dicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mixup data augmentation\nWhat is mixup?\nAs the name kind of suggests, the authors of the mixup article propose to train the model on a mix of the pictures of the training set. Let’s say we’re on CIFAR10 for instance, then instead of feeding the model the raw images, we take two (which could be in the same class or not) and do a linear combination of them: in terms of tensor it’s\n\nnew_image = t * image1 + (1-t) * image2\nwhere t is a float between 0 and 1. Then the target we assign to that image is the same combination of the original targets:\n\nnew_target = t * target1 + (1-t) * target2\nassuming your targets are one-hot encoded (which isn’t the case in pytorch usually). And that’s as simple as this.\n\nRefer to this link : [https://forums.fast.ai/t/mixup-data-augmentation/22764](http://)","metadata":{}},{"cell_type":"code","source":"import numpy\nimport six\nimport torch\nfrom torch.utils.data.dataset import Dataset\n\n\nclass DatasetMixin(Dataset):\n\n    def __init__(self, transform=None):\n        self.transform = transform\n\n    def __getitem__(self, index):\n        \"\"\"Returns an example or a sequence of examples.\"\"\"\n        #print(\"datamix___getitem\")\n        if torch.is_tensor(index):\n            index = index.tolist()\n        if isinstance(index, slice):\n            current, stop, step = index.indices(len(self))\n            return [self.get_example_wrapper(i) for i in\n                    six.moves.range(current, stop, step)]\n        elif isinstance(index, list) or isinstance(index, numpy.ndarray):\n            return [self.get_example_wrapper(i) for i in index]\n        else:\n            return self.get_example_wrapper(index)\n\n    def __len__(self):\n        \"\"\"Returns the number of data points.\"\"\"\n        raise NotImplementedError\n\n    def get_example_wrapper(self, i):\n        \"\"\"Wrapper of `get_example`, to apply `transform` if necessary\"\"\"\n        #print(\"datamix___get_example_wrapper\")\n        \n        example = self.get_example(i)\n        #print(\"get_example_wrapper_len\",len(example))\n        if self.transform:\n            #print(\"transform\")\n            example = self.transform(example)\n        return example\n\n    def get_example(self, i):\n        \"\"\"Returns the i-th example.\n\n        Implementations should override it. It should raise :class:`IndexError`\n        if the index is invalid.\n\n        Args:\n            i (int): The index of the example.\n\n        Returns:\n            The i-th example.\n\n        \"\"\"\n        raise NotImplementedError\n        \nimport cv2\nimport numpy as np\n\n\nclass VinbigdataTwoClassDataset(DatasetMixin):\n    def __init__(self, dataset_dicts, image_transform=None, transform=None, train: bool = True,\n                 mixup_prob: float = -1.0, label_smoothing: float = 0.0):\n        super(VinbigdataTwoClassDataset, self).__init__(transform=transform)\n        self.dataset_dicts = dataset_dicts\n        self.image_transform = image_transform\n        self.train = train\n        self.mixup_prob = mixup_prob\n        self.label_smoothing = label_smoothing\n\n    def _get_single_example(self, i):\n        #print('_get_single_example')\n        d = self.dataset_dicts[i]\n        filename = d[\"file_name\"]\n\n        img = cv2.imread(filename)\n        if self.image_transform:\n            img = self.image_transform(img)\n        img = torch.tensor(np.transpose(img, (2, 0, 1)).astype(np.float32))\n\n        if self.train:\n            label = int(len(d[\"annotations\"]) > 0)  # 0 normal, 1 abnormal\n            if self.label_smoothing > 0:\n                if label == 0:\n                    return img, float(label) + self.label_smoothing\n                else:\n                    return img, float(label) - self.label_smoothing\n            else:\n                return img, float(label)\n        else:\n            # Only return img\n            return img, None\n\n    def get_example(self, i):\n        #print('get_example')\n        \n        img, label = self._get_single_example(i)\n        if self.mixup_prob > 0. and np.random.uniform() < self.mixup_prob:\n            j = np.random.randint(0, len(self.dataset_dicts))\n            p = np.random.uniform()\n            img2, label2 = self._get_single_example(j)\n            img = img * p + img2 * (1 - p)\n            if self.train:\n                label = label * p + label2 * (1 - p)\n\n        if self.train:\n            label_logit = torch.tensor([1 - label, label], dtype=torch.float32)\n            return img, label_logit\n            #return img, torch.argmax(label_logit)\n        \n        else:\n            # Only return img\n            return img\n\n    def __len__(self):\n        return len(self.dataset_dicts)        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_dicts = get_vinbigdata_dicts(imgdir, train, debug=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = VinbigdataTwoClassDataset(dataset_dicts)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\n\n\nclass Transform:\n    def __init__(\n        self, hflip_prob: float = 0.5, ssr_prob: float = 0.5, random_bc_prob: float = 0.5\n    ):\n        self.transform = A.Compose(\n            [\n                A.HorizontalFlip(p=hflip_prob),\n                A.ShiftScaleRotate(\n                    shift_limit=0.0625, scale_limit=0.1, rotate_limit=10, p=ssr_prob\n                ),\n                A.RandomBrightnessContrast(p=random_bc_prob),\n            ]\n        )\n\n    def __call__(self, image):\n        image = self.transform(image=image)[\"image\"]\n        return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aug_dataset = VinbigdataTwoClassDataset(dataset_dicts, image_transform=Transform())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 1, figsize=(20,20))\nfor index in range(3):\n    img, label = dataset[index]\n    axes[index].imshow(img.cpu().numpy().transpose((1, 2, 0)) / 255.)\n    axes[index].set_title(f\"{index}-th image: label {label}\")\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install timm library \nrefer to this link for more info : [https://rwightman.github.io/pytorch-image-models/](http://)\n","metadata":{}},{"cell_type":"code","source":"!pip install timm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import nn\nfrom torch.nn import Linear\n\n\nclass CNNFixedPredictor(nn.Module):\n    def __init__(self, cnn: nn.Module, num_classes: int = 2):\n        super(CNNFixedPredictor, self).__init__()\n        self.cnn = cnn\n        self.lin = Linear(cnn.num_features, num_classes)\n        #self.lin = Linear(cnn.fc.in_features, num_classes)\n\n        print(\"cnn.num_features\", cnn.num_features)\n        #print(\"cnn.num_features\", cnn.fc.in_features)\n        # We do not learn CNN parameters.\n        # https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html\n        for param in self.cnn.parameters():\n            param.requires_grad = True\n\n    def forward(self, x):\n        feat = self.cnn(x)\n        return self.lin(feat)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the resnet18 model to be trained as classifer :\nrefer to this link for more info :[ https://rwightman.github.io/pytorch-image-models/feature_extraction/](http://)\n","metadata":{}},{"cell_type":"code","source":"import timm\ntimm_model = timm.create_model(flags.model_name, pretrained=True, num_classes=0, in_chans=3)\nmodel=CNNFixedPredictor(timm_model, num_classes=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_paramters_report(model):\n  pytorch_total_params = sum(p.numel() for p in model.parameters())\n\n  pytorch_total_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n  print(f\"number of parameters = {pytorch_total_params} number of trainable parameters = {pytorch_total_trainable_params} \")\n  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_paramters_report(model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for name, param in model.named_parameters():\n    #if 'cnn.layer4' in str(name) or 'cnn.layer3.1' in str(name) :\n    #print(name)\n    #param.requires_grad=True \n     ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Dict\n\nimport albumentations as A\n\n\nclass Transform:\n    def __init__(self, aug_kwargs: Dict):\n        self.transform = A.Compose(\n            [getattr(A, name)(**kwargs) for name, kwargs in aug_kwargs.items()]\n        )\n\n    def __call__(self, image):\n        image = self.transform(image=image)[\"image\"]\n        return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\n\n\ndef cross_entropy_with_logits(input, target, dim=-1):\n    loss = torch.sum(- target * F.log_softmax(input, dim), dim)\n    return loss.mean()\n\ndef accuracy(y: torch.Tensor, t: torch.Tensor) -> torch.Tensor:\n    \"\"\"Computes multi-class classification accuracy\"\"\"\n    assert y.shape[:-1] == t.shape, f\"y {y.shape}, t {t.shape} is inconsistent.\"\n    pred_label = torch.max(y.detach(), dim=-1)[1]\n    count = t.nelement()\n    correct = (pred_label == t).sum().float()\n    acc = correct / count\n    return acc\n\n\ndef accuracy_with_logits(y: torch.Tensor, t: torch.Tensor) -> torch.Tensor:\n    \"\"\"Computes multi-class classification accuracy\"\"\"\n    assert y.shape == t.shape\n    gt_label = torch.max(t.detach(), dim=-1)[1]\n    return accuracy(y, gt_label)\n\n\n\n\ndef predict_proba(model,data_loader):\n    #device: torch.device = next(self.parameters()).device\n    y_list = []\n    model.to(device).eval()\n    with torch.no_grad():\n        for batch in tqdm(data_loader):\n            if isinstance(batch, (tuple, list)):\n                # Assumes first argument is \"image\"\n                batch = batch[0].to(device)\n            else:\n                batch = batch.to(device)\n            y = model(batch)\n            y = torch.softmax(y, dim=-1)\n            y_list.append(y)\n    pred = torch.cat(y_list)\n    return pred\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\noptimizer = torch.optim.Adam([param for param in model.parameters() if param.requires_grad ], lr=0.001)\nloss_function=nn.BCEWithLogitsLoss()\nlr_sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=28125)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm_notebook\n\ndef epoch_training(model,train_loader,optimizer,scheduler,criterion):\n    model.train()\n    running_loss=0\n    tqdm_train = tqdm(train_loader, total=int(len(train_loader)),position=0, leave=True)\n    for counter,(input_,target) in enumerate(tqdm_train):\n        input_.to(device)\n        target.to(device)\n        output=model(input_)\n        optimizer.zero_grad()\n        loss=criterion(output,target)\n        loss.backward()\n        optimizer.step()\n        #lr_sched.step()\n        tqdm_train.set_postfix(loss=loss.item())\n        running_loss+=loss.item()\n    epoch_loss=running_loss/len(train_loader)  \n    \n    return epoch_loss\n\ndef validation_epoch(model,valid_loader,criterion,accuracy):\n    model.eval()\n    running_acc=0\n    tqdm_val = tqdm(valid_loader, total=int(len(valid_loader)),position=0, leave=True)\n    running_loss=0\n    labels=[]\n    model_outputs=[]\n    for counter,(input_,target) in enumerate(tqdm_val):\n        with torch.no_grad():\n            input_.to(device)\n            target.to(device)\n            output=model(input_)\n            model_outputs+=torch.max(output.detach(), dim=-1)[1].numpy().tolist()\n            labels+=torch.max(target.detach(), dim=-1)[1].numpy().tolist()\n            loss=criterion(output,target).item()\n            acc=accuracy(output,target)\n            tqdm_val.set_postfix(loss=loss,acc=acc)    \n            running_acc+=acc\n            running_loss+=loss\n    epoch_loss=running_loss/len(valid_loader)  \n    epoch_acc= running_acc/len(valid_loader)\n    \n    return epoch_loss,epoch_acc,model_outputs,labels\nfrom sklearn import metrics\n\ndef model_training(model,train_loader,valid_loader,optimizer,scheduler,criterion,accuracy,epochs):\n    best_val=np.inf \n    for epoch in range(epochs):\n        epoch_loss=epoch_training(model,train_loader,optimizer,scheduler,criterion)\n        print(f\"Epoch={epoch} , Loss={epoch_loss}\")\n        \n        if valid_loader!= None:\n           val_epoch_loss,val_epoch_acc,model_outputs,labels=validation_epoch(model,valid_loader,criterion,accuracy)\n           print(f\"Epoch={epoch} , val Loss={val_epoch_loss} ,Val Accuracy={val_epoch_acc}\")\n           print(metrics.confusion_matrix(labels, model_outputs).T)\n           print(metrics.classification_report(labels, model_outputs, digits=3))\n           metrics_=metrics.classification_report(labels, model_outputs, digits=3,output_dict=True)\n           acc=metrics_['accuracy']\n           f1_macro_avg=metrics_['macro avg']['f1-score']\n           f1_weighted_avg=metrics_['weighted avg']['f1-score']\n           if val_epoch_loss < best_val:\n              print(f\"---------saving model---------with val loss = {val_epoch_loss}\")\n              best_val=val_epoch_loss\n              checkpoint = {\n                            'epoch': epoch,\n                            'state_dict': model.state_dict(),\n                            'optimizer': optimizer.state_dict(),\n                            'Val Loss' : val_epoch_loss,\n                            'Val Accuracy' : acc,\n                            'f1 macro avg': f1_macro_avg,\n                            'f1 weighted avg':f1_weighted_avg\n\n                        }\n              torch.save(checkpoint, 'vinbigdata_checkpoint.pt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stratified K Fold  \nApplying Stratified K fold cross validation to check model performance because the dataset is relatively small   ","metadata":{}},{"cell_type":"code","source":"'''\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch import nn, optim\nfrom torch.utils.data.dataloader import DataLoader\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=flags.seed)\n# skf.get_n_splits(None, None)\ny = np.array([int(len(d[\"annotations\"]) > 0) for d in dataset_dicts])\nsplit_inds = list(skf.split(dataset_dicts, y))\n\nfor fold in range(len(split_inds)):\n    train_inds, valid_inds = split_inds[fold]  # 0th fold\n    train_dataset = VinbigdataTwoClassDataset(\n        [dataset_dicts[i] for i in train_inds],\n        image_transform=Transform(flags.aug_kwargs),\n        mixup_prob=flags.mixup_prob,\n        label_smoothing=flags.label_smoothing,\n    )\n    valid_dataset = VinbigdataTwoClassDataset([dataset_dicts[i] for i in valid_inds])\n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=flags.batchsize,\n        num_workers=flags.num_workers,\n        shuffle=True,\n        pin_memory=True,\n    )\n    valid_loader = DataLoader(\n        valid_dataset,\n        batch_size=flags.valid_batchsize,\n        num_workers=flags.num_workers,\n        shuffle=False,\n        pin_memory=True,\n    )\n    timm_model = timm.create_model(flags.model_name, pretrained=True, num_classes=0, in_chans=3)\n    model=CNNFixedPredictor(timm_model, num_classes=2)\n    \n    model_training(model,train_loader,valid_loader,optimizer,lr_sched,\n                   cross_entropy_with_logits,\n                   accuracy_with_logits,\n                   1\n                   )\n'''                   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom torch.utils.data.dataloader import DataLoader\n\ntrain_dicts,val_dicts = train_test_split(dataset_dicts, test_size=0.20, random_state=42)\n\n\ntrain_dataset = VinbigdataTwoClassDataset(\n        train_dicts,\n        #image_transform=Transform(flags.aug_kwargs),\n        mixup_prob=flags.mixup_prob,\n        label_smoothing=flags.label_smoothing,\n    )\n\ntrain_loader = DataLoader(\n        train_dataset,\n        batch_size=flags.batchsize,\n        num_workers=flags.num_workers,\n        shuffle=True,\n        pin_memory=True,\n    )\n\nvalid_dataset = VinbigdataTwoClassDataset(val_dicts)\n\n\nvalid_loader = DataLoader(\n        valid_dataset,\n        batch_size=flags.valid_batchsize,\n        num_workers=flags.num_workers,\n        shuffle=False,\n        pin_memory=True,\n    )\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"'''\nmodel_training(model,train_loader,valid_loader,optimizer,lr_sched,\n                   #loss_function,\n                   cross_entropy_with_logits,\n                   accuracy_with_logits,\n                   5\n                   )\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the trained model","metadata":{}},{"cell_type":"code","source":"checkpoint = torch.load('../input/vinbigdata-2-class-classifier/vinbigdata_checkpoint.pt')\nmodel.load_state_dict(checkpoint['state_dict'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evaluation using F1 Score ","metadata":{}},{"cell_type":"code","source":"from sklearn import metrics\n\nepoch_loss,epoch_acc,model_outputs,labels=validation_epoch(model,valid_loader,cross_entropy_with_logits,\n               accuracy_with_logits)\nprint(f\"Epoch={0} , Loss={epoch_loss} ,Accuracy={epoch_acc}\")\nprint(metrics.confusion_matrix(labels, model_outputs).T)\nprint(metrics.classification_report(labels, model_outputs, digits=3))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_meta = pd.read_csv(inputdir / \"vinbigdata-testmeta\" / \"test_meta.csv\")\ndataset_dicts_test = get_vinbigdata_dicts_test(imgdir, test_meta, debug=False)\ntest_dataset = VinbigdataTwoClassDataset(dataset_dicts_test, train=False)\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=flags.valid_batchsize,\n    num_workers=flags.num_workers,\n    shuffle=False,\n    pin_memory=True,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predicts=predict_proba(model,test_loader)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub=pd.read_csv('../input/vinbigdata/submission (11).csv')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"number of anomaly Xray images = {torch.argmax(test_predicts,-1).sum()}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['PredictionString']\npredicts=[]\nfor bounding_box,class_ in  zip(list(sub['PredictionString']),torch.argmax(test_predicts,-1).numpy().tolist()):\n    if class_ == 0:\n       predicts.append('14 1 0 0 1 1')\n    else:\n       predicts.append(bounding_box)\n                ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(predicts)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['PredictionString']=predicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('sub_02.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'sub_02.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}