{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom fastai import vision\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import transforms","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data processing"},{"metadata":{},"cell_type":"markdown","source":"Read data and gives index images"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = vision.Path(\"/kaggle/input/rsna-intracranial-hemorrhage-detection/stage_1_train_images\")\ntest_path = vision.Path(\"/kaggle/input/rsna-intracranial-hemorrhage-detection/stage_1_test_images\")\nall_classes = ['any', 'epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataframe = pd.read_csv(\"/kaggle/input/rsna-intracranial-hemorrhage-detection/stage_1_train.csv\")\ndataframe[\"Classes\"] = dataframe.ID.str.rsplit(pat=\"_\", expand=True)[2]\ndataframe[\"ID\"] = dataframe[\"ID\"].apply(lambda x: \"_\".join(x.split('_')[:-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataframe = pd.pivot_table(dataframe, index=\"ID\", columns=\"Classes\", values=\"Label\")\ndataframe = pd.DataFrame(dataframe.to_records())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for curr_class in all_classes:\n    dataframe[curr_class] = dataframe[curr_class].apply(\n        lambda x: curr_class if x == 1 else \"\")\ndataframe[\"Class\"] = dataframe[all_classes].apply(\n    lambda x: ' '.join(list(filter(None, x))), axis=1)\ndataframe = dataframe.drop(columns=all_classes)\ndataframe[\"ID\"] = dataframe[\"ID\"].apply(lambda x: x + \".dcm\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"broken_samples = [\"ID_6431af929.dcm\"]\nid_list = dataframe[\"ID\"].tolist()\nbroken_indexes = [id_list.index(sample) for sample in broken_samples]\ndataframe = dataframe.drop(broken_indexes).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, val_df = train_test_split(dataframe, test_size=0.2)\ntrain_df = train_df.reset_index(drop=True)\nval_df = val_df.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Class for DataLoader"},{"metadata":{},"cell_type":"markdown","source":"Work with data for dataloader - fast ai function, goal is split data to bunch"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Data(vision.Dataset):\n    \"\"\"This class loads one sample data using a DataFrame.\n    \n    Attributes:\n        df: Pandas DataFrame. DataFrame Structure:\n            ----------------------------------------\n            |      ID     |         Class          |\n            ----------------------------------------\n            | \"name1.dcm\" |          \"\"            | \n            ----------------------------------------\n            | \"name2.dcm\" | \"class1 class2 class3\" |\n            ----------------------------------------\n            |     ...     |          ...           |\n            ----------------------------------------\n        classes: A list containing all image classes\n        folder: Object of \"vision.Path\" class\n        transfrom: Transforms from PyTorch (more about this:\n            https://pytorch.org/docs/stable/torchvision/transforms.html)\n    \"\"\"\n    def __init__(self, df, classes, folder, transform):\n        self.df = df\n        self.df_cols_name = self.df.columns\n        self.classes = classes\n        self.folder = folder\n        self.transform = transform\n        self.c = 6\n\n    def __getitem__(self, idx):\n        path_to_img = self.folder/self.df[self.df_cols_name[0]][idx]\n        \n        # Image processing\n        img = pydicom.dcmread(str(path_to_img)).pixel_array\n        img = np.maximum(img,0) / img.max() * 255\n        img = img.astype(\"uint8\")\n        img = np.expand_dims(img, -1)\n        img = np.broadcast_to(img, [img.shape[0], img.shape[1], 3])\n        img = Image.fromarray(img)\n        img = self.transform(img)\n        \n        # Label processing\n        label = vision.torch.zeros(len(self.classes), dtype=torch.float if torch.cuda.is_available() else torch.long)\n        all_image_classes = self.df[self.df_cols_name[1]][idx].split(' ')\n        all_image_classes = list(filter(None, all_image_classes))\n        for curr_class in all_image_classes:\n            label[self.classes.index(curr_class)] = 1\n        return img, label\n    \n    def __len__(self):\n        return self.df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data to bunch"},{"metadata":{},"cell_type":"markdown","source":"Split data for datasets and doing data augmentation in fast ai"},{"metadata":{"trusted":true},"cell_type":"code","source":"transform = transforms.Compose([transforms.RandomHorizontalFlip(),\n                                transforms.Resize((224, 224)),\n                                transforms.ToTensor(),\n                                transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n                                ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = Data(df=train_df, classes=all_classes, folder=train_path, transform=transform)\ntrain_dataloader = vision.DataLoader(train_dataset, batch_size=16, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_dataset = Data(df=val_df, classes=all_classes, folder=train_path, transform=transform)\nval_dataloader = vision.DataLoader(val_dataset, batch_size=16, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"databunch = vision.DataBunch(train_dataloader, val_dataloader)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{},"cell_type":"markdown","source":"We use efficient net b7. For this thing, lets download git repository"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append(\"/kaggle/input/efficient-net-for-pytorch-or-fast-ai/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import model\nmodel_eff = model.EfficientNet.from_pretrained('efficientnet-b7')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Optimizer"},{"metadata":{},"cell_type":"markdown","source":"RAdam optimizer, the best solution https://medium.com/@lessw/new-state-of-the-art-ai-optimizer-rectified-adam-radam-5d854730807b\n![Radam](https://miro.medium.com/max/2118/1*BMwu8Km-CtPsvaH8OM5_-g.jpeg)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\nclass RAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, degenerated_to_sgd=True):\n        if not 0.0 <= lr:\n            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n        if not 0.0 <= eps:\n            raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n        if not 0.0 <= betas[0] < 1.0:\n            raise ValueError(\"Invalid beta parameter at index 0: {}\".format(betas[0]))\n        if not 0.0 <= betas[1] < 1.0:\n            raise ValueError(\"Invalid beta parameter at index 1: {}\".format(betas[1]))\n        \n        self.degenerated_to_sgd = degenerated_to_sgd\n        if isinstance(params, (list, tuple)) and len(params) > 0 and isinstance(params[0], dict):\n            for param in params:\n                if 'betas' in param and (param['betas'][0] != betas[0] or param['betas'][1] != betas[1]):\n                    param['buffer'] = [[None, None, None] for _ in range(10)]\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, buffer=[[None, None, None] for _ in range(10)])\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = group['buffer'][int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    elif self.degenerated_to_sgd:\n                        step_size = 1.0 / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = -1\n                    buffered[2] = step_size\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:\n                    if group['weight_decay'] != 0:\n                        p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)\n                    p.data.copy_(p_data_fp32)\n                elif step_size > 0:\n                    if group['weight_decay'] != 0:\n                        p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n                    p_data_fp32.add_(-step_size * group['lr'], exp_avg)\n                    p.data.copy_(p_data_fp32)\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optar = vision.partial(RAdam)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{},"cell_type":"markdown","source":"Efficient net"},{"metadata":{"trusted":true},"cell_type":"code","source":"len(dir(model_eff.parameters))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is deliting 26 layer because, this layer gives 1000 in output, but we need only 6"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, elem in enumerate(model_eff.parameters()):\n    if i == 26:\n        del elem","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = vision.nn.Sequential(\n                            model_eff,\n                            vision.nn.Linear(1000, 6),\n                            vision.nn.Sigmoid())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = vision.Learner(databunch, model,\n                        metrics=vision.error_rate, loss_func=vision.nn.BCEWithLogitsLoss(),\n                        opt_func=optar)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(1, callbacks=[vision.callbacks.SaveModelCallback(learn, every='improvement', monitor='loss_func', mode='min', name='best')]).cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Csv creation"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/rsna-intracranial-hemorrhage-detection/stage_1_sample_submission.csv\")\nall_test_images = test_df[\"ID\"].apply(lambda x: \"_\".join(\n    list(filter(None, x.split(\"_\")[:-1])))).drop_duplicates().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_transforms = [transforms.RandomVerticalFlip(p=1),\n                   transforms.RandomHorizontalFlip(p=1)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_path = \"../input/rsna-intracranial-hemorrhage-detection/stage_1_test_images/\"\nfor curr_img_path in all_test_images:\n    img = pydicom.dcmread(str(f\"{test_path}{curr_img_path}.dcm\")).pixel_array\n    img = np.maximum(img,0) / img.max() * 255\n    img = img.astype(\"uint8\")\n    img = np.expand_dims(img, -1)\n    img = np.broadcast_to(img, [img.shape[0], img.shape[1], 3])\n    img = Image.fromarray(img)\n    transformed_images = torch.zeros((0, 3, 224, 224))\n    for test_transform in test_transforms:\n        transformed_image = test_transform(img)\n        transformed_image = transforms.functional.resize(transformed_image, (224, 224))\n        transformed_image = np.array(transformed_image).reshape((3, 224, 224))\n        transformed_tensor_image = torch.from_numpy(transformed_image).float()\n        transformed_tensor_image /= 255.\n        transformed_tensor_image = transforms.Normalize(\n            [0.485, 0.456, 0.406],\n            [0.229, 0.224, 0.225])(transformed_tensor_image)\n        transformed_tensor_image = transformed_tensor_image.expand(1, -1, -1, -1)\n        transformed_images = np.append(\n            transformed_images, transformed_tensor_image, 0)\n    transformed_tensor = torch.from_numpy(transformed_images)\n    if torch.cuda.is_available():\n        transformed_tensor = transformed_tensor.to(\"cuda:0\")\n    preds = model(transformed_tensor)\n    preds = torch.mean(preds, 0).detach().cpu().numpy()\n    preds = np.where(preds > 0.5, 1, 0)\n    for i, elem in enumerate(preds):\n        col_name = f\"{curr_img_path}_{all_classes[i]}\"\n        img_idx = test_df.iloc[:, 0].tolist().index(col_name)\n        test_df.iloc[img_idx, 1] = elem","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}