{"cells":[{"metadata":{"id":"8SedvC2MX0LQ"},"cell_type":"markdown","source":"# EfficientNet Regression\nIn this Kernel I am taking the approach of an EfficientNet Architecture. There are some concepts included from iafoss' tiles. However, they are then aggregated to a single picture and fed through the network.\n\nAfterwards I use an optimized threshold to turn the regression into a classification again.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"id":"O3UKLUZ4X0LV"},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline\n\nimport fastai\nfrom fastai.vision import *\nfrom fastai.callbacks import SaveModelCallback\nimport os\n#from sklearn.model_selection import KFold\nfrom radam import *\nfrom csvlogger import *\nfrom mish_activation import *\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score,confusion_matrix\nimport warnings\nimport scipy as sp\nimport skimage.io\nimport cv2\n\nwarnings.filterwarnings(\"ignore\")\n\n# remove this cell if run locally\n!mkdir 'cache'\n!mkdir 'cache/torch'\n!mkdir 'cache/torch/checkpoints'\ntorch.hub.DEFAULT_CACHE_DIR = 'cache'\n\nimport PIL.Image\nPIL.Image.MAX_IMAGE_PIXELS = 933120000\n\n# EfficientNet imports\nimport sys\npackage_path = '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nsys.path.append(package_path)\nfrom efficientnet_pytorch import EfficientNet\n\nfrom albumentations import Compose, Normalize, HorizontalFlip, VerticalFlip, Rotate, RandomScale\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"rMfH0urgX0Li"},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nSEED = 42\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"ZZMUeIjPX0Lr","outputId":"43c066fa-873c-46d5-e36c-9774c2dba4c0"},"cell_type":"code","source":"PATH = \"../input/\"\nos.listdir(PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"eMWaXfV1X0L4"},"cell_type":"code","source":"tile_sz = 224\nbs = 8\nn_epochs = 8\nnfolds = 5\nN = 12\nLABELS = PATH + 'prostate-cancer-grade-assessment/train.csv'\nIMAGES = PATH + 'prostate-cancer-grade-assessment/train_images/'","execution_count":null,"outputs":[]},{"metadata":{"id":"1X2QGb0DX0L-"},"cell_type":"markdown","source":"# Data Preparation and Cleaning\nData cleaning is based on this Kernel: https://www.kaggle.com/tanulsingh077/prostate-cancer-in-depth-understanding-eda-model\n\nThank you a lot for your EDA!","execution_count":null},{"metadata":{"trusted":true,"id":"m-FkbN1OX0MB","outputId":"4098dd80-9a7b-4164-ad23-1f017d5a6ec7"},"cell_type":"code","source":"df = pd.read_csv(LABELS).set_index('image_id')\n\n# Wrongly labeled data\nwrong_label = df[(df['isup_grade'] == 2) & (df['gleason_score'] == '4+3')]\ndisplay(wrong_label)\ndf.drop([wrong_label.index[0]],inplace=True)\ndf = df.reset_index()\n\n# incosistency with \"0\" and \"negative\"\ndf['gleason_score'] = df['gleason_score'].apply(lambda x: \"0+0\" if x==\"negative\" else x)\n\nsplits = StratifiedKFold(n_splits=nfolds, random_state=SEED, shuffle=True)\nsplits = list(splits.split(df,df.isup_grade))\nfolds_splits = np.zeros(len(df)).astype(np.int)\n\nfor i in range(nfolds):\n    if i == nfolds-1:\n        folds_splits[splits[i][1]] = 0\n    else:    \n        folds_splits[splits[i][1]] = 1\n    \ndf['split'] = folds_splits\ndf.head(10)","execution_count":null,"outputs":[]},{"metadata":{"id":"1xEZukIgX0ML"},"cell_type":"markdown","source":"## Data Loaders","execution_count":null},{"metadata":{"trusted":true,"id":"v0cTVpNJX0MM"},"cell_type":"code","source":"def tile(img, sz=128, N=16):\n    \"\"\" Subdivide large image in tiles and return most significant squares\n    \n    Params:\n    img: large input image\n    sz: size of tiles\n    N: number of most important tiles\n    \n    Returns: list of N most significant tiles\n    \"\"\"\n    shape = img.shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    \n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    \n    if len(img) < N:\n        img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n        \n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N]\n    img = img[idxs]\n    return img","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"h6t2hj2MX0MX"},"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, df, labels, transform=None):\n        self.df = df\n        self.labels = labels\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        \"\"\"\n        Prepares Items before feeding them in a neural network\n        \n        Params:\n        self: class\n        idx: current index of dataset\n        \n        Returns: List of images (tiles) and its label\n        \"\"\"\n        file_name = self.df['image_id'].values[idx]\n        file_path = IMAGES + f'{file_name}.tiff'\n        images = skimage.io.MultiImage(file_path)[1]\n        images = tile(images, sz=tile_sz, N=N)\n        new_images = []\n        for i,img in enumerate(images):\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            # Transformations\n            if self.transform:\n                augmented = self.transform(image=img)\n                img = augmented['image']\n            \n            new_images.append(img)\n            \n        label = torch.tensor(self.labels[idx]).float()\n        new_images = torch.stack(new_images)\n        return new_images, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"8fMq2WDNX0Mf"},"cell_type":"code","source":"def get_transforms(*, data):\n    \"\"\"\n    Get image transformation of data\n    \"\"\"\n    assert data in ('train', 'valid')\n    \n    if data == 'train':\n        return Compose([\n            HorizontalFlip(p=0.25),\n            Rotate(border_mode = cv2.BORDER_CONSTANT, value=(255,255,255), p=0.25),\n            VerticalFlip(p=0.25),\n            Normalize(),\n            ToTensorV2(),\n        ])\n    \n    elif data == 'valid':\n        return Compose([\n            Normalize(),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"w6ehxhHjX0Mu","outputId":"b9479efe-3c5f-4b72-da0c-0416c697b515"},"cell_type":"code","source":"train_dataset = TrainDataset(df, df[\"isup_grade\"], transform=get_transforms(data='train'))\ntrain_loader = DataLoader(train_dataset, batch_size=bs, shuffle=False)\n\nfor x,y in train_loader:\n    print(x.shape)\n    fig, axs = plt.subplots(bs,N, figsize=(30, 12), facecolor='w', edgecolor='k')\n    axs = axs.ravel()\n    for j,batch in enumerate(x):\n        for i,t in enumerate(batch):\n            axs[j*N+i].imshow(t.permute(1,2,0).numpy())\n    break","execution_count":null,"outputs":[]},{"metadata":{"id":"ZmrU3wmmX0M2"},"cell_type":"markdown","source":"## Model","execution_count":null},{"metadata":{"trusted":true,"id":"om4k0Gx8X0M3"},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, pre=True):\n        super().__init__()\n        \n        # Load model backbone\n        backbone = EfficientNet.from_name('efficientnet-b0')\n        \n        # Get preloaded model\n        if pre:\n            backbone.load_state_dict(torch.load(PATH + 'efficientnet-pytorch/efficientnet-b0-08094119.pth'))\n        \n        # Encoder, runs through the pretrained efficientnet\n        self.backbone = backbone\n        \n        # Neural network head. After running through the neural network, this is the transfer\n        nc = list(backbone.children())[-1].in_features\n        self.head = nn.Sequential(AdaptiveConcatPool2d(),\n                                  Flatten(),\n                                  nn.Linear(2*nc,512), \n                                  Mish(),\n                                  nn.BatchNorm1d(512), \n                                  nn.Dropout(0.5),\n                                  nn.Linear(512,1))\n        \n        \n    def forward(self, x):\n        \"\"\" Forward run through the neural network\n        Params:\n        x: torch tensor - input\n        \n        returns: output after feed forward.\n        \"\"\"\n        # Reshape array\n        shape = x.shape\n        n = shape[1]\n        x = x.view(-1,shape[2],shape[3],shape[4])\n        \n        #x: bs*N x 3 x 224 x 224\n        # Go through convolutional layers\n        x = self.backbone.extract_features(x)\n        \n        shape = x.shape\n        x = x.view(-1,n,shape[1],shape[2],shape[3]).permute(0,2,1,3,4).contiguous()\\\n          .view(-1,shape[1],shape[2]*n,shape[3])\n        \n        # Go through classifier\n        x = self.head(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"id":"e94fyeIYX0NA"},"cell_type":"markdown","source":"### Regression to Classification conversion using the OptimizedRounder","execution_count":null},{"metadata":{"trusted":true,"id":"kakXgWpYX0NB"},"cell_type":"code","source":"# inspired by https://www.kaggle.com/tanlikesmath/intro-aptos-diabetic-retinopathy-eda-starter\nclass KappaOptimizer(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.coef = [0.5, 1.5, 2.5, 3.5, 4.5]\n        # define score function:\n        self.func = self.quad_kappa\n    \n    \n    def predict(self, preds):\n        return self._predict(self.coef, preds)\n\n    \n    @classmethod\n    def _predict(cls, coef, preds):\n        if type(preds).__name__ == 'Tensor':\n            y_hat = preds.clone().view(-1)\n        else:\n            y_hat = torch.FloatTensor(preds).view(-1)\n\n        for i,pred in enumerate(y_hat):\n            if   pred < coef[0]: y_hat[i] = 0\n            elif pred < coef[1]: y_hat[i] = 1\n            elif pred < coef[2]: y_hat[i] = 2\n            elif pred < coef[3]: y_hat[i] = 3\n            elif pred < coef[4]: y_hat[i] = 4\n            else:                y_hat[i] = 5\n        return y_hat.int()\n    \n    \n    def quad_kappa(self, preds, y):\n        return self._quad_kappa(self.coef, preds, y)\n\n    \n    @classmethod\n    def _quad_kappa(cls, coef, preds, y):\n        y_hat = cls._predict(coef, preds)\n        \n        try:\n            return cohen_kappa_score(y, y_hat, weights='quadratic')\n        except:\n            return cohen_kappa_score(y.cpu(), y_hat.cpu(), weights='quadratic')\n\n    \n    def fit(self, preds, y):\n        ''' maximize quad_kappa '''\n        neg_kappa = lambda coef: -self._quad_kappa(coef, preds, y)\n        opt_res = sp.optimize.minimize(neg_kappa, x0=self.coef, method='nelder-mead',\n                                       options={'maxiter':150, 'fatol':1e-20, 'xatol':1e-20})\n        self.coef = opt_res.x\n\n        \n    def forward(self, preds, y):\n        ''' the pytorch loss function '''\n        return torch.tensor(self.quad_kappa(preds, y))\n\nkappa_opt = KappaOptimizer()","execution_count":null,"outputs":[]},{"metadata":{"id":"l2-5gXROX0NJ"},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true,"id":"sXrpwtIFX0NL"},"cell_type":"code","source":"# Prepare databunch\nfold =  0\ntrain_idx = df[df['split'] != fold].index\nval_idx = df[df['split'] == fold].index\n\ntrain_dataset = TrainDataset(df.loc[train_idx].reset_index(drop=True), \n                             df.loc[train_idx].reset_index(drop=True)[\"isup_grade\"], \n                             transform=get_transforms(data='train'))\nvalid_dataset = TrainDataset(df.loc[val_idx].reset_index(drop=True), \n                             df.loc[val_idx].reset_index(drop=True)[\"isup_grade\"], \n                             transform=get_transforms(data='valid'))\n\ntrain_loader = DataLoader(train_dataset, batch_size=bs, shuffle=True, num_workers=8)\nvalid_loader = DataLoader(valid_dataset, batch_size=bs, shuffle=False, num_workers=8)\n\n\ndata = DataBunch(train_dl = train_loader, valid_dl = valid_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"yMwlAiQfX0Nd"},"cell_type":"code","source":"fname = 'EFFNETB0_REGRESSION'\nmodel = Model()\n\nlearn = Learner(data, \n                model, \n                loss_func=torch.nn.SmoothL1Loss(),\n                opt_func=RAdam, \n                metrics=[kappa_opt])\n\nlogger = CSVLogger(learn, f'log_{fname}_{fold}')\n\nlearn.split([model.backbone, model.head])\nlearn.freeze_to(0)","execution_count":null,"outputs":[]},{"metadata":{"id":"NuIMN3PmG1NZ","outputId":"75c1b47c-51eb-47f8-ffda-209614ee4362","trusted":true},"cell_type":"code","source":"# Fit for n_epochs cycles\nlearn.fit_one_cycle(n_epochs, \n                    (1e-4, 1e-3),\n                    callbacks = [SaveModelCallback(learn,\n                                          name=f'model',\n                                          mode='min',\n                                          monitor='valid_loss')])\n\n# Save model\ntorch.save(learn.model.state_dict(), f'{fname}_{fold}.pth')","execution_count":null,"outputs":[]},{"metadata":{"id":"A4zymJ29X0Nr"},"cell_type":"markdown","source":"## Evaluation","execution_count":null},{"metadata":{"trusted":true,"id":"FSdTE1V7X0Ns","outputId":"a944910c-c5dd-4920-ac3c-f271e649fee0"},"cell_type":"code","source":"train_pred,train_target, pred, target = [],[],[],[]\nlearn.model.eval()\nwith torch.no_grad():\n    \"\"\"for step, (x, y) in progress_bar(enumerate(data.dl(DatasetType.Train)),total=len(data.dl(DatasetType.Train))):\n        p = learn.model(x)\n        p = p.float().cpu()\n        train_pred.append(p)\n        train_target.append(y.cpu())\"\"\"\n        \n    for step, (x, y) in progress_bar(enumerate(data.dl(DatasetType.Valid)),total=len(data.dl(DatasetType.Valid))):\n        p = learn.model(x)\n        p = p.float().cpu()\n        pred.append(p)\n        target.append(y.cpu())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"id":"1x1SnZDfX0Ny"},"cell_type":"code","source":"p = torch.cat(pred)\nt = torch.cat(target)\np = kappa_opt.predict(p)\nprint(cohen_kappa_score(t,p,weights='quadratic'), \"\\n\")\nprint(confusion_matrix(t,p), \"\\n\")\nprint(kappa_opt.coef)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"Y1jdncQRX0N6"},"cell_type":"code","source":"p_train = torch.cat(train_pred)\nt_train = torch.cat(train_target)\nkappa_opt.fit(p_train,t_train)\n\np = kappa_opt.predict(p)\nprint(cohen_kappa_score(t,p,weights='quadratic'), \"\\n\")\nprint(confusion_matrix(t,p), \"\\n\")\nprint(kappa_opt.coef)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"id":"_a0S6HInX0OB"},"cell_type":"code","source":"!rm -r 'cache'","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}