{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/timm-pytorch-image-models/pytorch-image-models-master/')\nimport timm\n\nimport numpy as np\nimport pandas as pd\nimport glob\nimport os\nimport cv2\nimport random\nimport gc\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.preprocessing import LabelEncoder, normalize\n\nfrom tqdm import tqdm\n\nimport torch\nfrom torch.utils.data import Dataset,DataLoader\nimport torch.nn as nn\nfrom torch.optim import Adam\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:01.47016Z","iopub.execute_input":"2023-02-11T18:15:01.470756Z","iopub.status.idle":"2023-02-11T18:15:01.48332Z","shell.execute_reply.started":"2023-02-11T18:15:01.470711Z","shell.execute_reply":"2023-02-11T18:15:01.481809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"def f1score(labels, predictions, thresh=0.5):\n    labels=torch.tensor(labels)\n    predictions=(torch.tensor(predictions))\n    labels=labels.squeeze()\n    predictions=predictions.squeeze()>thresh\n    conf_matrix={'tp':0, 'fp':0, 'fn':0, 'tn':0}\n    conf_matrix['tp']=(predictions[labels==1]).sum().item()\n    conf_matrix['tn']=(predictions==labels).sum().item()-conf_matrix['tp']\n    conf_matrix['fp']=(predictions.sum()-conf_matrix['tp']).item()\n    conf_matrix['fn']=(labels.sum()-conf_matrix['tp']).item()\n    if (conf_matrix['tp']+0.5*(conf_matrix['fp']+conf_matrix['fn']))>0:\n        score=(conf_matrix['tp'])/(conf_matrix['tp']+0.5*(conf_matrix['fp']+conf_matrix['fn']))\n    else:\n        score=0\n    return score\n    \ndef optimize_thresh(labels, predictions, verbose=False):\n    scores=[f1score(labels, predictions, thresh/10) for thresh in range(0,10,1)]\n    if verbose: print(scores)\n    max_score=max(scores)\n    best_thresh=scores.index(max_score)\n    return best_thresh, max_score\n    ","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:01.487784Z","iopub.execute_input":"2023-02-11T18:15:01.488689Z","iopub.status.idle":"2023-02-11T18:15:01.508166Z","shell.execute_reply.started":"2023-02-11T18:15:01.488646Z","shell.execute_reply":"2023-02-11T18:15:01.5066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataFrame","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\nTRAIN_PATH = '/kaggle/input/rsna-cut-off-empty-space-from-images/'\ntrain[\"path\"] = TRAIN_PATH+train.patient_id.astype(str)+'/'+train.image_id.astype(str)+'.png'","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:01.510581Z","iopub.execute_input":"2023-02-11T18:15:01.511931Z","iopub.status.idle":"2023-02-11T18:15:01.840099Z","shell.execute_reply.started":"2023-02-11T18:15:01.511871Z","shell.execute_reply":"2023-02-11T18:15:01.838845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Impute and apply encodings","metadata":{}},{"cell_type":"code","source":"train['age'] = train['age'].fillna(train['age'].mean())\ncsv_columns = ['laterality', 'view', 'age', 'implant']\n\ntrain = train[[\"patient_id\", \"image_id\", \"laterality\", \"view\", \"age\", \"implant\", \"cancer\",\"path\"]]\nle_laterality = LabelEncoder()\nle_view = LabelEncoder()\n\ntrain['laterality'] = le_laterality.fit_transform(train['laterality'])\ntrain['view'] = le_view.fit_transform(train['view'])\n","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:01.843155Z","iopub.execute_input":"2023-02-11T18:15:01.848022Z","iopub.status.idle":"2023-02-11T18:15:01.925416Z","shell.execute_reply.started":"2023-02-11T18:15:01.847951Z","shell.execute_reply":"2023-02-11T18:15:01.923726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class RSNADataset(Dataset):\n    def __init__(self, df, transform=None): \n        self.df = df\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self,idx):\n        row = self.df.iloc[idx]\n        img_path =row['path']\n        image = cv2.imread(img_path).astype(np.float32)/255\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform is not None:\n            image = self.transform(image=image)['image']\n        target = torch.tensor(row.cancer).float()\n        return {'image':image,'target': target}","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:01.927689Z","iopub.execute_input":"2023-02-11T18:15:01.928744Z","iopub.status.idle":"2023-02-11T18:15:01.943527Z","shell.execute_reply.started":"2023-02-11T18:15:01.928673Z","shell.execute_reply":"2023-02-11T18:15:01.941468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformations","metadata":{}},{"cell_type":"code","source":"def get_transforms(data):\n    if data == 'train':\n        return A.Compose([\n            A.augmentations.geometric.resize.Resize(512,256),\n            A.HorizontalFlip(p=0.5),\n             A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n    elif data == 'valid':\n        return A.Compose([\n            A.augmentations.geometric.resize.Resize(512,256),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:01.945859Z","iopub.execute_input":"2023-02-11T18:15:01.948154Z","iopub.status.idle":"2023-02-11T18:15:01.966679Z","shell.execute_reply.started":"2023-02-11T18:15:01.948062Z","shell.execute_reply":"2023-02-11T18:15:01.965298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"code","source":"def view_img(idx, data = 'train'):\n    row = train.iloc[idx]\n    img_path =row['path']\n    image = cv2.imread(img_path).astype(np.float32)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    transform = get_transforms(data = data)\n    image = transform(image=image)['image']\n    plt.imshow(image.cpu().detach().numpy().transpose(1, 2, 0))\n    plt.show()\n    return\n\nidx = 0\nview_img(idx)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:01.969203Z","iopub.execute_input":"2023-02-11T18:15:01.971057Z","iopub.status.idle":"2023-02-11T18:15:02.346249Z","shell.execute_reply.started":"2023-02-11T18:15:01.970877Z","shell.execute_reply":"2023-02-11T18:15:02.344867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss","metadata":{}},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=(1,1), gamma=2, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = torch.tensor(alpha)\n        self.gamma = gamma\n        self.reduce = reduce\n\n    def forward(self, preds, targets):\n        BCE_loss = nn.BCEWithLogitsLoss(reduction='none')(torch.flatten(preds), torch.flatten(targets).float())\n        logits=nn.Sigmoid()(preds)\n        F_loss = (self.alpha[0]*(targets)*((1-logits)**self.gamma)+ self.alpha[1]*(1-targets)*((logits)**self.gamma)) * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:02.352526Z","iopub.execute_input":"2023-02-11T18:15:02.3534Z","iopub.status.idle":"2023-02-11T18:15:02.372729Z","shell.execute_reply.started":"2023-02-11T18:15:02.353346Z","shell.execute_reply":"2023-02-11T18:15:02.37086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class BreastCancerModel(torch.nn.Module):\n    def __init__(self, model_type, pretrained=True,dropout=0.):\n        super().__init__()        \n        self.model = timm.create_model(model_type, pretrained=True, num_classes=0, drop_rate=dropout)\n\n        self.backbone_dim = self.model(torch.randn(1, 3, 256, 256)).shape[-1]\n        self.nn_cancer = torch.nn.Sequential(\n            torch.nn.Linear(self.backbone_dim, 1),\n        )\n\n    def forward(self, x):\n        x = self.model(x)\n        cancer = self.nn_cancer(x).squeeze()\n        return cancer\n    \n    def predict(self, x):\n        preds=torch.sigmoid(self.forward(x))\n        return preds\n\nDEBUG=False\nif DEBUG:\n    with torch.no_grad():\n        model = BreastCancerModel(model_type='seresnext50_32x4d')\n        pred = model.predict(torch.randn(2, 3, 512, 512))\n        print('seresnext', pred.shape)\n\n        model = BreastCancerModel(model_type='efficientnet_b4')\n        pred = model.predict(torch.randn(2, 3, 512, 512))\n        print('efficientnet_b4', pred.shape)\n\n    del model","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:02.382498Z","iopub.execute_input":"2023-02-11T18:15:02.386693Z","iopub.status.idle":"2023-02-11T18:15:02.405462Z","shell.execute_reply.started":"2023-02-11T18:15:02.386625Z","shell.execute_reply":"2023-02-11T18:15:02.404102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def train_folds(train, CFG):\n    skf = StratifiedGroupKFold(CFG.folds)\n\n    #train & valid\n    for fold, (tr_idx, val_idx) in enumerate(skf.split(train, y=train.cancer, groups=train.patient_id)):\n        print(f'========= fold : {fold+1} =========')\n\n        train_dataset = RSNADataset(train.loc[tr_idx,:],transform=get_transforms(data='train'))\n        valid_dataset = RSNADataset(train.loc[val_idx,:],transform=get_transforms(data='valid'))\n\n        train_dataloader = DataLoader(train_dataset, batch_size=CFG.BATCH, shuffle=True, num_workers=4)\n        valid_dataloader = DataLoader(valid_dataset, batch_size=CFG.BATCH, shuffle=False, num_workers=4)\n\n        model = BreastCancerModel(CFG.model_name, pretrained=True)\n        model.to(device)\n        criterion = FocalLoss(alpha=[50,1])#nn.BCEWithLogitsLoss()\n        optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')\n\n        for epoch in range(CFG.epochs):\n            print(f'epoch:{epoch+1}')\n            model.train()\n\n            train_losses = 0\n            val_losses = 0\n            train_targets,train_preds = [],[]\n            val_targets,val_preds = [],[]\n\n            # Train 1 epoch\n            for data in tqdm(train_dataloader):\n                image=data['image']\n                target=data['target']\n                \n                image = image.to(device)\n                train_target = target.to(device)\n                \n                train_pred = model(image)\n                train_loss = criterion(train_pred.view(-1),train_target)\n                train_losses += train_loss.item()\n                \n                optimizer.zero_grad()\n                train_loss.backward()\n                optimizer.step()\n\n                train_targets.extend(train_target.cpu().detach().tolist())\n                train_preds.extend(torch.sigmoid(train_pred.view(-1)).cpu().detach().tolist())\n\n            avg_train_loss = train_losses / len(train_dataloader)\n            train_score = f1score(train_targets, train_preds, thresh=0.5)\n            \n            # Validate 1 epoch\n            model.eval()\n            with torch.no_grad():\n                for data in tqdm(valid_dataloader):\n                    img=data['image']\n                    target=data['target']\n                    \n                    img = img.to(device)\n                    val_target = target.to(device)\n                    \n                    val_pred = model(img)\n                    val_loss = criterion(val_pred.view(-1),val_target)\n                    val_losses += val_loss.item()\n\n                    val_targets.extend(val_target.cpu().detach().tolist())\n                    val_preds.extend(torch.sigmoid(val_pred.view(-1)).cpu().detach().tolist())\n\n            avg_val_loss = val_losses / len(valid_dataloader)\n            val_score = f1score(val_targets, val_preds, thresh=0.5)\n            \n            scheduler.step(val_loss)\n\n            #score\n            print('avg_train_loss:',avg_train_loss,'avg_val_loss:',avg_val_loss)\n            print('train_score:',train_score,'val_score:',val_score)\n            \n        print(\"FOR TRAIN:-\")\n        print('THE BEST THRESH and SCORE will be:-',optimize_thresh(train_targets, train_preds))\n        print(\"FOR VALIDATION:-\")\n        valid_thresh = optimize_thresh(val_targets, val_preds)\n        print('THE BEST THRESH and SCORE will be:-',valid_thresh)\n\n        #model save\n        torch.save({'state dict' : model.state_dict(), 'best threshold' : valid_thresh, 'train preds' : train_preds}, f'baseline_{CFG.model_name}_{fold}_.pth')\n        del model, train_dataset, valid_dataset","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:02.414721Z","iopub.execute_input":"2023-02-11T18:15:02.41839Z","iopub.status.idle":"2023-02-11T18:15:02.639873Z","shell.execute_reply.started":"2023-02-11T18:15:02.418319Z","shell.execute_reply":"2023-02-11T18:15:02.638432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG0:\n    folds = 4\n    epochs = 8\n    BATCH = 32\n    model_name = \"seresnext50_32x4d\"\n    lr = 3e-4\n    weight_decay = 1e-6\n    target_size = 1\n    size = 256\n    seed = 42\n\n# train_folds(train.iloc[0:100], CFG0)\ntrain_folds(train, CFG0)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T18:15:02.645393Z","iopub.execute_input":"2023-02-11T18:15:02.645855Z","iopub.status.idle":"2023-02-11T18:20:01.464565Z","shell.execute_reply.started":"2023-02-11T18:15:02.645817Z","shell.execute_reply":"2023-02-11T18:20:01.463128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}