{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\n# image manipulation\nimport cv2\nimport PIL\nfrom PIL import Image\n\n# visualisation\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# helpers\nfrom tqdm import tqdm\nimport time\nimport copy\nimport gc\nfrom enum import Enum\n\n\n# for cnn\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader, random_split, TensorDataset, Dataset, WeightedRandomSampler\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, StepLR\nfrom torchvision import models\nfrom torchmetrics.classification import BinaryF1Score, BinaryPrecision, BinaryRecall, BinaryAccuracy, BinaryROC, BinaryAUROC\nfrom torchvision import transforms","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-05T18:24:12.738111Z","iopub.execute_input":"2023-02-05T18:24:12.738501Z","iopub.status.idle":"2023-02-05T18:24:12.746488Z","shell.execute_reply.started":"2023-02-05T18:24:12.738452Z","shell.execute_reply":"2023-02-05T18:24:12.74514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"csvpathtrain = '/kaggle/input/rsna-breast-cancer-detection/train.csv'\n\ndftrain = pd.read_csv(csvpathtrain)\ndftrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:12.764115Z","iopub.execute_input":"2023-02-05T18:24:12.765001Z","iopub.status.idle":"2023-02-05T18:24:12.833369Z","shell.execute_reply.started":"2023-02-05T18:24:12.76496Z","shell.execute_reply":"2023-02-05T18:24:12.832168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(10, 5))\n########## PLOTING CANCER ################\nsplot = sns.countplot(ax = axes[0], x = dftrain['cancer'])\n\ns = dftrain['cancer'].value_counts()\naxes[1].pie(s, autopct=\"%.1f%%\", labels = s.keys())\nfig.suptitle('Cancer distribution')","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:12.835543Z","iopub.execute_input":"2023-02-05T18:24:12.836565Z","iopub.status.idle":"2023-02-05T18:24:13.080459Z","shell.execute_reply.started":"2023-02-05T18:24:12.836521Z","shell.execute_reply":"2023-02-05T18:24:13.079345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_samples = len(dftrain['cancer'])\npositive_samples = sum(dftrain['cancer'] == 1)\nnegative_samples = total_samples - positive_samples\nprint(f\"{total_samples}, {positive_samples}, {negative_samples}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:13.082285Z","iopub.execute_input":"2023-02-05T18:24:13.08267Z","iopub.status.idle":"2023-02-05T18:24:13.094713Z","shell.execute_reply.started":"2023-02-05T18:24:13.082632Z","shell.execute_reply":"2023-02-05T18:24:13.093741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples_weight = torch.Tensor([positive_samples / total_samples, negative_samples / total_samples]).type(dtype = torch.float32)\n\nsamples_weight","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:13.097428Z","iopub.execute_input":"2023-02-05T18:24:13.098085Z","iopub.status.idle":"2023-02-05T18:24:13.109106Z","shell.execute_reply.started":"2023-02-05T18:24:13.098047Z","shell.execute_reply":"2023-02-05T18:24:13.108003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RSNAMamographyDataset(Dataset):\n    def __init__(self, annotations_file, img_dir, transform=None):\n        self.df = pd.read_csv(annotations_file)\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n    \n\n\n    def __getitem__(self, ind):\n        \n        img_path = f\"{self.img_dir}/{self.df.iloc[ind].patient_id}_{self.df.iloc[ind].image_id}.png\"\n        img = Image.open(img_path).convert('RGB')\n        \n        label = self.df.iloc[ind].cancer\n        # there is no need to normalize data, it has already been normalized\n        if self.transform:\n            img = self.transform(img).to(torch.float32) \n        else:\n            default_transform = transforms.Compose([transforms.ToTensor()])\n            img = default_transform(img).to(torch.float32)\n            \n        #sample = {\"image\" : img, \"label\": label}\n        return img, label","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:13.111066Z","iopub.execute_input":"2023-02-05T18:24:13.111451Z","iopub.status.idle":"2023-02-05T18:24:13.121805Z","shell.execute_reply.started":"2023-02-05T18:24:13.111412Z","shell.execute_reply":"2023-02-05T18:24:13.1205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = '/kaggle/input/rsna-breast-cancer-detection/train.csv'\nimgs_dir = '/kaggle/input/rsnamamorgaphybreastcancerrecognition512x512'\n\naugmentator = transforms.Compose([\n    # input for augmentator is always PIL image\n    # transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.RandomVerticalFlip(0.5),\n    transforms.RandomRotation(5),\n    transforms.ToTensor(), # return it as a tensor and transforms it to [0, 1]\n])\ndataset = RSNAMamographyDataset(train_csv, imgs_dir, augmentator)","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:13.123303Z","iopub.execute_input":"2023-02-05T18:24:13.123985Z","iopub.status.idle":"2023-02-05T18:24:13.182639Z","shell.execute_reply.started":"2023-02-05T18:24:13.123926Z","shell.execute_reply":"2023-02-05T18:24:13.18158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use torch.utils.data to create a DataLoader \n# that will take care of creating batches \n\n# TODO, remove using half of dataset\n# dataset, _ = random_split(dataset, [int(len(dataset)*0.02), int(len(dataset)*0.98 + 1)])\n# split training into validation and train\nval_pct = 0.1\nval_size = int(val_pct * len(dataset))\ntrain_size = len(dataset) - val_size\ntrain_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n\n","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:13.183937Z","iopub.execute_input":"2023-02-05T18:24:13.184316Z","iopub.status.idle":"2023-02-05T18:24:13.195515Z","shell.execute_reply.started":"2023-02-05T18:24:13.18428Z","shell.execute_reply":"2023-02-05T18:24:13.194494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Class counting...\")\nlabels = dftrain['cancer'].values\nclass_sample_count = np.array([len(np.where(labels == l)[0]) for l in np.unique(labels)])\n\n\n# the trouble with this aproach is that it now has to load all images one by one and label them\n# but it saves RAM memory in training process\n#class_sample_count = np.zeros(2)\n\n#print(\"Class counting...\")\n#for _, label in tqdm(train_dataset):\n#    class_sample_count[label] += 1\n\nprint(class_sample_count)\n\n# This maybe apply, maybe not\n# since there is big class imbalance, we will not sample positive class THAT frequent\n# to be closer to 'reality, every fifth image will be cancer (instead of 50/50 distribution)'\nclass_sample_count[1] *= 5\nclass_weights = 1. / class_sample_count\n\nprint(\"Adding weights to each training sample...\")\nsample_weights = []\nfor _, label in tqdm(train_dataset):\n    sample_weights.append(class_weights[label])\n\nsample_weights = np.array(sample_weights)\nsample_weights = torch.from_numpy(sample_weights)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:13.196919Z","iopub.execute_input":"2023-02-05T18:24:13.197471Z","iopub.status.idle":"2023-02-05T18:24:27.734086Z","shell.execute_reply.started":"2023-02-05T18:24:13.197427Z","shell.execute_reply":"2023-02-05T18:24:27.733015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weighted_random_sampler = WeightedRandomSampler(sample_weights, len(sample_weights))","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:27.735731Z","iopub.execute_input":"2023-02-05T18:24:27.736474Z","iopub.status.idle":"2023-02-05T18:24:27.742656Z","shell.execute_reply.started":"2023-02-05T18:24:27.736428Z","shell.execute_reply":"2023-02-05T18:24:27.741401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nbatch_size = 32\n\n# Applying random sampler just tu train dataset, not for validation, since the validation dataset should be imitation of 'real' DS\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, num_workers = 2, pin_memory = True, sampler = weighted_random_sampler)\nval_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle = True, pin_memory = True)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:27.749003Z","iopub.execute_input":"2023-02-05T18:24:27.749738Z","iopub.status.idle":"2023-02-05T18:24:27.75662Z","shell.execute_reply.started":"2023-02-05T18:24:27.749695Z","shell.execute_reply":"2023-02-05T18:24:27.755387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataloaders = {'train' : train_dataloader, 'val' : val_dataloader}\ndataset_sizes = {'train': train_size, 'val' : val_size}","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:27.758598Z","iopub.execute_input":"2023-02-05T18:24:27.759161Z","iopub.status.idle":"2023-02-05T18:24:27.768561Z","shell.execute_reply.started":"2023-02-05T18:24:27.759118Z","shell.execute_reply":"2023-02-05T18:24:27.767439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_dataset), len(val_dataset))\nprint(len(train_dataloader), len(val_dataloader))","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:27.770359Z","iopub.execute_input":"2023-02-05T18:24:27.770917Z","iopub.status.idle":"2023-02-05T18:24:27.780643Z","shell.execute_reply.started":"2023-02-05T18:24:27.770877Z","shell.execute_reply":"2023-02-05T18:24:27.779498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = 5\ncols = 5\nplt.subplots(rows, cols, figsize = (20, 20))\n\nbatch_imgs, batch_labels = next(iter(train_dataloader))\ni = 0\nfor img in batch_imgs:\n    if i >= rows*cols:\n        break\n    plt.subplot(rows, cols, i + 1)\n    plt.title(\"Cancer\" if batch_labels[i] == 1 else \"No cancer\")\n    plt.imshow(img.permute(1, 2, 0))\n\n    i += 1\n\nlabels_count = np.zeros(2)\nfor l in batch_labels:\n    labels_count[l] += 1 \n    \nprint(f'There are {labels_count[0]} negative and {labels_count[1]} positive samples in this batch.')","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:27.782418Z","iopub.execute_input":"2023-02-05T18:24:27.782795Z","iopub.status.idle":"2023-02-05T18:24:32.646515Z","shell.execute_reply.started":"2023-02-05T18:24:27.782766Z","shell.execute_reply":"2023-02-05T18:24:32.645292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img.size()","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:32.648544Z","iopub.execute_input":"2023-02-05T18:24:32.648906Z","iopub.status.idle":"2023-02-05T18:24:32.656245Z","shell.execute_reply.started":"2023-02-05T18:24:32.648871Z","shell.execute_reply":"2023-02-05T18:24:32.655382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'Current device is {device}')","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:32.658147Z","iopub.execute_input":"2023-02-05T18:24:32.658989Z","iopub.status.idle":"2023-02-05T18:24:32.665996Z","shell.execute_reply.started":"2023-02-05T18:24:32.658932Z","shell.execute_reply":"2023-02-05T18:24:32.664793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass CNN(nn.Module):\n    def __init__(self):\n        super(CNN, self).__init__()\n        self.network = models.resnet18(pretrained=True)\n        n_features = self.network.fc.out_features\n        print(n_features)\n        # add additional layer that maps 2048 extracted features from resnet to 1 feature determining the class\n        self.classifier_layer = nn.Sequential(\n            nn.Linear(n_features , 256),\n            nn.Dropout(0.3),\n            nn.Linear(256 , 1)\n        )\n    \n    def forward(self, xb):        \n        xb = self.network(xb)\n        xb = self.classifier_layer(xb)\n        return torch.sigmoid(xb)","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:32.667632Z","iopub.execute_input":"2023-02-05T18:24:32.668258Z","iopub.status.idle":"2023-02-05T18:24:32.676501Z","shell.execute_reply.started":"2023-02-05T18:24:32.668225Z","shell.execute_reply":"2023-02-05T18:24:32.675278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create class for earlystopping\nclass EarlyStopper:\n    def __init__(self, patience=1, min_delta=0):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.counter = 0\n        self.min_loss = np.inf\n\n    def early_stop(self, loss):\n        if loss <= self.min_loss:\n            self.min_loss = loss\n            self.counter = 0\n        elif loss > (self.min_loss + self.min_delta):\n            self.counter += 1\n            if self.counter >= self.patience:\n                return True\n        return False","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:32.677956Z","iopub.execute_input":"2023-02-05T18:24:32.678729Z","iopub.status.idle":"2023-02-05T18:24:32.687211Z","shell.execute_reply.started":"2023-02-05T18:24:32.678694Z","shell.execute_reply":"2023-02-05T18:24:32.686245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def BCELoss_class_weighted(weights):\n    \"\"\"\n    weights[0] is weight for class 0 (negative class)\n    weights[1] is weight for class 1 (positive class)\n    \"\"\"\n    def loss(y_pred, target):\n        y_pred = torch.clamp(y_pred,min=1e-7,max=1-1e-7) # for numerical stability\n        bce = - weights[1] * target * torch.log(y_pred) - (1 - target) * weights[0] * torch.log(1 - y_pred)\n        return torch.mean(bce)\n\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:32.688416Z","iopub.execute_input":"2023-02-05T18:24:32.689179Z","iopub.status.idle":"2023-02-05T18:24:32.697433Z","shell.execute_reply.started":"2023-02-05T18:24:32.689144Z","shell.execute_reply":"2023-02-05T18:24:32.696456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# defining the model for determining LR\nmodel = CNN()\nmodel.to(device)\n# convrt weights to cuda.float if cuda is avaliable\n#if torch.cuda.is_available():\n#    model.cuda()\n\n# defining the optimizer\noptimizer = Adam(model.parameters(), lr=1e-07)\n\n\n# defining the loss function\n# Binary cross entropy is chosen because it is the classification problem\n#labels = dftrain['cancer'].values\n#w_neg = sum(labels == 0) / len(labels)\n#w_pos = sum(labels == 1) / len(labels)\n#print(f\"Class weight: {w_neg}\")\n#criterion = BCELoss_class_weighted(weights = [w_neg, w_pos])\n# criterion = nn.BCEWithLogitsLoss()\n\nw_pos = 3\nw_neg = 1\nprint(f\"Class weight for negative class: {w_neg}, and for positive {w_pos}\")\ncriterion = BCELoss_class_weighted(weights = [w_neg, w_pos])\n\nmetric = BinaryF1Score().to(device)\n\n# print(model)","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:32.698648Z","iopub.execute_input":"2023-02-05T18:24:32.699514Z","iopub.status.idle":"2023-02-05T18:24:32.990505Z","shell.execute_reply.started":"2023-02-05T18:24:32.699481Z","shell.execute_reply":"2023-02-05T18:24:32.989252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:32.992702Z","iopub.execute_input":"2023-02-05T18:24:32.994006Z","iopub.status.idle":"2023-02-05T18:24:33.003122Z","shell.execute_reply.started":"2023-02-05T18:24:32.993938Z","shell.execute_reply":"2023-02-05T18:24:33.001787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def determine_lrs_and_losses(model, criterion, optimizer, metric, num_epochs=25, final_lr = 1e-02, total_batches = 1):\n    since = time.time()\n    \n    lr_list = []\n    loss_list = []\n    \n    train_metrics = {'loss' : [], 'acc' : [], 'f1': []}\n    val_metrics = {'loss' : [], 'acc' : [], 'f1': []}\n    \n    \n    print('Starting training...')\n    print('-' * 20)\n    for epoch in range(num_epochs):\n        \n        # Each epoch has a training and validation phase\n        for phase in ['train']:\n            \n            \n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n            running_f1 = 0.0\n            \n           \n            gc.collect()\n            \n            current_batch = 0\n            # Iterate over data.\n            for inputs, labels in dataloaders[phase]:\n                \n                labels = torch.unsqueeze(labels.to(torch.float32), 1)\n                current_batch += 1\n                if current_batch > total_batches:\n                    break\n\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    # this was different, it took max of output and 1\n                    # output should never be higher than 1, so it is confusing\n                    preds = outputs > 0.5\n                    # _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs.double(), labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()  \n\n                    #print(labels.detach().numpy().type,  outputs.detach().numpy().type)\n                #running_f1 += f1_score(labels.detach().numpy(), outputs.detach().numpy())\n                running_f1 += metric(outputs, labels)\n\n                # statistics\n                running_loss += loss.item() \n                #print(f'{phase}, {inputs.size(0)}, {preds.size()} {torch.squeeze(labels.data).size()}')\n                running_corrects += torch.sum(preds == labels.data)\n                \n                gc.collect()\n\n                \n            \n            epoch_loss = running_loss / total_batches \n            epoch_acc = running_corrects.double() / (total_batches * batch_size)\n            epoch_f1 = running_f1 / total_batches\n            if phase == 'train':\n                train_metrics['loss'].append(epoch_loss)\n                train_metrics['acc'].append(epoch_acc)\n                train_metrics['f1'].append(epoch_f1)\n\n            else:\n                val_metrics['loss'].append(epoch_loss)\n                val_metrics['acc'].append(epoch_acc)\n                val_metrics['f1'].append(epoch_f1)\n\n                \n        train_loss_l, train_acc_l, train_f1_l = train_metrics['loss'][-1], train_metrics['acc'][-1], train_metrics['f1'][-1] # cant be formated in string, so should be segregated separately\n        lr = optimizer.param_groups[0]['lr']\n        print(f'Epoch {epoch + 1}/{num_epochs}, Train Loss: {train_loss_l:.4f}, Train Acc: {train_acc_l:.4f}, Train f1: {train_f1_l:.4f}, learning rate: {lr}')\n\n        # set learning rate for optimizer for determining initial learning rate\n        for g in optimizer.param_groups:\n            g['lr'] *= 4\n        \n        \n        loss_list.append(train_loss_l) # the goal is to determine which learning rate results\n        # in steepest training loss difference\n        lr_list.append(optimizer.param_groups[0]['lr'])\n        \n        if optimizer.param_groups[0]['lr'] > final_lr:\n            break\n\n        \n\n    return lr_list, loss_list","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:33.005495Z","iopub.execute_input":"2023-02-05T18:24:33.00637Z","iopub.status.idle":"2023-02-05T18:24:33.024985Z","shell.execute_reply.started":"2023-02-05T18:24:33.006331Z","shell.execute_reply":"2023-02-05T18:24:33.023992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function that finds steepest descent in training loss\ndef determine_init_lr(lr_list, loss_list):\n    # find difference beetwen succesive losses\n    diffs = [j-i for i, j in zip(loss_list[:-1], loss_list[1:])]\n    # find where loss change is maximum\n    max_value_ind = np.argmin(diffs) + 1\n    # get learning rate for that change\n    print(f\"Learning rate {lr_list[max_value_ind]} resulted in biggest loss decrease and should be starting learning rate for this neural net\")\n\n    init_lr = lr_list[max_value_ind]\n    return init_lr","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:33.028633Z","iopub.execute_input":"2023-02-05T18:24:33.028894Z","iopub.status.idle":"2023-02-05T18:24:33.038268Z","shell.execute_reply.started":"2023-02-05T18:24:33.02887Z","shell.execute_reply":"2023-02-05T18:24:33.037291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_over_loss(lr_list, loss_list, init_lr):\n    lr_ind = lr_list.index(init_lr)\n    plt.figure(figsize = (15, 7))\n    p1 = plt.plot(lr_list, loss_list)\n    p2 = plt.scatter(lr_list, loss_list)\n    p3 = plt.scatter(lr_list[lr_ind], loss_list[lr_ind], marker = 'D', s = 80, color = 'r')\n    plt.legend((p2, p3), (\"all considered learning rates\", \"best learning rate\"))\n    plt.xlabel(\"Learning rate\")\n    plt.ylabel(\"Loss\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:33.04147Z","iopub.execute_input":"2023-02-05T18:24:33.041782Z","iopub.status.idle":"2023-02-05T18:24:33.052373Z","shell.execute_reply.started":"2023-02-05T18:24:33.041758Z","shell.execute_reply":"2023-02-05T18:24:33.051439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_list, loss_list = determine_lrs_and_losses(model, criterion, optimizer, metric, num_epochs=25, final_lr = 1e-02, total_batches = 10)\ninit_lr = determine_init_lr(lr_list, loss_list)\nplot_lr_over_loss(lr_list, loss_list, init_lr)","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:24:33.053746Z","iopub.execute_input":"2023-02-05T18:24:33.056865Z","iopub.status.idle":"2023-02-05T18:25:39.973476Z","shell.execute_reply.started":"2023-02-05T18:24:33.056837Z","shell.execute_reply":"2023-02-05T18:25:39.971561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# defining the model\nmodel = CNN()\nmodel.to(device)\n# convrt weights to cuda.float if cuda is avaliable\n#if torch.cuda.is_available():\n#    model.cuda()\n\n# defining the optimizer\noptimizer = Adam(model.parameters(), lr=init_lr)\n# defining learning rate schedualer to fight plateues\n# TODO: figure out how to measure validation loss independently\n# scheduler = ReduceLROnPlateau(optimizer, 'min', patience = 5)\nscheduler = StepLR(optimizer, step_size=5, gamma=0.1)\n# defining the loss function\n# Binary cross entropy is chosen because it is the classification problem\nlabels = dftrain['cancer'].values\n# the weight should be smaller if class count is higher\nneg_count = sum(labels == 0)\npos_count = sum(labels == 1)\nw_pos = 2\nw_neg = 1\nprint(f\"Class weight for negative class: {w_neg}, and for positive {w_pos}\")\ncriterion = BCELoss_class_weighted(weights = [w_neg, w_pos])\n# criterion = nn.BCEWithLogitsLoss()\n# define early stopping\nearlystoper = EarlyStopper(patience = 3)\n\n\ncheckpoint = {'model': CNN(),\n          'state_dict': model.state_dict(),\n          'optimizer' : optimizer.state_dict(),\n             'threshold' : 0.5}\n\n\n# print(model)","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:25:39.97531Z","iopub.execute_input":"2023-02-05T18:25:39.975668Z","iopub.status.idle":"2023-02-05T18:25:40.653061Z","shell.execute_reply.started":"2023-02-05T18:25:39.975632Z","shell.execute_reply":"2023-02-05T18:25:40.651837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_optim_thres(fpr, tpr, thresholds):\n    optim_thres = thresholds[0]\n    inx = 0\n    min_dist = 1.0\n    for i in range(len(fpr)):\n        dist = np.linalg.norm(np.array([0.0, 1.0]) - np.array([fpr[i], tpr[i]]))\n        if dist < min_dist:\n            min_dist = dist\n            optim_thres = thresholds[i]\n            inx = i\n            \n    return optim_thres, inx\n        ","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:25:40.655129Z","iopub.execute_input":"2023-02-05T18:25:40.655826Z","iopub.status.idle":"2023-02-05T18:25:40.663881Z","shell.execute_reply.started":"2023-02-05T18:25:40.655786Z","shell.execute_reply":"2023-02-05T18:25:40.662632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n    since = time.time()\n    \n    metricf1 = BinaryF1Score()\n    precision = BinaryPrecision()\n    recall = BinaryRecall()\n    accuracy = BinaryAccuracy()\n    roc = BinaryROC()\n    auc = BinaryAUROC()\n    \n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_f1 = -1.0\n    \n    train_metrics = {'loss' : [], 'acc' : [], 'f1': [], 'precision': [], 'recall': [], 'auc': []}\n    val_metrics = {'loss' : [], 'acc' : [], 'f1': [], 'precision': [], 'recall': [], 'auc': []}\n    \n    \n    # inital threshold for first epoch, it will change afterwards\n    threshold = 0.5\n    \n    print('Starting training...')\n    print('-' * 20)\n    for epoch in range(num_epochs):\n        \n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            # empty 'all' tensors for saving\n            # for calculating aoc at the end of epoch, and for calculating new threshold\n            all_outputs = torch.Tensor([])\n            all_labels = torch.Tensor([])\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            n_samples = 0\n            \n            n_correct = 0\n            running_f1 = 0.0\n            # Iterate over data.\n            print(f'{phase} for epoch {epoch + 1}')\n            for inputs, labels in tqdm(dataloaders[phase]):\n                \n                labels = torch.unsqueeze(labels.to(torch.float32), 1)\n                \n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    preds = (outputs > threshold).double()\n                    #print(all_outputs)\n                    #print(outputs)\n                    # concatenating all outputs and labels for calculation aoc and new threshold\n                    all_outputs = torch.cat((all_outputs, outputs.to('cpu')))\n                    all_labels = torch.cat((all_labels, labels.to('cpu')))\n                    \n                    #print(labels)\n                    # _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                # n_samples += labels.size(0)\n                running_loss += loss.item()\n                # n_correct += (preds == labels).sum().item()\n                # running_f1 += metric(outputs, labels) \n\n\n                # collect any unused memmory\n                gc.collect()\n                torch.cuda.empty_cache()\n            \n            # statistics\n            epoch_loss = running_loss / len(dataloaders[phase])\n            \n            # find true positive and false positive rates for ROC curve\n            fpr, tpr, thresholds = roc(all_outputs, all_labels)\n            epoch_auc = auc(all_outputs, all_labels)\n            # find new threshold\n            threshold, _ = find_optim_thres(fpr, tpr, thresholds)\n            print(f'New threshold is {threshold}')\n            # calculate metrics using new optimized threshold\n            epoch_f1 = metricf1(all_outputs > threshold, all_labels)\n            epoch_acc = accuracy(all_outputs > threshold, all_labels)\n            epoch_precision = precision(all_outputs > threshold, all_labels)\n            epoch_recall = recall(all_outputs > threshold, all_labels)\n            \n            # save all of the statistics for latter analysis\n            if phase == 'train':\n                scheduler.step()\n                train_metrics['loss'].append(epoch_loss)\n                train_metrics['acc'].append(epoch_acc)\n                train_metrics['f1'].append(epoch_f1)\n                train_metrics['precision'].append(epoch_precision)\n                train_metrics['recall'].append(epoch_recall)\n                train_metrics['auc'].append(epoch_auc)\n\n\n            else:\n                val_metrics['loss'].append(epoch_loss)\n                val_metrics['acc'].append(epoch_acc)\n                val_metrics['f1'].append(epoch_f1)\n                val_metrics['precision'].append(epoch_precision)\n                val_metrics['recall'].append(epoch_recall)\n                val_metrics['auc'].append(epoch_auc)\n\n\n\n            # deep copy the model\n            if phase == 'val' and epoch_f1 > best_f1:\n                best_f1 = epoch_f1\n                best_model_wts = copy.deepcopy(model.state_dict())\n                checkpoint['threshold'] = threshold\n                torch.save(checkpoint, 'checkpoint.pth')\n\n                \n        # cant be formated in string\n        tr_loss, tr_acc, tr_f1, tr_prec, tr_rec, tr_auc = train_metrics['loss'][-1], train_metrics['acc'][-1],  train_metrics['f1'][-1], train_metrics['precision'][-1], train_metrics['recall'][-1], train_metrics['auc'][-1]\n        val_loss, val_acc, val_f1, val_prec, val_rec, val_auc = val_metrics['loss'][-1], val_metrics['acc'][-1], val_metrics['f1'][-1], val_metrics['precision'][-1], val_metrics['recall'][-1], val_metrics['auc'][-1]\n        lr = optimizer.param_groups[0]['lr']\n        print(f'Epoch {epoch + 1}/{num_epochs}, learning rate: {lr}')\n        print(f'Train Loss: {tr_loss:.4f}, Train Acc: {tr_acc:.4f}, Train f1: {tr_f1:.4f}, Train Precision: {tr_prec:.4f}, Train Recall: {tr_rec:.4f}, Train AUC: {tr_auc:.4f}')\n        print(f'Valitadion Loss: {val_loss:.4f}, Validation Acc: {val_acc:.4f}, Vall f1: {val_f1:.4f}, Val Precision: {val_prec:.4f}, Val Recall: {val_rec:.4f}, Val AUC: {val_auc:.4f}')\n        \n        if earlystoper.early_stop(val_loss):\n            break\n        \n        \n    time_elapsed = time.time() - since\n    print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')\n    print(f'Best val f1: {best_f1:4f}')\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, train_metrics, val_metrics","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:25:40.66596Z","iopub.execute_input":"2023-02-05T18:25:40.6667Z","iopub.status.idle":"2023-02-05T18:25:40.694402Z","shell.execute_reply.started":"2023-02-05T18:25:40.66664Z","shell.execute_reply":"2023-02-05T18:25:40.693414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, train_metrics, val_metrics = train_model(model, criterion, optimizer, scheduler, num_epochs=5)","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:25:40.70223Z","iopub.execute_input":"2023-02-05T18:25:40.703544Z","iopub.status.idle":"2023-02-05T18:26:20.499335Z","shell.execute_reply.started":"2023-02-05T18:25:40.703496Z","shell.execute_reply":"2023-02-05T18:26:20.497448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = plt.subplots(6, 2, figsize = (18, 12))\nkeys = ['loss', 'acc', 'f1', 'precision', 'recall', 'auc']\ni = 0\nfor key in keys:\n    metric = [x for x in train_metrics[key]]\n    plt.subplot(6, 2, 2*i + 1)\n    plt.plot(range(1, len(metric) + 1), metric)\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(f\"{key}\")\n    \n    \n    metric = [x for x in val_metrics[key]]\n    plt.subplot(6, 2, 2*i + 2)\n    plt.plot(range(1, len(metric) + 1), metric)\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(f\"{key}\")\n    i += 1\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:26:20.500121Z","iopub.status.idle":"2023-02-05T18:26:20.500463Z","shell.execute_reply.started":"2023-02-05T18:26:20.500297Z","shell.execute_reply":"2023-02-05T18:26:20.500314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_weights = '/kaggle/working/checkpoint.pth'\n\ncheckpoint = torch.load(path_to_weights)\nmodel, best_weights, optimizer, threshold = checkpoint['model'], checkpoint['state_dict'], checkpoint['optimizer'], checkpoint['threshold']\nmodel.load_state_dict(best_weights)\nmodel.to(device)\n\nwith torch.no_grad():\n    n_correct = 0\n    n_samples = 0\n    false_positives = []\n    false_negatives = []\n    y_pred, y_true = [], []\n\n    for images, labels in tqdm(val_dataloader):\n            images = images.to(device)\n            labels = labels.to(device)\n            outputs = model(images)\n\n            predicted = outputs > threshold\n            n_samples += labels.size(0)\n            n_correct += (torch.squeeze(predicted) == labels).sum().item()\n            y_pred.append(np.array(torch.squeeze(predicted.cpu()), dtype = 'int32'))\n            y_true.append(np.array(torch.squeeze(labels.cpu()), dtype = 'int32'))\n            \n\n            #if predicted != labels[i]:\n            #    if predicted == 1:\n            #        false_positives.append(images)\n            #    else:\n            #        false_negatives.append(images)\n            \n    acc = 100.0 * n_correct / n_samples\n    print(f'Accuracy of the network on the {n_samples} test images: {acc} %')\n\n    y_true = np.concatenate(y_true, axis = 0)\n    y_pred = np.concatenate(y_pred, axis = 0)","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:26:20.502831Z","iopub.status.idle":"2023-02-05T18:26:20.503695Z","shell.execute_reply.started":"2023-02-05T18:26:20.503434Z","shell.execute_reply":"2023-02-05T18:26:20.50346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:26:20.505164Z","iopub.status.idle":"2023-02-05T18:26:20.506084Z","shell.execute_reply.started":"2023-02-05T18:26:20.505802Z","shell.execute_reply":"2023-02-05T18:26:20.505828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm = confusion_matrix(np.squeeze(np.array(y_true, dtype = 'int32')), np.squeeze(np.array(y_pred, dtype = 'int32')))\ngroup_names = ['True Negatives','False Positives', 'False Negatives','True Positives']\ngroup_counts = [\"{0:0.0f}\".format(value) for value in\n                cm.flatten()]\ngroup_percentages = [\"{0:.2%}\".format(value) for value in\n                     cm.flatten()/np.sum(cm)]\nlabels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n          zip(group_names,group_counts,group_percentages)]\nlabels = np.asarray(labels).reshape(2,2)\nplt.figure(figsize = (12,7))\nsns.heatmap(cm, annot=labels, fmt='', cmap='Blues')\n\n","metadata":{"execution":{"iopub.status.busy":"2023-02-05T18:26:20.507587Z","iopub.status.idle":"2023-02-05T18:26:20.50836Z","shell.execute_reply.started":"2023-02-05T18:26:20.508101Z","shell.execute_reply":"2023-02-05T18:26:20.508125Z"},"trusted":true},"execution_count":null,"outputs":[]}]}