{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://www.kaggle.com/code/jainamshah17/pytorch-starter-image-classification\n\nDataset Credit\n\nhttps://www.kaggle.com/datasets/qcqced/mayo-clinic-strip-ai-competition-1k-png-data","metadata":{}},{"cell_type":"code","source":"import os,sys\nimport cv2\nimport time\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data import RandomSampler\n\nimport torchvision.transforms as T\nimport torchvision.models as models\nfrom torchvision.utils import make_grid\nfrom torchvision.datasets import ImageFolder\n\nfrom matplotlib import pyplot as plt\nimport skimage.io\nfrom skimage.io import imread\nimport openslide\nfrom sklearn.model_selection import train_test_split\nDIR_TRAIN = \"../input/mayo-clinic-strip-ai/train/\"\n#DIR_VALID = \"../input/100-bird-species/valid/\"\nDIR_TEST = \"../input/mayo-clinic-strip-ai/test/\"\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:07.319776Z","iopub.execute_input":"2022-09-23T10:18:07.32011Z","iopub.status.idle":"2022-09-23T10:18:10.754996Z","shell.execute_reply.started":"2022-09-23T10:18:07.320035Z","shell.execute_reply":"2022-09-23T10:18:10.75381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append(\"../input/pretrained-model-weights-pytorch/\")","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:10.761593Z","iopub.execute_input":"2022-09-23T10:18:10.764325Z","iopub.status.idle":"2022-09-23T10:18:10.771677Z","shell.execute_reply.started":"2022-09-23T10:18:10.764284Z","shell.execute_reply":"2022-09-23T10:18:10.770637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import rasterio\nfrom rasterio.enums import Resampling\nfrom rasterio.transform import Affine","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:10.777594Z","iopub.execute_input":"2022-09-23T10:18:10.780287Z","iopub.status.idle":"2022-09-23T10:18:11.29662Z","shell.execute_reply.started":"2022-09-23T10:18:10.780246Z","shell.execute_reply":"2022-09-23T10:18:11.295112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/mayo-clinic-strip-ai/train.csv')\ntest_df = pd.read_csv('../input/mayo-clinic-strip-ai/test.csv')\nother_df = pd.read_csv('../input/mayo-clinic-strip-ai/other.csv')\nss_df = pd.read_csv('../input/mayo-clinic-strip-ai/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.302608Z","iopub.execute_input":"2022-09-23T10:18:11.302903Z","iopub.status.idle":"2022-09-23T10:18:11.337638Z","shell.execute_reply.started":"2022-09-23T10:18:11.302876Z","shell.execute_reply":"2022-09-23T10:18:11.335709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_imgs = os.listdir( '../input/mayo-clinic-strip-ai-competition-1k-png-data')\ntest_imgs = os.listdir(DIR_TEST)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.339012Z","iopub.execute_input":"2022-09-23T10:18:11.339338Z","iopub.status.idle":"2022-09-23T10:18:11.640847Z","shell.execute_reply.started":"2022-09-23T10:18:11.339304Z","shell.execute_reply":"2022-09-23T10:18:11.639854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image_path'] =  train_df.apply(lambda row : ('../input/mayo-clinic-strip-ai-competition-1k-png-data'  + str(row['image_id']) + str('.png') ), axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.643182Z","iopub.execute_input":"2022-09-23T10:18:11.643603Z","iopub.status.idle":"2022-09-23T10:18:11.66482Z","shell.execute_reply.started":"2022-09-23T10:18:11.643562Z","shell.execute_reply":"2022-09-23T10:18:11.66355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.666556Z","iopub.execute_input":"2022-09-23T10:18:11.667136Z","iopub.status.idle":"2022-09-23T10:18:11.684247Z","shell.execute_reply.started":"2022-09-23T10:18:11.6671Z","shell.execute_reply":"2022-09-23T10:18:11.683231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_imgs, valid_imgs = train_test_split(train_imgs[:100], test_size = 0.3, random_state = 42, stratify = None)\n#len(train_imgs), len(valid_imgs)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.685354Z","iopub.execute_input":"2022-09-23T10:18:11.685626Z","iopub.status.idle":"2022-09-23T10:18:11.689891Z","shell.execute_reply.started":"2022-09-23T10:18:11.685602Z","shell.execute_reply":"2022-09-23T10:18:11.68888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ['CE','LAA']\nclass_to_int = {classes[i] : i for i in range(len(classes))}","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.691566Z","iopub.execute_input":"2022-09-23T10:18:11.692421Z","iopub.status.idle":"2022-09-23T10:18:11.697735Z","shell.execute_reply.started":"2022-09-23T10:18:11.692366Z","shell.execute_reply":"2022-09-23T10:18:11.696761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.702938Z","iopub.execute_input":"2022-09-23T10:18:11.703661Z","iopub.status.idle":"2022-09-23T10:18:11.71397Z","shell.execute_reply.started":"2022-09-23T10:18:11.703625Z","shell.execute_reply":"2022-09-23T10:18:11.712945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\n\nskf = StratifiedKFold(n_splits=5,\n                      shuffle=True,\n                      random_state=42)\nX = train_df.loc[:, train_df.columns != \"label\"]\ny = train_df.loc[:, train_df.columns == \"label\"]\ntrain_df['fold'] = -1","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.715496Z","iopub.execute_input":"2022-09-23T10:18:11.716291Z","iopub.status.idle":"2022-09-23T10:18:11.725573Z","shell.execute_reply.started":"2022-09-23T10:18:11.716256Z","shell.execute_reply":"2022-09-23T10:18:11.724591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.727011Z","iopub.execute_input":"2022-09-23T10:18:11.727599Z","iopub.status.idle":"2022-09-23T10:18:11.740877Z","shell.execute_reply.started":"2022-09-23T10:18:11.727564Z","shell.execute_reply":"2022-09-23T10:18:11.739875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (_, test_index) in enumerate(skf.split(X,y)):\n    train_df.iloc[test_index, -1] = i\n    \ntrain_df['fold'] = train_df['fold'].astype('int')\n\n\ntrain_df['is_valid'] = False\ntrain_df['is_valid'][train_df['fold'] == 0] = True","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.742456Z","iopub.execute_input":"2022-09-23T10:18:11.742927Z","iopub.status.idle":"2022-09-23T10:18:11.76115Z","shell.execute_reply.started":"2022-09-23T10:18:11.74289Z","shell.execute_reply":"2022-09-23T10:18:11.760192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.fold.unique()","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.762231Z","iopub.execute_input":"2022-09-23T10:18:11.762477Z","iopub.status.idle":"2022-09-23T10:18:11.770276Z","shell.execute_reply.started":"2022-09-23T10:18:11.762454Z","shell.execute_reply":"2022-09-23T10:18:11.769075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold, (train_index, valid_index) in enumerate(skf.split(X, y)): \n    \n    df_train = train_df.loc[train_index,:]\n    df_valid = train_df.loc[valid_index,:]\n    \n    print(f\"There are {df_train['label'].value_counts()[0]} CE labels and {df_train['label'].value_counts()[1]} LAA labels in train dataframe\")\n    print(f\"There are {df_valid['label'].value_counts()[0]} CE labels and {df_valid['label'].value_counts()[1]} LAA labels in validation dataframe\")\n    break","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.771923Z","iopub.execute_input":"2022-09-23T10:18:11.77272Z","iopub.status.idle":"2022-09-23T10:18:11.787076Z","shell.execute_reply.started":"2022-09-23T10:18:11.772684Z","shell.execute_reply":"2022-09-23T10:18:11.786194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_valid.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:18:11.788367Z","iopub.execute_input":"2022-09-23T10:18:11.788717Z","iopub.status.idle":"2022-09-23T10:18:11.796828Z","shell.execute_reply.started":"2022-09-23T10:18:11.788684Z","shell.execute_reply":"2022-09-23T10:18:11.7956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\npredictions = torch.zeros(size = (len(df_valid), 1), dtype=torch.float32, device=device)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:43:38.260259Z","iopub.execute_input":"2022-09-23T10:43:38.260637Z","iopub.status.idle":"2022-09-23T10:43:40.922174Z","shell.execute_reply.started":"2022-09-23T10:43:38.260605Z","shell.execute_reply":"2022-09-23T10:43:40.921211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Loading Classification Dataset - FOR METHOD 2: For multi-class data, by inheriting Dataset class\n\ndef get_transform():\n    return T.Compose([T.ToTensor()])\n\nclass MayoDataset(Dataset):\n    \n    def __init__(self, imgs_list, class_to_int, transforms = None, is_test = False):\n        \n        super().__init__()\n        self.imgs_list = imgs_list\n        self.class_to_int = class_to_int\n        self.transforms = transforms\n        self.is_test = is_test\n        \n    def __getitem__(self, index):\n        if self.is_test:\n            \n            image_path = self.imgs_list[index]\n            print (image_path)\n            #Reading image\n            image = rasterio.open(DIR_TEST+image_path)\n            image = image.read(out_shape=(3, int(1024), int(1024)),\n                              resampling=Resampling.bilinear).transpose(1,2,0)\n            patient_ids = image_path.split('.')[0]\n\n            if self.transforms is not None:\n                  image = self.transforms(image)\n\n            return image, patient_ids \n        \n        else:\n            \n            img_path = self.imgs_list[index]\n            img_path = '../input/mayo-clinic-strip-ai-competition-1k-png-data/'+img_path+'.png'\n            image = cv2.imread(img_path)\n            #print (img_path)\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            image = cv2.resize(image, (256,256))\n           \n\n            #Retriving class label\n            label = train_df.loc[train_df.image_id == self.imgs_list[index].split(\".\")[0],'label'].values[0]\n            label = self.class_to_int[label]\n        \n            #Applying transforms on image\n            if self.transforms:\n                image = self.transforms(image)\n\n            return image, label\n        \n        \n    def __len__(self):\n        return len(self.imgs_list)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:43:44.08617Z","iopub.execute_input":"2022-09-23T10:43:44.086652Z","iopub.status.idle":"2022-09-23T10:43:44.101028Z","shell.execute_reply.started":"2022-09-23T10:43:44.08661Z","shell.execute_reply":"2022-09-23T10:43:44.099983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Loading Classification Dataset\n\n\"\"\"\n# Method 1: For multi-class data directly from folders using ImageFolder\ntrain_dataset = ImageFolder(root = DIR_TRAIN, transform = T.ToTensor())\nvalid_dataset = ImageFolder(root = DIR_VALID, transform = T.ToTensor())\ntest_dataset = ImageFolder(root = DIR_TEST, transform = T.ToTensor())\n\"\"\"\ndef custom_loader(fold=0):\n        train_data = train_df.loc[train_df['fold']!=fold]\n        valid_data = train_df.loc[train_df['fold']==fold]\n        # Method 2: Using Dataset Class\n        print (train_data, valid_data)\n        train_dataset = MayoDataset(train_data['image_id'].values, class_to_int, get_transform())\n        valid_dataset = MayoDataset(valid_data['image_id'].values, class_to_int, get_transform())\n        #test_dataset = MayoDataset(test_imgs, class_to_int, get_transform())\n\n        #Data Loader  -  using Sampler (YT Video)\n        train_random_sampler = RandomSampler(train_dataset)\n        valid_random_sampler = RandomSampler(valid_dataset)\n        #test_random_sampler = RandomSampler(test_dataset)\n\n        #Shuffle Argument is mutually exclusive with Sampler!\n        train_data_loader = DataLoader(\n            dataset = train_dataset,\n            batch_size = 12,\n            sampler = train_random_sampler,\n            num_workers = 4,\n        )\n\n        valid_data_loader = DataLoader(\n            dataset = valid_dataset,\n            batch_size = 12,\n            sampler = valid_random_sampler,\n            num_workers = 4,\n        )\n        '''\n        test_data_loader = DataLoader(\n            dataset = test_dataset,\n            batch_size = 4,\n            sampler = test_random_sampler,\n            num_workers = 4,\n        )\n        '''\n        return train_data_loader, valid_data_loader","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:44:37.757349Z","iopub.execute_input":"2022-09-23T10:44:37.757807Z","iopub.status.idle":"2022-09-23T10:44:37.766593Z","shell.execute_reply.started":"2022-09-23T10:44:37.757768Z","shell.execute_reply":"2022-09-23T10:44:37.7651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader, valid_loader = custom_loader(fold = 1)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:44:40.600737Z","iopub.execute_input":"2022-09-23T10:44:40.60111Z","iopub.status.idle":"2022-09-23T10:44:40.625157Z","shell.execute_reply.started":"2022-09-23T10:44:40.601077Z","shell.execute_reply":"2022-09-23T10:44:40.624135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Visualize one training batch\nfor images, labels in train_loader:\n    fig, ax = plt.subplots(figsize = (10, 10))\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.imshow(make_grid(images, 4).permute(1,2,0))\n    break","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:44:45.942957Z","iopub.execute_input":"2022-09-23T10:44:45.943375Z","iopub.status.idle":"2022-09-23T10:44:47.967587Z","shell.execute_reply.started":"2022-09-23T10:44:45.943336Z","shell.execute_reply":"2022-09-23T10:44:47.96648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Define model\nmodel = models.vgg16(pretrained = False)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:44:56.369438Z","iopub.execute_input":"2022-09-23T10:44:56.37018Z","iopub.status.idle":"2022-09-23T10:44:58.484094Z","shell.execute_reply.started":"2022-09-23T10:44:56.370137Z","shell.execute_reply":"2022-09-23T10:44:58.483004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = efficientnet_pytorch.EfficientNet.from_name(\"vgg16-397923af.pth\")\ncheckpoint = torch.load('../input/pretrained-model-weights-pytorch/vgg16-397923af.pth')\nmodel.load_state_dict(checkpoint)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:45:01.305213Z","iopub.execute_input":"2022-09-23T10:45:01.3056Z","iopub.status.idle":"2022-09-23T10:45:07.655307Z","shell.execute_reply.started":"2022-09-23T10:45:01.305564Z","shell.execute_reply":"2022-09-23T10:45:07.654226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n### Modifying last few layers and no of classes\n# NOTE: cross_entropy loss takes unnormalized op (logits), then function itself applies softmax and calculates loss, so no need to include softmax here\nmodel.classifier = nn.Sequential(\n    nn.Linear(25088, 4096, bias = True),\n    nn.ReLU(inplace = True),\n    nn.Dropout(0.4),\n    nn.Linear(4096, 2048, bias = True),\n    nn.ReLU(inplace = True),\n    nn.Dropout(0.4),\n    nn.Linear(2048, 2)\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:45:09.581169Z","iopub.execute_input":"2022-09-23T10:45:09.58155Z","iopub.status.idle":"2022-09-23T10:45:10.611872Z","shell.execute_reply.started":"2022-09-23T10:45:09.581504Z","shell.execute_reply":"2022-09-23T10:45:10.610694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Get device\n\n#device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\ntorch.cuda.empty_cache()\n\nmodel.to(device)\n\n### Training Details\n\noptimizer = torch.optim.Adam(model.parameters(), lr = 0.0001)\nlr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size = 5, gamma = 0.75)\ncriterion = nn.CrossEntropyLoss()\n\ntrain_loss = []\ntrain_accuracy = []\n\nval_loss = []\nval_accuracy = []\n\nepochs = 25","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:45:22.635512Z","iopub.execute_input":"2022-09-23T10:45:22.635902Z","iopub.status.idle":"2022-09-23T10:45:22.781501Z","shell.execute_reply.started":"2022-09-23T10:45:22.635871Z","shell.execute_reply":"2022-09-23T10:45:22.780553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_accuracy(true,pred):\n    pred = F.softmax(pred, dim = 1)\n    true = torch.zeros(pred.shape[0], pred.shape[1]).scatter_(1, true.unsqueeze(1), 1.)\n    acc = (true.argmax(-1) == pred.argmax(-1)).float().detach().numpy()\n    acc = float((100 * acc.sum()) / len(acc))\n    return round(acc, 4)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:45:26.065121Z","iopub.execute_input":"2022-09-23T10:45:26.065729Z","iopub.status.idle":"2022-09-23T10:45:26.072221Z","shell.execute_reply.started":"2022-09-23T10:45:26.065691Z","shell.execute_reply":"2022-09-23T10:45:26.07123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:45:29.066396Z","iopub.execute_input":"2022-09-23T10:45:29.067364Z","iopub.status.idle":"2022-09-23T10:45:29.08212Z","shell.execute_reply.started":"2022-09-23T10:45:29.067319Z","shell.execute_reply":"2022-09-23T10:45:29.081077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold, (train_index, valid_index) in enumerate(skf.split(X, y)):\n    #print (train_index, valid_index)\n    print (fold)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:45:33.696392Z","iopub.execute_input":"2022-09-23T10:45:33.696982Z","iopub.status.idle":"2022-09-23T10:45:33.706503Z","shell.execute_reply.started":"2022-09-23T10:45:33.696943Z","shell.execute_reply":"2022-09-23T10:45:33.705373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://www.kaggle.com/code/alejopaullier/background-vs-clots-classifier","metadata":{}},{"cell_type":"code","source":"class config:\n    BATCH_SIZE_TRAIN = 32\n    BATCH_SIZE_VALIDATION = 16\n    BATCH_SIZE_TEST = 32\n    EPOCHS = 1\n    FOLDS = 5\n    LEARNING_RATE = 0.0005\n    LR_FACTOR = 0.4  # BY HOW MUCH THE LR IS DECREASING\n    LR_PATIENCE = 1  # 1 MODEL NOT IMPROVING UNTIL LR IS DECREASING\n    NUM_WORKERS = 1\n    OUTPUT_SIZE = 1\n    PATIENCE = 3\n    TTA = 3\n    WEIGHT_DECAY = 0.0","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:45:43.885348Z","iopub.execute_input":"2022-09-23T10:45:43.885733Z","iopub.status.idle":"2022-09-23T10:45:43.891517Z","shell.execute_reply.started":"2022-09-23T10:45:43.8857Z","shell.execute_reply":"2022-09-23T10:45:43.890552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim.lr_scheduler import ReduceLROnPlateau","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:45:50.784975Z","iopub.execute_input":"2022-09-23T10:45:50.785348Z","iopub.status.idle":"2022-09-23T10:45:50.790318Z","shell.execute_reply.started":"2022-09-23T10:45:50.785317Z","shell.execute_reply":"2022-09-23T10:45:50.789084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T12:05:36.864624Z","iopub.execute_input":"2022-09-23T12:05:36.865202Z","iopub.status.idle":"2022-09-23T12:05:36.8854Z","shell.execute_reply.started":"2022-09-23T12:05:36.865158Z","shell.execute_reply":"2022-09-23T12:05:36.883934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_function(predictions, train_df, valid_df, model, MayoDataset, device, version='v1'):\n    \"\"\"\n    This function iterates over folds. On each fold, the original train dataset is split into a new train subset and a\n    validation dataset. Test dataset is static through fold iterations. For each fold it trains the model for the\n    specified epochs. Hence, training as well as evaluation on validation dataset is performed at an epoch level.\n    The amount of iterations is therefore FOLDS*EPOCHS. After training is complete, the model is evaluated on the\n    validation dataset and the model artifacts are saved if the metric is improved. Finally, at the last of every fold\n    iteration the evaluation metric is computed for the test dataset. Please note that model selection is performed\n    with respect to the validation metric and that no retraining on the original train dataset is performed.\n\n    :param predictions: predictions for the test set.\n    :param model: model architecture. \n    :param StripAIDataset: a custom dataset for this problem. \n    :param version: model version. Each time we train a new model we must create a new version.\n    :return oof: Out of Fold predictions. In each fold we predict the Validation set, in consequence, as validation\n    sets are non overlapping we end up with predictions for the whole train set.\n    :return predictions: predictions for the test set\n    \"\"\"\n    # Creates a .txt file that will contain the logs\n    f = open(f\"./logs/logs_{version}.txt\", \"w+\")\n\n    # Out of Fold Predictions\n    oof = np.zeros(shape=(len(train_df), 1))\n    print(f\"OOF shape: {oof.shape}\")\n\n    # Iterate over folds\n    for fold, (train_index, valid_index) in enumerate(skf.split(X, y)):\n        # Append to .txt\n        with open(f\"./logs/logs_{version}.txt\", 'a+') as f:\n            print('-' * 10, 'Fold:', fold + 1, '-' * 10, file=f)\n        print('-' * 10, 'Fold:', fold + 1, '-' * 10)\n        \n        \n        # --- Create Instances ---\n        # Best ROC score in this fold\n        best_roc = None\n        # Reset patience before every fold.\n        patience_f = config.PATIENCE\n\n        # Initiate the model\n        model = model\n\n        # Create optimizer.\n        optimizer = torch.optim.Adam(model.parameters(),\n                                     lr=config.LEARNING_RATE,\n                                     weight_decay=config.WEIGHT_DECAY)\n\n        # Create scheduler.\n        scheduler = ReduceLROnPlateau(optimizer=optimizer,\n                                      mode='max',\n                                      patience=config.LR_PATIENCE,\n                                      verbose=True,\n                                      factor=config.LR_FACTOR)\n\n        # Create Loss. \n        criterion = nn.BCEWithLogitsLoss()\n\n        train_data = train_df.loc[train_df['fold']!=fold].values\n        valid_data = train_df.loc[train_df['fold']==fold].values\n        print (valid_data)\n        # Method 2: Using Dataset Class\n        #rint (train_data, valid_data)\n        train_dataset = MayoDataset(train_data, class_to_int, get_transform())\n        valid_dataset = MayoDataset(valid_data, class_to_int, get_transform())\n        #test_dataset = MayoDataset(test_imgs, class_to_int, get_transform())\n\n        #Data Loader  -  using Sampler (YT Video)\n        train_random_sampler = RandomSampler(train_dataset)\n        valid_random_sampler = RandomSampler(valid_dataset)\n        # Create Dataloaders\n        train_loader = DataLoader(train_dataset,\n                                  batch_size=config.BATCH_SIZE_TRAIN,\n                                  shuffle=True,\n                                  sampler = train_random_sampler,\n                                  num_workers=config.NUM_WORKERS)\n        # shuffle=False! Otherwise function won't work!!!\n        valid_loader = DataLoader(valid_dataset,\n                                  batch_size=config.BATCH_SIZE_VALIDATION,\n                                  shuffle=False,\n                                  sampler = valid_random_sampler,\n                                  num_workers=config.NUM_WORKERS)\n        test_loader = DataLoader(test,\n                                 batch_size=config.BATCH_SIZE_TEST,\n                                 shuffle=False,\n                                 num_workers=config.NUM_WORKERS)\n        print ('1......')\n\n        # === EPOCHS ===\n        epochs = config.EPOCHS\n        for epoch in range(epochs):\n            start_time = time.time()\n            correct = 0\n            train_losses = 0\n\n            # === TRAIN ===\n            # Sets the module in training mode.\n            model.train()\n\n            # === Iterate over batches ===\n            with tqdm(train_loader, unit=\"train_batch\") as tqdm_train_loader:\n                for images, labels in tqdm_train_loader:\n                    # Save them to device\n                    images = torch.tensor(images, device=device, dtype=torch.float32)\n                    labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n                    # Clear gradients first; very important, usually done BEFORE prediction\n                    optimizer.zero_grad()\n\n                    # Log Probabilities & Backpropagation\n                    out = model(images, verbose=False)\n                    loss = criterion(out, labels.unsqueeze(1))\n                    loss.backward()\n                    optimizer.step()\n\n                    # --- Save information after this batch ---\n                    # Save loss\n                    train_losses += loss.item()\n                    # From log probabilities to actual probabilities\n                    train_preds = torch.round(torch.sigmoid(out))  # 0 and 1\n                    # Number of correct predictions\n                    correct += (train_preds.cpu() == labels.cpu().unsqueeze(1)).sum().item()\n            # Compute Train Accuracy\n            train_acc = correct / len(train_index)\n\n            # === EVAL ===\n            # Sets the model in evaluation mode\n            model.eval()\n\n            # Create matrix to store evaluation predictions (for accuracy)\n            valid_preds = torch.zeros(size=(len(valid_index), 1), device=device, dtype=torch.float32)\n\n            # Disables gradients (we need to be sure no optimization happens)\n            with torch.no_grad():\n                for k, (images, labels) in enumerate(tqdm(valid_loader, unit=\"valid_batch\")):\n                    images = torch.tensor(images, device=device, dtype=torch.float32)\n                    out = model(images)\n                    pred = torch.sigmoid(out)\n                    valid_preds[k * images.shape[0]: k * images.shape[0] + images.shape[0]] = pred\n\n                # Compute accuracy\n                valid_acc = accuracy_score(valid_data['class'].values,\n                                           torch.round(valid_preds.cpu()))\n                # Compute ROC\n                valid_roc = roc_auc_score(valid_data['class'].values,\n                                          valid_preds.cpu())\n\n                # Compute time on Train + Eval\n                duration = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n\n                # PRINT INFO\n                # Append to .txt file\n                with open(f\"./logs/logs_{version}.txt\", 'a+') as f:\n                    print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'. \\\n                          format(duration, epoch + 1, epochs, train_losses, train_acc, valid_acc, valid_roc), file=f)\n                # Print to console\n                print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'. \\\n                      format(duration, epoch + 1, epochs, train_losses, train_acc, valid_acc, valid_roc))\n\n                # === SAVE MODEL ===\n\n                # Update scheduler (for learning_rate)\n                scheduler.step(valid_roc)\n\n                # Update best_roc\n                if not best_roc:  # If best_roc = None\n                    best_roc = valid_roc\n                    torch.save(model.state_dict(),\n                               f\"./saved_models/Fold{fold + 1}_Epoch{epoch + 1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n                    continue\n\n                if valid_roc > best_roc:\n                    best_roc = valid_roc\n                    # Reset patience (because we have improvement)\n                    patience_f = config.PATIENCE\n                    torch.save(model.state_dict(),\n                               f\"./saved_models/Fold{fold + 1}_Epoch{epoch + 1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n                else:\n                    # Decrease patience (no improvement in ROC)\n                    patience_f = patience_f - 1\n                    if patience_f == 0:\n                        with open(f\"./logs/logs_{version}.txt\", 'a+') as f:\n                            print('Early stopping (no improvement since 3 models) | Best ROC: {}'. \\\n                                  format(best_roc), file=f)\n                        print('Early stopping (no improvement since 3 models) | Best ROC: {}'. \\\n                              format(best_roc))\n                        break\n\n        # === INFERENCE ===\n        # Choose model with best_roc in this fold\n        best_model_path = './saved_models/' + [file for file in os.listdir('saved_models') if\n                                             str(round(best_roc, 3)) in file and 'Fold' + str(fold + 1) in file][0]\n        # Using best model from Epoch Train\n        model = EfficientNetwork(output_size=config.OUTPUT_SIZE,b4=False, b2=True).to(device)\n        model.load_state_dict(torch.load(best_model_path))\n        # Set the model in evaluation mode\n        model.eval()\n\n        with torch.no_grad():\n            # --- EVAL ---\n            # Predicting again on Validation data to get preds for OOF\n            valid_preds = torch.zeros(size=(len(valid_index), 1), device=device, dtype=torch.float32)\n\n            for k, (images, _) in enumerate(tqdm(valid_loader, unit=\"oof_batch\")):\n                images = torch.tensor(images, device=device, dtype=torch.float32)\n                out = model(images)\n                pred = torch.sigmoid(out)\n                valid_preds[k * images.shape[0]: k * images.shape[0] + images.shape[0]] = pred\n\n            # Save info to OOF\n            oof[valid_index] = valid_preds.cpu().numpy()\n            \n            # --- TEST ---\n            # Now (Finally) prediction for our TEST data\n            for i in range(config.TTA):\n                for k, images in enumerate(tqdm(test_loader, unit=f\"test_loader_TTA_{i}\")):\n                    images = torch.tensor(images, device=device, dtype=torch.float32)\n                    out = model(images)\n                    # Convert to probablities\n                    out = torch.sigmoid(out)\n                    # ADDS! the prediction to the matrix we already created\n                    predictions[k * images.shape[0]: k * images.shape[0] + images.shape[0]] += out\n\n            # Divide Predictions by TTA (to average the results during TTA)\n            predictions /= config.TTA\n\n        # === CLEANING ===\n        # Clear memory\n        del train, valid, train_loader, valid_loader, images, labels\n        # Garbage collector\n        gc.collect()\n\n    return oof, predictions","metadata":{"execution":{"iopub.status.busy":"2022-09-23T12:22:47.780492Z","iopub.execute_input":"2022-09-23T12:22:47.780868Z","iopub.status.idle":"2022-09-23T12:22:47.813626Z","shell.execute_reply.started":"2022-09-23T12:22:47.780836Z","shell.execute_reply":"2022-09-23T12:22:47.812585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!mkdir './logs'\n#!mkdir './saved_models'","metadata":{"execution":{"iopub.status.busy":"2022-09-23T10:46:04.298237Z","iopub.execute_input":"2022-09-23T10:46:04.298631Z","iopub.status.idle":"2022-09-23T10:46:06.249007Z","shell.execute_reply.started":"2022-09-23T10:46:04.298597Z","shell.execute_reply":"2022-09-23T10:46:06.247606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train\noof, predictions = train_function(predictions,\n                                   df_train,\n                                   df_valid,\n                                   model,\n                                   MayoDataset,\n                                   device)","metadata":{"execution":{"iopub.status.busy":"2022-09-23T12:24:00.802204Z","iopub.execute_input":"2022-09-23T12:24:00.802586Z","iopub.status.idle":"2022-09-23T12:24:00.846275Z","shell.execute_reply.started":"2022-09-23T12:24:00.802548Z","shell.execute_reply":"2022-09-23T12:24:00.839321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Training Code\n\nfor epoch in range(epochs):\n    \n    start = time.time()\n    \n    #Epoch Loss & Accuracy\n    train_epoch_loss = []\n    train_epoch_accuracy = []\n    _iter = 1\n    \n    #Val Loss & Accuracy\n    val_epoch_loss = []\n    val_epoch_accuracy = []\n    \n    # Training\n    for images, labels in train_loader:\n        \n        images = images.to(device)\n        labels = labels.to(device)\n        \n        #Reset Grads\n        optimizer.zero_grad()\n        \n        #Forward ->\n        preds = model(images)\n        \n        #Calculate Accuracy\n        acc = calc_accuracy(labels.cpu(), preds.cpu())\n        \n        #Calculate Loss & Backward, Update Weights (Step)\n        loss = criterion(preds, labels)\n        loss.backward()\n        optimizer.step()\n        \n        #Append loss & acc\n        loss_value = loss.item()\n        train_epoch_loss.append(loss_value)\n        train_epoch_accuracy.append(acc)\n        \n        if _iter % 500 == 0:\n            print(\"> Iteration {} < \".format(_iter))\n            print(\"Iter Loss = {}\".format(round(loss_value, 4)))\n            print(\"Iter Accuracy = {} % \\n\".format(acc))\n        \n        _iter += 1\n    \n    #Validation\n    for images, labels in valid_loader:\n        \n        images = images.to(device)\n        labels = labels.to(device)\n        \n        #Forward ->\n        preds = model(images)\n        \n        #Calculate Accuracy\n        acc = calc_accuracy(labels.cpu(), preds.cpu())\n        \n        #Calculate Loss\n        loss = criterion(preds, labels)\n        \n        #Append loss & acc\n        loss_value = loss.item()\n        val_epoch_loss.append(loss_value)\n        val_epoch_accuracy.append(acc)\n    \n    \n    train_epoch_loss = np.mean(train_epoch_loss)\n    train_epoch_accuracy = np.mean(train_epoch_accuracy)\n    \n    val_epoch_loss = np.mean(val_epoch_loss)\n    val_epoch_accuracy = np.mean(val_epoch_accuracy)\n    \n    end = time.time()\n    \n    train_loss.append(train_epoch_loss)\n    train_accuracy.append(train_epoch_accuracy)\n    \n    val_loss.append(val_epoch_loss)\n    val_accuracy.append(val_epoch_accuracy)\n    \n    #Print Epoch Statistics\n    print(\"** Epoch {} ** - Epoch Time {}\".format(epoch, int(end-start)))\n    print(\"Train Loss = {}\".format(round(train_epoch_loss, 4)))\n    print(\"Train Accuracy = {} % \\n\".format(train_epoch_accuracy))\n    print(\"Val Loss = {}\".format(round(val_epoch_loss, 4)))\n    print(\"Val Accuracy = {} % \\n\".format(val_epoch_accuracy))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.840049Z","iopub.status.idle":"2022-08-23T06:05:11.840866Z","shell.execute_reply.started":"2022-08-23T06:05:11.840583Z","shell.execute_reply":"2022-08-23T06:05:11.840608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(),'./vgg_model_epoch_5')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.842363Z","iopub.status.idle":"2022-08-23T06:05:11.843216Z","shell.execute_reply.started":"2022-08-23T06:05:11.84292Z","shell.execute_reply":"2022-08-23T06:05:11.842948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_dir = '../input/mayo-clinic-strip-ai/test'\ntest_dataset = MayoDataset(test_imgs, class_to_int, get_transform(), is_test= True)\ntest_random_sampler = RandomSampler(test_dataset)\n\n#Shuffle Argument is mutually exclusive with Sampler!\ntest_data_loader = DataLoader(\n    dataset = test_dataset,\n    batch_size = 4,\n    sampler = test_random_sampler,\n    num_workers = 4,\n)\n\ndef predict(model, test_data_loader): \n    #preds = np.zeros((4, 2))\n    ids = []\n    model.eval()\n    preds_part = []\n    \n    with torch.no_grad():\n        for images, patient_ids in test_data_loader:\n            images = images.to(device) #, dtype=torch.float32)\n            outputs = model(images)\n            \n            ids.append(patient_ids)\n            preds_part.append(torch.softmax(outputs.cpu(), dim=1).squeeze().numpy())\n            #preds[i*batch_size:(i+1)*batch_size] += preds_part\n            \n    return np.array(preds_part), ids\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.844768Z","iopub.status.idle":"2022-08-23T06:05:11.845531Z","shell.execute_reply.started":"2022-08-23T06:05:11.84526Z","shell.execute_reply":"2022-08-23T06:05:11.845283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds, ids = predict(model, test_data_loader)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.846901Z","iopub.status.idle":"2022-08-23T06:05:11.84764Z","shell.execute_reply.started":"2022-08-23T06:05:11.847373Z","shell.execute_reply":"2022-08-23T06:05:11.847407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = preds[-1]","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.849025Z","iopub.status.idle":"2022-08-23T06:05:11.849791Z","shell.execute_reply.started":"2022-08-23T06:05:11.849525Z","shell.execute_reply":"2022-08-23T06:05:11.849549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred[:,0], pred[:,1], ids","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.851134Z","iopub.status.idle":"2022-08-23T06:05:11.851875Z","shell.execute_reply.started":"2022-08-23T06:05:11.851602Z","shell.execute_reply":"2022-08-23T06:05:11.851626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.DataFrame({'patient_id' : ids[0][:], \"CE\": pred[:,0], \"LAA\" : pred[:,1]}).groupby(\"patient_id\").mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.853217Z","iopub.status.idle":"2022-08-23T06:05:11.85397Z","shell.execute_reply.started":"2022-08-23T06:05:11.853697Z","shell.execute_reply":"2022-08-23T06:05:11.853719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.855513Z","iopub.status.idle":"2022-08-23T06:05:11.856265Z","shell.execute_reply.started":"2022-08-23T06:05:11.856011Z","shell.execute_reply":"2022-08-23T06:05:11.856034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#result_table = pd.DataFrame({'patient_id' : ids, \"CE\" : preds[:,0], \"LAA\" : preds[:,1]}).groupby(\"patient_id\").mean()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.857628Z","iopub.status.idle":"2022-08-23T06:05:11.85839Z","shell.execute_reply.started":"2022-08-23T06:05:11.858123Z","shell.execute_reply":"2022-08-23T06:05:11.858147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm './vgg_model_epoch_5'","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.859751Z","iopub.status.idle":"2022-08-23T06:05:11.860527Z","shell.execute_reply.started":"2022-08-23T06:05:11.86024Z","shell.execute_reply":"2022-08-23T06:05:11.860263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nresult.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.861895Z","iopub.status.idle":"2022-08-23T06:05:11.862639Z","shell.execute_reply.started":"2022-08-23T06:05:11.862376Z","shell.execute_reply":"2022-08-23T06:05:11.862411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-08-23T06:05:11.86402Z","iopub.status.idle":"2022-08-23T06:05:11.864741Z","shell.execute_reply.started":"2022-08-23T06:05:11.864488Z","shell.execute_reply":"2022-08-23T06:05:11.864511Z"},"trusted":true},"execution_count":null,"outputs":[]}]}