{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#install openslide\n!apt update && apt install -y openslide-tools\n!pip install openslide-python","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-26T09:06:51.761004Z","iopub.execute_input":"2022-04-26T09:06:51.761332Z","iopub.status.idle":"2022-04-26T09:07:05.872195Z","shell.execute_reply.started":"2022-04-26T09:06:51.76126Z","shell.execute_reply":"2022-04-26T09:07:05.871295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import openslide\nfrom matplotlib import pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport os\nfrom sklearn.model_selection import StratifiedShuffleSplit \nimport torch\nimport torchvision\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset as D\nimport torchvision.transforms as transforms\nfrom torch import nn\n","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:08.456445Z","iopub.execute_input":"2022-04-26T09:07:08.456787Z","iopub.status.idle":"2022-04-26T09:07:08.466887Z","shell.execute_reply.started":"2022-04-26T09:07:08.456723Z","shell.execute_reply":"2022-04-26T09:07:08.465093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check train.csv\nDataset = pd.read_csv('../input/prostate-cancer-grade-assessment/train.csv')\nDataset","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:08.472478Z","iopub.execute_input":"2022-04-26T09:07:08.475132Z","iopub.status.idle":"2022-04-26T09:07:08.536189Z","shell.execute_reply.started":"2022-04-26T09:07:08.475092Z","shell.execute_reply":"2022-04-26T09:07:08.535523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test dataset is too small (consider only split train dataset to(training:60%,testing:20%,validation:20%))\ntestDataset = pd.read_csv('../input/prostate-cancer-grade-assessment/test.csv')\ntestDataset","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:08:09.44255Z","iopub.execute_input":"2022-04-25T16:08:09.442952Z","iopub.status.idle":"2022-04-25T16:08:09.458991Z","shell.execute_reply.started":"2022-04-25T16:08:09.442916Z","shell.execute_reply":"2022-04-25T16:08:09.458066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check if dataset contains any null value (no null value)\nDataset.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:12.459687Z","iopub.execute_input":"2022-04-26T09:07:12.460516Z","iopub.status.idle":"2022-04-26T09:07:12.471745Z","shell.execute_reply.started":"2022-04-26T09:07:12.460471Z","shell.execute_reply":"2022-04-26T09:07:12.470867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check how many classes are in gleason score (11 classes)\ngleason_score = Dataset.gleason_score.unique()\nprint('Classes in gleason score: ',len(gleason_score),'\\n')\nprint(Dataset.gleason_score.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:14.558309Z","iopub.execute_input":"2022-04-26T09:07:14.558586Z","iopub.status.idle":"2022-04-26T09:07:14.576598Z","shell.execute_reply.started":"2022-04-26T09:07:14.558555Z","shell.execute_reply":"2022-04-26T09:07:14.575843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n#visualize gleason score data\nglsply = sns.countplot(x='gleason_score',data=Dataset,color='dodgerblue') \nglsply.set(xlabel = None)\nglsply.set(title='gleason score')\nglsply.tick_params(axis='x', rotation=30)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T08:06:25.875585Z","iopub.execute_input":"2022-04-26T08:06:25.876283Z","iopub.status.idle":"2022-04-26T08:06:26.300667Z","shell.execute_reply.started":"2022-04-26T08:06:25.876228Z","shell.execute_reply":"2022-04-26T08:06:26.299981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check how many classes are in isUp grade(6 classes)\nisUp_grade= Dataset.isup_grade.unique()\nprint('Classes in isUp grade: ',len(isUp_grade),'\\n')\nprint(Dataset.isup_grade.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:17.543673Z","iopub.execute_input":"2022-04-26T09:07:17.544326Z","iopub.status.idle":"2022-04-26T09:07:17.553449Z","shell.execute_reply.started":"2022-04-26T09:07:17.544288Z","shell.execute_reply":"2022-04-26T09:07:17.552703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glsply = sns.countplot(x='isup_grade',data=Dataset,color='green') \nglsply.set(xlabel = None)\nglsply.set(title='isUp grade')\n#glsply.tick_params(axis='x', rotation=30)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T08:06:31.072308Z","iopub.execute_input":"2022-04-26T08:06:31.073187Z","iopub.status.idle":"2022-04-26T08:06:31.259249Z","shell.execute_reply.started":"2022-04-26T08:06:31.073144Z","shell.execute_reply":"2022-04-26T08:06:31.258571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glsply = sns.countplot(x='data_provider',data=Dataset) \nglsply.set(xlabel = None)\nglsply.set(title='Data Provider')","metadata":{"execution":{"iopub.status.busy":"2022-04-26T08:06:33.443127Z","iopub.execute_input":"2022-04-26T08:06:33.4434Z","iopub.status.idle":"2022-04-26T08:06:33.605845Z","shell.execute_reply.started":"2022-04-26T08:06:33.443368Z","shell.execute_reply":"2022-04-26T08:06:33.604987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#do we need to apply weights?","metadata":{"execution":{"iopub.status.busy":"2022-04-23T00:51:27.64605Z","iopub.execute_input":"2022-04-23T00:51:27.646802Z","iopub.status.idle":"2022-04-23T00:51:27.651713Z","shell.execute_reply.started":"2022-04-23T00:51:27.646768Z","shell.execute_reply":"2022-04-23T00:51:27.650903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ncheck what is gleason score when isUp grade is 0 \n(when isUpgrade == 2, 4+3 maybe a misdiagnosis)\n(to see if we need to remove it)\n'''\nfor i in range(6):\n    print('isUp grade = '+str(i)+': ')\n    print(Dataset.loc[Dataset.isup_grade == i,'gleason_score'].value_counts())\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:22.17234Z","iopub.execute_input":"2022-04-26T09:07:22.172605Z","iopub.status.idle":"2022-04-26T09:07:22.192056Z","shell.execute_reply.started":"2022-04-26T09:07:22.172574Z","shell.execute_reply":"2022-04-26T09:07:22.1913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gleason_score","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:25.515916Z","iopub.execute_input":"2022-04-26T09:07:25.516431Z","iopub.status.idle":"2022-04-26T09:07:25.523243Z","shell.execute_reply.started":"2022-04-26T09:07:25.516394Z","shell.execute_reply":"2022-04-26T09:07:25.522479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#relabel gleason score\ntarget = []\nfor i in range(len(Dataset)):\n    if Dataset.iloc[i]['gleason_score'] == '0+0':\n        target.append(0)\n    if Dataset.iloc[i]['gleason_score'] == 'negative':\n        target.append(0)\n    elif Dataset.iloc[i]['gleason_score'] == '3+3':\n        target.append(1)\n    elif Dataset.iloc[i]['gleason_score'] == '3+4':\n        target.append(2)\n    elif Dataset.iloc[i]['gleason_score'] == '4+3':\n        target.append(3)\n    elif Dataset.iloc[i]['gleason_score'] == '4+4':\n        target.append(4)\n    elif Dataset.iloc[i]['gleason_score'] == '3+5':\n        target.append(5)\n    elif Dataset.iloc[i]['gleason_score'] == '5+3':\n        target.append(6)\n    elif Dataset.iloc[i]['gleason_score'] == '4+5':\n        target.append(7)\n    elif Dataset.iloc[i]['gleason_score'] == '5+4':\n        target.append(8)\n    elif Dataset.iloc[i]['gleason_score'] == '5+5':\n        target.append(9)\n\nDataset['target'] = target\nDataset","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:26.796678Z","iopub.execute_input":"2022-04-26T09:07:26.797472Z","iopub.status.idle":"2022-04-26T09:07:33.322336Z","shell.execute_reply.started":"2022-04-26T09:07:26.797424Z","shell.execute_reply":"2022-04-26T09:07:33.321661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check how many cases are in dataset\nprint('Number of cases: ',len(Dataset))\n#check how many images are in train_image and train_label_masks folder\ntrain_path = '../input/prostate-cancer-grade-assessment/train_images'\nmask_path = '../input/prostate-cancer-grade-assessment/train_label_masks'\n\ntrain_image = 0\nfor path in os.listdir(train_path):\n    if os.path.isfile(os.path.join(train_path, path)):\n        train_image += 1\nprint('Number of train image: ',train_image)\n\nmask_image = 0\nfor path in os.listdir(mask_path):\n    if os.path.isfile(os.path.join(mask_path, path)):\n        mask_image += 1\nprint('Number of mask image: ',mask_image)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T06:49:51.157982Z","iopub.execute_input":"2022-04-26T06:49:51.158246Z","iopub.status.idle":"2022-04-26T06:50:31.827968Z","shell.execute_reply.started":"2022-04-26T06:49:51.158217Z","shell.execute_reply":"2022-04-26T06:50:31.827145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Problem here: missing some mask images!**","metadata":{}},{"cell_type":"code","source":"#check some images\n#1.select 2 cases per gleason score class\nTempImageDataset = pd.DataFrame()\nfor i in range(len(gleason_score)):\n    TempImageDataset = TempImageDataset.append(Dataset.loc[Dataset.gleason_score == gleason_score[i]][:2])\nTempImageDataset.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T16:08:25.51853Z","iopub.execute_input":"2022-04-25T16:08:25.519394Z","iopub.status.idle":"2022-04-25T16:08:25.57871Z","shell.execute_reply.started":"2022-04-25T16:08:25.519346Z","shell.execute_reply":"2022-04-25T16:08:25.577965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check images using openslide\ndef plotImageAndmasks(TempImageDataset):\n    fig = plt.figure(figsize=(40, 70))\n    rows = 11\n    columns = 4\n    k = 1\n    for i in range(len(TempImageDataset)):\n        #get each image id\n        image_id = TempImageDataset.iloc[i]['image_id']\n        #read image\n        img = openslide.OpenSlide('../input/prostate-cancer-grade-assessment/train_images/'+str(image_id)+'.tiff')\n        #print('Number of image levels: ',img.level_count)#----3\n        #print('dimensions of levels: ',img.level_dimensions)#----((27648, 29440), (6912, 7360), (1728, 1840))\n        img = img.read_region((0, 0), img.level_count-1, img.level_dimensions[2]).convert('RGB')\n        fig.add_subplot(rows, columns, k)\n        #plt.axis('off')\n        plt.title('isUp_grade: '+str(TempImageDataset.iloc[i]['isup_grade'])+\n                  ' gleason_score: '+str(TempImageDataset.iloc[i]['gleason_score'])+\n                  '\\ndata_provider: '+str(TempImageDataset.iloc[i]['data_provider']))\n        k+=1\n        plt.imshow(img)\n        # plot corresponding masks\n        #1.check if it's in mask folder\n        file_exists = os.path.exists('../input/prostate-cancer-grade-assessment/train_label_masks/'+str(image_id)+'_mask.tiff')\n        if file_exists == True:\n            #2.load masks\n            mask = openslide.OpenSlide('../input/prostate-cancer-grade-assessment/train_label_masks/'+str(image_id)+'_mask.tiff')\n            mask_img = mask.read_region((0,0), mask.level_count - 1, mask.level_dimensions[mask.level_count - 1]).convert(mode='RGB')\n            #split image to individual bands(a copy of one of original bands (red,green,blue))\n            #now mask is in red band.\n            mask_img = mask_img.split()[0]\n            #if the raw mode is “RGB”, then palette sequence must contain at most 768 values\n            palette = [0, 0, 0, 102, 102, 102, 255, 0, 0, 0, 251, 255, 246, 255, 0, 0, 255, 8]\n            palette.extend([0]*750)\n            mask_img.putpalette(data=palette, rawmode='RGB')\n            fig.add_subplot(rows, columns, k)\n            #plt.axis('off')\n            plt.title('isUp_grade: '+str(TempImageDataset.iloc[i]['isup_grade'])+\n                      ' gleason_score: '+str(TempImageDataset.iloc[i]['gleason_score'])+\n                      '\\ndata_provider: '+str(TempImageDataset.iloc[i]['data_provider']))\n            plt.imshow(mask_img)\n        k+=1\n\nplotImageAndmasks(TempImageDataset)","metadata":{"execution":{"iopub.status.busy":"2022-04-23T00:52:06.323034Z","iopub.execute_input":"2022-04-23T00:52:06.323743Z","iopub.status.idle":"2022-04-23T00:52:27.19536Z","shell.execute_reply.started":"2022-04-23T00:52:06.323695Z","shell.execute_reply":"2022-04-23T00:52:27.194079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**different data providers had different masks**","metadata":{}},{"cell_type":"code","source":"#overlay the mask on the image","metadata":{"execution":{"iopub.status.busy":"2022-04-23T00:52:27.196926Z","iopub.execute_input":"2022-04-23T00:52:27.197859Z","iopub.status.idle":"2022-04-23T00:52:27.20156Z","shell.execute_reply.started":"2022-04-23T00:52:27.19781Z","shell.execute_reply":"2022-04-23T00:52:27.200913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**What input we should use?  images or images with mask**","metadata":{}},{"cell_type":"markdown","source":"**What class attribute we want to predict? isUp_grade(6 classes) or Gleason_score(11 classes, if we use this, need to consider data ibalance)**","metadata":{}},{"cell_type":"code","source":"Dataset = Dataset.drop([Dataset.index[7273]])","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:37.996915Z","iopub.execute_input":"2022-04-26T09:07:37.997419Z","iopub.status.idle":"2022-04-26T09:07:38.004176Z","shell.execute_reply.started":"2022-04-26T09:07:37.997379Z","shell.execute_reply":"2022-04-26T09:07:38.003387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''In this part, we split dataset'''\n'''(only loaded 6 images to training dataset and 6 images to testing dataset for testing)'''\n\ndef splitData(Dataset):\n    image_id = Dataset.image_id.values\n    label = Dataset.target.values\n    '''\n    tg = Dataset['target'].unique()\n    image_id,label = [],[]\n    for l in tg:\n        da = Dataset.loc[Dataset.target == l,'image_id']\n        for i in range(10):\n            image_id.append(da.iloc[i])\n            label.append(l)\n    '''\n########tutorial: https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.StratifiedShuffleSplit.html########################\n    split = StratifiedShuffleSplit(n_splits=1, test_size=0.2)\n    split.get_n_splits(image_id, label)\n    #train_data,test_data = split.split(image_id, label)\n    image_id = np.array(image_id)\n    label = np.array(label)\n    for train_index, test_index in split.split(image_id, label):\n        print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n        X_train, X_test = image_id[train_index], image_id[test_index]\n        y_train, y_test = label[train_index], label[test_index]\n##########################################################################################################################\n    return X_train, X_test,y_train, y_test\n    \nX_train, X_test,y_train, y_test = splitData(Dataset)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:40.513201Z","iopub.execute_input":"2022-04-26T09:07:40.513452Z","iopub.status.idle":"2022-04-26T09:07:40.533079Z","shell.execute_reply.started":"2022-04-26T09:07:40.513423Z","shell.execute_reply":"2022-04-26T09:07:40.53186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Data Augmentation and create dataloader\n###############tutorial: https://androidkt.com/load-custom-image-datasets-into-pytorch-dataloader-without-using-imagefolder/#####################\nnorm_mean = (0.4914, 0.4822, 0.4465)\nnorm_std = (0.2023, 0.1994, 0.2010)\n\nclass imageDataset(D):\n    def __init__(self, X_train,y_train):\n        self.X_t = X_train\n        self.y_t = y_train\n        #make transformations (basic transformations)\n        self.transform =transforms.Compose([transforms.Resize((224,224)),\n                                            transforms.RandomHorizontalFlip(),\n                                            transforms.RandomRotation(degrees=60),\n                                            transforms.ToTensor(),\n                                            transforms.Normalize(norm_mean, norm_std)])\n        \n    def __len__(self):\n        return len(self.X_t)\n\n    def __getitem__(self, index):\n        img = openslide.OpenSlide('../input/prostate-cancer-grade-assessment/train_images/'+str(self.X_t[index])+'.tiff')\n        img = img.read_region((0, 0), img.level_count-1, img.level_dimensions[2]).convert('RGB')\n        #make transformations\n        image=self.transform(img)\n        target = self.y_t[index]\n        #write images and labels into a dictionary ()\n        sample = {'images': image,'labels':target}\n        return sample\n\n#dataloader \ntrain_dataset = imageDataset(X_train,y_train)\ntest_dataset = imageDataset(X_test,y_test)\ntrain_loader = DataLoader(train_dataset, batch_size=100, shuffle=True)\ntest_loader = DataLoader(test_dataset, batch_size=100, shuffle=False)\n#########################################################################################################","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:43.906202Z","iopub.execute_input":"2022-04-26T09:07:43.906451Z","iopub.status.idle":"2022-04-26T09:07:43.917626Z","shell.execute_reply.started":"2022-04-26T09:07:43.906422Z","shell.execute_reply":"2022-04-26T09:07:43.916656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n###############tutorial: https://androidkt.com/load-custom-image-datasets-into-pytorch-dataloader-without-using-imagefolder/#####################\n\n# Get a batch of train loader\n#Get some random training images\nimages = next(iter(train_loader))\ndef plot_image(img):\n    output = torchvision.utils.make_grid(img/ 2 + 0.5)\n    output = output.numpy().transpose((1, 2, 0))\n    output = np.clip(output, 0, 1)\n    return output\n# check if store images properly in a batch\n#Show images\n#output = torchvision.utils.make_grid(images['images'][0]/ 2 + 0.5)\n#output = output.numpy().transpose((1, 2, 0))\n#output = np.clip(output, 0, 1)\noutput = plot_image(images['images'])\n#plt.title('isup_grade: '+str(images['labels'][0].numpy()))\nplt.imshow(output)\n#########################################################################################################\n","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:07:52.217671Z","iopub.execute_input":"2022-04-26T09:07:52.218005Z","iopub.status.idle":"2022-04-26T09:08:00.885087Z","shell.execute_reply.started":"2022-04-26T09:07:52.217969Z","shell.execute_reply":"2022-04-26T09:08:00.883133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check device\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:08:04.7176Z","iopub.execute_input":"2022-04-26T09:08:04.718091Z","iopub.status.idle":"2022-04-26T09:08:04.78897Z","shell.execute_reply.started":"2022-04-26T09:08:04.718048Z","shell.execute_reply":"2022-04-26T09:08:04.787999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''from HW6 LeNet'''\nclass CNN(nn.Module):#(LeNet)\n    def __init__(self):\n        super(CNN, self).__init__()\n        self.layer1 = nn.Sequential(\n                        nn.Conv2d(3, 6, (5,5), padding=2),\n                        nn.ReLU(),\n                        nn.MaxPool2d((2, 2)))\n        self.layer2 = nn.Sequential(\n                        nn.Conv2d(6, 16, (5,5)),\n                        nn.ReLU(),\n                        nn.MaxPool2d((2, 2)))\n        self.seq = nn.Sequential(\n                nn.Linear(16*54*54, 120),\n                nn.ReLU(),\n                nn.Linear(120, 84),\n                nn.ReLU(),\n                nn.Linear(84, 10)\n                )\n    def forward(self, x):\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = x.reshape(x.shape[0], -1)\n        x = self.seq(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:08:07.231184Z","iopub.execute_input":"2022-04-26T09:08:07.231438Z","iopub.status.idle":"2022-04-26T09:08:07.24032Z","shell.execute_reply.started":"2022-04-26T09:08:07.23141Z","shell.execute_reply":"2022-04-26T09:08:07.239478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optim\nmodel = CNN().to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=1e-5)\n","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:08:10.917135Z","iopub.execute_input":"2022-04-26T09:08:10.917544Z","iopub.status.idle":"2022-04-26T09:08:13.688845Z","shell.execute_reply.started":"2022-04-26T09:08:10.917509Z","shell.execute_reply":"2022-04-26T09:08:13.688116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 10\n#for accuracy and losses record\ntrain_accuracy = []\nVal_accuracy = []\ntrain_losses = []\nVal_losses = []\n\n#start training\nfor epoch in range(num_epochs):\n    train_loss = 0.0\n    correct_total= 0.0\n    Vcorrect_total= 0.0\n    num_samples_total=0.0\n    Vnum_samples_total=0.0\n    valid_loss = 0.0\n    for i, data in enumerate(train_loader):\n        \n        # get the inputs\n        inputs, labels = data['images'],data['labels']\n        inputs, labels = inputs.to(device), labels.to(device)\n        # set parameter gradients to zero\n        optimizer.zero_grad()\n        # forward\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        #backward\n        loss.backward()\n        #optimize\n        optimizer.step()\n        \n        #accuracy for it\n        _, predicted = torch.max(outputs, 1)\n        correct = (predicted == labels).sum().item()\n        num_samples_total +=labels.size(0)\n        correct_total +=correct\n        train_loss += loss.item()\n    \n    train_losses.append(train_loss/len(train_loader))\n    train_accuracy.append(correct_total/num_samples_total)\n    \n    model.eval()\n \n    for l, testdata in enumerate(test_loader):\n        val_inputs, val_labels = testdata['images'],testdata['labels']\n        val_inputs, val_labels = val_inputs.to(device), val_labels.to(device)\n        val_outputs = model(val_inputs)\n    \n        vloss = criterion(val_outputs, val_labels)\n        valid_loss += vloss.item() \n        _, Vpredicted = torch.max(val_outputs, 1)\n        Vcorrect = (Vpredicted == val_labels).sum().item()\n        Vnum_samples_total +=val_labels.size(0)\n        Vcorrect_total +=Vcorrect\n        \n\n    print('Epoch: %d' %(epoch+1))\n    print('train loss: %.3f  train accuracy:%.3f  val loss: %.3f  val accuracy: %.3f ' %(train_loss/len(train_loader), correct_total/num_samples_total,valid_loss/len(test_loader),Vcorrect_total/Vnum_samples_total))\n\n    \n    Val_losses.append(valid_loss/len(test_loader))\n    Val_accuracy.append(Vcorrect/Vnum_samples_total)","metadata":{"execution":{"iopub.status.busy":"2022-04-26T09:08:16.57427Z","iopub.execute_input":"2022-04-26T09:08:16.574529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# MODEL EVALUATION\n\nplt.plot(train_accuracy, label='train_accuracy')\nplt.plot(Val_accuracy, label='val_accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(train_losses, label='train_loss')\nplt.plot(Val_losses, label='valid_loss')\nplt.xlabel('Epoch')\nplt.ylabel('loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-26T08:05:16.669108Z","iopub.execute_input":"2022-04-26T08:05:16.669589Z","iopub.status.idle":"2022-04-26T08:05:16.865635Z","shell.execute_reply.started":"2022-04-26T08:05:16.669545Z","shell.execute_reply":"2022-04-26T08:05:16.864944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import OrderedDict, Sequence\n\nclass _BaseWrapper(object):\n    \"\"\"\n    Please modify forward() and backward() according to your task.\n    \"\"\"\n\n    def __init__(self, model):\n        super(_BaseWrapper, self).__init__()\n        self.device = next(model.parameters()).device\n        self.model = model\n        self.handlers = []  # a set of hook function handlers\n\n    def _encode_one_hot(self, ids):\n        one_hot = torch.zeros_like(self.logits).to(self.device)\n        one_hot.scatter_(1, ids, 1.0)\n        return one_hot\n\n    def forward(self, image):\n        \"\"\"\n        Simple classification\n        \"\"\"\n        self.model.zero_grad()\n        self.logits = self.model(image)\n        self.probs = torch.nn.functional.softmax(self.logits, dim=1)\n        return self.probs.sort(dim=1, descending=True)\n\n    def backward(self, ids):\n        \"\"\"\n        Class-specific backpropagation\n        Either way works:\n        1. self.logits.backward(gradient=one_hot, retain_graph=True)\n        2. (self.logits * one_hot).sum().backward(retain_graph=True)\n        \"\"\"\n\n        one_hot = self._encode_one_hot(ids)\n        self.logits.backward(gradient=one_hot, retain_graph=True)\n\n    def generate(self):\n        raise NotImplementedError\n\n    def remove_hook(self):\n        \"\"\"\n        Remove all the forward/backward hook functions\n        \"\"\"\n        for handle in self.handlers:\n            handle.remove()\n\n\nclass GradCAM(_BaseWrapper):\n    \"\"\"\n    \"Grad-CAM: Visual Explanations from Deep Networks via Gradient-based Localization\"\n    https://arxiv.org/pdf/1610.02391.pdf\n    Look at Figure 2 on page 4\n    \"\"\"\n\n    def __init__(self, model, candidate_layers=None):\n        super(GradCAM, self).__init__(model)\n        self.fmap_pool = OrderedDict()\n        self.grad_pool = OrderedDict()\n        self.candidate_layers = candidate_layers  # list\n\n        def forward_hook(key):\n            def forward_hook_(module, input, output):\n                # Save featuremaps\n                self.fmap_pool[key] = output.detach()\n\n            return forward_hook_\n\n        def backward_hook(key):\n            def backward_hook_(module, grad_in, grad_out):\n                # Save the gradients correspond to the featuremaps\n                self.grad_pool[key] = grad_out[0].detach()\n\n            return backward_hook_\n\n        # If any candidates are not specified, the hook is registered to all the layers.\n        for name, module in self.model.named_modules():\n            if self.candidate_layers is None or name in self.candidate_layers:\n                self.handlers.append(module.register_forward_hook(forward_hook(name)))\n                self.handlers.append(module.register_backward_hook(backward_hook(name)))\n\n    def _find(self, pool, target_layer):\n        if target_layer in pool.keys():\n            return pool[target_layer]\n        else:\n            raise ValueError(\"Invalid layer name: {}\".format(target_layer))\n\n    def _compute_grad_weights(self, grads):\n        return torch.nn.functional.adaptive_avg_pool2d(grads, 1)\n\n    def forward(self, image):\n        self.image_shape = image.shape[2:]\n        return super(GradCAM, self).forward(image)\n\n    def generate(self, target_layer):\n        fmaps = self._find(self.fmap_pool, target_layer)\n        grads = self._find(self.grad_pool, target_layer)\n        weights = self._compute_grad_weights(grads)\n\n        gcam = torch.mul(fmaps, weights).sum(dim=1, keepdim=True)\n        gcam = torch.nn.functional.relu(gcam)\n\n        gcam = torch.nn.functional.interpolate(\n            gcam, self.image_shape, mode=\"bilinear\", align_corners=False\n        )\n\n        B, C, H, W = gcam.shape\n        gcam = gcam.view(B, -1)\n        gcam -= gcam.min(dim=1, keepdim=True)[0]\n        gcam /= gcam.max(dim=1, keepdim=True)[0]\n        gcam = gcam.view(B, C, H, W)\n\n        return gcam","metadata":{"execution":{"iopub.status.busy":"2022-04-26T06:35:43.312035Z","iopub.execute_input":"2022-04-26T06:35:43.312355Z","iopub.status.idle":"2022-04-26T06:35:43.332109Z","shell.execute_reply.started":"2022-04-26T06:35:43.312321Z","shell.execute_reply":"2022-04-26T06:35:43.331363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def demo2(image, label, model):\n    \"\"\"\n    Generate Grad-CAM\n    \"\"\"\n    # Model\n    model = model\n    model.to(device)\n    model.eval()\n\n    # The layers\n    target_layers = [\"layer2\"]\n    target_class = label\n\n    # Images\n    images = image.unsqueeze(0)\n    gcam = GradCAM(model=model)\n    probs, ids = gcam.forward(images)\n    ids_ = torch.LongTensor([[target_class]] * len(images)).to(device)\n    gcam.backward(ids=ids_)\n\n    for target_layer in target_layers:\n        print(\"Generating Grad-CAM @{}\".format(target_layer))\n\n        # Grad-CAM\n        regions = gcam.generate(target_layer=target_layer)\n        for j in range(len(images)):\n            #print(\"\\t#{}: {} ({:.5f})\".format(j, classes[target_class], float(probs[ids == target_class]))\n            \n            gcam=regions[j, 0]\n            plt.imshow(gcam.cpu())\n            plt.show()\n            \n#image, label = next(iter(test_loader))\nimages = next(iter(test_loader))\n# Load the model\nmodel = model\nimg = images['images']\nlabal = images['labels']\n# Grad cam\ndemo2(img[0].to(device), labal[0].to(device), model)\n\n\nimage = np.transpose(images['images'][0], (1,2,0))\nimage2  = np.add(np.multiply(image.numpy(), np.array(norm_std)) ,np.array(norm_mean))\n#print(\"True Class: \", classes[label[0].cpu()])\nplt.imshow(image)\nplt.show()\nplt.imshow(image2)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-26T06:36:34.259975Z","iopub.execute_input":"2022-04-26T06:36:34.260308Z","iopub.status.idle":"2022-04-26T06:36:36.069588Z","shell.execute_reply.started":"2022-04-26T06:36:34.260264Z","shell.execute_reply":"2022-04-26T06:36:36.068841Z"},"trusted":true},"execution_count":null,"outputs":[]}]}