{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Installing EfficientNet model without internet due to competition requisites\n\nimport os\nimport sys\n\nsys.path = [\n    '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',\n] + sys.path\n\nimport torch\nos.listdir(\"../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master\")\n\n\nfrom efficientnet_pytorch import EfficientNet\n\n\n#!pip install efficientnet_pytorch torchtoolbox","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\n# Imports here\nimport matplotlib.pyplot as plt\nimport torch\nfrom torch import nn\nfrom torch import optim\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms, models\nfrom torch.utils.data import Dataset, DataLoader\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport csv\nimport pandas as pd\nimport os\nimport random\nimport math\nimport skimage.io\n#from csv_loader import load_csv\n\n# Tiff visualisation imports and downloads\nimport numpy as np\nimport tifffile as tiff\n\n# For re-importing python modules\nimport importlib","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #use GPU if available\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ntorch.set_default_tensor_type(torch.cuda.FloatTensor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# These image augmentation functions have already been defined in another notebook.\n# Using these functions to pre-process test images before running them through the model.\n\ndef compute_statistics(img):\n    #img = skimage.io.MultiImage(image)[2]\n    width, height = img.shape[0], img.shape[1]\n    num_pixels = width * height\n    \n    num_white_pixels = 0\n    green_values_img = 0\n    blue_values_img = 0\n    \n    pixelsum=np.sum(img,axis=-1)\n    num_white_pixels = int((pixelsum>620).sum())\n    \n    green_values_img=np.sum(img[:,:,1])\n    blue_values_img=np.sum(img[:,:,2])\n    \n    ratio_white_pixels = num_white_pixels / num_pixels\n    green_concentration = green_values_img / num_pixels\n    blue_concentration = blue_values_img / num_pixels\n    \n    return ratio_white_pixels, green_concentration, blue_concentration\n\n\ndef select_k_best_regions(regions, k=16):\n    best_regions = [x for x in regions if x[3] > 0 and x[4] > 0]\n    k_best_regions = sorted(best_regions, key=lambda tup: tup[2])[:k]\n    \n    return k_best_regions\n\n\ndef get_k_best_regions(coordinates, image, window_size=512):\n    regions = {}\n    for i, tup in enumerate(coordinates):\n        x, y = tup[0], tup[1]\n        regions[i] = image[x : x+window_size, y : y+window_size, :]\n    \n    return regions\n\n\ndef generate_patches(image, window_size=200, stride=135, k=20):\n    #image = skimage.io.MultiImage(slide_path)[1]\n    image = np.array(image)\n    \n    max_width, max_height = image.shape[0], image.shape[1]\n    regions_container = []\n    i = 0\n    \n    while window_size + stride*i <= max_height:\n        j = 0\n        \n        while window_size + stride*j <= max_width:            \n            x_top_left_pixel = j * stride\n            y_top_left_pixel = i * stride\n            \n            patch = image[\n                x_top_left_pixel : x_top_left_pixel + window_size,\n                y_top_left_pixel : y_top_left_pixel + window_size,\n                :]\n            \n            ratio_white_pixels, green_concentration, blue_concentration = compute_statistics(patch)\n            \n            region_tuple = (x_top_left_pixel, y_top_left_pixel, ratio_white_pixels, green_concentration, blue_concentration)\n            regions_container.append(region_tuple)\n            \n            j += 1\n        \n        i += 1\n    \n    k_best_region_coordinates = select_k_best_regions(regions_container, k=k)\n    k_best_regions = get_k_best_regions(k_best_region_coordinates, image, window_size)\n    #excess_overlap_removed = remove_overlaps(k_best_regions, k_sample=k, k_final=16)\n    \n           \n    return image, k_best_region_coordinates, k_best_regions #, excess_overlap_removed\n\n\ndef display_images(regions, title):\n    fig, ax = plt.subplots(6, 6, figsize=(20, 20))\n    \n    for i, region in regions.items():\n        ax[i//6, i%6].imshow(region)\n    \n    fig.suptitle(title)\n\n    \ndef glue_to_one_picture(image_patches, window_size=200, k=16):\n    side = int(np.sqrt(k))\n    image = np.zeros((side*window_size, side*window_size, 3), dtype=np.int16)\n        \n    for i, patch in image_patches.items():\n        x = i // side\n        y = i % side\n        image[\n            x * window_size : (x+1) * window_size,\n            y * window_size : (y+1) * window_size,\n            :\n        ] = patch\n    \n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class load_csv(Dataset):\n    def __init__(self, csv_file, root_dir, transform=None):\n        self.annotations = pd.read_csv(csv_file)# todo remove sample for debug\n        self.root_dir = root_dir\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.annotations)\n        \n    \n    def __getitem__(self, index):\n        #img_path = os.path.join(self.root_dir, self.annotations.iloc[index, 0])\n        image_id = self.annotations.iloc[index, 0]\n        img_path = os.path.join(self.root_dir, str(image_id) +\".tiff\")\n        img = skimage.io.MultiImage(img_path)[1]\n        \n        image, best_coordinates, best_regions = generate_patches(img, window_size=128, stride=128, k=36)\n        glued_img = glue_to_one_picture(best_regions, window_size=128, k=36)\n        \n        image = torch.from_numpy(glued_img).permute(2,0,1).float()\n        \n        \n        \n        #Image.MAX_IMAGE_PIXELS = None\n                \n        #image.transform = transforms.RandomResizedCrop(224)\n        \n        #y_label = torch.tensor(int(self.annotations.iloc[index,:]['isup_grade']))\n        #isup_grade = int(self.annotations.iloc[index,:]['isup_grade'])\n        \n        #label = np.zeros(6).astype(np.float32)\n        #y_label = label[isup_grade] = 1.\n        #y_label = torch.tensor(y_label)\n        \n        self.transform= transforms.Compose([transforms.ToPILImage(),\n                                            #transforms.Resize((int(1840/2),int(1728/2))), \n                                            transforms.ToTensor()])\n                                            #transforms.Normalize([0.485, 0.456, 0.406],\n                                                             #      [0.229, 0.224, 0.225])])\n        if self.transform:\n            image = self.transform(image)\n        \n        return (image, np.array(0))#1840, 1728","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading model\n\nmodel = EfficientNet.from_name('efficientnet-b0')\nmodel.load_state_dict(torch.load(\"../input/efficientnet-pytorch/efficientnet-b0-08094119.pth\"))\nmodel._fc = nn.Sequential(nn.Linear(model._fc.in_features, 216),\n                          nn.ReLU(),\n                          nn.Linear(216, 36, bias=True),\n                          nn.ReLU(),\n                          nn.Linear(36, 5, bias=True))\n\n# Model path\nmodel_path = '../input/arutema-model-36x128x128/Arutema_base_model_w_logits_36x128x128.pth'\n\n# Load checkpoint\ncheckpoint = torch.load(model_path)\n\n# Load model parameters\nmodel.load_state_dict(checkpoint['model_state_dict'])\nmodel._fc.load_state_dict(checkpoint['classifier_state_dict'])\n\n# Load other model related components\ncriterion = nn.NLLLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\noptimizer.load_state_dict(checkpoint['optimizer_state_dict'])\nepoch = checkpoint['epoch']\nloss = checkpoint['loss']\n\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For submission\nsubmission_test_path = \"/kaggle/input/prostate-cancer-grade-assessment/test_images/\"\ncsv_file_dir = '/kaggle/input/prostate-cancer-grade-assessment/test.csv'\n\npred_y_excel = []\nsubmission=None\nif os.path.exists(submission_test_path):\n    #print(\"dir found\")\n    test_df = pd.read_csv(csv_file_dir)\n    test_data = load_csv(csv_file=csv_file_dir, root_dir=submission_test_path)\n    test_loader = torch.utils.data.DataLoader(test_data, batch_size=1, shuffle=False)\n    #model.eval()\n    for ii2, (inputs2, labels2) in enumerate(test_loader):\n        inputs2, labels2 = inputs2.to(device), labels2.to(device)\n    \n        output2 = model.forward(inputs2)\n        pred_y2 = output2.sigmoid().sum(1).round().detach()\n\n        pred_y_excel.append(int(pred_y2))\n    #print(\"writing csv\")\n    #pred_y_excel = [3 for i in range(test_df.shape[0])]\n    submission = pd.DataFrame({'image_id':test_df.image_id, 'isup_grade':pred_y_excel})\n    submission.to_csv(\"submission.csv\", sep=\",\", index=False)\nelse:\n    #print(\"dir not found, building dummy\")\n    test_df = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/sample_submission.csv\")\n#     test_df['isup_grade'] = 0\n#     fake_submission=test_df[['image_id','isup_grade']]\n    test_df.to_csv('submission.csv', index=False)\n    #print('submission saved')\n    #print(fake_submission)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}