{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade fastai > /dev/null\n!pip install --upgrade fastcore > /dev/null\n!pip install pretrainedmodels > /dev/null","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.all import *\nimport pretrainedmodels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!apt update && apt install -y openslide-tools\n!pip install openslide-python","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Load the dependancies\nfrom fastai.basics import *\nfrom fastai.callback.all import *\nfrom fastai.vision.all import *\n\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\nimport os\nimport cv2\n\nimport openslide\n\nsns.set(style=\"whitegrid\")\nsns.set_context(\"paper\")\n\nmatplotlib.rcParams['image.cmap'] = 'ocean_r'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"source = Path(\"../input/prostate-cancer-grade-assessment/\")\nfiles = os.listdir(source)\nsource.ls()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = source/'train_images'\nmask = source/'train_label_masks'\ntrain_labels = pd.read_csv(source/'train.csv')\ntrain_labels.head(), len(train_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels_dtypes = {'image_id': 'string', 'data_provider': 'string', \n                       'isup_grade': int, 'gleason_score': 'string' }\ntrain_labels = train_labels.astype(train_labels_dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"localfiles = list(train.glob('*.tiff'))\nlocalfiles = set([filename.stem[:32] for filename in localfiles])\nlen(localfiles)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = train_labels[train_labels.image_id.isin(localfiles)]\nprint(len(train_labels))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels[train_labels.image_id == \"b5db121ca6ba4d979a6bef814d5fdb17\"]\n'b5db121ca6ba4d979a6bef814d5fdb17' in localfiles","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def open_tiff_image(image_path):\n    f = openslide.OpenSlide(str(image_path))    \n\ndef check_tiff_images(path):\n    valid_files = L()\n    for file in path.glob(\"*.tiff\"):\n        try:\n            n = open_tiff_image(file)\n            valid_files.append(file.stem[:32])\n        except Exception as e:\n            print(str(file))\n            print(e)        \n    return valid_files","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_train_files = check_tiff_images(train)\nlen(valid_train_files)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_mask_files = check_tiff_images(mask)\nlen(valid_mask_files)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_mask_files[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_to_delete = (set(valid_train_files)).difference(set(valid_mask_files))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_train_files[:10], valid_mask_files[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(file_to_delete)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor file in list(file_to_delete):\n    train_labels = train_labels[train_labels.image_id != file]\n    #os.remove(file)\nlen(train_labels)    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_img(fn):\n    fn = f'{train}/{fn.image_id}.tiff'\n    #print(fn)\n    try:\n        file = openslide.OpenSlide(str(fn))        \n    except Exception as e:\n        print(fn)\n        print(e)\n    t = tensor(file.get_thumbnail(size=(255, 255)))\n    img_pil = PILImage.create(t)\n    return img_pil\n\ndef show_selective(p, scale=True, cmap=plt.cm.ocean_r, min_px=None, max_px=None):\n    px = tensor(p)\n    if min_px is not None: px[px<min_px] = float(min_px)\n    if max_px is not None: px[px>max_px] = float(max_px)\n    return px\n\ndef custom_selective_mask(fn):\n    fn = f'{mask}/{fn.image_id}_mask.tiff'\n    try:\n        file = openslide.OpenSlide(str(fn))  \n        #file = Image.open(str(fn))\n    except Exception as e:        \n        print(fn)\n        print(e)\n    t = tensor(file.get_thumbnail(size=(255, 255)))[:,:,0]\n    ts = show_selective(t, min_px=None, max_px=None)\n    return ts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"blocks = (ImageBlock,\n          ImageBlock,\n          CategoryBlock)\n\ngetters = [\n           custom_img,\n           custom_selective_mask,\n           ColReader('isup_grade')\n          ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dblock_model = DataBlock(blocks=blocks,\n                   getters=getters,\n                   splitter=RandomSplitter(0.1),\n                   item_tfms=[Resize(224), ToTensor],\n                   batch_tfms=[IntToFloatTensor, Normalize.from_stats(*imagenet_stats)])\n\ndl = dblock_model.dataloaders(train_labels, bs=128)\ndl.show_batch(max_n=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dl.train_ds.vocab","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ProstateCancerModel(Module):\n    def __init__(self, encoder, head):\n        self.encoder, self.head = encoder, head\n\n    def forward(self, x1, x2):\n        enc1 = self.encoder(x1)    \n        enc2 = self.encoder(x2)    \n        ftrs = torch.cat([enc1, enc2], dim=1)\n        return self.head(ftrs)\n\ndef loss_func(out, targ):\n    return CrossEntropyLossFlat()(out, targ.long())\n\ndef siamese_splitter(model):\n    return [params(model.encoder), params(model.head)]\n\nencoder = create_body(resnet34, cut=-2)\nhead = create_head(512*2, len(dl.vocab), ps=0.5)\nmodel = ProstateCancerModel(encoder, head)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prostate_cancer_splitter(model):\n    return [params(model.encoder), params(model.head)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kp = CohenKappa()\nkp.weights = 'quadratic'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner = Learner(dl,\n                  model,\n                  loss_func=loss_func,\n                  splitter=prostate_cancer_splitter,\n                  metrics=[accuracy, kp]\n                  )\nlearner.freeze()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.fine_tune(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.export(\"prostate_stage_1.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.unfreeze()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.lr_find()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.model = model.cuda()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.fit_one_cycle(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learner.export(\"prostate_stage_2.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}