{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Prostate cANcer graDe Assessment (PANDA) Challenge\n## Prostate cancer diagnosis using the Gleason grading system","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"![](https://storage.googleapis.com/kaggle-competitions/kaggle/18647/logos/header.png)","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"import os\nimport cv2\nimport sys\nimport random\nimport openslide\nimport matplotlib\nimport skimage.io\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nimport plotly.graph_objs as go\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, display\nfrom skimage.transform import resize, rescale\nsns.set_style(\"darkgrid\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"ROOT = \"/kaggle/input/prostate-cancer-grade-assessment/\"\n!ls {ROOT}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(ROOT+\"train.csv\")\ntest = pd.read_csv(ROOT+\"test.csv\")\nsub = pd.read_csv(ROOT+\"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(train.head())\nprint(\"shape : \", train.shape)\nprint(\"unique ids : \", len(train.image_id.unique()))\nprint(\"unique data provider : \", len(train.data_provider.unique()))\nprint(\"unique isup_grade(target) : \", len(train.isup_grade.unique()))\nprint(\"unique gleason_score : \", len(train.gleason_score.unique()))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"files = os.listdir(ROOT+\"train_images/\")\nprint(f\"there are {len(files)} tiff files in train_images folder\")\nfor i in train.image_id:\n    assert i+\".tiff\" in files\nprint(\"all training image_ids have their files in train_images folder\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(test.head())\nprint(\"shape : \", test.shape)\nprint(\"unique ids : \", len(test.image_id.unique()))\nprint(\"unique data provider : \", len(test.data_provider.unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['image_id'][0]+\".tiff\" in os.listdir(ROOT+\"train_images/\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"test images are not present in given folder. I think test_images folder is missing","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig= plt.figure(figsize=(10,6))\nax = sns.countplot(x=\"isup_grade\", data=train)\nplt.title(\"target distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig= plt.figure(figsize=(10,6))\nax = sns.countplot(x=\"isup_grade\", hue=\"data_provider\", data=train)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"target variable shows different distribution depending on data provider","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14,5))\nsns.countplot(ax=ax1, x=\"data_provider\", data=train)\nax1.set_title(\"data_provider distribution in training data\")\nsns.countplot(ax=ax2, x=\"data_provider\", data=test)\nax2.set_title(\"data_provider distribution in test data\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14,5))\nfig.suptitle(\"counts of different gleason_score\")\nsns.countplot(ax=ax1, y=\"gleason_score\", data=train)\nsns.countplot(ax=ax2, y=\"gleason_score\",hue='data_provider', data=train)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"print(\"isup_score ->  \", end='')\nfor j in train['isup_grade'].unique():\n    print(j, end=\"\\t\")\nprint(\"\\n\", \"gleason_score \\n\", \"-\"*60, sep='')\n\nfor i in train['gleason_score'].unique():\n    print(f\"{i:>10} |   \", end=\"\")\n    for j in train['isup_grade'].unique():\n        print(len(train[(train['gleason_score']==i) & (train['isup_grade']==j)]), end=\"\\t\")\n    print(\"\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**negative and 0+0 are same just different names from different center**","execution_count":null},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"train['gleason_score'] = train['gleason_score'].apply(lambda x: \"0+0\" if x==\"negative\" else x)\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14,5))\nfig.suptitle(\"counts of different gleason_score after correction\")\nsns.countplot(ax=ax1, y=\"gleason_score\", data=train)\nsns.countplot(ax=ax2, y=\"gleason_score\",hue='data_provider', data=train)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"slide = openslide.OpenSlide(ROOT+\"train_images/\"+files[0])\nspacing = 1 / (float(slide.properties['tiff.XResolution']) / 10000)\nprint(f\"File id: {slide}\")\nprint(f\"Dimensions: {slide.dimensions}\")\nprint(f\"Microns per pixel / pixel spacing: {spacing:.3f}\")\nprint(f\"Number of levels in the image: {slide.level_count}\")\nprint(f\"Downsample factor per level: {slide.level_downsamples}\")\nprint(f\"Dimensions of levels: {slide.level_dimensions}\")\npatch = slide.read_region((1780,1950), 0, (256, 256))\ndisplay(patch) # Display the image\nslide.close()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"dims, spacings, level_counts = [], [], []\ndown_levels, level_dims = [], []\n\n# train = train.sample(300)\nfor i in train.image_id:\n    slide = openslide.OpenSlide(ROOT+\"train_images/\"+i+\".tiff\")\n    spacing = 1 / (float(slide.properties['tiff.XResolution']) / 10000)\n    dims.append(slide.dimensions)\n    spacings.append(spacing)\n    level_counts.append(slide.level_count)\n    down_levels.append(slide.level_downsamples)\n    level_dims.append(slide.level_dimensions)\n    slide.close()\n    del slide\n\ntrain['width']  = [i[0] for i in dims]\ntrain['height'] = [i[1] for i in dims]\ntrain['spacing'] = spacings\ntrain['level_count'] = level_counts","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nax = sns.scatterplot(x='width', y='height', data=train, alpha=0.3)\nplt.title(\"height(y) width(x) scatter plot\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12, 6))\nax = sns.scatterplot(x='width', y='height', hue='isup_grade', data=train, alpha=0.6)\nplt.title(\"height(y) width(x) scatter plot with target\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13,5))\nsns.distplot(ax=ax1, a=train['width'])\nax1.set_title(\"width distribution\")\nsns.distplot(ax=ax2, a=train['height'])\nax2.set_title(\"height distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"shapes = [j for i in level_dims for j in i]\nlevel  = np.array([j for i in level_dims for j in range(len(i))])\nwidths  = np.array([i[0] for i in shapes])\nheights = np.array([i[1] for i in shapes])\nfig, axes = plt.subplots(1, 3 ,figsize=(10,4))\nfor i in range(3):\n    ax = sns.scatterplot(ax=axes[i], x=widths[level==i], y=heights[level==i], alpha=0.9)\n    axes[i].set_title(f\"level {i}\")\nplt.tight_layout()\nplt.show()\n\nfig = plt.figure(figsize=(12,6))\nsns.scatterplot(x=widths, y=heights,hue=level, alpha=0.9)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))\nfig.suptitle(\"available different levels\")\nsns.distplot(ax=ax1, a=widths)\nax1.set_title(\"width distribution\")\nsns.distplot(ax=ax2, a=heights)\nax2.set_title(\"height distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(f\"unique level counts : {train['level_count'].unique()}\")\nprint(f\"unique spacings     : {train['spacing'].unique()}\")\nprint(f\"unique down levels  : {pd.Series([round(j) for i in down_levels for j in i]).unique()}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"masks = os.listdir(ROOT+'train_label_masks/')\nprint(f\"masks available for {len(masks)} out of {len(files)}\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def print_mask_details(slide, center='radboud', show_thumbnail=True, max_size=(400,400)):\n    \"\"\"Print some basic information about a slide\"\"\"\n\n    if center not in ['radboud', 'karolinska']:\n        raise Exception(\"Unsupported palette, should be one of [radboud, karolinska].\")\n\n    # Generate a small image thumbnail\n    if show_thumbnail:\n        # Read in the mask data from the highest level\n        # We cannot use thumbnail() here because we need to load the raw label data.\n        mask_data = slide.read_region((0,0), slide.level_count - 1, slide.level_dimensions[-1])\n        # Mask data is present in the R channel\n        mask_data = mask_data.split()[0]\n\n        # To show the masks we map the raw label values to RGB values\n        preview_palette = np.zeros(shape=768, dtype=int)\n        if center == 'radboud':\n            # Mapping: {0: background, 1: stroma, 2: benign epithelium, 3: Gleason 3, 4: Gleason 4, 5: Gleason 5}\n            preview_palette[0:18] = (np.array([0, 0, 0, 0.5, 0.5, 0.5, 0, 1, 0, 1, 1, 0.7, 1, 0.5, 0, 1, 0, 0]) * 255).astype(int)\n        elif center == 'karolinska':\n            # Mapping: {0: background, 1: benign, 2: cancer}\n            preview_palette[0:9] = (np.array([0, 0, 0, 0.5, 0.5, 0.5, 1, 0, 0]) * 255).astype(int)\n        mask_data.putpalette(data=preview_palette.tolist())\n        mask_data = mask_data.convert(mode='RGB')\n        mask_data.thumbnail(size=max_size, resample=0)\n        display(mask_data)\n\n    # Compute microns per pixel (openslide gives resolution in centimeters)\n    spacing = 1 / (float(slide.properties['tiff.XResolution']) / 10000)\n    \n    print(f\"Dimensions: {slide.dimensions}\")\n    print(f\"Microns per pixel / pixel spacing: {spacing:.3f}\")\n    print(f\"Number of levels in the image: {slide.level_count}\")\n    print(f\"Downsample factor per level: {slide.level_downsamples}\")\n    print(f\"Dimensions of levels: {slide.level_dimensions}\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"mask = openslide.OpenSlide(os.path.join(ROOT+\"train_label_masks\", '08ab45297bfe652cc0397f4b37719ba1_mask.tiff'))\nprint_mask_details(mask, center='radboud')\nmask.close()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"mask = openslide.OpenSlide(os.path.join(ROOT+\"train_label_masks\", '090a77c517a7a2caa23e443a77a78bc7_mask.tiff'))\nprint_mask_details(mask, center='karolinska')\nmask.close()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"count = 5\ndef plot_with_images(masks):\n    fig, axes = plt.subplots(2, count, figsize=(4*count, 8))\n    cmap = matplotlib.colors.ListedColormap(['black', 'gray', 'green', 'yellow', 'orange', 'red'])\n    for i, j in enumerate(masks):\n        mask = openslide.OpenSlide(os.path.join(ROOT+\"train_label_masks\", j))\n        mask_data = mask.read_region((0,0), mask.level_count - 1, mask.level_dimensions[-1])\n        image = openslide.OpenSlide(os.path.join(ROOT+\"train_images\", j[:-10]+\".tiff\"))\n        patch = image.read_region((0, 0), image.level_count-1, image.level_dimensions[-1]) \n        axes[0, i].imshow(np.asarray(mask_data)[:,:,0], cmap=cmap, interpolation='nearest', vmin=0, vmax=5)\n        axes[1, i].imshow(patch)\n        mask.close()\n        image.close()\n    plt.tight_layout()\n    plt.show()\n\nfor i in range(7):\n    plot_with_images(masks[i*count: (i+1)*count])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Background","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image(id, level):\n    im = skimage.io.MultiImage(ROOT+\"train_images/\"+id+\".tiff\")\n    return im[level]\n\ndef get_mask(id, level):\n    mask = skimage.io.MultiImage(ROOT+\"train_label_masks/\"+id+\"_mask.tiff\")\n    return mask[level]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ids_with_mask = [i[:-10] for i in os.listdir(ROOT+\"train_label_masks/\")]\nids = [(i, j) for i,j in zip(train.image_id, train.data_provider) if i in ids_with_mask]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for t, loc in tqdm(ids):\n#     image = get_image(t, 2)\n#     mask = get_mask(t, 2)\n#     assert np.sum(mask[:, :, 1:]) == 0\n#     assert(image.shape == mask.shape)\n#     if loc == \"karolinska\":\n#         assert np.max(mask[:, :, 0]) <= 2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Last 2 channels of masks are empty also for karolinska values are 0, 1, 2 not 1, 2, 3 **","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set_style(\"whitegrid\")\n\ndef get_mask(id, level):\n    \"\"\"\n    updated mask to return only first channel\n    \"\"\"\n    mask = skimage.io.MultiImage(ROOT+\"train_label_masks/\"+id+\"_mask.tiff\")\n    assert np.sum(mask[level][:, :, 1:]) == 0\n    return mask[level][:, :, 0]\n\ndef get_background_mask(id, level):\n    mask = get_mask(id, level)\n    return (mask == 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(4):\n    t = random.randint(0, len(ids))\n    mask = get_background_mask(ids[t][0], 2)\n    image = get_image(ids[t][0], 2)\n    fig, axes = plt.subplots(1, 2, figsize=(6, 4))\n    axes[0].imshow(mask)\n    axes[1].imshow(image)\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pred_background(image, low=240, high=255):\n    \"\"\"\n    predicting background with threshold\n    \"\"\"\n    (r, g, b) = cv2.split(image)\n    ret0, thresh0 = cv2.threshold(r, low, high, cv2.THRESH_BINARY)\n    ret1, thresh1 = cv2.threshold(g, low, high, cv2.THRESH_BINARY)\n    ret2, thresh2 = cv2.threshold(b, low, high, cv2.THRESH_BINARY)\n    return (thresh0 & thresh1 & thresh2) == 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(5):\n    t = random.randint(0, len(ids))\n    mask = get_background_mask(ids[t][0], 2)\n    image = get_image(ids[t][0], 2)\n    fig, axes = plt.subplots(1, 3, figsize=(10, 4))\n    axes[0].imshow(mask)\n    axes[1].imshow(image)\n    axes[2].imshow(pred_background(image))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**We can predict background using thresholding as most of background is white **","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### do upvote if it helped :)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}