{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport os, glob, tifffile, cv2, gc\nimport openslide\nimport skimage.io\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"os.listdir('../input/prostate-cancer-grade-assessment')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## EDA - Tabular"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/prostate-cancer-grade-assessment'\ntrain = pd.read_csv(os.path.join(data_dir, 'train.csv'))\ntest = pd.read_csv(os.path.join(data_dir, 'test.csv'))\nsub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12, 5), facecolor='w')\nsns.countplot(train['data_provider'])\nplt.title('Data Provider')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12, 5), facecolor='w')\nsns.countplot(train['isup_grade'])\nplt.title('ISUP Grade (Target)')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12, 5), facecolor='w')\nsns.countplot(train['gleason_score'])\nplt.title('Gleason Score')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Distribution Target per Gleason Score"},{"metadata":{"trusted":true},"cell_type":"code","source":"score_num = train['gleason_score'].nunique()\nfig, axes = plt.subplots(ncols=4, nrows=int(np.ceil(score_num / 4)), figsize=(16, 10), facecolor='w')\n\nfor score, ax in zip(train['gleason_score'].unique(), axes.ravel()):\n    temp = train[train['gleason_score'] == score]\n    sns.countplot(temp['isup_grade'], ax=ax)\n    ax.set_title(score)\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"One-to-one relationship between \"isup_grade\" and \"gleason_score\"\n\nBut there is only one exception (probably wrong)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# isup_grade = 2 and gleason_score = \"4+3\"\na = train[train['isup_grade'] == 2]\na[a['gleason_score'] == '4+3']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Focus on \"0+0\" and \"negative\""},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['gleason_score'] == 'negative']['data_provider'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['gleason_score'] == '0+0']['data_provider'].unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Labeled 0 + 0 or negative depending on data provider\n\n    radboud -> \"negative\"\n    karolinska -> \"0+0\""},{"metadata":{"trusted":true},"cell_type":"code","source":"# Modify label\ntrain.loc[train['gleason_score'] == 'negative', 'gleason_score'] = '0+0'\ntrain.loc[train['isup_grade'] == 2, 'gleason_score'] = '3+4'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score_num = train['isup_grade'].nunique()\nfig, axes = plt.subplots(ncols=3, nrows=int(np.ceil(score_num / 4)), figsize=(16, 10), facecolor='w')\n\nfor score, ax in zip(sorted(train['isup_grade'].unique()), axes.ravel()):\n    temp = train[train['isup_grade'] == score]\n    sns.countplot(temp['gleason_score'], ax=ax)\n    ax.set_title(score)\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"isup_grades (< 4) are depended on Single \"gleason_score\".\n\nisup_grades (>= 4) are depended on Multi \"gleason_score\"."},{"metadata":{},"cell_type":"markdown","source":"---\n## EDA - Image\n\n\nFor image visualization, refer to the following kernel.\n\nhttps://www.kaggle.com/wouterbulten/getting-started-with-the-panda-dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_imgs = glob.glob(os.path.join(data_dir, 'train_images/*.tiff'))\ntrain_imgs = sorted(train_imgs)\nprint(len(train_imgs))\ntrain_imgs[:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_masks = glob.glob(os.path.join(data_dir, 'train_label_masks/*.tiff'))\ntrain_masks = sorted(train_masks)\nprint(len(train_masks))\ntrain_masks[:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_img(img_id, show_img=True, tiff_level=-1):\n    img_path = os.path.join(data_dir, 'train_images', f'{img_id}.tiff')\n    # Using Openslide\n    slide = openslide.OpenSlide(img_path)\n    # Set Properties  1: Point   2: Tiff Level   3: Viewing Dimension\n    # .level_count -> Get Tiff Level Count\n    # .level_dimensions -> Get Tiff Width, Height per Level\n    if tiff_level == -1:\n        patch = slide.read_region((0, 0), slide.level_count - 1, slide.level_dimensions[-1])\n    else:\n        patch = slide.read_region((0, 0), tiff_level, slide.level_dimensions[tiff_level])\n    \n    if show_img:\n        display(patch)\n        \n    # PIL -> ndarray\n    patch = np.asarray(patch)\n    # RGBA -> RGB\n    if patch.shape[-1] == 4:\n        patch = patch[:, :, :3]\n        \n    slide.close()\n    \n    return patch\n\n\ndef display_mask(img_id, center, show_img=True, tiff_level=-1):\n    assert center in ['radboud', 'karolinska'], \"Please Set center=['radboud', 'karolinska']\"\n    \n    img_path = os.path.join(data_dir, 'train_label_masks', f'{img_id}_mask.tiff')\n    # Using Openslide\n    slide = openslide.OpenSlide(img_path)\n    # Set Properties  1: Point   2: Tiff Level   3: Viewing Dimension\n    # .level_count -> Get Tiff Level Count\n    # .level_dimensions -> Get Tiff Width, Height per Level\n    if tiff_level == -1:\n        mask_data = slide.read_region((0, 0), slide.level_count - 1, slide.level_dimensions[-1])\n    else:\n        mask_data = slide.read_region((0, 0), tiff_level, slide.level_dimensions[tiff_level])\n    \n    mask_data = mask_data.split()[0]\n    # To show the masks we map the raw label values to RGB values\n    preview_palette = np.zeros(shape=768, dtype=int)\n    if center == 'radboud':\n        # Mapping: {0: background(Black), 1: stroma(Dark Gray), 2: benign epithelium(Light Gray), 3: Gleason 3(Pale Yellow), 4: Gleason 4(Orange), 5: Gleason 5(Red)}\n        preview_palette[0:18] = (np.array([0, 0, 0, 0.2, 0.2, 0.2, 0.4, 0.4, 0.4, 1, 1, 0.7, 1, 0.5, 0, 1, 0, 0]) * 255).astype(int)\n    elif center == 'karolinska':\n        # Mapping: {0: background(Black), 1: benign(Gray), 2: cancer(Red)}\n        preview_palette[0:9] = (np.array([0, 0, 0, 0.5, 0.5, 0.5, 1, 0, 0]) * 255).astype(int)\n    mask_data.putpalette(data=preview_palette.tolist())\n    mask_data = mask_data.convert(mode='RGB')\n    if show_img:\n        display(mask_data)\n        \n    # PIL -> ndarray\n    mask_data = np.asarray(mask_data)\n    # RGBA -> RGB\n    if mask_data.shape[-1] == 4:\n        mask_data = mask_data[:, :, :3]\n        \n    slide.close()\n    \n    return mask_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = display_img('0018ae58b01bdadc8e347995b69f99aa')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Mask Data: Those judged to be cancer\nmask = display_mask('0018ae58b01bdadc8e347995b69f99aa', 'radboud')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Mask Data: Those judged not to be cancer\nmask = display_mask('001d865e65ef5d2579c190a0e0350d8f', 'karolinska')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score_list = train['gleason_score'].unique()\nscore_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show mask data for each gleason_score (karolinska)\n\n# Extract only images with mask data\nmask_ids = glob.glob(os.path.join(data_dir, 'train_label_masks/*.tiff'))\nmask_ids = [id.split('/')[4].split('_')[0] for id in mask_ids]\nmask_ids[:6]\n_train = train[train['image_id'].isin(mask_ids)]\n\n_train = _train[_train['data_provider'] == 'karolinska']\n\nfor score in score_list:\n    temp = _train[_train['gleason_score'] == score].sample(frac=1.0)\n    \n    ids = temp.head(6)['image_id'].values\n    data_provider = temp.head(6)['data_provider'].values\n    \n    fig, axes = plt.subplots(ncols=6, nrows=1, figsize=(24, 12), facecolor='w')\n    for _id, provider, ax in zip(ids, data_provider, axes.ravel()):\n        mask = display_mask(_id, provider, show_img=False)\n        ax.imshow(mask)\n        ax.set_title('{} - {}'.format(score, _id))\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show mask data for each gleason_score (radboud)\n\n# Extract only images with mask data\nmask_ids = glob.glob(os.path.join(data_dir, 'train_label_masks/*.tiff'))\nmask_ids = [id.split('/')[4].split('_')[0] for id in mask_ids]\nmask_ids[:6]\n_train = train[train['image_id'].isin(mask_ids)]\n\n_train = _train[_train['data_provider'] == 'radboud']\n\nfor score in score_list:\n    temp = _train[_train['gleason_score'] == score].sample(frac=1.0)\n    \n    ids = temp.head(6)['image_id'].values\n    data_provider = temp.head(6)['data_provider'].values\n    \n    fig, axes = plt.subplots(ncols=6, nrows=1, figsize=(24, 12), facecolor='w')\n    for _id, provider, ax in zip(ids, data_provider, axes.ravel()):\n        mask = display_mask(_id, provider, show_img=False)\n        ax.imshow(mask)\n        ax.set_title('{} - {}'.format(score, _id))\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Hypothesis"},{"metadata":{},"cell_type":"markdown","source":"Relationship between gleason_score and isup_grade (hypothesis)\n\n\nMask (Radboud) Definition\n\nRadboud: Prostate glands are individually labelled. Valid values are:\n\n    0: background (non tissue) or unknown\n    1: stroma (connective tissue, non-epithelium tissue)\n    2: healthy (benign) epithelium\n    3: cancerous epithelium (Gleason 3)\n    4: cancerous epithelium (Gleason 4)\n    5: cancerous epithelium (Gleason 5)\nLooking at the mask data for each gleason_score, it is assumed that the gleason_score has the following definition.\n\n    3+3: Almost cancerous epithelium (Gleason 3)\n    3+4: Gleason 3 and Gleason 4 are mixed. The ratio is more Gleason 3\n    4+3: Gleason 3 and Gleason 4 are mixed. The ratio is more Gleason 4\nSummary:\n\n    A+B: Gleason A and Gleason B are mixed. The ratio is more Gleason A\nAs mentioned above, isup_grade is determined by gleason_score\n\n    3+3(gleason_score) = 1(isup_grade)\n    4+3(gleason_score) = 2(isup_grade)\n    5+5(gleason_score) = 5(isup_grade)"},{"metadata":{"trusted":true},"cell_type":"code","source":"score_num = train['isup_grade'].nunique()\nfig, axes = plt.subplots(ncols=3, nrows=int(np.ceil(score_num / 4)), figsize=(20, 10), facecolor='w')\n\nfor score, ax in zip(sorted(train['isup_grade'].unique()), axes.ravel()):\n    temp = train[train['isup_grade'] == score]\n    sns.countplot(temp['gleason_score'], ax=ax)\n    ax.set_title(score)\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"What you need to find is to cut one image data with a grid and calculate the gleason_score for each grid.\n\nCalculate isup_grade from the ratio of distribution based on the result calculated by it."},{"metadata":{},"cell_type":"markdown","source":"Convert RGB to glearson score\n\n    0: background (non tissue) or unknown\n    RGB: (0, 0, 0): Sum: 0 -> Gleason_score: 0\n\n    1: stroma (connective tissue, non-epithelium tissue)\n    RGB: (51, 51, 51): Sum: 153 -> Gleason_score: 1\n\n    2: healthy (benign) epithelium\n    RGB: (102, 102, 102): Sum: 306 -> Gleason_score: 1 (Meaning Healthy)\n\n    3: cancerous epithelium (Gleason 3)\n    RGB: (255, 255, 178): Sum: 688 -> Gleason_score: 3\n    \n    4: cancerous epithelium (Gleason 4)\n    RGB: (255, 127, 0): Sum: 382 -> Gleason_score: 4\n\n    5: cancerous epithelium (Gleason 5)\n    RGB: (255, 0, 0): Sum: 255 -> Gleason_score: 5"},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing_mask(mask):\n\n    # RBG -> gleason_score\n    _mask = np.sum(mask, axis=2)\n\n    # 0: Background\n    # 1: stroma = 153\n    _mask = np.where(_mask==153, 1, _mask)\n    # 2: benign epithelium = 306\n    _mask = np.where(_mask==306, 1, _mask)  # Healthy\n    # 3: Gleason 3 = 688  score=3\n    _mask = np.where(_mask==688, 3, _mask)\n    # 4: Gleason 4 = 382   score=4\n    _mask = np.where(_mask==382, 4, _mask)\n    # 5: Gleason 5 = 255   score=5\n    _mask = np.where(_mask==255, 5, _mask)\n\n    u, counts = np.unique(_mask, return_counts=True)\n    score_dict = {k:v for k, v in zip(u, counts)}\n    \n    return _mask, score_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate percentage for each gleason_score\nscore = '3+3'\nsearch_num = 5\n\ntemp = train[train['data_provider'] == 'radboud']\ntemp = temp[temp['gleason_score'] == score]\n\ntar_ids = temp.sample(frac=1.0)['image_id'].values[:search_num]\n\nfor id in tar_ids:\n    try:\n        mask = display_mask(id, 'radboud', show_img=False, tiff_level=0)\n    except:\n        continue\n        \n    _, score_dict = preprocessing_mask(mask)\n    \n    try:\n        rate = score_dict[3] / score_dict[1] * 100\n    except:\n        rate = 0\n        \n    print('#'*30)\n    print('Image_id: ', id)\n    print(score_dict)\n    print(f'Score 3 / All   Rate: {rate:.3f}%')\n    \n    del mask\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate percentage for each gleason_score\nscore = '3+4'\nsearch_num = 5\n\ntemp = train[train['data_provider'] == 'radboud']\ntemp = temp[temp['gleason_score'] == score]\n\ntar_ids = temp.sample(frac=1.0)['image_id'].values[:search_num]\n\nfor id in tar_ids:\n    try:\n        mask = display_mask(id, 'radboud', show_img=False, tiff_level=0)\n    except:\n        continue\n        \n    _, score_dict = preprocessing_mask(mask)\n    \n    try:\n        rate_3 = score_dict[3] / (score_dict[1] + score_dict[4]) * 100\n        rate_4 = score_dict[4] / (score_dict[1] + score_dict[3]) * 100\n    except:\n        rate_3 = 0\n        rate_4 = 0\n        \n    print('#'*30)\n    print('Image_id: ', id)\n    print(score_dict)\n    print(f'Score 3 / All   Rate: {rate_3:.3f}%')\n    print(f'Score 4 / All   Rate: {rate_4:.3f}%')\n    \n    del mask\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate percentage for each gleason_score\nscore = '4+3'\nsearch_num = 5\n\ntemp = train[train['data_provider'] == 'radboud']\ntemp = temp[temp['gleason_score'] == score]\n\ntar_ids = temp.sample(frac=1.0)['image_id'].values[:search_num]\n\nfor id in tar_ids:\n    try:\n        mask = display_mask(id, 'radboud', show_img=False, tiff_level=0)\n    except:\n        continue\n        \n    _, score_dict = preprocessing_mask(mask)\n    \n    try:\n        rate_3 = score_dict[3] / (score_dict[1] + score_dict[4]) * 100\n        rate_4 = score_dict[4] / (score_dict[1] + score_dict[3]) * 100\n    except:\n        rate_3 = 0\n        rate_4 = 0\n        \n    print('#'*30)\n    print('Image_id: ', id)\n    print(score_dict)\n    print(f'Score 3 / All   Rate: {rate_3:.3f}%')\n    print(f'Score 4 / All   Rate: {rate_4:.3f}%')\n    \n    del mask\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate percentage for each gleason_score\nscore = '5+4'\nsearch_num = 5\n\ntemp = train[train['data_provider'] == 'radboud']\ntemp = temp[temp['gleason_score'] == score]\n\ntar_ids = temp.sample(frac=1.0)['image_id'].values[:search_num]\n\nfor id in tar_ids:\n    try:\n        mask = display_mask(id, 'radboud', show_img=False, tiff_level=0)\n    except:\n        continue\n        \n    _, score_dict = preprocessing_mask(mask)\n    \n    try:\n        rate_4 = score_dict[4] / (score_dict[1] + score_dict[5]) * 100\n        rate_5 = score_dict[5] / (score_dict[1] + score_dict[4]) * 100\n    except:\n        rate_4 = 0\n        rate_5 = 0\n        \n    print('#'*30)\n    print('Image_id: ', id)\n    print(score_dict)\n    print(f'Score 4 / All   Rate: {rate_4:.3f}%')\n    print(f'Score 5 / All   Rate: {rate_5:.3f}%')\n    \n    del mask\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data Provider == 'radboud'\n# Calculate percentage for each gleason_score\n\nids = []\ngscore_list = []\nrate_3_list = []\nrate_4_list = []\nrate_5_list = []\nlimit = 800\n\ntemp = train[train['data_provider'] == 'radboud']\ntemp = temp.sample(frac=1.0).head(limit)\ntar_ids = temp['image_id'].values\ngleason_score = temp['gleason_score'].values\n\nfor i in tqdm(range(len(temp))):\n    id = tar_ids[i]\n    gscore = gleason_score[i]\n    \n    try:\n        mask = display_mask(id, 'radboud', show_img=False, tiff_level=0)\n    except:\n        continue\n        \n    _, score_dict = preprocessing_mask(mask)\n    \n    # Excluding background\n    del score_dict[0]\n    all_pix = np.sum([v for v in score_dict.values()])\n    \n    # Calculate percentage for each score\n    try:\n        rate_3 = score_dict[3] / all_pix * 100\n    except:\n        rate_3 = 0\n    \n    try:\n        rate_4 = score_dict[4] / all_pix * 100\n    except:\n        rate_4 = 0\n        \n    try:\n        rate_5 = score_dict[5] / all_pix * 100\n    except:\n        rate_5 = 0\n        \n    ids.append(id)\n    gscore_list.append(gscore)\n    rate_3_list.append(rate_3)\n    rate_4_list.append(rate_4)\n    rate_5_list.append(rate_5)\n    \n    del mask, score_dict\n    gc.collect()\n    \nrate_res = pd.DataFrame({\n    'image_id': ids,\n    'gleason_score': gscore_list,\n    'rate_3': rate_3_list,\n    'rate_4': rate_4_list,\n    'rate_5': rate_5_list\n})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ScatterPlot\nfig, axes = plt.subplots(nrows=2, ncols=3, figsize=(14, 8), facecolor='w')\nscores = ['3+4', '4+3', '4+5', '5+4', '3+5', '5+3']\nfor score, ax in zip(scores, axes.ravel()):\n    temp = rate_res[rate_res['gleason_score'] == score]\n    sns.scatterplot(x='rate_{}'.format(score.split('+')[0]), y='rate_{}'.format(score.split('+')[1]), data=temp, ax=ax)\n    ax.set_ylim(0, 50)\n    ax.set_xlim(0, 50)\n    ax.set_title(score)\n    \nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rate_res.to_csv('rate_res.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}