{"cells":[{"metadata":{},"cell_type":"markdown","source":"Source: https://www.kaggle.com/c/prostate-cancer-grade-assessment","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport openslide\nimport skimage.io\nimport matplotlib\nimport numpy as np \nimport pandas as pd \nimport deepdish as dd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n## print out the names of the first 3 image_files (total = 10 images for train_imgaes & train_label_masks) with the train, test, submission.csv files\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames[:3]:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Loading & viewing dataset","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BASE_PATH = '../input/prostate-cancer-grade-assessment'\n\ndata_dir = f'{BASE_PATH}/train_images'\nmask_dir = f'{BASE_PATH}/train_label_masks'\nh5_dir = '/kaggle/input/h5-files/full_data_coordinate.h5'\n\ntrain = pd.read_csv(f'{BASE_PATH}/train.csv').set_index('image_id')\ntest = pd.read_csv(f'{BASE_PATH}/test.csv')\nsubmission = pd.read_csv(f'{BASE_PATH}/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We want to seperate the dataset from `radboud` to build a model of $6$ classes, since almost data from `karolinska` has `isup_grade = {0, 1}`, ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_relative_distribution(df, feature, hue, title='', size=2):\n    f, ax = plt.subplots(1,1, figsize=(10*size, 4*size))\n    total = float(len(df))\n    sns.countplot(x=feature, hue=hue, data=df, palette='Set2')\n    plt.title(title)\n    for p in ax.patches:\n        height = p.get_height()\n        ax.text(p.get_x() + p.get_width()/2,  ## set the text in the middle of each column \n                height*0.7,                   ## set the text above 0.8*height of the column's height\n                '{:d} img_id \\n\\n approx {:1.2f}%'.format(height, (height*100/total)), ## content\n                ha=\"center\") \n    plt.show()\n    \nplot_relative_distribution(df=train, feature='isup_grade', hue='data_provider', title = 'relative count plot of isup_grade with data_provider', size=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"See how many `img_id` from `radboud`","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_count(df, feature, title='', size=2):\n    f, ax = plt.subplots(1,1, figsize=(5.5*size, 3.25*size))\n    total = float(len(df))\n    sns.countplot(df[feature],order = df[feature].value_counts().index, palette='Set2')\n    plt.title(title)\n    for p in ax.patches:\n        height = p.get_height()\n        ax.text(p.get_x() + p.get_width()/2,  ## set the text in the middle of each column \n                height*0.8,                   ## set the text above 0.8*height of the column's height\n                '{:d} img_id \\n\\n approx {:1.2f}%'.format(height, (height*100/total)), ## content\n                ha=\"center\") \n    plt.show()\n    \nplot_count(df=train, feature='data_provider', title = 'data_provider count_plot')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Load `h5.file`**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%time df = dd.io.load(h5_dir)\nlen(df), df[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### See something in the `h5.file`\n\nThis dataset is a list of `list(image_id, coordinates)`\n\nEach `coordinates` contains the values of `x_start, x_end, y_start, y_end` from the `data_image` and `mask_image`. For example","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_tiles_in_data(img_ID, level = 0):\n    idx = df[img_ID][0]\n    im = skimage.io.MultiImage(os.path.join(data_dir, f'{idx}.tiff'))[level]\n    plt.figure(figsize = (25, 25))\n    for k in range(36):\n        plt.subplot(6, 6, k + 1)\n        a, b, c, d = df[img_ID + k][1 :]\n        plt.imshow(im[a : b, c: d, :])\n        plt.title(k)\n        \ndisplay_tiles_in_data(720)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, display 36 tiles in the corresponding mask","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_tiles_in_mask(img_ID, level = 0):\n    idx = df[img_ID][0]\n    im = skimage.io.MultiImage(os.path.join(mask_dir, f'{idx}_mask.tiff'))[0]\n    cmap = matplotlib.colors.ListedColormap(['black', 'gray', 'green', 'yellow', 'orange', 'red'])\n    plt.figure(figsize = (25, 25))\n    for k in range(36):\n        plt.subplot(6, 6, k + 1)\n        a, b, c, d = df[img_ID + k][1 :]\n        plt.imshow(im[a : b, c: d, 0], cmap=cmap, interpolation='nearest', vmin=0, vmax=5)\n        plt.title(k)\n        \ndisplay_tiles_in_mask(720)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Randomly display tiles of image_mask & image_data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_tiles_df_h5_file(img_ID, tile_indexes, level = 0):\n    \"\"\"\n    Load & display the indexes of {image_id and tiles_coordinates} in h5.file\n    Input args:\n        img_ID (int : a multiple of 36, less than len of h5.tile): img_id from the data_dir \n        tile_indexes (list of int from [0, 36]): list of the indexes of 36 tiles per image from data_dir\n    output:\n        display mask_img & data_img coresponding to img_id and its indexes\n    \"\"\"\n    img_id = df[img_ID][0]\n    img_data = skimage.io.MultiImage(os.path.join(data_dir, f'{img_id}.tiff'))[level]\n    img_mask = skimage.io.MultiImage(os.path.join(mask_dir, f'{img_id}_mask.tiff'))[level]\n    cmap = matplotlib.colors.ListedColormap(['black', 'gray', 'green', 'yellow', 'orange', 'red'])\n    nrow = len(tile_indexes)\n    resize = 2**(2*level)\n    plt.figure(figsize = (10, 5.5*nrow))\n    for k in range(nrow):\n        coordinates = df[img_ID + tile_indexes[k]][1 : ]\n        x_start, x_end, y_start, y_end = [int(coordinates[j] / resize) for j in range(4)]\n        plt.subplot(nrow, 2, 2*(k+1) -1 )\n        plt.imshow(img_data[x_start: x_end, y_start: y_end, :])\n        plt.title('ID = %s \\n level = %s & tile_index = %s'%(img_id, level, tile_indexes[k]))\n        plt.subplot(nrow, 2, 2*(k+1))\n        plt.imshow(img_mask[x_start: x_end, y_start: y_end,0], cmap=cmap, interpolation='nearest', vmin=0, vmax=5)\n        plt.title('mask image')\n        \ndisplay_tiles_df_h5_file(1080, [15, 7, 12, 21, 28])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Now, extract the `img_id` in `h5.database` which is from `radboud`","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DF = []\nfor idx in range(len(df)):\n    img_id = df[idx][0]\n    data_provider = train.loc[img_id, 'data_provider']\n    if data_provider == 'radboud':\n        DF += [df[idx]]\nDF[0], DF[-1], len(DF)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Finally, save it as the new `h5.file`","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%time dd.io.save('radboud_tiles_coordinates.h5', data = DF)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(DF)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Choose what device to run**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train model","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}