{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip -q install tensorflow==2.3.0","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:43:21.195039Z","iopub.execute_input":"2022-02-28T17:43:21.195502Z","iopub.status.idle":"2022-02-28T17:44:27.111474Z","shell.execute_reply.started":"2022-02-28T17:43:21.195442Z","shell.execute_reply":"2022-02-28T17:44:27.110433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Basics / Data manipulation\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport zipfile\nimport os\n\n# Visualization\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport cv2\nimport skimage.io\nfrom IPython.display import display, HTML\n\n# ML\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:27.113877Z","iopub.execute_input":"2022-02-28T17:44:27.114224Z","iopub.status.idle":"2022-02-28T17:44:32.211882Z","shell.execute_reply.started":"2022-02-28T17:44:27.114179Z","shell.execute_reply":"2022-02-28T17:44:32.211203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data\n10k+ of .tiff images\n*    **90%** for training \n*    **10%** for internal testing\n            *  75% Validation\n            *  25% Testing","metadata":{}},{"cell_type":"code","source":"# Folder paths\nTRAIN = '../input/prostate-cancer-grade-assessment/train_images'\nMASKS = '../input/prostate-cancer-grade-assessment/train_label_masks'\n\nOUT_TRAIN = './train.zip'\nOUT_VALIDATION = './validation.zip'\nOUT_TEST = './test.zip'\nOUT_MASKS_TRAIN = './masks_train.zip'\nOUT_MASKS_VALIDATION = './masks_validation.zip'\nOUT_MASKS_TEST = './masks_test.zip'\n\nBASE_FOLDER = \"/kaggle/input/prostate-cancer-grade-assessment/\"\n!ls {BASE_FOLDER}\nBASE_FOLDER2 =\"/kaggle/input/panda-tiles/\"\n!ls {BASE_FOLDER2}","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:32.213226Z","iopub.execute_input":"2022-02-28T17:44:32.213512Z","iopub.status.idle":"2022-02-28T17:44:33.744329Z","shell.execute_reply.started":"2022-02-28T17:44:32.213482Z","shell.execute_reply":"2022-02-28T17:44:33.7434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(BASE_FOLDER + \"train.csv\")\ntrain.columns.name = \"train.csv\"\ntest = pd.read_csv(BASE_FOLDER + \"test.csv\")\ntest.columns.name = \"test.csv\"\nsub = pd.read_csv(BASE_FOLDER + \"sample_submission.csv\")\nsub.columns.name = \"sample_submission.csv\"\n\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.746935Z","iopub.execute_input":"2022-02-28T17:44:33.747357Z","iopub.status.idle":"2022-02-28T17:44:33.838767Z","shell.execute_reply.started":"2022-02-28T17:44:33.747292Z","shell.execute_reply":"2022-02-28T17:44:33.83792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking for all the \"negative\" labels in the label of gleason_score\ndisplay(train[train['gleason_score'] == 'negative'])","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.842557Z","iopub.execute_input":"2022-02-28T17:44:33.84282Z","iopub.status.idle":"2022-02-28T17:44:33.866476Z","shell.execute_reply.started":"2022-02-28T17:44:33.84279Z","shell.execute_reply":"2022-02-28T17:44:33.865749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Deleting from the dataset a mislabeled row and converting the \"negative\" labels to \"0+0\" in order to have an standard\ntrain.drop([7273],inplace=True)\ntrain['gleason_score'] = train['gleason_score'].apply(lambda x: \"0+0\" if x == \"negative\" else x)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.868738Z","iopub.execute_input":"2022-02-28T17:44:33.869108Z","iopub.status.idle":"2022-02-28T17:44:33.881412Z","shell.execute_reply.started":"2022-02-28T17:44:33.869079Z","shell.execute_reply":"2022-02-28T17:44:33.88053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = np.array(train) # Converting the DataFrame to an array to take the column\nlabels = data[:, 3] # Labels of interest (GLEASON SCORE)\n#labels","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.885397Z","iopub.execute_input":"2022-02-28T17:44:33.885704Z","iopub.status.idle":"2022-02-28T17:44:33.891863Z","shell.execute_reply.started":"2022-02-28T17:44:33.88567Z","shell.execute_reply":"2022-02-28T17:44:33.890934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = data[:, :3] # Features of interest (ID, PROVIDER, ISUP GRADE)\n#features","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.893658Z","iopub.execute_input":"2022-02-28T17:44:33.894295Z","iopub.status.idle":"2022-02-28T17:44:33.903231Z","shell.execute_reply.started":"2022-02-28T17:44:33.894249Z","shell.execute_reply":"2022-02-28T17:44:33.902524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = features\ny = labels","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.904962Z","iopub.execute_input":"2022-02-28T17:44:33.905312Z","iopub.status.idle":"2022-02-28T17:44:33.914897Z","shell.execute_reply.started":"2022-02-28T17:44:33.90527Z","shell.execute_reply":"2022-02-28T17:44:33.914278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.916133Z","iopub.execute_input":"2022-02-28T17:44:33.916664Z","iopub.status.idle":"2022-02-28T17:44:33.925936Z","shell.execute_reply.started":"2022-02-28T17:44:33.91662Z","shell.execute_reply":"2022-02-28T17:44:33.925151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_check, y_train, y_check = train_test_split(X, y, test_size=0.1, random_state = 42) ","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.927711Z","iopub.execute_input":"2022-02-28T17:44:33.928327Z","iopub.status.idle":"2022-02-28T17:44:33.940161Z","shell.execute_reply.started":"2022-02-28T17:44:33.928283Z","shell.execute_reply":"2022-02-28T17:44:33.93941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_validation, X_test, y_validation, y_test = train_test_split(X_check, y_check, test_size=0.25, random_state = 84)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.941609Z","iopub.execute_input":"2022-02-28T17:44:33.942131Z","iopub.status.idle":"2022-02-28T17:44:33.958032Z","shell.execute_reply.started":"2022-02-28T17:44:33.942096Z","shell.execute_reply":"2022-02-28T17:44:33.957197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train-Validation-Test","metadata":{}},{"cell_type":"code","source":"X_train = pd.DataFrame(X_train, columns=[\"image_id\", \"data_provider\", \"isup_grade\"])\nX_train[\"gleason_score\"] = y_train\n\nX_validation = pd.DataFrame(X_validation, columns=[\"image_id\", \"data_provider\", \"isup_grade\"])\nX_validation[\"gleason_score\"] = y_validation\n\nX_test = pd.DataFrame(X_test, columns=[\"image_id\", \"data_provider\", \"isup_grade\"])\nX_test[\"gleason_score\"] = y_test","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.959722Z","iopub.execute_input":"2022-02-28T17:44:33.960192Z","iopub.status.idle":"2022-02-28T17:44:33.975699Z","shell.execute_reply.started":"2022-02-28T17:44:33.960159Z","shell.execute_reply":"2022-02-28T17:44:33.974907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_eda = X_train.groupby(\"gleason_score\").count()[\"image_id\"].reset_index().sort_values(by=\"image_id\", ascending=False)\ntrain_eda.style.background_gradient(cmap=\"Greens\")\ntrain_eda.style.set_caption(\"Train\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:33.977031Z","iopub.execute_input":"2022-02-28T17:44:33.977542Z","iopub.status.idle":"2022-02-28T17:44:34.088427Z","shell.execute_reply.started":"2022-02-28T17:44:33.97751Z","shell.execute_reply":"2022-02-28T17:44:34.087579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_eda = X_validation.groupby(\"gleason_score\").count()[\"image_id\"].reset_index().sort_values(by=\"image_id\", ascending=False)\nvalidation_eda.style.background_gradient(cmap=\"Reds\")\ntrain_eda.style.set_caption(\"Validation\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:34.089832Z","iopub.execute_input":"2022-02-28T17:44:34.090107Z","iopub.status.idle":"2022-02-28T17:44:34.110585Z","shell.execute_reply.started":"2022-02-28T17:44:34.090077Z","shell.execute_reply":"2022-02-28T17:44:34.109689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_eda = X_test.groupby(\"gleason_score\").count()[\"image_id\"].reset_index().sort_values(by=\"image_id\", ascending=False)\ntest_eda.style.background_gradient(cmap=\"Blues\")\ntrain_eda.style.set_caption(\"Test\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:34.111871Z","iopub.execute_input":"2022-02-28T17:44:34.112229Z","iopub.status.idle":"2022-02-28T17:44:34.131984Z","shell.execute_reply.started":"2022-02-28T17:44:34.112199Z","shell.execute_reply":"2022-02-28T17:44:34.130935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\n\nfig = go.Figure(data=[\n    go.Bar(name=\"Test\", x=train_eda[\"gleason_score\"], y=train_eda[\"image_id\"]),\n    go.Bar(name=\"Validation\", x=validation_eda[\"gleason_score\"], y=validation_eda[\"image_id\"]),\n    go.Bar(name=\"Train\", x=test_eda[\"gleason_score\"], y=test_eda[\"image_id\"]),\n])\n\n# Change the bar mode\nfig.update_layout(barmode='group')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:34.133146Z","iopub.execute_input":"2022-02-28T17:44:34.133529Z","iopub.status.idle":"2022-02-28T17:44:37.081284Z","shell.execute_reply.started":"2022-02-28T17:44:34.133492Z","shell.execute_reply":"2022-02-28T17:44:37.079159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:37.083434Z","iopub.execute_input":"2022-02-28T17:44:37.083807Z","iopub.status.idle":"2022-02-28T17:44:37.241093Z","shell.execute_reply.started":"2022-02-28T17:44:37.083766Z","shell.execute_reply":"2022-02-28T17:44:37.240369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_eda\nfig = px.pie(df, values='image_id', names='gleason_score', title = 'Training Images')\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:37.24352Z","iopub.execute_input":"2022-02-28T17:44:37.243871Z","iopub.status.idle":"2022-02-28T17:44:40.309433Z","shell.execute_reply.started":"2022-02-28T17:44:37.243825Z","shell.execute_reply":"2022-02-28T17:44:40.308483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = validation_eda\nfig = px.pie(df, values='image_id', names='gleason_score', title = 'Validation Images')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:40.311373Z","iopub.execute_input":"2022-02-28T17:44:40.311723Z","iopub.status.idle":"2022-02-28T17:44:40.838409Z","shell.execute_reply.started":"2022-02-28T17:44:40.311682Z","shell.execute_reply":"2022-02-28T17:44:40.83747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test_eda\nfig = px.pie(df, values='image_id', names='gleason_score', title = 'Testing Images')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:40.84008Z","iopub.execute_input":"2022-02-28T17:44:40.840618Z","iopub.status.idle":"2022-02-28T17:44:41.18881Z","shell.execute_reply.started":"2022-02-28T17:44:40.84057Z","shell.execute_reply":"2022-02-28T17:44:41.18795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = 'Training Images','Validation Images', 'Testing Images'\nsizes_features = [len(X_train), len(X_validation), len(X_test)]\n# sizes_labels = [len(y_train), len(y_validation)]\n\nfig, ax = plt.subplots(figsize=(30,7))\n\nax.pie(sizes_features, labels=labels, autopct='%1.1f%%',\n          shadow=True, startangle=60)\nax.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle\nax.set_title(f\"Distribution of the dataset\\n Total Images - {(len(X) / len(X) * 100)}%: {len(X)}\\n Training images - {(len(X_train) / len(X) * 100)}%: {len(X_train)}\\n Validation images - {(len(X_validation) / len(X) * 100)}%: {len(X_validation)}\\n Testing images - {(len(X_test) / len(X) * 100)}%: {len(X_test)} \\n \"\n                                                                          ,weight=\"bold\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:41.190092Z","iopub.execute_input":"2022-02-28T17:44:41.190351Z","iopub.status.idle":"2022-02-28T17:44:41.360599Z","shell.execute_reply.started":"2022-02-28T17:44:41.190305Z","shell.execute_reply":"2022-02-28T17:44:41.359737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(X_train.head())","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-02-28T17:44:51.344399Z","iopub.execute_input":"2022-02-28T17:44:51.344759Z","iopub.status.idle":"2022-02-28T17:44:51.377045Z","shell.execute_reply.started":"2022-02-28T17:44:51.344726Z","shell.execute_reply":"2022-02-28T17:44:51.375852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(X_validation.head())","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-02-28T17:44:51.378944Z","iopub.execute_input":"2022-02-28T17:44:51.379291Z","iopub.status.idle":"2022-02-28T17:44:51.407206Z","shell.execute_reply.started":"2022-02-28T17:44:51.37925Z","shell.execute_reply":"2022-02-28T17:44:51.406577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(X_test.head())","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-02-28T17:44:51.408614Z","iopub.execute_input":"2022-02-28T17:44:51.408878Z","iopub.status.idle":"2022-02-28T17:44:51.436204Z","shell.execute_reply.started":"2022-02-28T17:44:51.408847Z","shell.execute_reply":"2022-02-28T17:44:51.435305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Saving the datasets\nX_train = pd.DataFrame(X_train, columns=[\"image_id\", \"variant\", \"data_provider\", \"isup_grade\", \"gleason_score\"])\nX_validation = pd.DataFrame(X_validation, columns=[\"image_id\", \"variant\", \"data_provider\", \"isup_grade\", \"gleason_score\"])\nX_test = pd.DataFrame(X_test, columns=[\"image_id\", \"variant\", \"data_provider\", \"isup_grade\", \"gleason_score\"])\n\nX_train.to_csv(\"./training.csv\")\nX_validation.to_csv(\"./validation.csv\")\nX_test.to_csv(\"./testing.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:51.43792Z","iopub.execute_input":"2022-02-28T17:44:51.438574Z","iopub.status.idle":"2022-02-28T17:44:52.107128Z","shell.execute_reply.started":"2022-02-28T17:44:51.438527Z","shell.execute_reply":"2022-02-28T17:44:52.10628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SIZE_IMG = 112\nN = 16\ndef tile(img, mask):\n    result = []\n    shape = img.shape\n    pad0,pad1 = (SIZE_IMG - shape[0]%SIZE_IMG)%SIZE_IMG, (SIZE_IMG - shape[1]%SIZE_IMG)%SIZE_IMG\n    img = np.pad(img, [[pad0//2, pad0-pad0//2], [pad1//2, pad1 - pad1//2],[0,0]],\n                constant_values=255)\n    mask = np.pad(mask,[[pad0//2, pad0-pad0//2], [pad1//2,pad1-pad1//2], [0,0]],\n                constant_values=0)\n    img = img.reshape(img.shape[0]//SIZE_IMG, SIZE_IMG, img.shape[1]//SIZE_IMG,SIZE_IMG, 3)\n    img = img.transpose(0, 2, 1, 3, 4).reshape(-1, SIZE_IMG,SIZE_IMG,3)\n    mask = mask.reshape(mask.shape[0]//SIZE_IMG, SIZE_IMG,mask.shape[1]//SIZE_IMG, SIZE_IMG, 3)\n    mask = mask.transpose(0, 2, 1, 3, 4).reshape(-1, SIZE_IMG,SIZE_IMG, 3)\n    if len(img) < N:\n        mask = np.pad(mask, [[0, N-len(img)], [0, 0], [0, 0],[0, 0]], constant_values=0)\n        img = np.pad(img, [[0, N-len(img)],[0, 0],[0, 0], [0, 0]], constant_values=255)\n    idxs = np.argsort(img.reshape(img.shape[0], -1).sum(-1))[: N]\n    img = img[idxs]\n    mask = mask[idxs]\n    \n    for i in range(len(img)):\n        result.append({'img':img[i], 'mask':mask[i], 'idx':i})\n\n    return result","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-02-28T17:44:52.108458Z","iopub.execute_input":"2022-02-28T17:44:52.108732Z","iopub.status.idle":"2022-02-28T17:44:52.126506Z","shell.execute_reply.started":"2022-02-28T17:44:52.108702Z","shell.execute_reply":"2022-02-28T17:44:52.125444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def multiplyTiles(tiles):\n    variationA = []\n    variationB = []\n    variationC = []\n    variationD = []\n    variationE = []\n    variationF = []\n    variationG = []\n    variationH = []\n    for t in range(len(tiles)):\n        \n        # Original Tile (A)\n        tile_a_img = tiles[t]['img']\n        tile_a_mask = tiles[t]['mask']\n        tile_a_idx = tiles[t]['idx']\n        tile_a = {\"img\": tile_a_img, \"mask\": tile_a_mask, \"idx\": tile_a_idx}\n        \n        # Rotated Tiles (B, C, D)\n        tile_b_img = np.rot90(tile_a_img)\n        tile_b_mask = np.rot90(tile_a_mask)\n        tile_b_idx = tile_a_idx\n        tile_b = {\"img\": tile_b_img, \"mask\": tile_b_mask, \"idx\": tile_b_idx}\n        \n        tile_c_img = np.rot90(tile_b_img)\n        tile_c_mask = np.rot90(tile_b_mask)\n        tile_c_idx = tile_b_idx\n        tile_c = {\"img\": tile_c_img, \"mask\": tile_c_mask, \"idx\": tile_c_idx}\n        \n        tile_d_img = np.rot90(tile_c_img)\n        tile_d_mask = np.rot90(tile_c_mask)\n        tile_d_idx = tile_c_idx\n        tile_d = {\"img\": tile_d_img, \"mask\": tile_d_mask, \"idx\": tile_d_idx}\n        \n        # Mirrored Original Tile (A:E)\n        tile_e_img = np.fliplr(tile_a_img)\n        tile_e_mask = np.fliplr(tile_a_mask)\n        tile_e_idx = tile_a_idx\n        tile_e = {\"img\": tile_e_img, \"mask\": tile_e_mask, \"idx\": tile_e_idx}        \n        \n        # Mirrored Rotated Tiles (B:F, C:G, D:H)\n        tile_f_img = np.fliplr(tile_b_img)\n        tile_f_mask = np.fliplr(tile_b_mask)\n        tile_f_idx = tile_a_idx\n        tile_f = {\"img\": tile_f_img, \"mask\": tile_f_mask, \"idx\": tile_f_idx}\n        \n        tile_g_img = np.fliplr(tile_c_img)\n        tile_g_mask = np.fliplr(tile_c_mask)\n        tile_g_idx = tile_c_idx\n        tile_g = {\"img\": tile_g_img, \"mask\": tile_g_mask, \"idx\": tile_g_idx}\n        \n        tile_h_img = np.fliplr(tile_d_img)\n        tile_h_mask = np.fliplr(tile_d_mask)\n        tile_h_idx = tile_d_idx\n        tile_h = {\"img\": tile_h_img, \"mask\": tile_h_mask, \"idx\": tile_h_idx}        \n        \n        \n        variationA.append(tile_a)\n        variationB.append(tile_b)\n        variationC.append(tile_c)\n        variationD.append(tile_d)\n        variationE.append(tile_e)\n        variationF.append(tile_f)\n        variationG.append(tile_g)\n        variationH.append(tile_h)\n        \n        tile_bulk = [variationA, variationB, variationC, variationD, variationE, variationF, variationG, variationH]\n\n    return tile_bulk","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:52.128031Z","iopub.execute_input":"2022-02-28T17:44:52.12831Z","iopub.status.idle":"2022-02-28T17:44:52.148622Z","shell.execute_reply.started":"2022-02-28T17:44:52.128281Z","shell.execute_reply":"2022-02-28T17:44:52.147785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import openslide\nimg=openslide.OpenSlide('/kaggle/input/prostate-cancer-grade-assessment/train_images/0005f7aaab2800f6170c399693a96917.tiff')\ndisplay(img.get_thumbnail(size=(512,512)))\n","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-28T17:44:52.149971Z","iopub.execute_input":"2022-02-28T17:44:52.150258Z","iopub.status.idle":"2022-02-28T17:44:52.633964Z","shell.execute_reply.started":"2022-02-28T17:44:52.150227Z","shell.execute_reply":"2022-02-28T17:44:52.633094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Preview\n","metadata":{}},{"cell_type":"code","source":"train_dataset = pd.read_csv(\"./training.csv\", usecols=[\"image_id\", \"data_provider\", \"isup_grade\", \"gleason_score\"])\nvalidation_dataset = pd.read_csv(\"./validation.csv\", usecols=[\"image_id\", \"data_provider\", \"isup_grade\", \"gleason_score\"])\ntest_dataset = pd.read_csv(\"./testing.csv\", usecols=[\"image_id\", \"data_provider\", \"isup_grade\", \"gleason_score\"])\n\n# Mapping to the original dataset\n# ../input/prostate-cancer-grade-assessment/train_images/0005f7aaab2800f6170c399693a96917.tiff\n# ../input/prostate-cancer-grade-assessment/train_label_masks/0005f7aaab2800f6170c399693a96917_mask.tiff\nimg = skimage.io.MultiImage(os.path.join(TRAIN,\"0005f7aaab2800f6170c399693a96917\"+'.tiff'))[1]\nmask = skimage.io.MultiImage(os.path.join(MASKS,\"0005f7aaab2800f6170c399693a96917\"+'_mask.tiff'))[1]\ntiles = tile(img, mask)\n[tiles_A, tiles_B, tiles_C, tiles_D, tiles_E, tiles_F, tiles_G, tiles_H] = multiplyTiles(tiles)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:52.635278Z","iopub.execute_input":"2022-02-28T17:44:52.635562Z","iopub.status.idle":"2022-02-28T17:44:53.739675Z","shell.execute_reply.started":"2022-02-28T17:44:52.635531Z","shell.execute_reply":"2022-02-28T17:44:53.738964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:53.743192Z","iopub.execute_input":"2022-02-28T17:44:53.743703Z","iopub.status.idle":"2022-02-28T17:44:53.763321Z","shell.execute_reply.started":"2022-02-28T17:44:53.743669Z","shell.execute_reply":"2022-02-28T17:44:53.762429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#To Display The Variations\nf_A, ax_A = plt.subplots(4,4, figsize=(10, 10))\nf_B, ax_B = plt.subplots(4,4, figsize=(10, 10))\nf_C, ax_C = plt.subplots(4,4, figsize=(10, 10))\nf_D, ax_D = plt.subplots(4,4, figsize=(10, 10))\nf_E, ax_E = plt.subplots(4,4, figsize=(10, 10))\nf_F, ax_F = plt.subplots(4,4, figsize=(10, 10))\nf_G, ax_G = plt.subplots(4,4, figsize=(10, 10))\nf_H, ax_H = plt.subplots(4,4, figsize=(10, 10))\n\n#Display Variations\nfor t in range(len(tiles_A)):\n        ax_A[t//4, t%4].imshow(tiles_A[t][\"img\"]) # Displaying Image    \n        ax_A[t//4, t%4].axis('off')\nf_A.suptitle('Variation A')\n\nfor t in range(len(tiles_B)):\n        ax_B[t//4, t%4].imshow(tiles_B[t][\"img\"]) # Displaying Image    \n        ax_B[t//4, t%4].axis('off')  \nf_B.suptitle('Variation B')\n\nfor t in range(len(tiles_C)):\n        ax_C[t//4, t%4].imshow(tiles_C[t][\"img\"]) # Displaying Image    \n        ax_C[t//4, t%4].axis('off')  \nf_C.suptitle('Variation C') \n\nfor t in range(len(tiles_D)):\n        ax_D[t//4, t%4].imshow(tiles_D[t][\"img\"]) # Displaying Image    \n        ax_D[t//4, t%4].axis('off') \nf_D.suptitle('Variation D')\n\nfor t in range(len(tiles_E)):\n        ax_E[t//4, t%4].imshow(tiles_E[t][\"img\"]) # Displaying Image    \n        ax_E[t//4, t%4].axis('off')\nf_E.suptitle('Variation E')\n\nfor t in range(len(tiles_F)):\n        ax_F[t//4, t%4].imshow(tiles_F[t][\"img\"]) # Displaying Image    \n        ax_F[t//4, t%4].axis('off')  \nf_F.suptitle('Variation F')\n\nfor t in range(len(tiles_G)):\n        ax_G[t//4, t%4].imshow(tiles_G[t][\"img\"]) # Displaying Image    \n        ax_G[t//4, t%4].axis('off')  \nf_G.suptitle('Variation G')\n\nfor t in range(len(tiles_H)):\n        ax_H[t//4, t%4].imshow(tiles_H[t][\"img\"]) # Displaying Image    \n        ax_H[t//4, t%4].axis('off') \nf_H.suptitle('Variation H')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-02-28T17:44:53.765202Z","iopub.execute_input":"2022-02-28T17:44:53.765534Z","iopub.status.idle":"2022-02-28T17:44:59.657882Z","shell.execute_reply.started":"2022-02-28T17:44:53.765499Z","shell.execute_reply":"2022-02-28T17:44:59.65709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Concatenate Images - 16:1\nThe function ```concat_tile()``` concatenates 16 tiles in one single image, which will be saved later on.","metadata":{}},{"cell_type":"code","source":"id_train = train_dataset[\"image_id\"][163]\nid_validation = validation_dataset[\"image_id\"][163]\nid_test = test_dataset[\"image_id\"][163]\n\n# Testing the function\ndef concat_tile(im_list_2d):\n    return cv2.vconcat([cv2.hconcat(im_list_h) for im_list_h in im_list_2d])\n\ndef mosaic(tiles):\n\n    im1 = tiles[0][\"img\"]\n    im2 = tiles[1][\"img\"]\n    im3 = tiles[2][\"img\"]\n    im4 = tiles[3][\"img\"]\n\n    im5 = tiles[4][\"img\"]\n    im6 = tiles[5][\"img\"]\n    im7 = tiles[6][\"img\"]\n    im8 = tiles[7][\"img\"]\n\n    im9 = tiles[8][\"img\"]\n    im10 = tiles[9][\"img\"]\n    im11 = tiles[10][\"img\"]\n    im12 = tiles[11][\"img\"]\n\n    im13 = tiles[12][\"img\"]\n    im14 = tiles[13][\"img\"]\n    im15 = tiles[14][\"img\"]\n    im16 = tiles[15][\"img\"]\n\n    im_tile = concat_tile([[im1, im2, im3, im4],\n                           [im5, im6, im7, im8],\n                           [im9, im10, im11, im12],\n                           [im13, im14, im15, im16]])\n    return im_tile\n\nimg = skimage.io.MultiImage(os.path.join(TRAIN, f\"{id_train}.tiff\"))[1]\nmask = skimage.io.MultiImage(os.path.join(MASKS, f\"{id_train}_mask.tiff\"))[1]\n\n\nmosaic_img = mosaic(tiles)\n\nplt.title(f\"ID: {id_train}\")\nplt.imshow(mosaic_img)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:44:59.659531Z","iopub.execute_input":"2022-02-28T17:44:59.659789Z","iopub.status.idle":"2022-02-28T17:45:00.07279Z","shell.execute_reply.started":"2022-02-28T17:44:59.65976Z","shell.execute_reply":"2022-02-28T17:45:00.071903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generating the Dataset\n\n* Iterate through the train and test dataset\n    * Map the for both the train and the test dataset to the base folder\n    * Zip the 16 subimages\n    * Save the 16 subimages in their correspondant GLEASON_SCORE folder","metadata":{}},{"cell_type":"code","source":"train_IDs = train_dataset[\"image_id\"]\nvalidation_IDs = validation_dataset[\"image_id\"]\ntest_IDs = test_dataset[\"image_id\"]\n\nnot_found_train = []\nnot_found_validation = []\nnot_found_test = []\ndef generate_dataset(ids, dataset_type):\n    if dataset_type == \"train\":\n        x_tot,x2_tot = [], []\n        with zipfile.ZipFile(OUT_TRAIN, 'w') as img_out,\\\n         zipfile.ZipFile(OUT_MASKS_TRAIN, 'w') as mask_out:\n            for gleason_score, id in enumerate(tqdm(ids)):\n                try:\n                    img = skimage.io.MultiImage(os.path.join(TRAIN,id+'.tiff'))[1]\n                    mask = skimage.io.MultiImage(os.path.join(MASKS,id+'_mask.tiff'))[1]\n                    tiles = tile(img,mask)\n                    [tiles_A, tiles_B, tiles_C, tiles_D, tiles_E, tiles_F, tiles_G, tiles_H] = multiplyTiles(tiles)\n\n                    #tiles_A:    \n                    img = mosaic(tiles_A)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0))\n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'train/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'train/GLEASON_SCORE_{train_dataset[\"gleason_score\"][gleason_score]}/{id}-variationA.png', img)\n                    \n                    #tiles_B:    \n                    img = mosaic(tiles_B)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0))\n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'train/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'train/GLEASON_SCORE_{train_dataset[\"gleason_score\"][gleason_score]}/{id}-variationB.png', img)\n                    \n                    #tiles_C:    \n                    img = mosaic(tiles_C)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0))\n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'train/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'train/GLEASON_SCORE_{train_dataset[\"gleason_score\"][gleason_score]}/{id}-variationC.png', img)\n                    \n                    #tiles_D:    \n                    img = mosaic(tiles_D)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0))\n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'train/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'train/GLEASON_SCORE_{train_dataset[\"gleason_score\"][gleason_score]}/{id}-variationD.png', img)\n\n                    #tiles_E:    \n                    img = mosaic(tiles_E)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0))\n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'train/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'train/GLEASON_SCORE_{train_dataset[\"gleason_score\"][gleason_score]}/{id}-variationE.png', img)\n                    \n                    #tiles_F:    \n                    img = mosaic(tiles_F)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0))\n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'train/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'train/GLEASON_SCORE_{train_dataset[\"gleason_score\"][gleason_score]}/{id}-variationF.png', img)\n                    \n                    #tiles_G:    \n                    img = mosaic(tiles_G)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0))\n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'train/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'train/GLEASON_SCORE_{train_dataset[\"gleason_score\"][gleason_score]}/{id}-variationG.png', img)\n                    \n                    #tiles_H:    \n                    img = mosaic(tiles_H)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0))\n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'train/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'train/GLEASON_SCORE_{train_dataset[\"gleason_score\"][gleason_score]}/{id}-variationH.png', img)\n                        \n                except Exception as e:\n                    not_found_train.append(id)\n        print(f\"INFO: Not images found in train: {len(not_found_train)}\")\n        \n    elif dataset_type == \"valid\": \n        x_tot,x2_tot = [], []\n        with zipfile.ZipFile(OUT_VALIDATION, 'w') as img_out,\\\n         zipfile.ZipFile(OUT_MASKS_VALIDATION, 'w') as mask_out:\n            for gleason_score, id in enumerate(tqdm(ids)):\n                try:\n                    img = skimage.io.MultiImage(os.path.join(TRAIN,id+'.tiff'))[1]\n                    mask = skimage.io.MultiImage(os.path.join(MASKS,id+'_mask.tiff'))[1]\n                    tiles = tile(img,mask)\n                    img = mosaic(tiles)\n                    \n                    [tiles_A, tiles_B, tiles_C, tiles_D, tiles_E, tiles_F, tiles_G, tiles_H] = multiplyTiles(tiles)\n                    \n                    #tiles_A:    \n                    img = mosaic(tiles_A)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'validation/GLEASON_SCORE_{validation_dataset[\"gleason_score\"][gleason_score]}/{id}-variationA.png', img)\n                    \n                    #tiles_B:    \n                    img = mosaic(tiles_B)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'validation/GLEASON_SCORE_{validation_dataset[\"gleason_score\"][gleason_score]}/{id}-variationB.png', img)     \n                    \n                    #tiles_C:    \n                    img = mosaic(tiles_C)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'validation/GLEASON_SCORE_{validation_dataset[\"gleason_score\"][gleason_score]}/{id}-variationC.png', img)\n                    \n                    #tiles_D:    \n                    img = mosaic(tiles_D)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'validation/GLEASON_SCORE_{validation_dataset[\"gleason_score\"][gleason_score]}/{id}-variationD.png', img)\n\n                    #tiles_E:    \n                    img = mosaic(tiles_E)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'validation/GLEASON_SCORE_{validation_dataset[\"gleason_score\"][gleason_score]}/{id}-variationE.png', img)\n                    \n                    #tiles_F:    \n                    img = mosaic(tiles_F)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'validation/GLEASON_SCORE_{validation_dataset[\"gleason_score\"][gleason_score]}/{id}-variationF.png', img)     \n                    \n                    #tiles_G:    \n                    img = mosaic(tiles_G)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'validation/GLEASON_SCORE_{validation_dataset[\"gleason_score\"][gleason_score]}/{id}-variationG.png', img)\n                    \n                    #tiles_H:    \n                    img = mosaic(tiles_H)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'validation/GLEASON_SCORE_{validation_dataset[\"gleason_score\"][gleason_score]}/{id}-variationH.png', img)\n                    \n                except Exception as e:\n                    not_found_validation.append(id)\n\n        print(f\"INFO: Not images found in validation: {len(not_found_validation)}\")\n        \n    elif dataset_type == \"test\":  \n        x_tot,x2_tot = [], []\n        with zipfile.ZipFile(OUT_TEST, 'w') as img_out,\\\n         zipfile.ZipFile(OUT_MASKS_TEST, 'w') as mask_out:\n            for gleason_score, id in enumerate(tqdm(ids)):\n                try:\n                    img = skimage.io.MultiImage(os.path.join(TRAIN,id+'.tiff'))[1]\n                    mask = skimage.io.MultiImage(os.path.join(MASKS,id+'_mask.tiff'))[1]\n                    tiles = tile(img,mask)\n                    [tiles_A, tiles_B, tiles_C, tiles_D, tiles_E, tiles_F, tiles_G, tiles_H] = multiplyTiles(tiles)\n\n                    #tiles_A:\n                    img = mosaic(tiles_A)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'test/GLEASON_SCORE_{test_dataset[\"gleason_score\"][gleason_score]}/{id}-variationA.png', img)\n                \n                    #tiles_B:\n                    img = mosaic(tiles_B)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'test/GLEASON_SCORE_{test_dataset[\"gleason_score\"][gleason_score]}/{id}-variationB.png', img)\n                    \n                    #tiles_C:\n                    img = mosaic(tiles_C)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'test/GLEASON_SCORE_{test_dataset[\"gleason_score\"][gleason_score]}/{id}-variationC.png', img)\n                    \n                    #tiles_D:\n                    img = mosaic(tiles_D)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'test/GLEASON_SCORE_{test_dataset[\"gleason_score\"][gleason_score]}/{id}-variationD.png', img)\n                    \n                    #tiles_E:\n                    img = mosaic(tiles_E)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'test/GLEASON_SCORE_{test_dataset[\"gleason_score\"][gleason_score]}/{id}-variationE.png', img)\n                \n                    #tiles_F:\n                    img = mosaic(tiles_F)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'test/GLEASON_SCORE_{test_dataset[\"gleason_score\"][gleason_score]}/{id}-variationF.png', img)\n                    \n                    #tiles_G:\n                    img = mosaic(tiles_G)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'test/GLEASON_SCORE_{test_dataset[\"gleason_score\"][gleason_score]}/{id}-variationG.png', img)\n                    \n                    #tiles_H:\n                    img = mosaic(tiles_H)\n                    x_tot.append((img/255.0).reshape(-1,3).mean(0))\n                    x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n                    # If read with PIL RGB turns into BGR\n                    img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n                    # Uncomment to classify by ISUP GRADE \n                    # img_out.writestr(f'test/ISUP_GRADE_{train_dataset[\"isup_grade\"][isup_grade]}/{id}_{idx}.png', img)\n                    img_out.writestr(f'test/GLEASON_SCORE_{test_dataset[\"gleason_score\"][gleason_score]}/{id}-variationH.png', img)                        \n                except Exception as e:\n                    not_found_test.append(id)\n\n        print(f\"INFO: Not images found in test: {len(not_found_test)}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:45:00.07418Z","iopub.execute_input":"2022-02-28T17:45:00.074489Z","iopub.status.idle":"2022-02-28T17:45:00.174531Z","shell.execute_reply.started":"2022-02-28T17:45:00.074458Z","shell.execute_reply":"2022-02-28T17:45:00.173579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"generate_dataset(train_IDs, dataset_type='train')\ngenerate_dataset(validation_IDs, dataset_type='valid')\ngenerate_dataset(test_IDs, dataset_type='test')","metadata":{"execution":{"iopub.status.busy":"2022-02-28T17:45:00.176301Z","iopub.execute_input":"2022-02-28T17:45:00.176603Z","iopub.status.idle":"2022-02-28T17:45:05.496255Z","shell.execute_reply.started":"2022-02-28T17:45:00.176573Z","shell.execute_reply":"2022-02-28T17:45:05.495458Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lost/Corrupted Data","metadata":{}},{"cell_type":"code","source":"labels = \"Training Images\", \"Validation Images\",\"Testing Images\", \"Loss\"\nsizes_features = [len(X_train), len(X_validation), len(X_test), len(not_found_train) + len(not_found_validation) +len(not_found_test)]\n# sizes_labels = [len(X_train), len(X_validation), 20]\n\nfig, ax = plt.subplots(figsize=(30,7))\n\nax.pie(sizes_features, labels=labels, autopct='%1.1f%%',\n          shadow=True, startangle=60)\nax.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle\nax.set_title(f\"Distribution of the dataset\\n Total Images - {(len(X) / len(X) * 100)}%: {len(X)} \\n Training images - {(len(X_train) / len(X) * 100)}%: {len(X_train)} \\n Validation images - {(len(X_validation) / len(X) * 100)}%: {len(X_validation)} \\n Testing images - {(len(X_test) / len(X) * 100)}%: {len(X_test)} \\n Loss - : {len(not_found_train) + len(not_found_validation) +len(not_found_test)} / {len(X)} images\", weight=\"bold\")\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Removing Lost/Corrupted Data","metadata":{}},{"cell_type":"code","source":"not_found_train_eda = []\nif not_found_train:\n    for not_found in not_found_train:\n        not_found_train_eda.append(train_dataset[train_dataset[\"image_id\"] == not_found])\n    not_found_train_eda = pd.concat(not_found_train_eda)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"not_found_validation_eda = []\nif not_found_validation:\n    for not_found in not_found_validation:\n        not_found_validation_eda.append(validation_dataset[validation_dataset[\"image_id\"] == not_found])\n    not_found_validation_eda = pd.concat(not_found_validation_eda)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"not_found_test_eda = []\nif not_found_test:\n    for not_found in not_found_test:\n        not_found_test_eda.append(test_dataset[test_dataset[\"image_id\"] == not_found])\n    not_found_test_eda = pd.concat(not_found_test_eda)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not_found_train_eda.empty:\nnot_found_train_eda = not_found_train_eda.groupby('gleason_score').count()['image_id'].reset_index().sort_values(by='image_id', ascending=False)\nnot_found_train_eda.style.background_gradient(cmap='Greens')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not_found_validation_eda.empty:\nnot_found_validation_eda = not_found_validation_eda.groupby('gleason_score').count()['image_id'].reset_index().sort_values(by='image_id', ascending=False)\nnot_found_validation_eda.style.background_gradient(cmap='Reds')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not not_found_test_eda.empty:\nnot_found_test_eda = not_found_test_eda.groupby('gleason_score').count()['image_id'].reset_index().sort_values(by='image_id', ascending=False)\nnot_found_test_eda.style.background_gradient(cmap='Blues')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not_found_train_eda or not_found_validation_eda or not_found_test_eda:\nfig = go.Figure(data=[\n    go.Bar(name=\"Not found test\", x=not_found_train_eda[\"gleason_score\"], y=not_found_train_eda[\"image_id\"]),\n    go.Bar(name=\"Not found validation\", x=not_found_validation_eda[\"gleason_score\"], y=not_found_validation_eda[\"image_id\"]),\n    go.Bar(name=\"Not found train\", x=not_found_test_eda[\"gleason_score\"], y=not_found_test_eda[\"image_id\"])\n])\n\n# Change the bar mode\nfig.update_layout(barmode='group')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not_found_train_eda or not_found_test_eda:\nfig = go.Figure(data=[\n    go.Bar(name=\"Not found train\", x=not_found_train_eda[\"gleason_score\"], y=not_found_train_eda[\"image_id\"]),\n    go.Bar(name=\"Not found validation\", x=not_found_validation_eda[\"gleason_score\"], y=not_found_validation_eda[\"image_id\"]),\n    go.Bar(name=\"Not found test\", x=not_found_test_eda[\"gleason_score\"], y=not_found_test_eda[\"image_id\"]),\n    go.Bar(name=\"Found test\", x=train_eda[\"gleason_score\"], y=train_eda[\"image_id\"]),\n    go.Bar(name=\"Found validation\", x=validation_eda[\"gleason_score\"], y=validation_eda[\"image_id\"]),\n    go.Bar(name=\"Found train\", x=test_eda[\"gleason_score\"], y=test_eda[\"image_id\"])\n])\n\n# Change the bar mode\nfig.update_layout(barmode='group')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution of the loss images","metadata":{}},{"cell_type":"code","source":"# if not_found_train_eda:\ndf = not_found_train_eda\nfig = px.pie(df, values='image_id', names='gleason_score')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not_found_validation_eda:\ndf = not_found_validation_eda\nfig = px.pie(df, values='image_id', names='gleason_score')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not_found_test_eda:\ndf = not_found_test_eda\nfig = px.pie(df, values='image_id', names='gleason_score')\nfig.show()","metadata":{},"execution_count":null,"outputs":[]}]}