{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook creates a CSV file labelling iafoss' tiles. It uses the 12 tiles with the most tissues and creates a row per tile, with image id, tile index, data provider, gleason score, % background, % benign, % gleason 3, % gleason 4, % gleason 5 for each tile.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom subprocess import check_output\nimport skimage.io\nimport matplotlib\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir('/kaggle/working/masks/')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import zipfile\nwith zipfile.ZipFile(\"../input/panda-step-1-tiling/masks.zip\",\"r\") as z:\n    z.extractall(\"../masks\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading and using our training data\nMASKS = '../input/prostate-cancer-grade-assessment/train_label_masks/'\nMAIN_DIR = '../input/prostate-cancer-grade-assessment'\n# load data\ntrain = pd.read_csv(os.path.join(MAIN_DIR, 'train.csv'))\nK_0 = train[(train.data_provider == 'karolinska') & (train.gleason_score == '0+0')].iloc[:,0]\nK_1 = train[(train.data_provider == 'karolinska') & (train.gleason_score == '3+3')].iloc[:,0]\nK_2 = train[(train.data_provider == 'karolinska') & (train.gleason_score == '4+4')].iloc[:,0]\nK_3 = train[(train.data_provider == 'karolinska') & (train.gleason_score == '5+5')].iloc[:,0]\nR_All = train[train.data_provider == 'radboud'].iloc[:,0]\nlist_of_dfs = [K_0, K_1, K_2, K_3, R_All]\nfor item in list_of_dfs:\n    print(len(item))\nnames = [name[:-10] for name in os.listdir(MASKS)]\ndef drop_id(df_of_ids):\n    for i in df_of_ids.index:\n        if df_of_ids[i] not in names:\n            df_of_ids = df_of_ids.drop([i])\n    return(df_of_ids)\nnew_list_of_dfs = [drop_id(item) for item in list_of_dfs]            \nfor item in new_list_of_dfs:\n    print(len(item))\nKar_0 = pd.Series(['karolinska'])\nRad = pd.Series(['radboud'])\nRad = Rad.repeat(len(new_list_of_dfs[4])*12)\nKar_0 = Kar_0.repeat((len(new_list_of_dfs[0])+len(new_list_of_dfs[1])+len(new_list_of_dfs[2])+len(new_list_of_dfs[3]))*12)\ndp_s = pd.concat([Kar_0, Rad])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images = [skimage.io.imread(\"../masks/\"+K_3.iloc[0]+\"_\"+str(i)+\".png\") for i in range(10,22)]\nplt.imshow(images[10])\nnum_canc = [(image == 2).sum() for image in images]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nK_0_S = pd.Series(new_list_of_dfs[0])\nK_0_S = K_0_S.repeat(12)\nK_1_S = pd.Series(new_list_of_dfs[1])\nK_1_S = K_1_S.repeat(12)\nK_2_S = pd.Series(new_list_of_dfs[2])\nK_2_S = K_2_S.repeat(12)\nK_3_S = pd.Series(new_list_of_dfs[3])\nK_3_S = K_3_S.repeat(12)\nR_All_S = pd.Series(new_list_of_dfs[4])\nR_All_S = R_All_S.repeat(12)\n\nids_s = pd.concat([K_0_S, K_1_S, K_2_S, K_3_S, R_All_S])\n\ntile_ind = [i for i in range(0,12)] * (int(len(ids_s)/12))\ntile_ind_s = pd.Series(tile_ind)\nnon_existent = []\nvals = np.zeros((len(ids_s), 5))\nfor i in range(0,len(ids_s)):\n        idx = tile_ind_s[i] + 10\n        if os.path.isfile(\"../masks/\"+ids_s.iloc[i]+\"_\"+str(idx)+\".png\"):\n            image = skimage.io.imread(\"../masks/\"+ids_s.iloc[i]+\"_\"+str(idx)+\".png\")\n            if i < len(K_0_S):\n                for j in range(0,2):\n                    vals[i,j] = (image == j).sum()\n            else: \n                if i < (len(K_0_S)+len(K_1_S)):\n                    for j in range(0,3):\n                        vals[i,j] = (image == j).sum()\n                else: \n                    if i < (len(K_0_S)+len(K_1_S)+len(K_2_S)):\n                        for j in range(0,2):\n                            vals[i,j] = (image == j).sum()\n                        vals[i,3] = (image == 2).sum()\n                    else:\n                        if i < (len(K_0_S)+len(K_1_S)+len(K_2_S)+len(K_3_S)):\n                            for j in range(0,2):\n                                vals[i,j] = (image == j).sum()\n                            vals[i,4] = (image == 2).sum()\n                        else:\n                            for j in [3,4,5]:\n                                vals[i,(j-1)] = (image == j).sum()\n                            vals[i,0] = (image == 0).sum()\n                            vals[i,1] = (image == 1).sum() + (image == 2).sum()\n        else:\n            non_existent.append(ids_s.iloc[i]+\"_\"+str(idx)+\".png does not exist\")\n                                \nvals = (vals * 100) / (128 ** 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Radboudumc: Prostate glands are individually labelled. Valid values are:\n\n0: background (non tissue) or unknown 1: stroma (connective tissue, non-epithelium tissue) 2: healthy (benign) epithelium 3: cancerous epithelium (Gleason 3) 4: cancerous epithelium (Gleason 4) 5: cancerous epithelium (Gleason 5)\n\nKarolinska: Regions are labelled. Valid values: 0: background (non tissue) or unknown 1: benign tissue (stroma and epithelium combined) 2: cancerous tissue (stroma and epithelium combined)\n\nThe label masks of Radboudumc were semi-automatically generated by several deep learning algorithms, contain noise, and can be considered as weakly-supervised labels. The label masks of Karolinska were semi-autotomatically generated based on annotations by a pathologist.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df2 = pd.DataFrame(vals, columns=['%background', '%benign', '%Gleason3', '%Gleason4', '%Gleason5'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gs_l = [train[(train.image_id == ids_s.iloc[i])].iloc[0]['gleason_score'] for i in range(0,len(ids_s))]\nids_l = ids_s.tolist()\ndp_l = dp_s.tolist()\ntile_ind_l = tile_ind_s.tolist()\n\n# maybe lists instead of series maybe multi index is goin mad","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nd = {'image_id': ids_l, 'data_provider': dp_l, 'Gleason_score': gs_l, 'tile_index': tile_ind_s}\ndf1 = pd.DataFrame(data=d)\ndf = pd.concat([df1, df2], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('tile_info.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nshutil.rmtree(\"../masks\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}