{"cells":[{"metadata":{},"cell_type":"markdown","source":"Filter out images where gleason pattern in mask match more or less isup grade of train.csv.\nNo clusterization done here. Only look if percentage of masked areas could in principle describe the isup grade.\nFor example, if a mask has no gleason pattern 5, it is not possible to reach isup grade 5.","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport skimage.io\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/prostate-cancer-grade-assessment/train.csv')\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_dicts = []\nfor i, row in df_train.iterrows():\n    if i%1000 == 0:\n        for i in range(10):\n            gc.collect()  # to be really effective need to run it several times... \n    \n    # new columns in csv with default values\n    row[\"size\"] = -1\n    for j in range(6):\n        row[f\"gleason_{j}_size\"] = -1\n\n    image_id = row[\"image_id\"]\n    mimg_mask = skimage.io.MultiImage(\"/kaggle/input/prostate-cancer-grade-assessment/train_label_masks/\" + image_id + \"_mask.tiff\")\n    if len(mimg_mask) < 1:\n        result_dicts += [row]\n        continue\n              \n    # for this rough estiamte take only smallest zoom\n    mask = mimg_mask[2][...,0]\n    # be conservative and dilate pixels\n    mask = cv2.morphologyEx(mask, cv2.MORPH_DILATE, np.ones((21,21),np.uint8))\n    row[\"size\"] = mask.size\n    for j in range(6):\n        row[f\"size_gleason_{j}\"] = np.sum(mask==j)\n    result_dicts += [row]\n    \n    del mimg_mask\n    del mask\n\ndf_train = pd.DataFrame(result_dicts)\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[\"size_tissue\"] = df_train[[f\"size_gleason_{i}\" for i in range(6)]].sum(axis=1)\nfor i in range(6):\n    df_train[f\"percent_gleason_{i}\"] = df_train[f\"size_gleason_{i}\"] / df_train[\"size_tissue\"] * 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_mask(df, query, comment):\n    print(f\"{comment}: {len(df.query(query))}\")\n    if \"good_mask\" not in df:\n        df.loc[:, \"good_mask\"] = True\n    df_good = df.query(f\"not ({query})\")\n    df_bad = df.query(query)\n    df_bad.loc[:, \"good_mask\"] = False\n    return pd.concat([df_good, df_bad], sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# no data\ndf_train = test_mask(df_train, \"size < 1\", \"no mask at all\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# gleason-score (most common + second most common pattern) | ISUP Grade\n# 3+3 | 1\n# 3+4 | 2\n# 4+3 | 3\n# 4+4 | 4\n# 3+5 | 4\n# 5+3 | 4\n# 4+5 | 5\n# 5+4 | 5\n# 5+5 | 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# isup 1\ndf_train = test_mask(df_train, \"(percent_gleason_4 > 5 or percent_gleason_5 > 5) and isup_grade == 1\", \"gleason pattern too large for isup 1\")\ndf_train = test_mask(df_train, \"percent_gleason_3 < 5 and isup_grade == 1\", \"gleason pattern too small for isup 1\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# isup 2\ndf_train = test_mask(df_train, \"(percent_gleason_5 > 5) and isup_grade == 2\", \"gleason pattern too large for isup 2\")\ndf_train = test_mask(df_train, \"(percent_gleason_3 < 5 or percent_gleason_4 < 5) and isup_grade == 2\", \"gleason pattern too small for isup 2\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# isup 3\ndf_train = test_mask(df_train, \"(percent_gleason_5 > 5) and isup_grade == 3\", \"gleason pattern too large for isup 3\")\ndf_train = test_mask(df_train, \"(percent_gleason_3 < 5 and percent_gleason_4 < 5) and isup_grade == 3\", \"gleason pattern too small for isup 3\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# isup 4\ndf_train = test_mask(df_train, \"(percent_gleason_5 > 10) and isup_grade == 4\", \"gleason pattern too large for isup 4\")\ndf_train = test_mask(df_train, \"(percent_gleason_4 < 10 or (percent_gleason_3 < 5 and percent_gleason_5 < 5)) and isup_grade == 4\", \"gleason pattern too small for isup 4\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# isup 5\ndf_train = test_mask(df_train, \"(percent_gleason_5 < 5) and isup_grade == 5\", \"gleason pattern too small for isup 5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Good masks: {np.sum(df_train.good_mask)}/{len(df_train)} ({round(100*np.sum(df_train.good_mask)/len(df_train),1)}%)\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.to_csv(\"/kaggle/working/train_mask_cleanup.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}