{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Collection of Suspicious Slides\n\nThis notebook will be a growing list of those slides that for some reason are not ideal for training the model.\n\n## [Discussion Post](https://www.kaggle.com/c/prostate-cancer-grade-assessment/discussion/148060)\n\nI will continue to update this model with items placed in the above discussion post, or the discussion on this notebook.\n\n## Output and [Dataset](https://www.kaggle.com/dannellyz/panda-analysis/)\n\nI uploaded the output of this notebook to a dataset that is the full collection of suspicious slides.\n\n### Last Updated MAY14\n\n# Code","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nfrom glob import glob\n\n# Get those with markings\n# Credit https://www.kaggle.com/rohitsingh9990\nmarkings_path = \"../input/marker-images/marker_images/\"\nimg_ext = 'png'\nglob_obj = glob('{}*.{}'.format(markings_path,img_ext))\nmarker_ids = [file.split(\"/\")[-1].strip(\".png\") \n              for file in glob_obj]\nmarker_dict = [{\"image_id\":img_id, \"reason\":\"marks\"} \n               for img_id in marker_ids]\nmarker_df = pd.DataFrame(marker_dict)\n\n# Get those that are suspicious for mask/classification\n# Credit https://www.kaggle.com/iamleonie\nmask_df = pd.read_csv(\"../input/panda-suspicious-slides/suspicious_test_cases.csv\")\nmask_df.columns = [\"image_id\", \"reason\"]\n\n# Get those that are blank\n# Credit https://www.kaggle.com/yuvaramsing\nblank_slides = [\"3790f55cad63053e956fb73027179707\"]\nblank_masks = [\"4a2ca53f240932e46eaf8959cb3f490a\", \n               \"3790f55cad63053e956fb73027179707\", \n               \"e4215cfc8c41ec04a55431cc413688a9\", \n               \"aaa5732cd49bffddf0d2b7d36fbb0a83\"]\nblanks = blank_slides + blank_masks\nblanks_dict = [{\"image_id\":img_id, \"reason\":\"blank\"} \n               for img_id in blanks]\nblanks_df = pd.DataFrame(blanks_dict)\n\n#Get those with <3% tissue\n# Credit https://www.kaggle.com/dannellyz\nlow_tiss_slides = [ '033e39459301e97e457232780a314ab7',\n                    '0b6e34bf65ee0810c1a4bf702b667c88',\n                    '3385a0f7f4f3e7e7b380325582b115c9',\n                    '3790f55cad63053e956fb73027179707',\n                    '5204134e82ce75b1109cc1913d81abc6',\n                    'a08e24cff451d628df797efc4343e13c']\nlow_tiss_dict = [{\"image_id\":img_id, \"reason\":\"tiss\"} \n                 for img_id in low_tiss_slides]\ntiss_df = pd.DataFrame(low_tiss_dict)\n\n#Combine\nall_suspicious = pd.concat([marker_df, mask_df, tiss_df, blanks_df])\nprint(f\"There are a total of {len(all_suspicious)} slides.\\nBreakdown:\")\nprint(all_suspicious.reason.value_counts())\nall_suspicious.to_csv(\"PANDA_Suspicious_Slides.csv\", index=False)\nall_suspicious.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Usage: Dataloader example\n\nI went ahead and included the dataloader I use in my modeling in order to show the best use of the list.\n\n### In order for this to work you have to `+ Add data` from this [dataset](https://www.kaggle.com/dannellyz/panda-analysis/)","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class PandaData:\n    \"\"\"\n    Description\n    class for loading and sampling the PANDA dataset\n    __________\n    \n    init:\n    pct: float\n        percentage of the dataset to sample\n    cat_col: str\n        this is the name of the column to sample equally from \n        based on observations. If this is left as none then a \n        simple pd.DataFrame.sample() is used\n    sample: bool\n        if false then sampling will not take place and instead\n        the desired percentage will be returned with \n        pd.DataFrame.head()\n    \"\"\"\n    \n    #Establish Globals\n    SLIDE_DIR = \"../input/prostate-cancer-grade-assessment/train_images/\"\n    MASK_DIR = \"../input/prostate-cancer-grade-assessment/train_label_masks/\"\n    TRAIN_LOC = \"../input/prostate-cancer-grade-assessment/train.csv\"\n    SKIP_LOC = \"../input/panda-analysis/PANDA_Suspicious_Slides.csv\"\n    \n    def __init__(self, pct=.1, cat_col=None, sample=True):\n        #Set up init variables\n        self.cat_col = cat_col\n        start_df = pd.read_csv(PandaData.TRAIN_LOC)\n        skip_df = pd.read_csv(PandaData.SKIP_LOC)\n        skip_ids = list(skip_df[\"image_id\"])\n        start_df = start_df[~start_df[\"image_id\"].isin(skip_ids)]\n        \n        #Take percentages and turn into number\n        self.n_sample = int(len(start_df) * pct)\n        \n        #Get subsample if desired\n        if pct < 1:\n            if sample:\n                self.train_df = self.categorical_sample(start_df,\n                                                       self.cat_col,\n                                                       self.n_sample)\n            else:\n                self.train_df = start_df.head(self.n_sample)\n        else:\n            self.train_df = start_df.copy()\n        \n        #Clean up\n        del start_df\n        del skip_df\n                \n        \n    def categorical_sample(self, df, cat_col, N):\n        \"\"\"\n        Description\n        __________\n        sample evenly based on observations from categorical column\n       \n\n        Parameters\n        __________\n        df: Panda DataFrame\n            df to sample from\n        cat_col: str\n            name of categorical to pull observations from\n        N: int\n            number to return from sample\n        \n        Returns (1)\n        __________\n        - Sampled Pandas DataFrame\n        \"\"\"\n        \n        if cat_col in df.columns:\n            group = df.groupby(cat_col, group_keys=False)\n            sample_df = group.apply(lambda g: g.sample(int(N * len(g) / len(df))))\n        else:\n            print(\"Col not found in df. Normal sample used.\")\n            sample_df = df.sample(N)\n        return sample_df\n    \n#Sample run\npanda_data = PandaData(pct=.1, cat_col=\"isup_grade\", sample=True)\nprint(f\"Total number sampled: {panda_data.n_sample}\")\npanda_data.train_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}