{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom IPython.display import HTML\nfrom glob import glob\nimport matplotlib.pyplot as plt\nfrom collections import defaultdict\nfrom openslide import OpenSlide\nfrom PIL import Image\nimport os\nimport cv2\nimport gc\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-23T20:16:06.465832Z","iopub.execute_input":"2022-08-23T20:16:06.46626Z","iopub.status.idle":"2022-08-23T20:16:06.473419Z","shell.execute_reply.started":"2022-08-23T20:16:06.466227Z","shell.execute_reply":"2022-08-23T20:16:06.472183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HTML('<iframe width=\"675\" height=\"506\" src=\"https://www.youtube.com/embed/3CInkjVReDA\" title=\"Stroke Prevention & Transient Ischemic Attack (TIA)\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen></iframe>')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:06.486772Z","iopub.execute_input":"2022-08-23T20:16:06.487992Z","iopub.status.idle":"2022-08-23T20:16:06.494035Z","shell.execute_reply.started":"2022-08-23T20:16:06.487952Z","shell.execute_reply":"2022-08-23T20:16:06.493149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/mayo-clinic-strip-ai/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:06.511081Z","iopub.execute_input":"2022-08-23T20:16:06.511476Z","iopub.status.idle":"2022-08-23T20:16:06.525729Z","shell.execute_reply.started":"2022-08-23T20:16:06.511444Z","shell.execute_reply":"2022-08-23T20:16:06.524902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"There are {} unique patients\".format(train_df[\"patient_id\"].nunique()))\n\ncount_patients = train_df.groupby(\"patient_id\")[\"patient_id\"].count()\n\nnum_images = count_patients[count_patients > 1]\n\nprint(\"There are {} unique patients who have images greater than 1\".format(num_images.shape[0]))\n\nprint(\"Lets Visulaize patients who have images > 2\")\n\nnum_images = count_patients[count_patients > 2]\n\n\nfig=go.Figure()\nfig.add_trace(go.Pie(labels=num_images.index, values=num_images.values, hole=.25,sort=True, textinfo='value'))\n\nfig.update_layout(title='Images Distribution',width=700)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:06.584262Z","iopub.execute_input":"2022-08-23T20:16:06.585018Z","iopub.status.idle":"2022-08-23T20:16:06.70211Z","shell.execute_reply.started":"2022-08-23T20:16:06.584962Z","shell.execute_reply":"2022-08-23T20:16:06.700922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"center_id\"] = train_df[\"center_id\"].astype(\"string\")\n\nnum_centers = train_df.groupby(\"center_id\")[\"center_id\"].agg(\"count\")\n\nfig=go.Figure()\nfig.add_trace(go.Bar(x=num_centers.index, y=num_centers.values,\n                     text=num_centers.values, texttemplate='%{text:.0f}', \n                     textposition='inside',insidetextanchor=\"middle\"))\n\nfig.update_layout(xaxis_title='center_id', yaxis_title='Number of Images',\n                  xaxis={'categoryorder':'total descending'})\n              \nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:06.704215Z","iopub.execute_input":"2022-08-23T20:16:06.704561Z","iopub.status.idle":"2022-08-23T20:16:06.765531Z","shell.execute_reply.started":"2022-08-23T20:16:06.704533Z","shell.execute_reply":"2022-08-23T20:16:06.764295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Lets check how many unique patients are in centers\")\n\nnum_patients = train_df.groupby(\"center_id\")[\"patient_id\"].apply(lambda x: len(np.unique(x)))\n\nfig=go.Figure()\nfig.add_trace(go.Bar(x=num_patients.index, y=num_patients.values,\n                     text=num_patients.values, texttemplate='%{text:.0f}', \n                     textposition='inside',insidetextanchor=\"middle\"))\n\nfig.update_layout(xaxis_title='center_id', yaxis_title='Number of Patients',\n                  xaxis={'categoryorder':'total descending'})\n              \nfig.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:06.767022Z","iopub.execute_input":"2022-08-23T20:16:06.767812Z","iopub.status.idle":"2022-08-23T20:16:06.789937Z","shell.execute_reply.started":"2022-08-23T20:16:06.767775Z","shell.execute_reply":"2022-08-23T20:16:06.788763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Lets see how many patients are with CE and LAA\")\n\nCE_pateints = train_df[train_df[\"label\"]==\"CE\"]\n\nCE_count = CE_pateints.groupby(\"center_id\")[\"patient_id\"].apply(lambda x: len(np.unique(x)))\n\nLAA_pateints = train_df[train_df[\"label\"]==\"LAA\"]\n\nLAA_count = LAA_pateints.groupby(\"center_id\")[\"patient_id\"].apply(lambda x: len(np.unique(x)))\n\nfig = go.Figure(data=[\n    go.Bar(name='CE', x=CE_count.index, y=CE_count.values, hovertemplate = \"%{value}\"),\n    go.Bar(name='LAA', x=LAA_count.index, y=LAA_count.values, hovertemplate = \"%{value}\")\n])\n\nfig.update_layout(barmode='group', xaxis={'categoryorder':'total descending'})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:06.791777Z","iopub.execute_input":"2022-08-23T20:16:06.792786Z","iopub.status.idle":"2022-08-23T20:16:06.824039Z","shell.execute_reply.started":"2022-08-23T20:16:06.792739Z","shell.execute_reply":"2022-08-23T20:16:06.822749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = glob(\"/kaggle/input/mayo-clinic-strip-ai/train/*.tif\")","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:06.825709Z","iopub.execute_input":"2022-08-23T20:16:06.82692Z","iopub.status.idle":"2022-08-23T20:16:06.902011Z","shell.execute_reply.started":"2022-08-23T20:16:06.826872Z","shell.execute_reply":"2022-08-23T20:16:06.900935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_dict = defaultdict(list)\n\nfor path in train_images:\n    \n    slide = OpenSlide(path)\n    \n    width = slide.dimensions[0]\n    height = slide.dimensions[1]\n    \n    img_dict['image_id'].append(path[-12:-4])\n    img_dict['pixels'].append(width * height)\n    img_dict['aspect_ratio'].append(width / height)\n    img_dict['size_MB'].append(round(os.path.getsize(path) / 1e6, 2))\n    img_dict['path'].append(path)\n    \n\nimage_data = pd.DataFrame(img_dict)\nimage_data.sort_values(by='size_MB',ascending=False, inplace=True)\nimage_data.reset_index(inplace=True, drop=True)\n\nimage_data = image_data.merge(train_df, on='image_id')\nimage_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:06.905041Z","iopub.execute_input":"2022-08-23T20:16:06.905536Z","iopub.status.idle":"2022-08-23T20:16:26.062756Z","shell.execute_reply.started":"2022-08-23T20:16:06.905488Z","shell.execute_reply":"2022-08-23T20:16:26.061909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(go.Histogram(name=\"images\",x=image_data['size_MB'],\n                           hovertemplate = \"Number of images: %{y}, Range: %{x}\", opacity=0.65))\n\nfig.update_layout(xaxis_title='size of images', yaxis_title='Number of images')\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:26.065866Z","iopub.execute_input":"2022-08-23T20:16:26.066927Z","iopub.status.idle":"2022-08-23T20:16:26.090222Z","shell.execute_reply.started":"2022-08-23T20:16:26.066891Z","shell.execute_reply":"2022-08-23T20:16:26.089093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_max_image_pixels = 1073741824   #This is max image pixels cv2 can afford to read\n\nmin_pxl_imgs = image_data[image_data[\"pixels\"] <= cv_max_image_pixels]\nprint(\"There are {} images which is less than cv2 threshold\".format(min_pxl_imgs.shape[0]))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:26.09161Z","iopub.execute_input":"2022-08-23T20:16:26.092028Z","iopub.status.idle":"2022-08-23T20:16:26.100629Z","shell.execute_reply.started":"2022-08-23T20:16:26.091998Z","shell.execute_reply":"2022-08-23T20:16:26.099407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CE_paths = min_pxl_imgs.loc[min_pxl_imgs[\"label\"]==\"CE\", \"path\"]\nLAA_paths = min_pxl_imgs.loc[min_pxl_imgs[\"label\"]==\"LAA\", \"path\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:26.102157Z","iopub.execute_input":"2022-08-23T20:16:26.102568Z","iopub.status.idle":"2022-08-23T20:16:26.109734Z","shell.execute_reply.started":"2022-08-23T20:16:26.102536Z","shell.execute_reply":"2022-08-23T20:16:26.108719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(20, 20))\nimg_path = CE_paths.iloc[0]\n\nimg = cv2.imread(img_path)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\naxes.imshow(img)\n\ndel img\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:16:26.111289Z","iopub.execute_input":"2022-08-23T20:16:26.111889Z","iopub.status.idle":"2022-08-23T20:18:26.524084Z","shell.execute_reply.started":"2022-08-23T20:16:26.111818Z","shell.execute_reply":"2022-08-23T20:18:26.523017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from openslide.deepzoom import DeepZoomGenerator\n\nimg_path = CE_paths.iloc[0]\nslide = OpenSlide(img_path)\n\nLEVEL = 16\nROW = 2\nCOLUMN = 3\n\ndeep_tile = DeepZoomGenerator(slide, tile_size=2000, overlap=0, limit_bounds=False)\n\nprint(\"There are {} Levels\".format(deep_tile.level_count))\n\nprint(\"Total number of dimensions in each level are\\n{} \".format(deep_tile.level_dimensions))\n\nprint(\"There are {} rows and {} columns in tiled image at {}th level\".format(deep_tile.level_tiles[LEVEL][0],\n                                                                           deep_tile.level_tiles[LEVEL][1],\n                                                                           LEVEL + 1))\n\nprint(\"Below image is placed at {} row and {} column\".format(ROW, COLUMN))\n\ntile_image = deep_tile.get_tile(LEVEL, (ROW, COLUMN))\ntile_image = tile_image.convert('RGB')\n\nplt.figure(figsize=(20, 20))\nplt.imshow(tile_image)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:18:26.5255Z","iopub.execute_input":"2022-08-23T20:18:26.526438Z","iopub.status.idle":"2022-08-23T20:18:28.42659Z","shell.execute_reply.started":"2022-08-23T20:18:26.526404Z","shell.execute_reply":"2022-08-23T20:18:28.425141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/code/jirkaborovec/bloodclots-eda-load-wsi-prune-background\n\ndef prune_image_rows_cols(im, mask, thr=0.960):\n    \n    for l in reversed(range(im.shape[1])):\n        if (np.sum(mask[:, l]) / float(mask.shape[0])) > thr:\n            im = np.delete(im, l, 1)\n   \n    for l in reversed(range(im.shape[0])):\n        if (np.sum(mask[l, :]) / float(mask.shape[1])) > thr:\n            im = np.delete(im, l, 0)\n            \n    return im\n\ndef mask_median(im, val=255):\n    masks = [None] * 3\n    for c in range(3):\n        masks[c] = im[..., c] >= np.median(im[:, :, c]) - 5\n    mask = np.logical_and(*masks)\n    im[mask, :] = val\n    return im, mask","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:18:28.428064Z","iopub.execute_input":"2022-08-23T20:18:28.428413Z","iopub.status.idle":"2022-08-23T20:18:28.438325Z","shell.execute_reply.started":"2022-08-23T20:18:28.428384Z","shell.execute_reply":"2022-08-23T20:18:28.437147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets see CE images and compare them with normalized images","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=10, ncols=1, figsize=(20, 20))\nfig.tight_layout()\n\nfor k, (i, j) in enumerate(zip(range(0, 10, 2), range(1, 10, 2))):\n    \n    img_path = cv2.imread(CE_paths.iloc[k])\n    img = cv2.cvtColor(img_path, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (0,0), fx=0.05, fy=0.05)\n\n    if img.shape[0] > img.shape[1]:\n        img = np.rollaxis(img, 0, 2)\n    axes[i].imshow(img)\n    axes[i].set_title(\"Original\")\n    \n    img, mask = mask_median(np.array(img))\n    img = prune_image_rows_cols(img, mask)\n    \n    axes[j].imshow(img)\n    axes[j].set_title(\"Normalized\")\n\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:18:28.43993Z","iopub.execute_input":"2022-08-23T20:18:28.440354Z","iopub.status.idle":"2022-08-23T20:20:22.664701Z","shell.execute_reply.started":"2022-08-23T20:18:28.440325Z","shell.execute_reply":"2022-08-23T20:20:22.663434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### Lets see LAA images and compare them with normalized images","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=10, ncols=1, figsize=(20, 20))\nfig.tight_layout()\n\nfor k, (i, j) in enumerate(zip(range(0, 10, 2), range(1, 10, 2))):\n    \n    img_path = cv2.imread(LAA_paths.iloc[k])\n    img = cv2.cvtColor(img_path, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (0,0), fx=0.05, fy=0.05)\n\n    if img.shape[0] > img.shape[1]:\n        img = np.rollaxis(img, 0, 2)\n    axes[i].imshow(img)\n    axes[i].set_title(\"Original\")\n    \n    img, mask = mask_median(np.array(img))\n    img = prune_image_rows_cols(img, mask)\n    \n    axes[j].imshow(img)\n    axes[j].set_title(\"Normalized\")\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T20:20:22.666352Z","iopub.execute_input":"2022-08-23T20:20:22.666715Z","iopub.status.idle":"2022-08-23T20:21:59.254558Z","shell.execute_reply.started":"2022-08-23T20:20:22.666683Z","shell.execute_reply":"2022-08-23T20:21:59.253004Z"},"trusted":true},"execution_count":null,"outputs":[]}]}