{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{"_uuid":"08daa41f-987f-4186-bf63-c574ceeb6a3c","_cell_guid":"65a3367e-4b93-4e9b-b07d-2c602a4f340f","trusted":true}},{"cell_type":"code","source":"!pip -q install skimpy colorama \nimport os \nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt \nimport plotly_express as px \nimport plotly\nimport plotly.io as pio\nplotly.offline.init_notebook_mode (connected = True)\nfrom glob import glob\nfrom PIL import Image\nimport skimpy \nimport random\nfrom colorama import Fore, Back, Style ","metadata":{"_uuid":"0f4aa051-5c54-4649-b5e0-1d18e3c68232","_cell_guid":"09f3fffd-490b-465f-bf4a-089dd7200e5d","collapsed":false,"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-23T18:02:22.582042Z","iopub.execute_input":"2022-07-23T18:02:22.582724Z","iopub.status.idle":"2022-07-23T18:02:33.556696Z","shell.execute_reply.started":"2022-07-23T18:02:22.582687Z","shell.execute_reply":"2022-07-23T18:02:33.555435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading data","metadata":{"_uuid":"ad4bc82e-6bcf-4204-b620-07a32f620990","_cell_guid":"e6f8ebcf-2178-46bc-8913-a6194983d795","trusted":true}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/mayo-clinic-strip-ai/train.csv\")\ntest = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\nother = pd.read_csv(\"../input/mayo-clinic-strip-ai/other.csv\")\nss = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")","metadata":{"_uuid":"3d604ad2-a5ef-43b4-bd4b-04545b3c7f14","_cell_guid":"3c97d7b1-32c3-4a28-ae12-542c49952338","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-23T18:02:33.558273Z","iopub.execute_input":"2022-07-23T18:02:33.558612Z","iopub.status.idle":"2022-07-23T18:02:33.580517Z","shell.execute_reply.started":"2022-07-23T18:02:33.558578Z","shell.execute_reply":"2022-07-23T18:02:33.57942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring the data","metadata":{"_uuid":"ec0c6162-12aa-4a73-99b9-dd2105de34ef","_cell_guid":"127026ab-b55a-497a-a63a-c3a734ceb944","trusted":true}},{"cell_type":"code","source":"# exploring the train df \nprint(Fore.BLACK + Back.CYAN + \"skimpy summary for train_df\")\nskimpy.skim(train)","metadata":{"_uuid":"8682e39a-07e5-4ae1-87a8-a92b11567151","_cell_guid":"63509fc0-7508-408d-bb12-35f2890f0234","execution":{"iopub.status.busy":"2022-07-23T18:02:33.582393Z","iopub.execute_input":"2022-07-23T18:02:33.582809Z","iopub.status.idle":"2022-07-23T18:02:33.659648Z","shell.execute_reply.started":"2022-07-23T18:02:33.582769Z","shell.execute_reply":"2022-07-23T18:02:33.658415Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exploring test datafrmae \nprint(Fore.BLACK + Back.CYAN + \"skimpy summary for test_df\")\nskimpy.skim(test)","metadata":{"_uuid":"6f94c639-9d7a-42e9-9f5a-e1f0cc5ff076","_cell_guid":"00b27516-d1fa-4663-ae04-fb2d698aa18b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-23T18:02:22.487693Z","iopub.execute_input":"2022-07-23T18:02:22.488023Z","iopub.status.idle":"2022-07-23T18:02:22.579869Z","shell.execute_reply.started":"2022-07-23T18:02:22.487992Z","shell.execute_reply":"2022-07-23T18:02:22.578806Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exploring others dataframe \nprint(Fore.BLACK + Back.CYAN + \"skimpy summary for other_df\")\nskimpy.skim(other)","metadata":{"_uuid":"ce38b1ae-f85b-4d03-b162-fe128c4dcf2d","_cell_guid":"8bfb8bc3-3b40-4148-81db-a8c0fd8607fb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-23T18:02:33.661637Z","iopub.execute_input":"2022-07-23T18:02:33.661944Z","iopub.status.idle":"2022-07-23T18:02:33.724929Z","shell.execute_reply.started":"2022-07-23T18:02:33.661917Z","shell.execute_reply":"2022-07-23T18:02:33.724038Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.head()","metadata":{"_uuid":"aadfc8d0-791d-45b7-9d01-14561762d6b9","_cell_guid":"d103779b-eba1-4aab-8d67-dfb33d30033e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-23T16:26:02.206466Z","iopub.execute_input":"2022-07-23T16:26:02.207302Z","iopub.status.idle":"2022-07-23T16:26:02.221686Z","shell.execute_reply.started":"2022-07-23T16:26:02.207258Z","shell.execute_reply":"2022-07-23T16:26:02.220474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# what do we exactly have to do in this competition \n* there are digital patholab images given to us and the patients information is given in short \n* we have to classify between if the patient is having CE or having LAA","metadata":{"_uuid":"afbb0d48-fd24-43cb-bedc-644d7ca7d40b","_cell_guid":"43568a43-d1fd-4873-9671-127a5774fb49","trusted":true}},{"cell_type":"code","source":"# let's see the directory structure( how many data we have and other exploration ) \nlen(glob(\"../input/mayo-clinic-strip-ai/train/*\"))","metadata":{"_uuid":"f2244ffd-14fa-4489-9145-b0641415c160","_cell_guid":"3c8ffeb8-ebb6-4c18-82dc-95795bebb538","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-23T16:26:02.652985Z","iopub.execute_input":"2022-07-23T16:26:02.653416Z","iopub.status.idle":"2022-07-23T16:26:02.663377Z","shell.execute_reply.started":"2022-07-23T16:26:02.653381Z","shell.execute_reply":"2022-07-23T16:26:02.662335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's visualize some of the tiff images \n# reading the image using Immage.open function \nprint(Fore.BLACK + Back.CYAN + \"visualizing tiff images in dataset\")\nim = Image.open('../input/mayo-clinic-strip-ai/test/008e5c_0.tif')\n# converting the image to the numpy array \nnp_img = np.array(im)\n# showimg image\nplt.imshow(np_img)","metadata":{"_uuid":"ad41288f-b4b7-4b85-9f1e-0ca70f1d59d7","_cell_guid":"53187e7b-db9d-4dc5-b612-ae8c09afbe21","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-23T18:02:59.927836Z","iopub.execute_input":"2022-07-23T18:02:59.928267Z","iopub.status.idle":"2022-07-23T18:03:24.725353Z","shell.execute_reply.started":"2022-07-23T18:02:59.92823Z","shell.execute_reply":"2022-07-23T18:03:24.72387Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```\nAs we see the number of pixels in above image is really large and the time image takes in for loading is high it'll requires us high processing power if we didn't converted this tiff images to png format or some other lightweight format. \nTIFF format stores all information of image by using lossless compresssion techniques the images stored by tiff image format do have high resolution but they occupy compared to other image saving formats\n```","metadata":{"_uuid":"76512c3c-047a-47d6-a26c-8666e0b88dc8","_cell_guid":"663d8e7e-0043-488e-abca-cfa5b555e553","trusted":true}},{"cell_type":"markdown","source":"# using the cropped dataset saved in png format \nusing this dataset by https://www.kaggle.com/competitions/mayo-clinic-strip-ai/discussion/338299\nthanks @yasufuminakama for providing this dataset","metadata":{"_uuid":"0cddf0a2-71b6-45c8-a383-2a712594a90b","_cell_guid":"c010b48a-b4e2-4b7b-8777-828888469573","trusted":true}},{"cell_type":"code","source":"print(Fore.YELLOW + Back.BLACK)\nprint(\"Checking if the there are any duplicate patients\")\nprint(Fore.WHITE + Back.BLACK)\nfor x in train.patient_id.unique():\n    if x in test.patient_id.unique():\n        print(x)\n\nprint(\"We have same patients data into test data as well \")\nprint(Style.RESET_ALL)","metadata":{"_uuid":"d9aaadd0-95b1-431f-84f9-c1dcae048143","_cell_guid":"f973a554-9633-4dec-8522-0183a9a4ee26","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-23T18:03:24.727575Z","iopub.execute_input":"2022-07-23T18:03:24.727956Z","iopub.status.idle":"2022-07-23T18:03:24.776416Z","shell.execute_reply.started":"2022-07-23T18:03:24.727921Z","shell.execute_reply":"2022-07-23T18:03:24.775515Z"},"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizations ","metadata":{}},{"cell_type":"code","source":"# reading the image from converted and croppoed images \ntrain_1 = glob(\"../input/mayo-train-images-size1024-n16/train_images_1/*.jpg\")\nprint(Fore.BLACK + Back.CYAN + \"images from croppoed dataset\")\nfor x in range(9):\n    plt.subplot(3,3,x+1)\n    ch = random.choice(train_1)\n    plt.imshow(plt.imread(ch))\n    plt.axis(False)\n    plt.tight_layout()","metadata":{"_uuid":"8a93aa93-69a8-42fb-9443-833de9aad8d6","_cell_guid":"8a3ac0d3-9702-480d-a740-b6b1ea6ba155","collapsed":false,"jupyter":{"outputs_hidden":false},"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-23T18:04:00.972098Z","iopub.execute_input":"2022-07-23T18:04:00.972896Z","iopub.status.idle":"2022-07-23T18:04:03.095058Z","shell.execute_reply.started":"2022-07-23T18:04:00.972857Z","shell.execute_reply":"2022-07-23T18:04:03.093864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(train , x = \"image_num\", template = \"plotly_dark\")\nfig.update_layout(\n{\n    \"title\" : \"no of samples available per image_num\"\n})\nprint(Fore.BLACK + Back.CYAN + \"visualizing samples for image_num on train_df\")\nfig.show()","metadata":{"_uuid":"5ef4bfd9-8c3b-46d9-84d6-2995d9d5073e","_cell_guid":"eff728ac-b256-4485-8c9c-90a73cabf4a9","collapsed":false,"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-23T18:04:25.002066Z","iopub.execute_input":"2022-07-23T18:04:25.003339Z","iopub.status.idle":"2022-07-23T18:04:25.065922Z","shell.execute_reply.started":"2022-07-23T18:04:25.003281Z","shell.execute_reply":"2022-07-23T18:04:25.064862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(train , x = \"label\", template = \"plotly_dark\")\nfig.update_layout({\n    \"title\": \"value counts for the labels\",\n})\nprint(Fore.BLACK + Back.CYAN + \"visualizing the labels in train_df\")\nfig.show()","metadata":{"_uuid":"a2de3b9f-eeae-46e3-bd5d-173b10796181","_cell_guid":"71ed47ea-a6d2-4a38-b036-e276f2c16fc5","collapsed":false,"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-23T18:13:38.024366Z","iopub.execute_input":"2022-07-23T18:13:38.024831Z","iopub.status.idle":"2022-07-23T18:13:38.095669Z","shell.execute_reply.started":"2022-07-23T18:13:38.024795Z","shell.execute_reply":"2022-07-23T18:13:38.094428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(train , x = \"center_id\", template = \"plotly_dark\")\nfig.update_layout(\n{\n    \"title\": \"No of samples taken on various center_id\"\n})\nprint(Fore.BLACK + Back.CYAN + \"samples taken on particular center train_df\")\nfig.show()","metadata":{"_uuid":"c7963089-006d-414f-966a-eebfb28308cd","_cell_guid":"4cd234df-2676-47c8-96da-1d1109c4bff6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-23T18:05:59.064775Z","iopub.execute_input":"2022-07-23T18:05:59.065185Z","iopub.status.idle":"2022-07-23T18:05:59.135925Z","shell.execute_reply.started":"2022-07-23T18:05:59.065152Z","shell.execute_reply":"2022-07-23T18:05:59.134817Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizations for other df","metadata":{"execution":{"iopub.status.busy":"2022-07-23T16:30:25.191103Z","iopub.execute_input":"2022-07-23T16:30:25.191507Z","iopub.status.idle":"2022-07-23T16:30:25.210087Z","shell.execute_reply.started":"2022-07-23T16:30:25.191476Z","shell.execute_reply":"2022-07-23T16:30:25.209278Z"}}},{"cell_type":"code","source":"fig = px.histogram(other , x = \"image_num\", template = \"plotly_dark\" )\nfig.update_layout({\n    \"title\": \"image_num count for other dataframe \"\n})\nprint(Fore.BLACK + Back.CYAN + \"image_num for others_df\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T18:06:30.263362Z","iopub.execute_input":"2022-07-23T18:06:30.263784Z","iopub.status.idle":"2022-07-23T18:06:30.326595Z","shell.execute_reply.started":"2022-07-23T18:06:30.263748Z","shell.execute_reply":"2022-07-23T18:06:30.325465Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(other, x= \"label\", template = \"plotly_dark\")\nfig.update_layout({\n    \"title\": \"labels for other_df\"\n})\nprint(Fore.BLACK + Back.CYAN + \"labels count for others_df\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T18:07:54.482608Z","iopub.execute_input":"2022-07-23T18:07:54.483316Z","iopub.status.idle":"2022-07-23T18:07:54.546402Z","shell.execute_reply.started":"2022-07-23T18:07:54.483279Z","shell.execute_reply":"2022-07-23T18:07:54.545214Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(other, x= \"other_specified\", template = \"plotly_dark\")\nfig.update_layout({\n    \"title\": \"labels for other_df\"\n})\nprint(Fore.BLACK + Back.CYAN + \"the patient having other symptoms\")\nfig.show()\nprint(Fore.BLACK + Back.CYAN +\"The patient can show other also this visualization describes the same\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T18:11:40.663081Z","iopub.execute_input":"2022-07-23T18:11:40.663489Z","iopub.status.idle":"2022-07-23T18:11:40.732043Z","shell.execute_reply.started":"2022-07-23T18:11:40.663461Z","shell.execute_reply":"2022-07-23T18:11:40.73119Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WORK IN PROGRESS 🚧","metadata":{"_uuid":"7d228447-8ce0-4b73-97a4-1b07580ad330","_cell_guid":"3e69a2ed-528a-4d66-b8f0-dbeedb4a64ef","jupyter":{"outputs_hidden":false}}}]}