{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Basic EDA + albumentations augs"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from glob import glob\nimport os\nimport pandas as pd\nimport numpy as np\nimport re\nfrom PIL import Image\nimport seaborn as sns\nfrom random import randrange\n\nfrom albumentations import (\n    HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, RandomBrightnessContrast, IAAPiecewiseAffine,\n    IAASharpen, IAAEmboss, Flip, OneOf, Compose, PadIfNeeded, RandomGamma\n)\n#checnking the input files\nprint(os.listdir(\"../input/rsna-intracranial-hemorrhage-detection/\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"#reading all dcm files into train and text\ntrain = sorted(glob(\"../input/rsna-intracranial-hemorrhage-detection/stage_1_train_images/*.dcm\"))\ntest = sorted(glob(\"../input/rsna-intracranial-hemorrhage-detection/stage_1_test_images/*.dcm\"))\nprint(\"train files: \", len(train))\nprint(\"test files: \", len(test))\n\npd.reset_option('max_colwidth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/rsna-intracranial-hemorrhage-detection/stage_1_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stage_1_sample_submission = pd.read_csv('../input/rsna-intracranial-hemorrhage-detection/stage_1_sample_submission.csv') ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import pydicom\nimport matplotlib.pyplot as plt\n\n#displaying the image\nimg = pydicom.read_file(train[0]).pixel_array\nplt.imshow(img, cmap=plt.cm.bone)\nplt.grid(False)\n\n#displaying metadata\ndata = pydicom.dcmread(train[0])\nprint(data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Explore Data"},{"metadata":{},"cell_type":"markdown","source":"Explore the number of examples in train and test sets:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualize pie chart\nlabels = 'Train', 'Test'\nsizes = [len(train), len(test)]\nexplode = (0, 0.1)  # \"explode\" the 2nd slice\n\nfig, ax = plt.subplots(figsize=(7,7))\nax.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',\n        shadow=True, startangle=90)\nax.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nplt.title(\"Number of images in train/test sets\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Transform the dataset to devote the number of diagnoses"},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df['ID'].str.find('_', 3) \ntrain_df['image'] = train_df['ID'].str.slice(stop=12)\ntrain_df['diagnosis'] = train_df['ID'].str.slice(start=13)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby('diagnosis').sum().plot(kind='bar',figsize = (10, 5));\nplt.title('Class counts');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is very good, but let's look at the number of people with different class of Hemorrhage"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_lable = train_df.groupby('image').sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_lable['Label'].value_counts().plot(kind='bar',figsize = (10, 5));\nplt.title('Number of people with different class of Hemorrhage');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\"Any\" occurs in anyone who has even one of the diagnoses. Remove it."},{"metadata":{"trusted":true},"cell_type":"code","source":"image_lable = train_df.query('diagnosis!=\"any\"').groupby('image').sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_lable['Label'].value_counts().plot(kind='bar',figsize = (10, 5));\nplt.title('Class with labels');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize Sample Images"},{"metadata":{},"cell_type":"markdown","source":"Visualize Sample Images with different diagnosis"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_IMG_PATH = \"../input/rsna-intracranial-hemorrhage-detection/stage_1_train_images/\"\nTEST_IMG_PATH = \"../input/rsna-intracranial-hemorrhage-detection/stage_1_test_images/\"\n\ndef view_images(images, title = '', aug = None):\n    width = 5\n    height = 2\n    fig, axs = plt.subplots(height, width, figsize=(15,5))\n    \n    for im in range(0, height * width):\n        image = pydicom.read_file(os.path.join(TRAIN_IMG_PATH,images[im]+ '.dcm')).pixel_array\n        i = im // width\n        j = im % width\n        axs[i,j].imshow(image, cmap=plt.cm.bone) \n        axs[i,j].axis('off')\n        \n    plt.suptitle(title)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_images(train_df[(train_df['diagnosis'] == 'epidural') & (train_df['Label'] == 1)][:10].image.values, title = 'Images with epidural')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_images(train_df[(train_df['diagnosis'] == 'intraparenchymal') & (train_df['Label'] == 1)][:10].image.values, title = 'Images with intraparenchymal')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_images(train_df[(train_df['diagnosis'] == 'intraventricular')& (train_df['Label'] == 1)][:10].image.values, title = 'Images with intraventricular')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_images(train_df[(train_df['diagnosis'] == 'subarachnoid')& (train_df['Label'] == 1)][:10].image.values, title = 'Images with subarachnoid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_images(train_df[(train_df['diagnosis'] == 'subdural') & (train_df['Label'] == 1)][:10].image.values, title = 'Images with subarachnoid')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Analyze Image Sizes"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image_sizes(df, train = True):\n    if train:\n        path = TRAIN_IMG_PATH\n    else:\n        path = TEST_IMG_PATH\n        \n    widths = []\n    heights = []\n    \n    images = df.image.values\n    #print(images)\n    max_im = pydicom.read_file(os.path.join(path,images[0]+ '.dcm')).pixel_array\n    min_im = pydicom.read_file(os.path.join(path,images[0]+ '.dcm')).pixel_array\n        \n    for im in range(0, len(images)):\n        image = pydicom.read_file(os.path.join(path,images[im]+ '.dcm')).pixel_array\n        \n        width = image.shape[0]\n        height = image.shape[1]\n        \n        if len(widths) > 0:\n            if width > max(widths):\n                max_im = image\n\n            if width < min(widths):\n                min_im = image\n\n        widths.append(width)\n        heights.append(height)\n        \n    return widths, heights, max_im, min_im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stage_1_sample_submission['image'] = stage_1_sample_submission['ID'].str.slice(stop=12)\nstage_1_sample_submission['diagnosis'] = stage_1_sample_submission['ID'].str.slice(start=13)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stage_1_sample_submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Drop images duplicates "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_d = train_df.drop_duplicates(subset='image')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_d.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stage_1_sample_submission_d = stage_1_sample_submission.drop_duplicates(subset='image')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stage_1_sample_submission_d.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"A lot of images, let's take some sample - 10000"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_widths, train_heights, max_train, min_train = get_image_sizes(train_df_d.sample(10000), train = True)\ntest_widths, test_heights, max_test, min_test = get_image_sizes(stage_1_sample_submission_d.sample(10000), train = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Maximum width for training set is {}'.format(max(train_widths)))\nprint('Minimum width for training set is {}'.format(min(train_widths)))\nprint('Maximum height for training set is {}'.format(max(train_heights)))\nprint('Minimum height for training set is {}'.format(min(train_heights)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Maximum width for test set is {}'.format(max(test_widths)))\nprint('Minimum width for test set is {}'.format(min(test_widths)))\nprint('Maximum height for test set is {}'.format(max(test_heights)))\nprint('Minimum height for test set is {}'.format(min(test_heights)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(14,6))\nplt.subplot(121)\nsns.distplot(train_widths, kde=False, label='Train Width')\nsns.distplot(train_heights, kde=False, label='Train Height')\nplt.legend()\nplt.title('Training Image Dimension Histogram', fontsize=15)\n\nplt.subplot(122)\nsns.kdeplot(train_widths, label='Train Width')\nsns.kdeplot(train_heights, label='Train Height')\nplt.legend()\nplt.title('Train Image Dimension KDE Plot', fontsize=15)\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(14,6))\nplt.subplot(121)\nsns.distplot(test_widths, kde=False, label='Test Width')\nsns.distplot(test_heights, kde=False, label='Test Height')\nplt.legend()\nplt.title('Test Image Dimension Histogram', fontsize=15)\n\nplt.subplot(122)\nsns.kdeplot(test_widths, label='Test Width')\nsns.kdeplot(test_heights, label='Test Height')\nplt.legend()\nplt.title('Test Image Dimension KDE Plot', fontsize=15)\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We see that we have some different distributions of image sizes for sample of train datasets."},{"metadata":{},"cell_type":"markdown","source":"### Plot largest and smallest images"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.axis('off')\nplt.imshow(max_train, cmap=plt.cm.bone) #plot the data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.axis('off')\nplt.imshow(min_train, cmap=plt.cm.bone) #plot the data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augmentations by albumentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get some random image indices from the training set\nrand_indices = [randrange(len(train_df_d)) for x in range(0,10)]\nrand_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_aug_images(train, rand_indices, aug = None, title = ''):\n    width = 5\n    height = 2\n    counter = 0\n    fig, axs = plt.subplots(height, width, figsize=(15,5))\n    \n    for im in rand_indices:\n        image = pydicom.read_file(os.path.join(TRAIN_IMG_PATH,train.iloc[im].image+ '.dcm')).pixel_array\n        if aug is not None:\n            image = aug(image=np.array(image))['image']\n        \n        i = counter // width\n        j = counter % width\n        axs[i,j].imshow(image, cmap=plt.cm.bone) #plot the data\n        axs[i,j].axis('off')\n        \n        diagnosis = train[train['image'] == train.iloc[im].image].diagnosis.values[0]\n        \n        axs[i,j].set_title(diagnosis)\n        counter += 1\n\n    plt.suptitle(title)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_aug_images(train_df_d, rand_indices, title = 'Original images')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug = GaussNoise(p=1)\nview_aug_images(train_df_d, rand_indices, aug, title= 'GaussNoise')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug = RandomBrightnessContrast(brightness_limit=1, contrast_limit=1, p = 1)\nview_aug_images(train_df_d, rand_indices, aug, title = 'RandomBrightnessContrast')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug = RandomGamma(gamma_limit=[80,120], p = 1)\nview_aug_images(train_df_d, rand_indices, aug, title = 'RandomGamma')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug = GridDistortion(num_steps =5, distort_limit=[-0.3,0.3], interpolation=1, border_mode= 4, p = 1)\nview_aug_images(train_df_d, rand_indices, aug, title = 'GridDistortion')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug = OpticalDistortion(shift_limit =[-0.5, 0.5], distort_limit=[-2,2], interpolation=1, border_mode= 4, p = 1)\nview_aug_images(train_df_d, rand_indices, aug, title = 'OpticalDistortion')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Conclusion"},{"metadata":{},"cell_type":"markdown","source":"1. The dataset is imbalanced. \n1. Need to play with data augmentation\n1. The distribution of sizes of images from train and test sets is small different\n"},{"metadata":{},"cell_type":"markdown","source":"baseline model is ongoing ..."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}