{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"## Code to extract features from images with abnormalities...","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport random\nimport cv2\nimport warnings\nimport torch\nimport numpy as np\nimport pandas as pd\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\nimport os.path\n\n#from SSIM_PIL import compare_ssim\nfrom PIL import Image\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.loggers import CSVLogger\n\nfrom sklearn.model_selection import GroupKFold\n#from icevision.all import *\n\nwarnings.filterwarnings('ignore')\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Load train csv data file\ndf = pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\nprint(df.head())\nprint(df.shape)\n# The total number of unique images \ncount = pd.Series(os.listdir('../input/vinbigdata-chest-xray-abnormalities-detection/train/'))\nprint(count.nunique())\nprint(len(count))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Additional features \ndf['image_name'] = '../input/vinbigdata-chest-xray-abnormalities-detection/train/' + df.image_id + '.dicom'\ndf['xd'] = df.x_max - df.x_min\ndf['yd'] = df.y_max - df.y_min\ndf['area'] = df.xd*df.yd\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df['image_id'].nunique())\nprint(len(df['image_id']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tab = pd.DataFrame(df['image_id'].value_counts())\ntab.columns = ['count']\ntab['image_id'] = tab.index\ntab1= tab.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tab1.drop('index', axis =1,  inplace = True)\ntab1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tab.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(df[df['image_id'] == tab1.image_id[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.area.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['class_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.crosstab(df['class_id'],df['rad_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"s = np.array(df['class_id'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.sum(s)-s[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\n%matplotlib inline\n\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"fig=plt.figure(figsize=(12, 10))\ncolumns = 2\nrows = 2\nfor i in range(1, columns*rows +1):\n    p = df.image_name[i]\n    img = read_xray(p)\n    print(img.shape)\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(img, 'gray')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"def plot_class_id(class_id, col, row):\n    \"\"\"\n    This function plots x-ray images of selected class. Pass class_id an integer from 0 to 14.\n    The number of images to plot depends in col and row input which is integer.\n    \"\"\"\n    fig=plt.figure(figsize=(12, 10))\n    columns = col\n    rows = row\n    from random import sample\n    ids = list(df.index[df.class_id == class_id])\n    to_plot = sample(ids,row*col)\n    counter = 1\n    for i in to_plot:\n        p = df.image_name[i]\n        #print(p)\n        img = read_xray(p)\n        print(img.shape)\n        fig.add_subplot(rows, columns, counter)\n        plt.imshow(img, 'gray')\n        counter +=1\n    plt.show()"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"def plot_class_box(class_id, col, row):\n    \"\"\"\n    This function plots x-ray images of selected class and box identifying abnormality. \n    Pass class_id an integer from 0 to 14.\n    The number of images to plot depends in col and row input which is integer.\n    \"\"\"\n    fig=plt.figure(figsize=(12, 10))\n    columns = col\n    rows = row\n    from random import sample\n    ids = list(df.index[df.class_id == class_id])\n    to_plot = sample(ids,row*col)\n    counter = 1\n    for i in to_plot:\n        p = df.image_name[i]\n        #print(p)\n        img = read_xray(p)\n        print(img.shape)\n        fig.add_subplot(rows, columns, counter)\n        plt.imshow(img, 'gray')\n        plt.gca().add_patch(Rectangle((df.x_max[i],df.y_max[i]),df.xd[i],df.yd[i],linewidth=1,edgecolor='r',facecolor='none'))\n        counter +=1\n    plt.show()"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"def plot_class_crop(class_id, col, row):\n    \"\"\"\n    This function plots x-ray images of selected class and box identifying abnormality. \n    Pass class_id an integer from 0 to 14.\n    The number of images to plot depends in col and row input which is integer.\n    \"\"\"\n    fig=plt.figure(figsize=(12, 10))\n    columns = col\n    rows = row\n    from random import sample\n    ids = list(df.index[df.class_id == class_id])\n    to_plot = sample(ids,row*col)\n    counter = 1\n    for i in to_plot:\n        p = df.image_name[i]\n        #print(p)\n        img = read_xray(p)\n        fig.add_subplot(rows, columns, counter)\n        im = img[int(df.x_min[i]):int(df.x_max[i]),int(df.y_min[i]):int(df.y_max[i])]\n        print(im.shape)\n        plt.imshow(im, 'gray')\n        counter +=1\n    plt.show()"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"def plot_class_crop_head(class_id, col, row):\n    \"\"\"\n    This function plots x-ray images of selected class and box identifying abnormality. \n    Pass class_id an integer from 0 to 14.\n    The number of images to plot depends in col and row input which is integer.\n    \"\"\"\n    columns = col\n    rows = row\n    from random import sample\n    ids = list(df.index[df.class_id == class_id])\n    to_plot = sample(ids,row*col)\n    counter = 1\n    for i in to_plot:\n        p = df.image_name[i]\n        #print(p)\n        img = read_xray(p)\n        im = img[int(df.x_min[i]):int(df.x_max[i]),int(df.y_min[i]):int(df.y_max[i])]\n        print(im[0:10,0:10])"},{"metadata":{"trusted":true},"cell_type":"code","source":"def features(class_id, df):\n    gb = df.groupby('class_id')\n    indx = df[df.class_id == class_id].index\n    df1 = df.iloc[indx,:]\n    minareaindx = df1[df1.area == df1.area.min()].index\n    out = {'dataframe': df1, 'minareaindx': minareaindx}\n    return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"out = features(0,df)\nout['minareaindx']\n#out['dataframe'].head(1)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#pip install opencv_python==3.4.2.16 \n#pip install opencv-contrib-python==3.4.2.16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"img = read_xray('../input/vinbigdata-chest-xray-abnormalities-detection/train/0108949daa13dc94634a7d650a05c0bb.dicom')\nim = img[int(df.x_min[2]):int(df.x_max[2]),int(df.y_min[2]):int(df.y_max[2])]"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"out['dataframe'].loc[np.asscalar(out['minareaindx']),'image_name']"},{"metadata":{"trusted":true},"cell_type":"code","source":"def similar_array(df, class_id, min_area_index):\n    k = np.asscalar(min_area_index)\n    out = features(class_id,df)\n    #print('Base image is {}'.format(out['dataframe'].loc[k,'image_name']))\n    img_base = read_xray(out['dataframe'].loc[k,'image_name'])\n    im = img_base[int(df.x_min[k]):int(df.x_max[k]),int(df.y_min[k]):int(df.y_max[k])]\n    #plt.imshow(im, 'gray')\n    nrow = im.shape[0]\n    ncol = im.shape[1]\n    img_index_oth = df[df.class_id == class_id].index[0:200]\n    \n    for index in img_index_oth:\n        current_directory = os.getcwd()\n        final_directory = os.path.join(current_directory, r'features'+str(class_id)+'_')\n        if not os.path.exists(final_directory):\n            os.makedirs(final_directory)\n        #print(out['dataframe'].loc[index,'image_name'])\n        img_trial = read_xray(out['dataframe'].loc[index,'image_name'])\n        img_trial = img_trial[int(df.x_min[index]):int(df.x_max[index]),int(df.y_min[index]):int(df.y_max[index])]\n        #plt.imshow(img_trial, 'gray')\n        ssi = []\n        idx = []\n        for r in list(range(img_trial.shape[0]-nrow)):            #img_trial.shape[0]-1\n            for c in list(range(img_trial.shape[1]-ncol)):          #img_trial.shape[1]-1\n                cropped = img_trial[r:(nrow+r),c:(ncol+c)]\n                idx.append((r,c))\n                ssi.append(np.linalg.norm(im - cropped))\n        #print('Done for {}'.format(out['dataframe'].loc[index,'image_name']))\n        if len(ssi)>0:\n            minssi_id = np.array(ssi).argmin()\n            idx_min = idx[minssi_id]\n            min_crop = img_trial[idx_min[0]:(nrow+idx_min[0]),idx_min[1]:(ncol+idx_min[1])]\n            #cv2.imwrite(\"features\"+str(class_id)+str(r)+str(c)+'.png', min_crop)\n            #dct = {\"minssi_id\":minssi_id,\"idx_min\":idx_min,\"base_shape\":im.shape,\n            #      \"cropped.shape\":cropped.shape,\"fitted_crop\":min_crop.shape}\n            #print(final_directory+str(idx_min[0])+str(idx_min[1])+'.jpg')\n            #to_dir.save(\"features\"+str(class_id)+str(idx_min[0])+str(idx_min[1])+'.jpg')\n            cv2.imwrite(final_directory+str(idx_min[0])+str(idx_min[1])+'.jpg', min_crop)\n    #print(img_base, img_index_oth)\n    #return ssi, idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(img.shape)\n#print(list(range(img.shape[0] - im.shape[0])))\n#class_id=12;r=2;c=25\n#\"features\"+str(class_id)+str(r)+str(c)+'.jpg'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"current_directory = os.getcwd()\nclass_id=12\nfinal_directory = os.path.join(current_directory, r'features'+str(class_id))\nfinal_directory"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"out = features(12,df)\nsimilar_array(df,12,out['minareaindx'])"},{"metadata":{"trusted":true},"cell_type":"code","source":"for class_id in list(range(14)):\n    print('working on class id = {}'.format(class_id))\n    out = features(class_id,df)\n    similar_array(df,class_id,out['minareaindx'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}