{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\ndef crop_image_from_gray(img,tol=9):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n\ndef circle_crop(img):\n    \"\"\"\n    Create circular crop around image centre\n    \"\"\"\n    #img = cv2.imread(img)\n    img = crop_image_from_gray(img)\n\n    height, width, depth = img.shape\n    largest_side = np.max((height, width))\n    img = cv2.resize(img, (largest_side, largest_side))\n\n    height, width, depth = img.shape\n\n    x = int(width / 2)\n    y = int(height / 2)\n    r = np.amin((x, y))\n\n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x, y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image_from_gray(img)\n\n    return Pimage.fromarray(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l /kaggle/input/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf  /kaggle/working/filter\n!mkdir filter\n#!rm -rf /kaggle/working/circle_crop_atos\n#!ls -hs /kaggle/working/circle_crop_atos\n#file_names=os.listdir(\"../input/severstal-steel-defect-detection/train_images\")\n#os.listdir(\"../input/aptos2019-blindness-detection/train_images\")\n#file_names\n#!ls -l /kaggle/input/ext-data/external_train/\n#os.remove('/kaggle/working/filter/8cb6b5b2f19c.png')\n\n!ls -l /kaggle/working/filter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!ls -l /kaggle/input/rsna-hemorrhage-png/meta/meta/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport os\nfrom fastai import *\nfrom fastai.vision import *\nimport pydicom as pic\npath_meta = Path('../input/rsna-hemorrhage-png/meta/meta/')\npath = Path('../input/rsna-intracranial-hemorrhage-detection/')\npath_str='../input/rsna-intracranial-hemorrhage-detection/stage_1_train_images/'\npath_trn = path/'stage_1_train_images'\npath_tst = path/'stage_1_test_images'\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_comb = pd.read_feather(path_meta/'comb.fth').set_index('SOPInstanceUID')\ndf_tst  = pd.read_feather(path_meta/'df_tst.fth').set_index('SOPInstanceUID')\ndf_samp = pd.read_feather(path_meta/'wgt_sample.fth').set_index('SOPInstanceUID')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport glob\nfrom joblib import Parallel, delayed\nfrom PIL import Image as Pimage\nimport matplotlib.pyplot as plt\nimport zipfile\n#import pylot as plt\nin_dir = 'train_orig/'\nout_dir = 'train_96/'\nIMAGE_SIZE = 96\n\nfrom PIL import Image, ImageChops\n#JPEG_FILES = glob.glob(in_dir+'*.jpeg')\ndef convert(img_file,path):\n    ext='../input/ext-data/external_train/'\n    #print('img',img_file)\n    if os.path.isfile(ext+img_file):\n        im = Pimage.open(ext+img_file)\n        #print(im.shape)\n    else:\n        \n        im = Pimage.open(path+img_file)\n    #im=circle_crop(np.asarray(im))\n    #print(im.size)\n    #plt.imshow(im)\n    #im.thumbnail((512,512),Pimage.ANTIALIAS)\n    #im = crop_image_from_gray(np.asarray(im))\n    #im  =cv2.bilateralFilter(np.asarray(im) ,8,10,10  )\n    #kernel = np.ones((7,7),np.float32)/30\n    #im = cv2.filter2D(im,-1,kernel)\n    im=Pimage.fromarray(im)\n    im.resize((328,328),Pimage.ANTIALIAS).save('/kaggle/working/filter/' + img_file, 'PNG')\n    z.write('/kaggle/working/filter/' + img_file )\n    os.remove('/kaggle/working/filter/' + img_file )\n\n#file_names=os.listdir(\"../input/aptos2019-blindness-detection/train_images\")\n#file_names.append(os.listdir('../input/ext-data/external_train/'))\n#file_names= np.hstack(file_names ).tolist()\n#z = zipfile.ZipFile(\"/kaggle/working/filter/filter.zip\", \"w\")\n#Parallel(n_jobs=1, verbose=10)(delayed(convert)(f) for f in file_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from skimage import exposure\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\ndef fix_pxrepr(dcm):\n    if dcm.PixelRepresentation != 0 or dcm.RescaleIntercept<-100: return dcm\n    x = dcm.pixel_array + 1000\n    px_mode = 4096\n    #print('shape',x.shape,dcm.pixel_array.shape)\n    x[x>=px_mode] = x[x>=px_mode] - px_mode\n    dcm.PixelData = x.tobytes()\n    dcm.RescaleIntercept = -1000\n    return dcm\ndef window_image(pixel_array, window_center, window_width, is_normalize=True):\n    image_min = window_center - window_width // 2\n    image_max = window_center + window_width // 2\n    image = np.clip(pixel_array, image_min, image_max)\n\n    if is_normalize:\n        \n        image =exposure.equalize_adapthist(image)\n        #image=exposure.adjust_gamma(image, gamma=1.5)\n    else:\n        \n        image=(image-image_min)/(image_max-image_min)\n    return image\ndef open_dicom(fname):\n    fname = str(fname)\n    dcm = pic.dcmread(fname) #'../input/stage_2_train_images/'\n    #print(dcm['WindowCenter'].value,fname)\n    #print(dcm.pixel_array.shape)\n    '''  \n    centre= (dcm['WindowCenter'].value)\n    width= (dcm['WindowWidth'].value)\n    \n    if type(centre) in [pic.multival.MultiValue, tuple]:\n        centre=float(centre[0])\n    else :\n        centre=float(centre)\n    \n    if type(width) in [pic.multival.MultiValue, tuple]:\n        width=int(width[0])\n    else :\n        width=int(width)\n    \n   '''  \n        \n    dcm=fix_pxrepr(dcm)\n    img= dcm.pixel_array \n    #level = 40; window = 80\n    #\n    img = img * int(dcm.RescaleSlope) + int(dcm.RescaleIntercept)\n    #img = np.clip(img, centre - width // 2, centre + width // 2)\n    img = crop_image_from_gray(np.asarray(img))  \n    brain       = window_image(img, 40,  80)\n    subdural    = window_image(img, 80, 200)\n    soft_tissue = window_image(img, 40, 380)\n    #brain = crop_image_from_gray(np.asarray(brain)).astype(np.uint8) \n    #subdural = crop_image_from_gray(np.asarray(subdural)).astype(np.uint8) \n    #soft_tissue = crop_image_from_gray(np.asarray(soft_tissue)).astype(np.uint8)\n    #img= np.stack((img,)*3, axis=-1)\n     \n    #img = PIL.Image.fromarray(img).convert('RGB')\n    im = crop_image_from_gray(np.asarray(img))  \n    im = np.dstack([brain,subdural,soft_tissue])\n    #im = crop_image_from_gray(np.asarray(im))\n     \n    #p2,p98=np.percentile(im,(1,98))\n    #img=exposure.rescale_intensity(im,(p2,p98))\n    \n    #img=np.moveaxis(img,2,0 )\n    #print(img.shape)\n                             \n    #return Image(pil2tensor(np.asarray(im)/255, np.float32).float())\n    return im*255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#file_names=os.listdir(\"../input/rsna-intracranial-hemorrhage-detection/stage_1_train_images/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!mkdir /kaggle/working/filter\n#!ls -hs  /kaggle/working/filter/ ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#df_samp.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport glob\nfrom joblib import Parallel, delayed\nfrom PIL import Image as Pimage\nimport matplotlib.pyplot as plt\nimport zipfile\n#import pylot as plt\nin_dir = 'train_orig/'\nout_dir = 'train_96/'\nIMAGE_SIZE = 96\n\nfrom PIL import Image, ImageChops\n#JPEG_FILES = glob.glob(in_dir+'*.jpeg')\ndef convert(img_file,path):\n    ext='../input/ext-data/external_train/'\n    #print('img',img_file)\n    if os.path.isfile(ext+img_file):\n        im = Pimage.open(ext+img_file+'.dcm')\n        #print(im.shape)\n    else:\n        \n        im = open_dicom(path+img_file+'.dcm')\n        #Pimage.open(path+img_file)\n    #im=circle_crop(np.asarray(im))\n    #print(im.size)\n    #plt.imshow(im)\n    #im.thumbnail((512,512),Pimage.ANTIALIAS)\n    #im = crop_image_from_gray(np.asarray(im))\n    #im  =cv2.bilateralFilter(np.asarray(im) ,8,10,10  )\n    #kernel = np.ones((7,7),np.float32)/30\n    #im = cv2.filter2D(im,-1,kernel)\n    #print(im.shape,im.min(),im.max())\n    im=Pimage.fromarray(im.astype(np.uint8))\n    im.resize((488,488),Pimage.ANTIALIAS).save('/kaggle/working/filter/' + img_file+'.jpg', 'JPEG')\n    z.write('/kaggle/working/filter/' + img_file+'.jpg' )\n    os.remove('/kaggle/working/filter/' +  img_file+'.jpg' )\n\n\n#file_names.append(os.listdir('../input/ext-data/external_train/'))\n#file_names= np.hstack(file_names ).tolist()\nfile_names=df_samp.index.values\nz = zipfile.ZipFile(\"/kaggle/working/filter/filter_rsna.zip\", \"w\")\nParallel(n_jobs=1, verbose=1)(delayed(convert)(f,path_str) for f in file_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#a=open_dicom(path_str+file_names[16:50][16]) \n#plt.hist(a.flatten(),256)\n#a=open_dicom(path_str+'ID_1bb6286d2'+'.dcm') \n#plt.imshow(a)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a=open_dicom(path_str+file_names[16:50][16]+'.dcm') \nplt.imshow(a)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#file_names=os.listdir(\"../input/aptos2019-blindness-detection/train_images\")\n#file_names.append(os.listdir('../input/ext-data/external_train/'))\n#len(np.hstack(file_names).tolist() )\n#file_names[-1]\n#file_names[0:1]\n\n#a=open_dicom(path_str+file_names[16:50][14]) \n#im=Pimage.fromarray(a.astype(np.uint8) )\n#plt.imshow(a)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#a=open_dicom(path_str+file_names[0:1][0]) \n#im=Pimage.fromarray(a.astype(np.uint8) )\n#plt.imshow(a)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#im2 = Pimage.open('/kaggle/working/filter/'+'ID_1bb6286d2.jpg')\n\n#plt.imshow(im2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#file_names=os.listdir(\"../input/aptos-preprocessed-420x420/val-imgs\")\n#file_names[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#im = Pimage.open('/kaggle/working/circle_crop_atos/'+'a87f53bc984a.png')\n\n#plt.imshow(im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#im = Pimage.open('/kaggle/working/circle_crop_atos/'+'a87f53bc984a.png')\n\n#plt.imshow(im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def __butterworth_filter( I_shape, filter_params=[30,2]):\n        P = I_shape[0]/2\n        Q = I_shape[1]/2\n        U, V = np.meshgrid(range(I_shape[0]), range(I_shape[1]), sparse=False, indexing='ij')\n        Duv = (((U-P)**2+(V-Q)**2)).astype(float)\n        H = 1/(1+(Duv/filter_params[0]**2)**filter_params[1])\n        return (1 - H)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def __apply_filter( I, H):\n        H = np.fft.fftshift(H)\n        I_filtered = (0.75 + 1.25*H)*I\n        return I_filtered\n  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import pandas as pd\n#df_orig=pd.read_csv('../input/severstal-steel-defect-detection/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom fastai import * \nfrom fastai.vision import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df_orig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\ndf_orig['ImageId']=df_orig.ImageId_ClassId.apply(lambda x : x.split('_')[0])\ndf_orig['class_id']=df_orig.ImageId_ClassId.apply(lambda x : x.split('_')[1])\n#df_train=df_orig.loc[~df_train.EncodedPixels.isnull()]\ndf_train_final=pd.DataFrame({'ImageId':[]})\ndf_train_final['ImageId']=df_orig[~df_orig.EncodedPixels.isnull()].ImageId.unique() \n#df_train_final['ImageId']=df_train.ImageId.unique() \n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df_orig.loc[df_orig.ImageId_ClassId=='0002cc93b.jpg_1',['EncodedPixels','class_id']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\n#df_orig\nl_1=[]\nl_2=[]\nl_3=[]\nl_4=[]\n\nfor i in df_train_final.ImageId:\n    \n\n#df_orig[df_orig.ImageId_Classid=='0002cc93b.jpg_1'].EncodedPixels.values\n    for rle in (df_orig.loc[df_orig.ImageId==i,['EncodedPixels','class_id']].values):\n        #print(rle[0])\n        if isinstance(rle[0], str):\n            \n            mask=open_mask_rle( rle[0],(256,1600)).px.permute(0,2,1) \n        #print(mask.shape)\n        #print(mask.sum())\n            if rle[1]=='1':\n                l_1.append(mask.sum())\n            if rle[1]=='2':\n                l_2.append(mask.sum())\n            if rle[1]=='3':\n                l_3.append(mask.sum())\n            if rle[1]=='4':\n                l_4.append(mask.sum())\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" #plt.hist(l_2)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nimport time\nimg1 = cv2.imread('../input/severstal-steel-defect-detection/train_images/0007a71bf.jpg' )\na=time.time()\nimg = cv2.imread('../input/severstal-steel-defect-detection/train_images/fff02e9c5.jpg',0)\nprint(img.shape)\n#img = np.float32(img)\n#img = img/255\n#img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n \n#img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\nI_log = np.log1p(np.array(img, dtype=\"float\"))\nI_fft = np.fft.fft2(I_log)\n\n\n        # Filters\n\nH = __butterworth_filter(I_shape = I_fft.shape, filter_params = [30,2])\n\nI_fft_filt = __apply_filter(I = I_fft, H = H)\nI_filt = np.fft.ifft2(I_fft_filt)\nI = np.exp(np.real(I_filt))-1\nI = cv2.cvtColor(np.uint8(I), cv2.COLOR_GRAY2RGB)\nfrom skimage import exposure\n \np2,p98=np.percentile(I,(2,99))\nI=exposure.rescale_intensity(I,(p2,p98))\nb=time.time()\nprint(b-a)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.imshow(np.uint8(I))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.imshow(img1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom fastai import * \nfrom fastai.vision import *\nopen_mask_rle(df[df.ImageId_ClassId=='fff02e9c5.jpg_3'].EncodedPixels.values[0],(256,1600))\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nimport cv2\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimg = cv2.imread('../input/severstal-steel-defect-detection/train_images/ebce68542.jpg',-1)\nprint(img.shape)\n#img = np.float32(img)\n#img = img/255\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nimg=cv2.bilateralFilter(img ,10,10,10  )\nrows,cols,dim=img.shape\n\nrh, rl, cutoff = 2.5,0.5,32\n\nimgYCrCb = cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)\ny,cr,cb = cv2.split(imgYCrCb)\n\ny_log = np.log(y+0.01)\n\ny_fft = np.fft.fft2(y_log)\n\ny_fft_shift = np.fft.fftshift(y_fft)\n\n\nDX = cols/cutoff\nG = np.ones((rows,cols))\nfor i in range(rows):\n    for j in range(cols):\n        G[i][j]=((rh-rl)*(1-np.exp(-((i-rows/2)**2+(j-cols/2)**2)/(2*DX**2))))+rl\n\nresult_filter = G * y_fft_shift\n\nresult_interm = np.real(np.fft.ifft2(np.fft.ifftshift(result_filter)))\n\nresult = np.exp(result_interm)\n'''\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.imshow(imgYCrCb )\n#im  =cv2.median\n#cv2.bilateralFilter(np.asarray(img) ,10,10,10  )\n#plt.imshow(im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.imshow(img)#","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}