{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install python-gdcm\n%pip install pylibjpeg[all]\n!pip install --extra-index-url https://developer.download.nvidia.com/compute/redist --upgrade nvidia-dali-cuda110","metadata":{"execution":{"iopub.status.busy":"2023-02-11T02:38:41.31089Z","iopub.execute_input":"2023-02-11T02:38:41.311355Z","iopub.status.idle":"2023-02-11T02:39:34.759142Z","shell.execute_reply.started":"2023-02-11T02:38:41.311316Z","shell.execute_reply":"2023-02-11T02:39:34.757948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport time\nimport shutil\nimport pydicom\nfrom tqdm import tqdm\nimport gc\nimport nvidia.dali as dali\nfrom skimage.transform import resize\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-02-11T02:39:34.761121Z","iopub.execute_input":"2023-02-11T02:39:34.761512Z","iopub.status.idle":"2023-02-11T02:39:35.705613Z","shell.execute_reply.started":"2023-02-11T02:39:34.761469Z","shell.execute_reply":"2023-02-11T02:39:35.704656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torchvision.transforms import RandomRotation,RandomAutocontrast,RandomHorizontalFlip","metadata":{"execution":{"iopub.status.busy":"2023-02-11T02:39:35.708148Z","iopub.execute_input":"2023-02-11T02:39:35.7087Z","iopub.status.idle":"2023-02-11T02:39:37.653336Z","shell.execute_reply.started":"2023-02-11T02:39:35.708664Z","shell.execute_reply":"2023-02-11T02:39:37.652312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Pre Process**","metadata":{}},{"cell_type":"code","source":"def img_preprocess(df):\n    image = []\n    \n    for t,d in df.iterrows():\n        dcm_file = f'{main_path}/{int(d.patient_id)}/{int(d.image_id)}.dcm'\n        img_arr = resize(pydicom.dcmread(dcm_file).pixel_array,((1536,768)))\n        m = norm_data(img_arr)\n        if pydicom.dcmread(dcm_file).PhotometricInterpretation == 'MONOCHROME1': \n            m = 1-m\n        m = croper(m)\n        m = resize(m,(1024,512))\n        image.append(m)\n        \n    image = np.stack(image)\n    image = torch.from_numpy(image)\n    image= RandomRotation(30)(image)\n    #image = RandomAutocontrast(p=0.3)(image)\n    #image = RandomHorizontalFlip(p=0.5)(image)\n    #image = torch.swapaxes(image[:,:,:,None], 1, 3)\n    #image = torch.cat([image, image, image], dim=1)\n    \n    return image","metadata":{"execution":{"iopub.status.busy":"2023-02-11T03:11:01.516551Z","iopub.execute_input":"2023-02-11T03:11:01.51693Z","iopub.status.idle":"2023-02-11T03:11:01.524484Z","shell.execute_reply.started":"2023-02-11T03:11:01.516898Z","shell.execute_reply":"2023-02-11T03:11:01.523453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ExternalInputIterator(object):\n    def __init__(self,jpeg_stream):\n        self.jpeg_stream = jpeg_stream\n    def __iter__(self):\n        return self\n    def __next__(self):\n        return self.jpeg_stream\n    \ndef norm_data(m):\n    max=m.max()\n    min=m.min()\n    m = (m-min)/(max-min+1e-6) #float32\n    m = m.astype(np.float16)\n    return m\n\ndef croper(input_image):\n    try :\n        t = np.mean(input_image)\n        threshh = input_image > t\n        cols = np.sum(threshh, axis = 0)\n        rows = np.sum(threshh, axis = 1)\n        def getPoint(hist, stride=50):\n            M = np.argmax(hist)\n            LR = [M, M]\n            while True:\n                temp = LR\n                if hist[LR[0]] != 0 and hist[LR[1]] != 0:\n                    LR = np.clip([LR[0]-stride, LR[1]+stride], a_min = 0, a_max = len(hist)-1)\n                if temp[0] == LR[0] and temp[1]==LR[1]:\n                    return LR\n        p_row = getPoint(rows)\n        p_col = getPoint(cols)\n        crop = input_image[p_row[0]: p_row[1], p_col[0]:p_col[1]]\n    except KeyError as e :\n        print(e)\n        crop = input_image\n    return crop\n\n@dali.pipeline_def\ndef j2k_decode_pipeline(eii):\n    jpeg  = dali.fn.external_source(source=eii, dtype=dali.types.UINT8)\n    image = dali.fn.experimental.decoders.image(jpeg, device='mixed', output_type=dali.types.ANY_DATA, dtype=dali.types.UINT16)\n    image = dali.fn.resize(image, size=[1536, 768])\n    return image\n\ndef img_preprocess2(df):\n    jpeg_stream = []\n    invert = []\n    for t,d in df.iterrows():\n        dcm_file = f'{main_path}/{int(d.patient_id)}/{int(d.image_id)}.dcm'\n        dicom = pydicom.dcmread(dcm_file)\n        s = dicom.PixelData\n        offset = s.find(b'\\x00\\x00\\x00\\x0C')  # <---- the jpeg2000 header info we're looking for\n        jpeg_stream.append(np.array(bytearray(s[offset:]), np.uint8))\n        invert.append(dicom.PhotometricInterpretation == 'MONOCHROME1')\n\n    pipe = j2k_decode_pipeline(ExternalInputIterator(jpeg_stream), batch_size=len(jpeg_stream), num_threads=2, device_id=0, debug=True)\n    pipe.build()\n    pipe_out = pipe.run()\n\n    #todo : use feed_ndarray() /nvidia/dali/plugin/pytorch.py to directly convert to gpu pytorch tensor\n    pipe_out = pipe_out[0].as_cpu().as_array().squeeze(-1)\n\n    image = []\n    for i in range(len(df)):\n        m = pipe_out[i]\n        m = norm_data(m)\n        if invert[i]:\n            m = 1-m\n        m = croper(m)\n        m = resize(m,(1024,512))\n        image.append(m)\n    \n    image = np.stack(image)\n    image = torch.from_numpy(image)\n    image= RandomRotation(30)(image)\n    #image = RandomAutocontrast(p=0.3)(image)\n    #image = RandomHorizontalFlip(p=0.5)(image)\n    #image = torch.swapaxes(image[:,:,:,None], 1, 3)\n    #image = torch.cat([image, image, image], dim=1)\n    \n    return image","metadata":{"execution":{"iopub.status.busy":"2023-02-11T03:10:53.627546Z","iopub.execute_input":"2023-02-11T03:10:53.628098Z","iopub.status.idle":"2023-02-11T03:10:53.659495Z","shell.execute_reply.started":"2023-02-11T03:10:53.628048Z","shell.execute_reply":"2023-02-11T03:10:53.658305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Save Function**","metadata":{}},{"cell_type":"code","source":"def data_saver(df):\n\n    t1=time.perf_counter()\n    \n    batch_size_big=256\n    batch_size_small=64\n    \n    for i in tqdm(range(0,len(df),batch_size_big)):\n        \n        x=torch.from_numpy(np.zeros((batch_size_big,1024,512)))\n        y=torch.from_numpy(np.zeros((batch_size_big)))\n\n        if (i+batch_size_big) > len(df):\n            break\n\n        try :\n            for k in range(0,batch_size_big,batch_size_small):\n                if (k+batch_size_small)>(batch_size_big) : break\n                try : \n                    temp=img_preprocess2(df[i:i+batch_size_small])\n                except : \n                    temp=img_preprocess(df[i:i+batch_size_small])\n                x[k:k+batch_size_small]=temp\n                y[k:k+batch_size_small]=torch.from_numpy(df['cancer'][i:i+batch_size_small].to_numpy()).float()\n                print(str(i+k+batch_size_small)+'Files loaded in '+str(int(time.perf_counter()-t1)))\n            x = torch.swapaxes(x[:,:,:,None], 1, 3)\n            x = RandomAutocontrast(p=0.3)(x)\n            torch.save(x,'ex/cc_img_%i-%i.pt' %(i,i+batch_size_big))\n            torch.save(y,'ex/cc_data_%i-%i.pt' %(i,i+batch_size_big))\n            print('Save Successful.')\n        except KeyError as e :\n            print(e)\n            print('Fail to load data %i to %i' %(i,i+batch_size_big))\n            continue","metadata":{"execution":{"iopub.status.busy":"2023-02-11T03:12:58.942909Z","iopub.execute_input":"2023-02-11T03:12:58.943682Z","iopub.status.idle":"2023-02-11T03:12:58.955512Z","shell.execute_reply.started":"2023-02-11T03:12:58.943643Z","shell.execute_reply":"2023-02-11T03:12:58.954461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_run(s,e):\n    if os.path.exists('/kaggle/working/ex') : shutil.rmtree('/kaggle/working/ex')\n    os.mkdir('/kaggle/working/ex')\n    data_saver(train_df[s,e])\n    filename=str(s)+'-'+str(e)\n    shutil.make_archive(filename, format='zip', root_dir='/kaggle/working/ex')\n    shutil.rmtree(\"/kaggle/working/ex\")","metadata":{"execution":{"iopub.status.busy":"2023-02-11T03:23:14.498821Z","iopub.execute_input":"2023-02-11T03:23:14.49947Z","iopub.status.idle":"2023-02-11T03:23:14.506137Z","shell.execute_reply.started":"2023-02-11T03:23:14.499431Z","shell.execute_reply":"2023-02-11T03:23:14.505002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **RUN**","metadata":{}},{"cell_type":"code","source":"main_path='/kaggle/input/rsna-breast-cancer-detection/train_images'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.read_csv('/kaggle/input/modefiedcsv/mod3.csv')\ntrain_df.drop(columns=train_df.columns[0], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T02:39:37.715573Z","iopub.execute_input":"2023-02-11T02:39:37.716Z","iopub.status.idle":"2023-02-11T02:39:37.795853Z","shell.execute_reply.started":"2023-02-11T02:39:37.715949Z","shell.execute_reply":"2023-02-11T02:39:37.79487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save_run(0,8500)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data Checker**","metadata":{}},{"cell_type":"code","source":"'''\nv=147\n\ntemp=torch.load('/kaggle/working/ex/cc_img_0-256.pt')\ntemp2=torch.load('/kaggle/working/ex/cc_data_0-256.pt')\nprint(train_df[v:v+1])\nprint(temp2[v])\nplt.imshow(temp[v,0,:,:])\n'''","metadata":{"execution":{"iopub.status.busy":"2023-02-11T03:25:22.589888Z","iopub.execute_input":"2023-02-11T03:25:22.590653Z","iopub.status.idle":"2023-02-11T03:25:23.25647Z","shell.execute_reply.started":"2023-02-11T03:25:22.590615Z","shell.execute_reply":"2023-02-11T03:25:23.255305Z"},"trusted":true},"execution_count":null,"outputs":[]}]}