{"cells":[{"metadata":{},"cell_type":"markdown","source":"source: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nfrom PIL import Image\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_xray(path):\n    dicom = pydicom.read_file(path)\n    return dicom\n\ndef xray_to_nparray(dicom,voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef nparray_to_img(array, size = None, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    if size == None:\n        return im\n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im\n\ndef xray_to_img(path,size = None,keep_ratio=False,voi_lut = True, fix_monochrome = True, resample=Image.LANCZOS):\n    dicom = get_xray(path)\n    data = xray_to_nparray(dicom,voi_lut=voi_lut,fix_monochrome=fix_monochrome)\n    img = nparray_to_img(data,size = size, keep_ratio=keep_ratio, resample=resample)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# process csv\ndef split_train_csv(load_path,save_dir):\n    data = pd.read_csv(load_path)\n    #split train to each class\n    class_ids = data['class_id'].unique()\n    class_ids.sort()\n    os.makedirs(save_dir,exist_ok=True)\n    file_names = []\n    for class_id in tqdm(class_ids):\n        file_name = 'train_{:02d}.csv'.format(class_id)\n        file_names.append(file_name)\n        data[data['class_id']==class_id].reset_index().to_csv(os.path.join(save_dir,file_name))\n    return save_dir, file_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Process dicom\ndef get_raw_data_by_class(df,class_id,load_dir,save_dir = None):\n    file_names = df[df['class_id']==class_id]['image_id'].unique()    \n    data = []\n    for file_name in tqdm(file_names):\n        path_dicom = os.path.join(load_dir,file_name+'.dicom')\n        xray = get_xray(path_dicom).pixel_array\n        if save_dir is not None:\n            os.makedirs(save_dir,exist_ok=True)\n            path_npy = os.path.join(save_dir,file_name+'.npy')\n            if os.path.isfile(path_npy) is False:\n                np.save(path_npy,xray)\n        data.append([file_name,xray])        \n    return data\ndef get_png_by_class(df,class_id,load_dir,save_dir = None):\n    file_names = df[df['class_id']==class_id]['image_id'].unique()\n    data = []\n    for file_name in tqdm(file_names):\n        path_dicom = os.path.join(load_dir,file_name+'.dicom')\n        img = xray_to_img(path_dicom)\n        if save_dir is not None:\n            os.makedirs(save_dir,exist_ok=True)\n            path_png = os.path.join(save_dir,file_name+'.png')\n            if os.path.isfile(path_png) is False:\n                img.save(path_png)\n        data.append([file_name,img])        \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from shutil import copyfile\ndef copy_data_by_class(df,class_id,load_dir,save_dir):\n    file_names = df[df['class_id']==class_id]['image_id'].unique()\n    os.makedirs(save_dir,exist_ok=True)\n    for file_name in tqdm(file_names):\n        copy_dicom = os.path.join(load_dir,file_name+'.dicom')\n        paste_dicom = os.path.join(save_dir,file_name+'.dicom')\n        copyfile(copy_dicom,paste_dicom)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = f'../input/vinbigdata-chest-xray-abnormalities-detection/'\ntrain_csv_path = os.path.join(data_dir,'train.csv')\ntrain_folder_path = os.path.join(data_dir,'train')\nsplit_train_csv_path = './train_csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csv_folder, csv_names= split_train_csv(train_csv_path,split_train_csv_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_14 = pd.read_csv(os.path.join(csv_folder,csv_names[14]))\nclass_14.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_12 = pd.read_csv(os.path.join(csv_folder,csv_names[12]))\nclass_12.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get number image as class_12\ndf_class_14 = class_14[class_14['image_id'].isin(class_14['image_id'].unique()[:len(class_12['image_id'].unique())])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_class_14.to_csv('./train_csv/class_14_12')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_raw_data_by_class(df_class_14,14,train_folder_path,'./train/npy_class_14_12')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_png_by_class(df_class_14,14,train_folder_path,'./train/png_class_14_12')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_raw_data_by_class(class_12,12,train_folder_path,'./train/npy_class_12')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_png_by_class(class_12,12,train_folder_path,'./train/png_class_12')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"copy_data_by_class(class_12,12,train_folder_path,'./train_dicom_12')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"copy_data_by_class(df_class_14,14,train_folder_path,'./train_dicom_14')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n!zip -r train_csv.zip ./train_csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!zip -r png_class_14_12.zip ./train/png_class_14_12","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!zip -r npy_class_14_12.zip ./train/npy_class_14_12","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!zip -r dicom_class_12.zip ./train_dicom_12","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!zip -r dicom_class_14_12.zip ./train_dicom_14","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}