{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Dataset Links :\n\n - [256x256 pngs](https://www.kaggle.com/datasets/theoviel/rsna-breast-cancer-256-pngs)\n - [512x512 pngs](https://www.kaggle.com/datasets/theoviel/rsna-breast-cancer-512-pngs)\n - [768x768 pngs](https://www.kaggle.com/datasets/theoviel/rsna-breast-cancer-768-pngs)\n - [1024x1024 pngs](https://www.kaggle.com/datasets/theoviel/rsna-breast-cancer-1024-pngs)\n\n**Changes :**\n- Invert images with PhotometricInterpretation == \"MONOCHROME1\" ","metadata":{}},{"cell_type":"markdown","source":"## Initialization","metadata":{}},{"cell_type":"code","source":"!pip install -qU python-gdcm pydicom pylibjpeg","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-06T14:21:55.67238Z","iopub.execute_input":"2022-12-06T14:21:55.672994Z","iopub.status.idle":"2022-12-06T14:22:12.769792Z","shell.execute_reply.started":"2022-12-06T14:21:55.672889Z","shell.execute_reply":"2022-12-06T14:22:12.768066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport gdcm\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tqdm.notebook import tqdm\nfrom joblib import Parallel, delayed","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:22:12.772874Z","iopub.execute_input":"2022-12-06T14:22:12.773763Z","iopub.status.idle":"2022-12-06T14:22:13.857383Z","shell.execute_reply.started":"2022-12-06T14:22:12.773709Z","shell.execute_reply":"2022-12-06T14:22:13.856194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = glob.glob(\"/kaggle/input/rsna-breast-cancer-detection/train_images/*/*.dcm\")\n\nlen(train_images)  # 54706","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:22:13.859068Z","iopub.execute_input":"2022-12-06T14:22:13.86008Z","iopub.status.idle":"2022-12-06T14:22:43.577382Z","shell.execute_reply.started":"2022-12-06T14:22:13.860031Z","shell.execute_reply":"2022-12-06T14:22:43.575754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examples","metadata":{}},{"cell_type":"markdown","source":"## Save the processed data\n**Images are quite big so resizing them is necessary.**\n  - Use 256 to train your first models, or if you don't have a lot of compute\n  - use 512 to have competitive models\n  - Check if 768/1024 is better, if you have the compute power\n\n**I advise using the `png` format because the jpg compression can be annoying during inference.**","metadata":{}},{"cell_type":"code","source":"SAVE_FOLDER = \"output/\"\nSIZE = 512\nEXTENSION = \"png\"\n\nos.makedirs(SAVE_FOLDER, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:22:43.593761Z","iopub.execute_input":"2022-12-06T14:22:43.594796Z","iopub.status.idle":"2022-12-06T14:22:43.629166Z","shell.execute_reply.started":"2022-12-06T14:22:43.594747Z","shell.execute_reply":"2022-12-06T14:22:43.627777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\nreadS,processS,writeS, count = 0,0,0,0\ndef process(f, size=512, save_folder=\"\", extension=\"png\"):\n    readS,pixelArrayS, resizeS,writeS = 0,0,0,0\n    patient = f.split('/')[-2]\n    image = f.split('/')[-1][:-4]\n    st = time.time()\n    dicom = pydicom.dcmread(f)\n    readS = time.time() - st\n    st = time.time()\n    \n    img = dicom.pixel_array\n    pixelArrayS = time.time() - st\n    img = (img - img.min()) / (img.max() - img.min())\n\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        img = 1 - img\n    st = time.time()\n    img = cv2.resize(img, (size, size))\n    resizeS =  time.time() - st\n    st = time.time()\n    \n    cv2.imwrite(save_folder + f\"{patient}_{image}.{extension}\", (img * 255).astype(np.uint8))\n    writeS =  time.time() - st\n    with open(\"times.csv\",\"a\") as f:\n        f.write(f\"{readS}, {pixelArrayS}, {resizeS}, {writeS}\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:52:40.49261Z","iopub.execute_input":"2022-12-06T14:52:40.493366Z","iopub.status.idle":"2022-12-06T14:52:40.505643Z","shell.execute_reply.started":"2022-12-06T14:52:40.493325Z","shell.execute_reply":"2022-12-06T14:52:40.504109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time #74 with write, 76 with 4, 89 with 1, 4 with 54, 52 with 2, 47 with 4, 2 with 54\n#219 with 2 jobs, 288 with 3 jobs, 216 with 4 jobs, 210 with 5 jobs\n\n\n\nst = time.time()\nprint(\"st \", st)\n_ = Parallel(n_jobs=4)(\n    delayed(process)(uid, size=SIZE, save_folder=SAVE_FOLDER, extension=EXTENSION)\n    for uid in tqdm(train_images[:500])    \n)\nprint(\"end \", time.time() - st)\nnp.average(pd.read_csv(\"times.csv\"), axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:53:12.934274Z","iopub.execute_input":"2022-12-06T14:53:12.934716Z","iopub.status.idle":"2022-12-06T14:53:17.937652Z","shell.execute_reply.started":"2022-12-06T14:53:12.934662Z","shell.execute_reply":"2022-12-06T14:53:17.936667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:48:39.579437Z","iopub.execute_input":"2022-12-06T14:48:39.579874Z","iopub.status.idle":"2022-12-06T14:48:39.590131Z","shell.execute_reply.started":"2022-12-06T14:48:39.579837Z","shell.execute_reply":"2022-12-06T14:48:39.588845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat /proc/cpuinfo","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Done !","metadata":{}}]}