{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a notebook to test the issue of Out of Memory reported by Mark Wijkhuizen here,\nhttps://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/382270","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"execution":{"iopub.status.busy":"2023-02-01T12:26:32.398239Z","iopub.execute_input":"2023-02-01T12:26:32.398942Z","iopub.status.idle":"2023-02-01T12:26:58.645537Z","shell.execute_reply.started":"2023-02-01T12:26:32.398792Z","shell.execute_reply":"2023-02-01T12:26:58.644132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import psutil\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport pydicom\nimport dicomsdl\nimport gc\nimport ctypes\n\n# Read Train DataFrame\ntrain = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv').head(1000)\n\n# Get File Path to DICOM files\ndef get_file_path(args):\n    patient_id, image_id = args\n    return f'/kaggle/input/rsna-breast-cancer-detection/train_images/{patient_id}/{image_id}.dcm'\n\n# Assign DICOM file path for each training sample\ntrain['file_path'] = train[['patient_id', 'image_id']].apply(get_file_path, axis=1)\n\n# Function which simply reads a DICOM file\ndef read_dcm_by_dicomsdl(fp):\n    dicom = dicomsdl.open(fp)\n    image = dicom.pixelData()\n    return image, dicom\n\ndef read_dcm_by_pydicom(fp):\n    dicom = pydicom.read_file(fp)\n    image = dicom.pixel_array\n    return image, dicom\n\ndef read_dcm_by_open(fp):\n    with open(fp, \"rb\") as read_file:\n        image = read_file.read()\n    return image, image\n\n# For 1000 training samples, read the DICOM file\npbar = tqdm(train['file_path'])\nfor fp in pbar:    \n    #image, dicom = read_dcm_by_dicomsdl(fp)\n    #image, dicom = read_dcm_by_pydicom(fp)\n    image, dicom = read_dcm_by_open(fp)    \n    del image, dicom\n    gc.collect()\n\n    libc = ctypes.CDLL(\"libc.so.6\") # clearing cache \n    libc.malloc_trim(0)\n\n    pbar.set_postfix({'RAM Used (GB)': psutil.virtual_memory()[3]/1000000000})","metadata":{"execution":{"iopub.status.busy":"2023-02-01T12:26:58.648235Z","iopub.execute_input":"2023-02-01T12:26:58.648588Z"},"trusted":true},"execution_count":null,"outputs":[]}]}