{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install opencv-python torch torchvision tqdm torchxrayvision\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T19:10:47.935229Z","iopub.execute_input":"2026-06-29T19:10:47.935976Z","iopub.status.idle":"2026-06-29T19:10:51.32785Z","shell.execute_reply.started":"2026-06-29T19:10:47.935939Z","shell.execute_reply":"2026-06-29T19:10:51.326794Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import subprocess\nimport os\nimport shutil\nfrom pathlib import Path\nfrom joblib import Parallel, delayed\n\nWORKING = \"/kaggle/working\"\nTMP = \"/kaggle/tmp\"\n\nSCRIPT_SRC = \"/kaggle/input/datasets/joshuaramnauth/preprocessing-script/split_parcellate_argparse.py\"\nSCRIPT_DST = f\"{WORKING}/split_parcellate_argparse.py\"\n\nOUTPUT_DIR = f\"{TMP}/dataset_parcellated\"  # 60GB scratch space\n\n# NIH\nNIH_BASE = \"/kaggle/input/datasets/organizations/nih-chest-xrays/data\"\nNIH_RAW_DATA_DIR = f\"{TMP}/nih_raw_data\"  # also in tmp\n\n\n\n# Copy script\nshutil.copy(SCRIPT_SRC, SCRIPT_DST)\nprint(\"Script copied.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T23:42:31.62991Z","iopub.execute_input":"2026-06-28T23:42:31.630291Z","iopub.status.idle":"2026-06-28T23:42:31.638335Z","shell.execute_reply.started":"2026-06-28T23:42:31.630261Z","shell.execute_reply":"2026-06-28T23:42:31.637591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- NIH setup ---\nos.makedirs(f\"{NIH_RAW_DATA_DIR}/images\", exist_ok=True)\nos.makedirs(f\"{NIH_RAW_DATA_DIR}/Data_Entry_2017\", exist_ok=True)\n\ndef symlink_dir(src):\n    subprocess.run(f\"ln -sf {src}/*.png {NIH_RAW_DATA_DIR}/images/ 2>/dev/null || true\", shell=True)\n\nnih_dirs = [f\"{NIH_BASE}/images_{i:03d}/images\" for i in range(1, 13)]\nParallel(n_jobs=-1, backend=\"threading\")(delayed(symlink_dir)(d) for d in nih_dirs)\n\nnih_csv_link = Path(f\"{NIH_RAW_DATA_DIR}/Data_Entry_2017/Data_Entry_2017.csv\")\nif not nih_csv_link.exists():\n    nih_csv_link.symlink_to(f\"{NIH_BASE}/Data_Entry_2017.csv\")\n\nprint(\"NIH setup done.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T23:42:31.639173Z","iopub.execute_input":"2026-06-28T23:42:31.639706Z","iopub.status.idle":"2026-06-28T23:42:37.249242Z","shell.execute_reply.started":"2026-06-28T23:42:31.639683Z","shell.execute_reply":"2026-06-28T23:42:37.248507Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# NIH\nsubprocess.run([\"python\", SCRIPT_DST,\n    \"--raw_data_dir\", NIH_RAW_DATA_DIR,\n    \"--output_dir\", OUTPUT_DIR,\n    \"--dataset_name\", \"NIH\"\n], check=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T23:42:37.250701Z","iopub.execute_input":"2026-06-28T23:42:37.251033Z","iopub.status.idle":"2026-06-29T01:59:15.680437Z","shell.execute_reply.started":"2026-06-28T23:42:37.251011Z","shell.execute_reply":"2026-06-29T01:59:15.67959Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result = subprocess.run([\"du\", \"-sh\", \"/kaggle/tmp/dataset_parcellated\"], capture_output=True, text=True)\nprint(result.stdout)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-29T19:12:24.255753Z","iopub.execute_input":"2026-06-29T19:12:24.256779Z","iopub.status.idle":"2026-06-29T19:12:24.276099Z","shell.execute_reply.started":"2026-06-29T19:12:24.256735Z","shell.execute_reply":"2026-06-29T19:12:24.275178Z"}},"outputs":[],"execution_count":null}]}