{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# $\\text{Import libraries}$","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom matplotlib import pyplot as plt\nimport polars as pl\nimport pydicom as pdc\nfrom PIL import Image\nimport pathlib\nfrom multiprocessing import Pool, cpu_count\nimport random\nimport time\nimport datetime\nfrom pathlib import Path\nimport random","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.__version__","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# $\\text{Make config}$","metadata":{}},{"cell_type":"code","source":"class Config:\n    def __init__(self):\n        self.columns = [\n            'patient_id',\n            'kidney_healthy',\n             'kidney_low',\n             'kidney_high',\n             'liver_healthy',\n             'liver_low',\n             'liver_high',\n             'spleen_healthy',\n             'spleen_low',\n             'spleen_high',\n             'any_injury'\n        ]\n        self.train_ratio = 0.7\n        self.val_ratio = 0.2\n    def __repr__(self):\n        return f\"{self.columns = }\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = Config()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# $\\text{Make small dataset only do this once, if marked then dont run the cell}$","metadata":{}},{"cell_type":"markdown","source":"# $\\text{Dataset created DO NOT run below code unless needed to make new dataset}$","metadata":{}},{"cell_type":"markdown","source":"## $\\text{Make dataframe}$","metadata":{}},{"cell_type":"code","source":"data_parquet = pl.read_parquet(\"/kaggle/input/rsna-2023-abdominal-trauma-detection/train_dicom_tags.parquet\")","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"link_images = data_parquet.select(\n    pl.col('path').str.split('/').list.to_struct()\n) \\\n.unnest('path') \\\n.drop(\"field_0\")\nlink_images = link_images.rename(\n    {\n        \"field_1\":\"patient_ID\",\n        \"field_2\": \"series_ID\",\n        \"field_3\":\"instance_ID\"\n    }\n)\nlink_images = link_images.with_columns(\n    pl.col(\"patient_ID\").cast(pl.Int64),\n    pl.col(\"series_ID\").cast(pl.Int64),\n    pl.col(\"instance_ID\").str.split(\".\").list.to_struct()\n)\nlink_images = link_images.unnest(\"instance_ID\") \\\n.drop(\"field_1\") \\\n.rename({\"field_0\": \"instance_ID\"})\nlink_images = link_images.with_columns(\n    pl.col(\"instance_ID\").cast(pl.Int64),\n)\nlink_images = link_images.with_columns(\n    pl.concat_str(\n        [\n            pl.lit(\"/kaggle/input/rsna-2023-abdominal-trauma-detection\"),\n            data_parquet[\"path\"]\n        ], separator = \"/\"\n    ).alias(\"link_dcom\")\n)\nlink_images.rows","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = link_images.group_by(\"patient_ID\", \"series_ID\").all()[:, :-1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"instance_ID_per_patient = x[\"instance_ID\"].to_numpy()\n# series_ID_per_patient = x[\"series_ID\"].to_numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"150000/4711","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:03:41.367469Z","iopub.execute_input":"2023-11-11T06:03:41.367921Z","iopub.status.idle":"2023-11-11T06:03:41.375812Z","shell.execute_reply.started":"2023-11-11T06:03:41.367879Z","shell.execute_reply":"2023-11-11T06:03:41.374623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rng = np.random.default_rng()\nsample_idx = np.array([\n    rng.choice(\n        range(len(ID)),\n        size = 32, # 150000/4711 ~= 32 (31.840373593716834)\n        replace = 0\n    ) for ID in instance_ID_per_patient\n])","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:10.311044Z","iopub.execute_input":"2023-11-11T06:04:10.311943Z","iopub.status.idle":"2023-11-11T06:04:10.659216Z","shell.execute_reply.started":"2023-11-11T06:04:10.311896Z","shell.execute_reply":"2023-11-11T06:04:10.657919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# series_ID_sample = np.array([series_ID_per_patient[i][ID] for (i, ID) in enumerate(sample_idx)])\ninstance_ID_sample = np.array([instance_ID_per_patient[i][ID] for (i, ID) in enumerate(sample_idx)])","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:16.641423Z","iopub.execute_input":"2023-11-11T06:04:16.641895Z","iopub.status.idle":"2023-11-11T06:04:16.660089Z","shell.execute_reply.started":"2023-11-11T06:04:16.641846Z","shell.execute_reply":"2023-11-11T06:04:16.658698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"instance_ID_sample.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:17.12265Z","iopub.execute_input":"2023-11-11T06:04:17.12308Z","iopub.status.idle":"2023-11-11T06:04:17.1308Z","shell.execute_reply.started":"2023-11-11T06:04:17.123045Z","shell.execute_reply":"2023-11-11T06:04:17.129419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = x.with_columns(\n    pl.Series(instance_ID_sample).alias(\"instance_ID\"),\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:19.331423Z","iopub.execute_input":"2023-11-11T06:04:19.332833Z","iopub.status.idle":"2023-11-11T06:04:19.417247Z","shell.execute_reply.started":"2023-11-11T06:04:19.332788Z","shell.execute_reply":"2023-11-11T06:04:19.416183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = sample_df.explode([\"instance_ID\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:19.660507Z","iopub.execute_input":"2023-11-11T06:04:19.6616Z","iopub.status.idle":"2023-11-11T06:04:19.668115Z","shell.execute_reply.started":"2023-11-11T06:04:19.661564Z","shell.execute_reply":"2023-11-11T06:04:19.667009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.rows","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:19.93499Z","iopub.execute_input":"2023-11-11T06:04:19.936273Z","iopub.status.idle":"2023-11-11T06:04:19.943768Z","shell.execute_reply.started":"2023-11-11T06:04:19.936229Z","shell.execute_reply":"2023-11-11T06:04:19.942449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = sample_df.join(link_images, on=[\"patient_ID\", \"series_ID\", \"instance_ID\"])","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:23.433194Z","iopub.execute_input":"2023-11-11T06:04:23.433667Z","iopub.status.idle":"2023-11-11T06:04:23.542389Z","shell.execute_reply.started":"2023-11-11T06:04:23.433628Z","shell.execute_reply":"2023-11-11T06:04:23.541447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_full_csv = pl.read_csv(\"/kaggle/input/rsna-2023-abdominal-trauma-detection/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:23.684976Z","iopub.execute_input":"2023-11-11T06:04:23.685429Z","iopub.status.idle":"2023-11-11T06:04:23.694533Z","shell.execute_reply.started":"2023-11-11T06:04:23.685394Z","shell.execute_reply":"2023-11-11T06:04:23.693275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_select = pl.DataFrame().with_columns(\n    data_full_csv[\"patient_id\"],\n).hstack(\n    pl.from_numpy(\n        injured_data, schema =[\"kidney\", \"liver\", \"spleen\"]\n    )\n)\ndata_select = data_select.rename({\"patient_id\":\"patient_ID\"}).cast({\"patient_ID\":pl.Int64})","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:23.973764Z","iopub.execute_input":"2023-11-11T06:04:23.974473Z","iopub.status.idle":"2023-11-11T06:04:23.981197Z","shell.execute_reply.started":"2023-11-11T06:04:23.974437Z","shell.execute_reply":"2023-11-11T06:04:23.980006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_select.rows","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:25.168262Z","iopub.execute_input":"2023-11-11T06:04:25.168856Z","iopub.status.idle":"2023-11-11T06:04:25.179566Z","shell.execute_reply.started":"2023-11-11T06:04:25.168806Z","shell.execute_reply":"2023-11-11T06:04:25.178502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = sample_df.join(data_select, on = \"patient_ID\")","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:04:28.362175Z","iopub.execute_input":"2023-11-11T06:04:28.362646Z","iopub.status.idle":"2023-11-11T06:04:28.418886Z","shell.execute_reply.started":"2023-11-11T06:04:28.36261Z","shell.execute_reply":"2023-11-11T06:04:28.41798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.rows","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:27:25.316991Z","iopub.execute_input":"2023-11-11T07:27:25.317564Z","iopub.status.idle":"2023-11-11T07:27:25.337213Z","shell.execute_reply.started":"2023-11-11T07:27:25.31752Z","shell.execute_reply":"2023-11-11T07:27:25.336307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## $\\text{Convert image}$","metadata":{}},{"cell_type":"code","source":"# !mkdir -p Abdoment_image/images","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:05:37.310503Z","iopub.execute_input":"2023-11-11T06:05:37.311003Z","iopub.status.idle":"2023-11-11T06:05:38.474612Z","shell.execute_reply.started":"2023-11-11T06:05:37.310961Z","shell.execute_reply":"2023-11-11T06:05:38.472704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !ls -la","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:05:50.471055Z","iopub.execute_input":"2023-11-11T06:05:50.471544Z","iopub.status.idle":"2023-11-11T06:05:51.628547Z","shell.execute_reply.started":"2023-11-11T06:05:50.471506Z","shell.execute_reply":"2023-11-11T06:05:51.627037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def standardize_pixel_array(dcm):\n    # Correct DICOM pixel_array if PixelRepresentation == 1.\n    pixel_array = dcm.pixel_array\n    if dcm.PixelRepresentation == 1:\n        bit_shift = dcm.BitsAllocated - dcm.BitsStored\n        dtype = pixel_array.dtype \n        new_array = (pixel_array << bit_shift).astype(dtype) >>  bit_shift\n        pixel_array = pdc.pixel_data_handlers.util.apply_modality_lut(new_array, dcm)\n    return pixel_array\n\ndef read_xray(path, fix_monochrome=True):\n    dicom = pdc.dcmread(path)\n    data = standardize_pixel_array(dicom)\n    data = data - np.min(data)\n    data = data / (np.max(data) + 1e-5)\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = 1.0 - data\n    return data\n\ndef resize_and_save(file_path):\n    img_array = read_xray(file_path)\n    img_array = (img_array * 255).astype(np.uint8)\n    \n    PIL_image = Image.fromarray(img_array)\n    path_component = file_path.rsplit(\"/\", 3)\n    img_dir = pathlib.Path(f\"./{path_component[1]}/{path_component[2]}\")\n    img_dir.mkdir(parents=True, exist_ok = True)\n    img_name = path_component[-1].replace(\"dcm\", \"png\")\n    PIL_image.save(img_dir/img_name)\n    return img_dir","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:06:08.920066Z","iopub.execute_input":"2023-11-11T06:06:08.920562Z","iopub.status.idle":"2023-11-11T06:06:08.934226Z","shell.execute_reply.started":"2023-11-11T06:06:08.920522Z","shell.execute_reply":"2023-11-11T06:06:08.93292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"link_dcom_list = sample_df[\"link_dcom\"].to_list()","metadata":{"execution":{"iopub.status.busy":"2023-11-11T06:06:11.358649Z","iopub.execute_input":"2023-11-11T06:06:11.359786Z","iopub.status.idle":"2023-11-11T06:06:11.450962Z","shell.execute_reply.started":"2023-11-11T06:06:11.359742Z","shell.execute_reply":"2023-11-11T06:06:11.449572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/Abdoment_image/images\nprocessing_dir = []\nwith Pool() as pool:\n    result = pool.imap_unordered(resize_and_save, link_dcom_list)\n    t_begin = time.time()\n    for img_dir in result:\n        if img_dir not in processing_dir:\n            processing_dir.append(img_dir)\n            processing_count = len(processing_dir)\n            t_append = time.time()\n            dif_time = t_append-t_begin\n            ratio = processing_count/4711\n            time_between = time.strftime('%H:%M:%S', time.gmtime(dif_time))\n            time_remaning = time.strftime('%H:%M:%S', time.gmtime(dif_time*(1/ratio-1)))\n            print(f\"Finished {processing_count}/4711 series ID - {ratio*100:.4f}%\")\n            print(f\"Time runnning: {time_between} | Time remaning: {time_remaning}\\n\")\n%cd /kaggle/working","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## $\\text{Save new dataset with csv}$","metadata":{}},{"cell_type":"code","source":"sample_df.write_csv(\"./Abdoment_image/dataset_150k.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:28:24.667209Z","iopub.execute_input":"2023-11-11T07:28:24.667715Z","iopub.status.idle":"2023-11-11T07:28:24.785197Z","shell.execute_reply.started":"2023-11-11T07:28:24.667669Z","shell.execute_reply":"2023-11-11T07:28:24.784199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /root/.kaggle","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:28:29.956804Z","iopub.execute_input":"2023-11-11T07:28:29.95764Z","iopub.status.idle":"2023-11-11T07:28:29.967129Z","shell.execute_reply.started":"2023-11-11T07:28:29.957597Z","shell.execute_reply":"2023-11-11T07:28:29.965654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!touch kaggle.json","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat kaggle.json","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:28:31.861574Z","iopub.execute_input":"2023-11-11T07:28:31.861997Z","iopub.status.idle":"2023-11-11T07:28:33.0291Z","shell.execute_reply.started":"2023-11-11T07:28:31.861965Z","shell.execute_reply":"2023-11-11T07:28:33.027779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile kaggle.json\n{\"username\":\"theunkovvn\",\"key\":\"ba064e9386e385cc1042c624ed089930\"}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/ ","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:28:45.653616Z","iopub.execute_input":"2023-11-11T07:28:45.655413Z","iopub.status.idle":"2023-11-11T07:28:45.665981Z","shell.execute_reply.started":"2023-11-11T07:28:45.655331Z","shell.execute_reply":"2023-11-11T07:28:45.664482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! kaggle datasets init -p \"/kaggle/working/Abdoment_image\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/Abdoment_image","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:28:49.113199Z","iopub.execute_input":"2023-11-11T07:28:49.114621Z","iopub.status.idle":"2023-11-11T07:28:49.121451Z","shell.execute_reply.started":"2023-11-11T07:28:49.114573Z","shell.execute_reply":"2023-11-11T07:28:49.120495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat *.json","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:28:50.975111Z","iopub.execute_input":"2023-11-11T07:28:50.976385Z","iopub.status.idle":"2023-11-11T07:28:52.133165Z","shell.execute_reply.started":"2023-11-11T07:28:50.976321Z","shell.execute_reply":"2023-11-11T07:28:52.13123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile dataset-metadata.json\n{\n  \"title\": \"Abdoment Dataset\",\n  \"id\": \"theunkovvn/abdoment-dataset\",\n  \"licenses\": [\n    {\n      \"name\": \"CC0-1.0\"\n    }\n  ]\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pwd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/Abdoment_image","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:29:46.895984Z","iopub.execute_input":"2023-11-11T07:29:46.896541Z","iopub.status.idle":"2023-11-11T07:29:46.907828Z","shell.execute_reply.started":"2023-11-11T07:29:46.896496Z","shell.execute_reply":"2023-11-11T07:29:46.906583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:29:47.782532Z","iopub.execute_input":"2023-11-11T07:29:47.783475Z","iopub.status.idle":"2023-11-11T07:29:48.986144Z","shell.execute_reply.started":"2023-11-11T07:29:47.783436Z","shell.execute_reply":"2023-11-11T07:29:48.984531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm -rf *.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!kaggle datasets version -r zip -m \"Remake dataset from scratch\"","metadata":{"execution":{"iopub.status.busy":"2023-11-11T07:29:55.023025Z","iopub.execute_input":"2023-11-11T07:29:55.02356Z","iopub.status.idle":"2023-11-11T07:48:00.199546Z","shell.execute_reply.started":"2023-11-11T07:29:55.023508Z","shell.execute_reply":"2023-11-11T07:48:00.197827Z"},"trusted":true},"execution_count":null,"outputs":[]}]}