{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, glob\nimport numpy as np\nimport pandas as pd\nimport shutil\nimport joblib\n\nfrom sklearn.model_selection import GroupKFold, GroupShuffleSplit\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/train.csv\")\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gkf = GroupKFold(n_splits=4)\n\ntrain_df[\"kfold\"] = -1\n\ny = train_df[\"StudyInstanceUID\"].values\n\nfor i, (train_idx, valid_idx) in enumerate(gkf.split(train_df, train_df, groups=y)):\n    train_df.loc[valid_idx, \"kfold\"] = i\n\ntrain_df[\"kfold\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fold3 = train_df[train_df.kfold == 3].copy()\ntrain_fold3.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /root/.kaggle/\n!cp ../input/mykaggleapi/kaggle.json /root/.kaggle/\n!chmod 600 /root/.kaggle/kaggle.json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p \"/tmp/RSNA\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAVE_PATH = \"/tmp/RSNA/train_3\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_dataset(file_name1, file_name2):\n    \n    image_paths = glob.glob(f\"../input/rsna-str-pe-detection-jpeg-256/train-jpegs/{file_name1}/{file_name2}/*.jpg\")\n    \n    save_path = f'{SAVE_PATH}/{file_name1}/{file_name2}'\n    os.makedirs(save_path, exist_ok=True)\n    \n    for f in image_paths:\n        shutil.copy(f, save_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"drop_df = train_fold3[['StudyInstanceUID', 'SeriesInstanceUID']]\ndrop_df = drop_df.drop_duplicates()\ndrop_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file_paths = drop_df[['StudyInstanceUID', 'SeriesInstanceUID']].values.tolist()\nlen(file_paths)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = Parallel(n_jobs=8, backend=\"multiprocessing\")(\n    delayed(create_dataset)(path[0], path[1]) for path in tqdm(file_paths, total=len(file_paths))\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls \"/tmp/RSNA/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fold3.to_csv(\"/tmp/RSNA/train_fold3.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls \"/tmp/RSNA/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!zip -r \"/tmp/RSNA/train_3.zip\" \"/tmp/RSNA/train_3\" >> quit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l \"/tmp/RSNA/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = '''{\n  \"title\": \"rsna-str-fold3-jpeg-256\",\n  \"id\": \"gopidurgaprasad/rsna-str-fold3-jpeg-256\",\n  \"licenses\": [\n    {\n      \"name\": \"CC0-1.0\"\n    }\n  ]\n}\n'''\ntext_file = open(\"/tmp/RSNA/dataset-metadata.json\", 'w+')\nn = text_file.write(data)\ntext_file.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l \"/tmp/RSNA/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!kaggle datasets create -p \"/tmp/RSNA/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}