{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Fazendo uma nova DS\nPelo add data, consigo adicionar a competição rsna-intracranial-hemorrhage-detection ao notebook e acessar os dados sem precisar baixar!","metadata":{"id":"1v0OFdO-M6Oi"}},{"cell_type":"markdown","source":"## Imports e Instalações","metadata":{"id":"ehKS4k_-Gqzl"}},{"cell_type":"code","source":"!mkdir ~/.kaggle\n!cp ../input/kagglejson/kaggle.json ~/.kaggle\n!chmod 600 /root/.kaggle/kaggle.json","metadata":{"execution":{"iopub.status.busy":"2022-05-02T14:54:53.56707Z","iopub.execute_input":"2022-05-02T14:54:53.568539Z","iopub.status.idle":"2022-05-02T14:54:55.971245Z","shell.execute_reply.started":"2022-05-02T14:54:53.56837Z","shell.execute_reply":"2022-05-02T14:54:55.970136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport kaggle\nimport shutil","metadata":{"id":"SPyR0ssr6fEh","outputId":"2e75bbbc-9932-4198-a299-db85ddb007aa","execution":{"iopub.status.busy":"2022-05-02T14:54:59.315985Z","iopub.execute_input":"2022-05-02T14:54:59.316463Z","iopub.status.idle":"2022-05-02T14:54:59.388351Z","shell.execute_reply.started":"2022-05-02T14:54:59.316416Z","shell.execute_reply":"2022-05-02T14:54:59.387637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_path = \"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection\"\ntrain_images_dir = \"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/\"\n#!ls {input_path}","metadata":{"execution":{"iopub.status.busy":"2022-05-02T14:55:02.288863Z","iopub.execute_input":"2022-05-02T14:55:02.289509Z","iopub.status.idle":"2022-05-02T14:55:03.0949Z","shell.execute_reply.started":"2022-05-02T14:55:02.289454Z","shell.execute_reply":"2022-05-02T14:55:03.093949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Criando os diretorios","metadata":{}},{"cell_type":"code","source":"!mkdir small_rsna_ds\n!mkdir small_rsna_ds/yes\n!mkdir small_rsna_ds/no","metadata":{"id":"Y9jJHKOi8vVo","outputId":"0aaf965e-cf61-40ce-af98-53b9e71fd2a8","execution":{"iopub.status.busy":"2022-05-02T14:55:05.284796Z","iopub.execute_input":"2022-05-02T14:55:05.285875Z","iopub.status.idle":"2022-05-02T14:55:07.683884Z","shell.execute_reply.started":"2022-05-02T14:55:05.285815Z","shell.execute_reply":"2022-05-02T14:55:07.682637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Selecionando os arquivos\n*   https://www.kaggle.com/code/akensert/rsna-inceptionv3-keras-tf1-14-0?scriptVersionId=39570832 secção 4\n\n","metadata":{"id":"A7pL9jwn6Gi6"}},{"cell_type":"markdown","source":"### Definições","metadata":{"id":"fTvVuL0GHhTO"}},{"cell_type":"code","source":"def read_trainset(filename=input_path+\"/stage_2_train.csv\"):\n    df = pd.read_csv(filename)\n    df[\"Image\"] = df[\"ID\"].str.slice(stop=12)\n    df[\"Diagnosis\"] = df[\"ID\"].str.slice(start=13)\n    \n    duplicates_to_remove = [\n        56346, 56347, 56348, 56349,\n        56350, 56351, 1171830, 1171831,\n        1171832, 1171833, 1171834, 1171835,\n        3705312, 3705313, 3705314, 3705315,\n        3705316, 3705317, 3842478, 3842479,\n        3842480, 3842481, 3842482, 3842483\n    ]\n    \n    df = df.drop(index=duplicates_to_remove)\n    df = df.reset_index(drop=True)\n    \n    df = df.loc[:, [\"Label\", \"Diagnosis\", \"Image\"]]\n    df = df.set_index(['Image', 'Diagnosis']).unstack(level=-1)\n    \n    return df\n\ndf = read_trainset()","metadata":{"id":"B5-UAxRb6dVL","execution":{"iopub.status.busy":"2022-05-02T14:55:20.63328Z","iopub.execute_input":"2022-05-02T14:55:20.633867Z","iopub.status.idle":"2022-05-02T14:55:37.121664Z","shell.execute_reply.started":"2022-05-02T14:55:20.633807Z","shell.execute_reply":"2022-05-02T14:55:37.119683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"id":"omMBN_jQ8qJk","outputId":"ab3a36f8-18b2-4dbe-edd0-ea8a3cf9b7dc","execution":{"iopub.status.busy":"2022-05-02T14:55:42.398563Z","iopub.execute_input":"2022-05-02T14:55:42.399087Z","iopub.status.idle":"2022-05-02T14:55:42.427549Z","shell.execute_reply.started":"2022-05-02T14:55:42.399028Z","shell.execute_reply":"2022-05-02T14:55:42.426854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Selecionando aleatoriamente","metadata":{"id":"JBCO9BqCHj48"}},{"cell_type":"code","source":"df_shuffled = df.sample(frac=1)\ndf_shuffled","metadata":{"execution":{"iopub.status.busy":"2022-05-02T14:55:45.368805Z","iopub.execute_input":"2022-05-02T14:55:45.369109Z","iopub.status.idle":"2022-05-02T14:55:45.621921Z","shell.execute_reply.started":"2022-05-02T14:55:45.369077Z","shell.execute_reply":"2022-05-02T14:55:45.620823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_shorten = df_shuffled[:35000]\ndf_shorten","metadata":{"execution":{"iopub.status.busy":"2022-05-02T14:55:48.053806Z","iopub.execute_input":"2022-05-02T14:55:48.054349Z","iopub.status.idle":"2022-05-02T14:55:48.145081Z","shell.execute_reply.started":"2022-05-02T14:55:48.05431Z","shell.execute_reply":"2022-05-02T14:55:48.144041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Criando o novo Dataset","metadata":{}},{"cell_type":"code","source":"n = 0\nwhile n < 34999:\n    id = df_shorten.index[n]\n    hemorrhage = df.loc[[id]].iat[0,0]\n    \n    src = train_images_dir+id+'.dcm'\n    \n    if(hemorrhage == 1):\n        dst = './small_rsna_ds/yes/'\n    else:\n        dst = './small_rsna_ds/no/'\n    \n    shutil.copy(src, dst)\n    n = n+1","metadata":{"id":"Xp5Oy6Ih9nCZ","outputId":"a6ee5ea3-0a6a-4e8a-d242-26d140bb9336","execution":{"iopub.status.busy":"2022-05-02T15:01:36.438366Z","iopub.execute_input":"2022-05-02T15:01:36.438743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!zip -rm ds_35000.zip ./small_rsna_ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}