{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras import datasets, layers, models\nimport os\nimport PIL\nimport PIL.Image\nfrom pathlib import Path\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_file = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ntrain_file.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_file['image_id'].value_counts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_file.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_file['cancer'] = train_file['cancer'].astype('float32')\ntrain_file.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pathlib\ndataset_path = \"/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_cv2_256/train_images_processed_cv2_256\"\ndata_dir = pathlib.Path(dataset_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for f in data_dir.iterdir():\n#     for i in f.iterdir():\n#         print(i)\n#         break\n#     break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pos_or_neg(img_directory):\n    img_id = str(img_directory.stem)\n    diagnosis = train_file.loc[train_file['image_id']==int(img_id), 'cancer'].values[0]\n    if diagnosis == 0:\n        return \"negative\"\n    else:\n        return \"positive\"\n    \n!mkdir -p /kaggle/working/train_images_processed_cv2_dicomsdl_256/positive/\n!mkdir -p /kaggle/working/train_images_processed_cv2_dicomsdl_256/negative/","metadata":{"execution":{"iopub.status.busy":"2023-03-16T16:50:21.592016Z","iopub.execute_input":"2023-03-16T16:50:21.592542Z","iopub.status.idle":"2023-03-16T16:50:21.598489Z","shell.execute_reply.started":"2023-03-16T16:50:21.592505Z","shell.execute_reply":"2023-03-16T16:50:21.597504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\n#moving training images into one folder\n\npos_path = \"/kaggle/working/train_images_processed_cv2_dicomsdl_256/positive/\"\nneg_path = \"/kaggle/working/train_images_processed_cv2_dicomsdl_256/negative/\"\n\npos_path_obj = pathlib.Path(pos_path)\nneg_path_obj = pathlib.Path(neg_path)\n\ndone_dir = Path(\"/kaggle/input/mammography-comp-pngs-256/train_images_processed_cv2_dicomsdl_256/\")\ndone_ids = [x.stem for x in list(done_dir.glob('*/*.png'))]\n# print(len(done_ids))\n# print(done_ids[0])\n# print(type(done_ids[0]))\nimage_directories = []\nfor patient_dir in Path('/kaggle/input/rsna-breast-cancer-detection/train_images/').iterdir():\n    for pic_dir in patient_dir.iterdir():\n        if str(pic_dir.stem) not in done_ids:\n            image_directories.append(pic_dir)\n            \nprint(len(image_directories))\nprint(image_directories[0])\n\ndef move_file(img_dir):\n    if pos_or_neg(img_dir) == \"negative\":\n        shutil.copy(str(img_dir),neg_path + img_dir.stem + '.png')\n    else:\n        shutil.copy(str(img_dir),pos_path + img_dir.stem + '.png')\n        \nimport multiprocessing as mp\n\nwith mp.Pool(64) as p:\n    p.map(move_file, image_directories)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:00:05.648917Z","iopub.execute_input":"2023-03-16T17:00:05.649409Z","iopub.status.idle":"2023-03-16T17:00:16.969712Z","shell.execute_reply.started":"2023-03-16T17:00:05.649368Z","shell.execute_reply":"2023-03-16T17:00:16.968164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_count = len(list(new_dir.glob('*/*.png')))\nprint(image_count)\n# print(str([str(x) for x in new_path_obj.glob('*')][0]))\n# print(str([str(x) for x in new_path_obj.glob('*')][1]))\n# print(str([str(x) for x in new_path_obj.glob('*')][2]))\n# print(str([str(x) for x in new_path_obj.glob('*')][3]))\n# PIL.Image.open(str(list(new_path_obj.glob('*'))[0]))","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:14:25.062497Z","iopub.execute_input":"2023-02-26T22:14:25.06311Z","iopub.status.idle":"2023-02-26T22:14:26.732009Z","shell.execute_reply.started":"2023-02-26T22:14:25.063072Z","shell.execute_reply":"2023-02-26T22:14:26.730225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# patient_example = list(data_dir.glob('10006/*'))\n# PIL.Image.open(str(patient_example[0]))","metadata":{"execution":{"iopub.status.busy":"2023-02-24T22:23:00.062926Z","iopub.execute_input":"2023-02-24T22:23:00.063706Z","iopub.status.idle":"2023-02-24T22:23:00.076922Z","shell.execute_reply.started":"2023-02-24T22:23:00.063665Z","shell.execute_reply":"2023-02-24T22:23:00.075796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list_ds = tf.matching_files(str(data_dir/'*/*.png'))\n# for f in list_ds.take(5):\n#     print(f.numpy())","metadata":{"execution":{"iopub.status.busy":"2023-02-24T22:41:10.085935Z","iopub.execute_input":"2023-02-24T22:41:10.086374Z","iopub.status.idle":"2023-02-24T22:41:21.067278Z","shell.execute_reply.started":"2023-02-24T22:41:10.08634Z","shell.execute_reply":"2023-02-24T22:41:21.066003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# patient_example[0]","metadata":{"execution":{"iopub.status.busy":"2023-02-24T22:55:03.328771Z","iopub.execute_input":"2023-02-24T22:55:03.329472Z","iopub.status.idle":"2023-02-24T22:55:03.337076Z","shell.execute_reply.started":"2023-02-24T22:55:03.329434Z","shell.execute_reply":"2023-02-24T22:55:03.33586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels = []\n# for img_id in [str(x).split('/')[-1][:-4] for x in new_path_obj.glob('*')]:\n#     label = train_file.loc[train_file['image_id']==int(img_id), 'cancer'].values[0]\n#     train_labels.append(label)\n\n# print(len(train_labels))\n# print(train_labels[:5])","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:26:03.561606Z","iopub.execute_input":"2023-02-26T22:26:03.56302Z","iopub.status.idle":"2023-02-26T22:26:20.501562Z","shell.execute_reply.started":"2023-02-26T22:26:03.562968Z","shell.execute_reply":"2023-02-26T22:26:20.500351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import shutil\n# #moving training images into one folder\n\n# new_path = \"/kaggle/working/train_images_256_all/\"\n# new_path_obj = pathlib.Path(new_path)\n# new_path_obj.mkdir()\n# for folder in data_dir.iterdir():\n#     for file in folder.iterdir():\n#         shutil.copy(str(file),new_path + str(file).split('/')[-1])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels = list(train_file['cancer'])\n# print(len(train_labels))\n# print(train_labels[:5])","metadata":{"execution":{"iopub.status.busy":"2023-02-24T22:23:08.64925Z","iopub.execute_input":"2023-02-24T22:23:08.650533Z","iopub.status.idle":"2023-02-24T22:23:08.66267Z","shell.execute_reply.started":"2023-02-24T22:23:08.650479Z","shell.execute_reply":"2023-02-24T22:23:08.661444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\nimg_height = 256\nimg_width = 256","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:26:47.966351Z","iopub.execute_input":"2023-02-26T22:26:47.966731Z","iopub.status.idle":"2023-02-26T22:26:47.97191Z","shell.execute_reply.started":"2023-02-26T22:26:47.966696Z","shell.execute_reply":"2023-02-26T22:26:47.970623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_ds = tf.keras.utils.image_dataset_from_directory(\n#     new_path_obj,\n#     labels=train_labels,\n#     label_mode = 'binary',\n#     validation_split=0.2,\n#     subset='training',\n#     seed=123,\n#     image_size=(img_height,img_width),\n#     batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:29:13.790943Z","iopub.execute_input":"2023-02-26T22:29:13.791357Z","iopub.status.idle":"2023-02-26T22:29:14.559636Z","shell.execute_reply.started":"2023-02-26T22:29:13.791322Z","shell.execute_reply":"2023-02-26T22:29:14.558216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_ds = tf.keras.utils.image_dataset_from_directory(\n#     data_dir,\n#     labels=train_labels,\n#     validation_split=0.2,\n#     subset='validation',\n#     seed=123,\n#     image_size=(img_height,img_width),\n#     batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-02-24T22:24:23.668071Z","iopub.execute_input":"2023-02-24T22:24:23.668784Z","iopub.status.idle":"2023-02-24T22:24:35.167123Z","shell.execute_reply.started":"2023-02-24T22:24:23.668745Z","shell.execute_reply":"2023-02-24T22:24:35.16598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class_names = train_ds.class_names\n# class_names[:5]","metadata":{"execution":{"iopub.status.busy":"2023-02-24T22:24:35.169267Z","iopub.execute_input":"2023-02-24T22:24:35.16994Z","iopub.status.idle":"2023-02-24T22:24:35.178621Z","shell.execute_reply.started":"2023-02-24T22:24:35.169895Z","shell.execute_reply":"2023-02-24T22:24:35.177253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-02-24T22:24:41.387081Z","iopub.execute_input":"2023-02-24T22:24:41.387847Z","iopub.status.idle":"2023-02-24T22:24:41.411783Z","shell.execute_reply.started":"2023-02-24T22:24:41.387807Z","shell.execute_reply":"2023-02-24T22:24:41.40943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}