{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":1873742,"sourceType":"datasetVersion","datasetId":1115384},{"sourceId":8923884,"sourceType":"datasetVersion","datasetId":5335303}],"dockerImageVersionId":30775,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom pathlib import Path\n# df1 =pd.read_csv(\"/kaggle/input/cbis-ddsm-breast-cancer-image-dataset/csv/dicom_info.csv\")\n# df1.head()\n# base_df =pd.read_csv(\"/kaggle/input/cbis-ddsm-breast-cancer-image-dataset/csv/calc_case_description_train_set.csv\")\n# base_df.head()\n# dataset_dir = '/kaggle/input/cbis-ddsm-breast-cancer-image-dataset'\n# dicom_data = pd.read_csv('/kaggle/input/cbis-ddsm-breast-cancer-image-dataset/csv/dicom_info.csv')\n# image_dir = '/kaggle/input/cbis-ddsm-breast-cancer-image-dataset/jpeg'\n# dicom_data['image_path'] = dicom_data.image_path.apply(lambda x: x.replace('CBIS-DDSM', dataset_dir))\n# dicom_data.head()\n# dicom_data.iloc[0]\n# base_df.iloc[0]\n\n# def count_images(folder_path):\n#     image_extensions = ['.jpg']\n#     image_count = 0\n\n#     for root, dirs, files in os.walk(folder_path):\n#         for file in files:\n#             if Path(file).suffix.lower() in image_extensions:\n#                 image_count += 1\n\n#     return image_count\n\n# # Replace 'path/to/your/folder' with the actual path to your folder\n# folder_path = '/kaggle/input/cbis-ddsm-breast-cancer-image-dataset/jpeg'\n# total_images = count_images(folder_path)\n\n# print(f\"Total number of images: {total_images}\")\n# cropped_images = dicom_data[dicom_data.SeriesDescription == 'cropped images'].image_path\n# cropped_images.shape\n# dicom_data.head()\n# dicom_data.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-30T06:43:57.08952Z","iopub.execute_input":"2024-09-30T06:43:57.089928Z","iopub.status.idle":"2024-09-30T06:43:57.496313Z","shell.execute_reply.started":"2024-09-30T06:43:57.08986Z","shell.execute_reply":"2024-09-30T06:43:57.495373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## dataset -Vin-dataset","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/vin-dataset-for-research/finding_annotations.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-09-30T06:43:57.599613Z","iopub.execute_input":"2024-09-30T06:43:57.600173Z","iopub.status.idle":"2024-09-30T06:43:57.733743Z","shell.execute_reply.started":"2024-09-30T06:43:57.600128Z","shell.execute_reply":"2024-09-30T06:43:57.732722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()\ndf['image_path'] = df['study_id'] + '/' + df['image_id'] + '.jpg'\ndf.iloc[0]\nfrom tqdm import tqdm\nbase_dir = \"/kaggle/input/vin-dataset-for-research/Vin_mammo\"\ncount = 0\nfor i,row in tqdm(df.iterrows()):\n    if os.path.exists(f\"{base_dir}/{row['image_path']}\"):\n        count+=1\ndf.shape, count\npath = f\"{base_dir}/{row['image_path']}\"\nfrom PIL import Image\nim = Image.open(path)","metadata":{"execution":{"iopub.status.busy":"2024-09-30T06:43:57.886756Z","iopub.execute_input":"2024-09-30T06:43:57.887671Z","iopub.status.idle":"2024-09-30T06:44:54.307524Z","shell.execute_reply.started":"2024-09-30T06:43:57.887628Z","shell.execute_reply":"2024-09-30T06:44:54.306491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## merging with metadata csv","metadata":{"execution":{"iopub.status.busy":"2024-09-30T05:10:36.526075Z","iopub.status.idle":"2024-09-30T05:10:36.526526Z","shell.execute_reply.started":"2024-09-30T05:10:36.526305Z","shell.execute_reply":"2024-09-30T05:10:36.52633Z"}}},{"cell_type":"code","source":"df_updated = df\nmetadata = pd.read_csv(\"/kaggle/input/vin-dataset-for-research/metadata.csv\")\ndf[df['series_id']==\"b36517b9cbbcfd286a7ae04f643af97a\"]\nmetadata.shape, df.shape\nmetadata['SOP Instance UID'].equals(metadata['SOP Instance UID.1'])\nmerged_df = pd.merge(df, metadata, left_on=['series_id', 'image_id'], right_on=['Series Instance UID', 'SOP Instance UID'])\ncolumns = ['study_id','series_id','image_id', 'laterality', 'view_position',\n       'height', 'width', 'breast_birads', 'breast_density',\n       'image_path', \"Patient's Age\", 'Photometric Interpretation']\nfinal_df = merged_df[columns]","metadata":{"execution":{"iopub.status.busy":"2024-09-30T06:44:54.309594Z","iopub.execute_input":"2024-09-30T06:44:54.31014Z","iopub.status.idle":"2024-09-30T06:44:54.523688Z","shell.execute_reply.started":"2024-09-30T06:44:54.310085Z","shell.execute_reply":"2024-09-30T06:44:54.522852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df['series_id'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-09-30T06:44:54.524921Z","iopub.execute_input":"2024-09-30T06:44:54.525242Z","iopub.status.idle":"2024-09-30T06:44:54.538246Z","shell.execute_reply.started":"2024-09-30T06:44:54.525209Z","shell.execute_reply":"2024-09-30T06:44:54.537332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\n\ndef split_csv_by_group(pandas_csv, group_column, test_size=0.3, random_state=42):\n    # Read the CSV file\n    df = pandas_csv\n    \n    # Get unique groups\n    groups = df[group_column].unique()\n    \n    # Split the groups into train and test\n    train_groups, test_groups = train_test_split(groups, test_size=test_size, random_state=random_state)\n    \n    # Create train and test dataframes\n    train_df = df[df[group_column].isin(train_groups)]\n    test_df = df[df[group_column].isin(test_groups)]\n    \n    return train_df, test_df\n\n# Example usage\npandas_csv = final_df\ngroup_column = 'series_id'\n\ntrain_data, test_data = split_csv_by_group(pandas_csv, group_column)\n\n# Save the split datasets\ntrain_data.to_csv('train_data.csv', index=False)\ntest_data.drop(['breast_birads', 'breast_density'], axis=1)\ntest_data.to_csv('test_data.csv', index=False)\n\nprint(f\"Train data shape: {train_data.shape}\")\nprint(f\"Test data shape: {test_data.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-30T06:44:54.540299Z","iopub.execute_input":"2024-09-30T06:44:54.540602Z","iopub.status.idle":"2024-09-30T06:44:55.298072Z","shell.execute_reply.started":"2024-09-30T06:44:54.54057Z","shell.execute_reply":"2024-09-30T06:44:55.297032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-30T06:45:26.643303Z","iopub.execute_input":"2024-09-30T06:45:26.644083Z","iopub.status.idle":"2024-09-30T06:45:26.648211Z","shell.execute_reply.started":"2024-09-30T06:45:26.644042Z","shell.execute_reply":"2024-09-30T06:45:26.647166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir comp_dataset\n!cp -r /kaggle/input/vin-dataset-for-research/Vin_mammo/* comp_dataset/","metadata":{"execution":{"iopub.status.busy":"2024-09-30T06:45:29.763602Z","iopub.execute_input":"2024-09-30T06:45:29.764147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}